From 86573d2778cb064fb7a930dfe08e84465084523f Mon Sep 17 00:00:00 2001 From: Anush Date: Sun, 27 Oct 2024 10:05:22 +0530 Subject: [PATCH 01/65] feat: Qdrant vectorstore support (#4689) * feat: Qdrant vectorstore support Signed-off-by: Anush008 * chore: make build-ui again Signed-off-by: Anush008 --------- Signed-off-by: Anush008 --- Makefile | 8 + docs/reference/alpha-vector-database.md | 22 +- docs/reference/online-stores/qdrant.md | 81 +++++ .../feast.infra.online_stores.contrib.rst | 16 + .../trino_offline_store/connectors/upload.py | 1 + .../infra/online_stores/contrib/qdrant.py | 311 ++++++++++++++++++ .../contrib/qdrant_repo_configuration.py | 12 + .../feast/infra/online_stores/vector_store.py | 5 +- sdk/python/feast/repo_config.py | 1 + .../requirements/py3.10-ci-requirements.txt | 29 +- .../requirements/py3.11-ci-requirements.txt | 29 +- .../requirements/py3.9-ci-requirements.txt | 29 +- sdk/python/tests/conftest.py | 4 +- .../universal/online_store/qdrant.py | 28 ++ .../online_store/test_universal_online.py | 2 +- setup.py | 4 + 16 files changed, 558 insertions(+), 24 deletions(-) create mode 100644 docs/reference/online-stores/qdrant.md create mode 100644 sdk/python/feast/infra/online_stores/contrib/qdrant.py create mode 100644 sdk/python/feast/infra/online_stores/contrib/qdrant_repo_configuration.py create mode 100644 sdk/python/tests/integration/feature_repos/universal/online_store/qdrant.py diff --git a/Makefile b/Makefile index 22d4d25f4e3..30ac86e8919 100644 --- a/Makefile +++ b/Makefile @@ -351,6 +351,14 @@ test-python-universal-singlestore-online: not test_snowflake" \ sdk/python/tests + test-python-universal-qdrant-online: + PYTHONPATH='.' \ + FULL_REPO_CONFIGS_MODULE=sdk.python.feast.infra.online_stores.contrib.qdrant_repo_configuration \ + PYTEST_PLUGINS=sdk.python.tests.integration.feature_repos.universal.online_store.qdrant \ + python -m pytest -n 8 --integration \ + -k "test_retrieve_online_documents" \ + sdk/python/tests/integration/online_store/test_universal_online.py + test-python-universal: python -m pytest -n 8 --integration sdk/python/tests diff --git a/docs/reference/alpha-vector-database.md b/docs/reference/alpha-vector-database.md index 06909bd5654..fca31ee4780 100644 --- a/docs/reference/alpha-vector-database.md +++ b/docs/reference/alpha-vector-database.md @@ -14,8 +14,9 @@ Below are supported vector databases and implemented features: | Milvus | [ ] | [ ] | | Faiss | [ ] | [ ] | | SQLite | [x] | [ ] | +| Qdrant | [x] | [x] | -Note: SQLite is in limited access and only working on Python 3.10. It will be updated as [sqlite_vec](https://github.com/asg017/sqlite-vec/) progresses. +Note: SQLite is in limited access and only working on Python 3.10. It will be updated as [sqlite_vec](https://github.com/asg017/sqlite-vec/) progresses. ## Example @@ -113,9 +114,11 @@ print_online_features(features) ``` ### Configuration -We offer two Online Store options for Vector Databases. PGVector and SQLite. + +We offer [PGVector](https://github.com/pgvector/pgvector), [SQLite](https://github.com/asg017/sqlite-vec), [Elasticsearch](https://www.elastic.co) and [Qdrant](https://qdrant.tech/) as Online Store options for Vector Databases. #### Installation with SQLite + If you are using `pyenv` to manage your Python versions, you can install the SQLite extension with the following command: ```bash PYTHON_CONFIGURE_OPTS="--enable-loadable-sqlite-extensions" \ @@ -124,6 +127,19 @@ PYTHON_CONFIGURE_OPTS="--enable-loadable-sqlite-extensions" \ pyenv install 3.10.14 ``` And you can the Feast install package via: + ```bash pip install feast[sqlite_vec] -``` \ No newline at end of file +``` + +#### Installation with Elasticsearch + +```bash +pip install feast[elasticsearch] +``` + +#### Installation with Qdrant + +```bash +pip install feast[qdrant] +``` diff --git a/docs/reference/online-stores/qdrant.md b/docs/reference/online-stores/qdrant.md new file mode 100644 index 00000000000..66f4a59d24e --- /dev/null +++ b/docs/reference/online-stores/qdrant.md @@ -0,0 +1,81 @@ +# Qdrant online store (contrib) + +## Description + +[Qdrant](http://qdrant.tech) is a vector similarity search engine. It provides a production-ready service with a convenient API to store, search, and manage vectors with additional payload and extended filtering support. It makes it useful for all sorts of neural network or semantic-based matching, faceted search, and other applications. + +## Getting started + +In order to use this online store, you'll need to run `pip install 'feast[qdrant]'`. + +## Example + +{% code title="feature_store.yaml" %} + +```yaml +project: my_feature_repo +registry: data/registry.db +provider: local +online_store: + type: qdrant + host: localhost + port: 6333 + vector_len: 384 + write_batch_size: 100 +``` + +{% endcode %} + +The full set of configuration options is available in [QdrantOnlineStoreConfig](https://rtd.feast.dev/en/master/#feast.infra.online_stores.contrib.qdrant.QdrantOnlineStoreConfig). + +## Functionality Matrix + +| | Qdrant | +| :-------------------------------------------------------- | :------- | +| write feature values to the online store | yes | +| read feature values from the online store | yes | +| update infrastructure (e.g. tables) in the online store | yes | +| teardown infrastructure (e.g. tables) in the online store | yes | +| generate a plan of infrastructure changes | no | +| support for on-demand transforms | yes | +| readable by Python SDK | yes | +| readable by Java | no | +| readable by Go | no | +| support for entityless feature views | yes | +| support for concurrent writing to the same key | no | +| support for ttl (time to live) at retrieval | no | +| support for deleting expired data | no | +| collocated by feature view | yes | +| collocated by feature service | no | +| collocated by entity key | no | + +To compare this set of functionality against other online stores, please see the full [functionality matrix](overview.md#functionality-matrix). + +## Retrieving online document vectors + +The Qdrant online store supports retrieving document vectors for a given list of entity keys. The document vectors are returned as a dictionary where the key is the entity key and the value is the document vector. The document vector is a dense vector of floats. + +{% code title="python" %} + +```python +from feast import FeatureStore + +feature_store = FeatureStore(repo_path="feature_store.yaml") + +query_vector = [1.0, 2.0, 3.0, 4.0, 5.0] +top_k = 5 + +# Retrieve the top k closest features to the query vector +# Since Qdrant supports multiple vectors per entry, +# the vector to use can be specified in the repo config. +# Reference: https://qdrant.tech/documentation/concepts/vectors/#named-vectors +feature_values = feature_store.retrieve_online_documents( + feature="my_feature", + query=query_vector, + top_k=top_k +) +``` + +{% endcode %} + +These APIs are subject to change in future versions of Feast to improve performance and usability. diff --git a/sdk/python/docs/source/feast.infra.online_stores.contrib.rst b/sdk/python/docs/source/feast.infra.online_stores.contrib.rst index 8c9dd7e5491..2403b5b8d48 100644 --- a/sdk/python/docs/source/feast.infra.online_stores.contrib.rst +++ b/sdk/python/docs/source/feast.infra.online_stores.contrib.rst @@ -40,6 +40,22 @@ feast.infra.online\_stores.contrib.elasticsearch\_repo\_configuration module :undoc-members: :show-inheritance: +feast.infra.online\_stores.contrib.qdrant module +------------------------------------------------------- + +.. automodule:: feast.infra.online_stores.contrib.qdrant + :members: + :undoc-members: + :show-inheritance: + +feast.infra.online\_stores.contrib.qdrant\_repo\_configuration module +---------------------------------------------------------------------------- + +.. automodule:: feast.infra.online_stores.contrib.qdrant_repo_configuration + :members: + :undoc-members: + :show-inheritance: + feast.infra.online\_stores.contrib.hazelcast\_repo\_configuration module ------------------------------------------------------------------------ diff --git a/sdk/python/feast/infra/offline_stores/contrib/trino_offline_store/connectors/upload.py b/sdk/python/feast/infra/offline_stores/contrib/trino_offline_store/connectors/upload.py index 1b551991932..1cdbf7f01e6 100644 --- a/sdk/python/feast/infra/offline_stores/contrib/trino_offline_store/connectors/upload.py +++ b/sdk/python/feast/infra/offline_stores/contrib/trino_offline_store/connectors/upload.py @@ -45,6 +45,7 @@ "thrift", "tpcds", "tpch", + "qdrant", } CONNECTORS_WITHOUT_WITH_STATEMENTS: Set[str] = { "bigquery", diff --git a/sdk/python/feast/infra/online_stores/contrib/qdrant.py b/sdk/python/feast/infra/online_stores/contrib/qdrant.py new file mode 100644 index 00000000000..074c52ba5e8 --- /dev/null +++ b/sdk/python/feast/infra/online_stores/contrib/qdrant.py @@ -0,0 +1,311 @@ +from __future__ import absolute_import + +import base64 +import json +import logging +import uuid +from datetime import datetime +from typing import Any, Callable, Dict, List, Optional, Sequence, Tuple + +from qdrant_client import QdrantClient, models + +from feast import Entity, FeatureView, RepoConfig +from feast.infra.key_encoding_utils import ( + get_list_val_str, + serialize_entity_key, +) +from feast.infra.online_stores.online_store import OnlineStore +from feast.infra.online_stores.vector_store import VectorStoreConfig +from feast.protos.feast.types.EntityKey_pb2 import EntityKey as EntityKeyProto +from feast.protos.feast.types.Value_pb2 import Value as ValueProto +from feast.repo_config import FeastConfigBaseModel +from feast.utils import _build_retrieve_online_document_record, to_naive_utc + +SCROLL_SIZE = 1000 + +DISTANCE_MAPPING = { + "cosine": models.Distance.COSINE, + "l2": models.Distance.EUCLID, + "dot": models.Distance.DOT, + "l1": models.Distance.MANHATTAN, +} + + +class QdrantOnlineStoreConfig(FeastConfigBaseModel, VectorStoreConfig): + """ + Configuration for the Qdrant online store. + """ + + type: str = "qdrant" + + location: Optional[str] = None + url: Optional[str] = None + port: Optional[int] = 6333 + grpc_port: int = 6334 + prefer_grpc: bool = False + https: Optional[bool] = None + api_key: Optional[str] = None + prefix: Optional[str] = None + timeout: Optional[int] = None + host: Optional[str] = None + path: Optional[str] = None + + # The name of the vector to use. + # Defaults to the single, unnamed vector + # Reference: https://qdrant.tech/documentation/concepts/vectors/#named-vectors + vector_name: str = "" + # The number of point to write in a single request + write_batch_size: Optional[int] = 64 + # Await for the upload results to be applied on the server side. + # If `true`, each request will explicitly wait for the confirmation of completion. Might be slower. + # If `false`, each reequest will return immediately after receiving an acknowledgement. + upload_wait: bool = True + + +class QdrantOnlineStore(OnlineStore): + _client: Optional[QdrantClient] = None + + def _get_client(self, config: RepoConfig) -> QdrantClient: + if self._client: + return self._client + online_store_config = config.online_store + assert isinstance( + online_store_config, QdrantOnlineStoreConfig + ), "Invalid type for online store config" + + assert online_store_config.similarity and ( + online_store_config.similarity.lower() in DISTANCE_MAPPING + ), f"Unsupported distance metric {online_store_config.similarity}" + + self._client = QdrantClient( + location=online_store_config.location, + url=online_store_config.url, + port=online_store_config.port, + grpc_port=online_store_config.grpc_port, + prefer_grpc=online_store_config.prefer_grpc, + https=online_store_config.https, + api_key=online_store_config.api_key, + prefix=online_store_config.prefix, + timeout=online_store_config.timeout, + host=online_store_config.host, + path=online_store_config.path, + ) + return self._client + + def online_write_batch( + self, + config: RepoConfig, + table: FeatureView, + data: List[ + Tuple[EntityKeyProto, Dict[str, ValueProto], datetime, Optional[datetime]] + ], + progress: Optional[Callable[[int], Any]], + ) -> None: + points = [] + for entity_key, values, timestamp, created_ts in data: + entity_key_bin = serialize_entity_key( + entity_key, + entity_key_serialization_version=config.entity_key_serialization_version, + ) + + timestamp = to_naive_utc(timestamp) + if created_ts is not None: + created_ts = to_naive_utc(created_ts) + for feature_name, value in values.items(): + encoded_value = base64.b64encode(value.SerializeToString()).decode( + "utf-8" + ) + vector_val = json.loads(get_list_val_str(value)) + points.append( + models.PointStruct( + id=uuid.uuid4().hex, + payload={ + "entity_key": entity_key_bin, + "feature_name": feature_name, + "feature_value": encoded_value, + "timestamp": timestamp, + "created_ts": created_ts, + }, + vector={config.online_store.vector_name: vector_val}, + ) + ) + + self._get_client(config).upload_points( + collection_name=table.name, + batch_size=config.online_store.write_batch_size, + points=points, + wait=True, + ) + + def online_read( + self, + config: RepoConfig, + table: FeatureView, + entity_keys: List[EntityKeyProto], + requested_features: Optional[List[str]] = None, + ) -> List[Tuple[Optional[datetime], Optional[Dict[str, ValueProto]]]]: + conditions: List[models.Condition] = [] + if entity_keys: + conditions.append( + models.FieldCondition( + key="entity_key", + match=models.MatchAny(any=entity_keys), # type: ignore + ) + ) + + if requested_features: + conditions.append( + models.FieldCondition( + key="feature_name", match=models.MatchAny(any=requested_features) + ) + ) + points = [] + next_offset = None + stop_scrolling = False + while not stop_scrolling: + records, next_offset = self._get_client(config).scroll( + collection_name=config.online_store.collection_name, + limit=SCROLL_SIZE, + offset=next_offset, + with_payload=True, + scroll_filter=models.Filter(must=conditions), + ) + stop_scrolling = next_offset is None + + points.extend(records) + + results = [] + for point in points: + assert isinstance(point.payload, Dict), "Invalid value of payload" + results.append( + ( + point.payload["timestamp"], + {point.payload["feature_name"]: point.payload["feature_value"]}, + ) + ) + + return results # type: ignore + + def create_collection(self, config: RepoConfig, table: FeatureView): + """ + Create a collection in Qdrant for the given table. + Args: + config: Feast repo configuration object. + table: FeatureView table for which the index needs to be created. + """ + + client: QdrantClient = self._get_client(config) + + client.create_collection( + collection_name=table.name, + vectors_config={ + config.online_store.vector_name: models.VectorParams( + size=config.online_store.vector_len, + distance=DISTANCE_MAPPING[config.online_store.similarity.lower()], + ) + }, + ) + client.create_payload_index( + collection_name=table.name, + field_name="entity_key", + field_schema=models.PayloadSchemaType.KEYWORD, + ) + client.create_payload_index( + collection_name=table.name, + field_name="feature_name", + field_schema=models.PayloadSchemaType.KEYWORD, + ) + + def update( + self, + config: RepoConfig, + tables_to_delete: Sequence[FeatureView], + tables_to_keep: Sequence[FeatureView], + entities_to_delete: Sequence[Entity], + entities_to_keep: Sequence[Entity], + partial: bool, + ): + for table in tables_to_delete: + self._get_client(config).delete_collection(collection_name=table.name) + for table in tables_to_keep: + self.create_collection(config, table) + + def teardown( + self, + config: RepoConfig, + tables: Sequence[FeatureView], + entities: Sequence[Entity], + ): + project = config.project + try: + for table in tables: + self._get_client(config).delete_collection(collection_name=table.name) + except Exception as e: + logging.exception(f"Error deleting collection in project {project}: {e}") + raise + + def retrieve_online_documents( + self, + config: RepoConfig, + table: FeatureView, + requested_feature: str, + embedding: List[float], + top_k: int, + distance_metric: Optional[str] = "cosine", + ) -> List[ + Tuple[ + Optional[datetime], + Optional[EntityKeyProto], + Optional[ValueProto], + Optional[ValueProto], + Optional[ValueProto], + ] + ]: + result: List[ + Tuple[ + Optional[datetime], + Optional[EntityKeyProto], + Optional[ValueProto], + Optional[ValueProto], + Optional[ValueProto], + ] + ] = [] + + if distance_metric and distance_metric.lower() not in DISTANCE_MAPPING: + raise ValueError(f"Unsupported distance metric: {distance_metric}") + points = ( + self._get_client(config) + .query_points( + collection_name=table.name, + query=embedding, + limit=top_k, + with_payload=True, + with_vectors=True, + using=config.online_store.vector_name or None, + ) + .points + ) + for point in points: + payload = point.payload or {} + entity_key = str(payload.get("entity_key")) + feature_value = str(payload.get("feature_value")) + timestamp_str = str(payload.get("timestamp")) + timestamp = datetime.strptime(timestamp_str, "%Y-%m-%dT%H:%M:%S.%f") + distance = point.score + vector_value = str( + point.vector[config.online_store.vector_name] + if isinstance(point.vector, Dict) + else point.vector + ) + + result.append( + _build_retrieve_online_document_record( + entity_key, + base64.b64decode(feature_value), + vector_value, + distance, + timestamp, + config.entity_key_serialization_version, + ) + ) + return result diff --git a/sdk/python/feast/infra/online_stores/contrib/qdrant_repo_configuration.py b/sdk/python/feast/infra/online_stores/contrib/qdrant_repo_configuration.py new file mode 100644 index 00000000000..eee77bb8775 --- /dev/null +++ b/sdk/python/feast/infra/online_stores/contrib/qdrant_repo_configuration.py @@ -0,0 +1,12 @@ +from tests.integration.feature_repos.integration_test_repo_config import ( + IntegrationTestRepoConfig, +) +from tests.integration.feature_repos.universal.online_store.qdrant import ( + QdrantOnlineStoreCreator, +) + +FULL_REPO_CONFIGS = [ + IntegrationTestRepoConfig( + online_store="qdrant", online_store_creator=QdrantOnlineStoreCreator + ), +] diff --git a/sdk/python/feast/infra/online_stores/vector_store.py b/sdk/python/feast/infra/online_stores/vector_store.py index 051f9bcaedd..f071cd4347d 100644 --- a/sdk/python/feast/infra/online_stores/vector_store.py +++ b/sdk/python/feast/infra/online_stores/vector_store.py @@ -11,6 +11,9 @@ class VectorStoreConfig: # The vector similarity metric to use in KNN search # It is helpful for vector database that does not support config at retrieval runtime - # E.g. Elasticsearch dense_vector field at + # E.g. + # Elasticsearch: # https://www.elastic.co/guide/en/elasticsearch/reference/current/dense-vector.html + # Qdrant: + # https://qdrant.tech/documentation/concepts/search/#metrics similarity: Optional[str] = "cosine" diff --git a/sdk/python/feast/repo_config.py b/sdk/python/feast/repo_config.py index 0a5b484e8c7..b2b9374aa97 100644 --- a/sdk/python/feast/repo_config.py +++ b/sdk/python/feast/repo_config.py @@ -67,6 +67,7 @@ "elasticsearch": "feast.infra.online_stores.contrib.elasticsearch.ElasticSearchOnlineStore", "remote": "feast.infra.online_stores.remote.RemoteOnlineStore", "singlestore": "feast.infra.online_stores.contrib.singlestore_online_store.singlestore.SingleStoreOnlineStore", + "qdrant": "feast.infra.online_stores.contrib.qdrant.QdrantOnlineStore", } OFFLINE_STORE_CLASS_FOR_TYPE = { diff --git a/sdk/python/requirements/py3.10-ci-requirements.txt b/sdk/python/requirements/py3.10-ci-requirements.txt index bc29f696718..8c940ba84e2 100644 --- a/sdk/python/requirements/py3.10-ci-requirements.txt +++ b/sdk/python/requirements/py3.10-ci-requirements.txt @@ -273,6 +273,7 @@ grpcio==1.67.0 # grpcio-status # grpcio-testing # grpcio-tools + # qdrant-client grpcio-health-checking==1.62.3 # via feast (setup.py) grpcio-reflection==1.62.3 @@ -282,7 +283,9 @@ grpcio-status==1.62.3 grpcio-testing==1.62.3 # via feast (setup.py) grpcio-tools==1.62.3 - # via feast (setup.py) + # via + # feast (setup.py) + # qdrant-client gunicorn==23.0.0 # via # feast (setup.py) @@ -291,21 +294,28 @@ h11==0.14.0 # via # httpcore # uvicorn +h2==4.1.0 + # via httpx happybase==1.2.0 # via feast (setup.py) hazelcast-python-client==5.5.0 # via feast (setup.py) hiredis==2.4.0 # via feast (setup.py) +hpack==4.0.0 + # via h2 httpcore==1.0.6 # via httpx httptools==0.6.4 # via uvicorn -httpx==0.27.2 +httpx[http2]==0.27.2 # via # feast (setup.py) # jupyterlab # python-keycloak + # qdrant-client +hyperframe==6.0.1 + # via h2 ibis-framework[duckdb]==9.5.0 # via # feast (setup.py) @@ -499,6 +509,7 @@ numpy==1.26.4 # ibis-framework # pandas # pyarrow + # qdrant-client # scipy oauthlib==3.2.2 # via requests-oauthlib @@ -564,7 +575,9 @@ pluggy==1.5.0 ply==3.11 # via thriftpy2 portalocker==2.10.1 - # via msal-extensions + # via + # msal-extensions + # qdrant-client pre-commit==3.3.1 # via feast (setup.py) prometheus-client==0.21.0 @@ -646,6 +659,7 @@ pydantic==2.9.2 # feast (setup.py) # fastapi # great-expectations + # qdrant-client pydantic-core==2.23.4 # via pydantic pygments==2.18.0 @@ -747,6 +761,8 @@ pyzmq==26.2.0 # ipykernel # jupyter-client # jupyter-server +qdrant-client==1.12.0 + # via feast (setup.py) redis==4.6.0 # via feast (setup.py) referencing==0.35.1 @@ -840,7 +856,7 @@ sniffio==1.3.1 # httpx snowballstemmer==2.2.0 # via sphinx -snowflake-connector-python[pandas]==3.12.2 +snowflake-connector-python[pandas]==3.12.3 # via feast (setup.py) sortedcontainers==2.4.0 # via snowflake-connector-python @@ -864,7 +880,7 @@ sqlalchemy[mypy]==2.0.36 # via feast (setup.py) sqlglot==25.20.2 # via ibis-framework -sqlite-vec==0.1.3 +sqlite-vec==0.1.1 # via feast (setup.py) sqlparams==6.1.0 # via singlestoredb @@ -1010,6 +1026,7 @@ urllib3==2.2.3 # great-expectations # kubernetes # minio + # qdrant-client # requests # responses # testcontainers @@ -1041,7 +1058,7 @@ websocket-client==1.8.0 # kubernetes websockets==13.1 # via uvicorn -werkzeug==3.0.4 +werkzeug==3.0.5 # via moto wheel==0.44.0 # via diff --git a/sdk/python/requirements/py3.11-ci-requirements.txt b/sdk/python/requirements/py3.11-ci-requirements.txt index a75b57f48eb..4d5d8a71885 100644 --- a/sdk/python/requirements/py3.11-ci-requirements.txt +++ b/sdk/python/requirements/py3.11-ci-requirements.txt @@ -266,6 +266,7 @@ grpcio==1.67.0 # grpcio-status # grpcio-testing # grpcio-tools + # qdrant-client grpcio-health-checking==1.62.3 # via feast (setup.py) grpcio-reflection==1.62.3 @@ -275,7 +276,9 @@ grpcio-status==1.62.3 grpcio-testing==1.62.3 # via feast (setup.py) grpcio-tools==1.62.3 - # via feast (setup.py) + # via + # feast (setup.py) + # qdrant-client gunicorn==23.0.0 # via # feast (setup.py) @@ -284,21 +287,28 @@ h11==0.14.0 # via # httpcore # uvicorn +h2==4.1.0 + # via httpx happybase==1.2.0 # via feast (setup.py) hazelcast-python-client==5.5.0 # via feast (setup.py) hiredis==2.4.0 # via feast (setup.py) +hpack==4.0.0 + # via h2 httpcore==1.0.6 # via httpx httptools==0.6.4 # via uvicorn -httpx==0.27.2 +httpx[http2]==0.27.2 # via # feast (setup.py) # jupyterlab # python-keycloak + # qdrant-client +hyperframe==6.0.1 + # via h2 ibis-framework[duckdb]==9.5.0 # via # feast (setup.py) @@ -490,6 +500,7 @@ numpy==1.26.4 # ibis-framework # pandas # pyarrow + # qdrant-client # scipy oauthlib==3.2.2 # via requests-oauthlib @@ -555,7 +566,9 @@ pluggy==1.5.0 ply==3.11 # via thriftpy2 portalocker==2.10.1 - # via msal-extensions + # via + # msal-extensions + # qdrant-client pre-commit==3.3.1 # via feast (setup.py) prometheus-client==0.21.0 @@ -637,6 +650,7 @@ pydantic==2.9.2 # feast (setup.py) # fastapi # great-expectations + # qdrant-client pydantic-core==2.23.4 # via pydantic pygments==2.18.0 @@ -738,6 +752,8 @@ pyzmq==26.2.0 # ipykernel # jupyter-client # jupyter-server +qdrant-client==1.12.0 + # via feast (setup.py) redis==4.6.0 # via feast (setup.py) referencing==0.35.1 @@ -831,7 +847,7 @@ sniffio==1.3.1 # httpx snowballstemmer==2.2.0 # via sphinx -snowflake-connector-python[pandas]==3.12.2 +snowflake-connector-python[pandas]==3.12.3 # via feast (setup.py) sortedcontainers==2.4.0 # via snowflake-connector-python @@ -855,7 +871,7 @@ sqlalchemy[mypy]==2.0.36 # via feast (setup.py) sqlglot==25.20.2 # via ibis-framework -sqlite-vec==0.1.3 +sqlite-vec==0.1.1 # via feast (setup.py) sqlparams==6.1.0 # via singlestoredb @@ -986,6 +1002,7 @@ urllib3==2.2.3 # great-expectations # kubernetes # minio + # qdrant-client # requests # responses # testcontainers @@ -1017,7 +1034,7 @@ websocket-client==1.8.0 # kubernetes websockets==13.1 # via uvicorn -werkzeug==3.0.4 +werkzeug==3.0.5 # via moto wheel==0.44.0 # via diff --git a/sdk/python/requirements/py3.9-ci-requirements.txt b/sdk/python/requirements/py3.9-ci-requirements.txt index 5b8086099c2..2ba384e205e 100644 --- a/sdk/python/requirements/py3.9-ci-requirements.txt +++ b/sdk/python/requirements/py3.9-ci-requirements.txt @@ -275,6 +275,7 @@ grpcio==1.67.0 # grpcio-status # grpcio-testing # grpcio-tools + # qdrant-client grpcio-health-checking==1.62.3 # via feast (setup.py) grpcio-reflection==1.62.3 @@ -284,7 +285,9 @@ grpcio-status==1.62.3 grpcio-testing==1.62.3 # via feast (setup.py) grpcio-tools==1.62.3 - # via feast (setup.py) + # via + # feast (setup.py) + # qdrant-client gunicorn==23.0.0 # via # feast (setup.py) @@ -293,21 +296,28 @@ h11==0.14.0 # via # httpcore # uvicorn +h2==4.1.0 + # via httpx happybase==1.2.0 # via feast (setup.py) hazelcast-python-client==5.5.0 # via feast (setup.py) hiredis==2.4.0 # via feast (setup.py) +hpack==4.0.0 + # via h2 httpcore==1.0.6 # via httpx httptools==0.6.4 # via uvicorn -httpx==0.27.2 +httpx[http2]==0.27.2 # via # feast (setup.py) # jupyterlab # python-keycloak + # qdrant-client +hyperframe==6.0.1 + # via h2 ibis-framework[duckdb]==9.0.0 # via # feast (setup.py) @@ -508,6 +518,7 @@ numpy==1.26.4 # ibis-framework # pandas # pyarrow + # qdrant-client # scipy oauthlib==3.2.2 # via requests-oauthlib @@ -572,7 +583,9 @@ pluggy==1.5.0 ply==3.11 # via thriftpy2 portalocker==2.10.1 - # via msal-extensions + # via + # msal-extensions + # qdrant-client pre-commit==3.3.1 # via feast (setup.py) prometheus-client==0.21.0 @@ -654,6 +667,7 @@ pydantic==2.9.2 # feast (setup.py) # fastapi # great-expectations + # qdrant-client pydantic-core==2.23.4 # via pydantic pygments==2.18.0 @@ -755,6 +769,8 @@ pyzmq==26.2.0 # ipykernel # jupyter-client # jupyter-server +qdrant-client==1.12.0 + # via feast (setup.py) redis==4.6.0 # via feast (setup.py) referencing==0.35.1 @@ -848,7 +864,7 @@ sniffio==1.3.1 # httpx snowballstemmer==2.2.0 # via sphinx -snowflake-connector-python[pandas]==3.12.2 +snowflake-connector-python[pandas]==3.12.3 # via feast (setup.py) sortedcontainers==2.4.0 # via snowflake-connector-python @@ -872,7 +888,7 @@ sqlalchemy[mypy]==2.0.36 # via feast (setup.py) sqlglot==23.12.2 # via ibis-framework -sqlite-vec==0.1.3 +sqlite-vec==0.1.1 # via feast (setup.py) sqlparams==6.1.0 # via singlestoredb @@ -1020,6 +1036,7 @@ urllib3==1.26.20 # great-expectations # kubernetes # minio + # qdrant-client # requests # responses # snowflake-connector-python @@ -1052,7 +1069,7 @@ websocket-client==1.8.0 # kubernetes websockets==13.1 # via uvicorn -werkzeug==3.0.4 +werkzeug==3.0.5 # via moto wheel==0.44.0 # via diff --git a/sdk/python/tests/conftest.py b/sdk/python/tests/conftest.py index 08b8757b955..3b706d3c274 100644 --- a/sdk/python/tests/conftest.py +++ b/sdk/python/tests/conftest.py @@ -183,7 +183,9 @@ def start_test_local_server(repo_path: str, port: int): @pytest.fixture def environment(request, worker_id): e = construct_test_environment( - request.param, worker_id=worker_id, fixture_request=request + request.param, + worker_id=worker_id, + fixture_request=request, ) e.setup() diff --git a/sdk/python/tests/integration/feature_repos/universal/online_store/qdrant.py b/sdk/python/tests/integration/feature_repos/universal/online_store/qdrant.py new file mode 100644 index 00000000000..f65725f41d5 --- /dev/null +++ b/sdk/python/tests/integration/feature_repos/universal/online_store/qdrant.py @@ -0,0 +1,28 @@ +from typing import Any, Dict + +from testcontainers.qdrant import QdrantContainer + +from tests.integration.feature_repos.universal.online_store_creator import ( + OnlineStoreCreator, +) + + +class QdrantOnlineStoreCreator(OnlineStoreCreator): + def __init__(self, project_name: str, **kwargs): + super().__init__(project_name) + self.container = QdrantContainer( + "qdrant/qdrant", + ) + + def create_online_store(self) -> Dict[str, Any]: + self.container.start() + return { + "host": self.container.get_container_host_ip(), + "type": "qdrant", + "port": self.container.exposed_rest_port, + "vector_len": 2, + "similarity": "cosine", + } + + def teardown(self): + self.container.stop() diff --git a/sdk/python/tests/integration/online_store/test_universal_online.py b/sdk/python/tests/integration/online_store/test_universal_online.py index a5493fbdb13..4074dcb194e 100644 --- a/sdk/python/tests/integration/online_store/test_universal_online.py +++ b/sdk/python/tests/integration/online_store/test_universal_online.py @@ -857,7 +857,7 @@ def assert_feature_service_entity_mapping_correctness( @pytest.mark.integration -@pytest.mark.universal_online_stores(only=["pgvector", "elasticsearch"]) +@pytest.mark.universal_online_stores(only=["pgvector", "elasticsearch", "qdrant"]) def test_retrieve_online_documents(vectordb_environment, fake_document_data): fs = vectordb_environment.feature_store df, data_source = fake_document_data diff --git a/setup.py b/setup.py index 96a6f311e57..b335d39c2b3 100644 --- a/setup.py +++ b/setup.py @@ -146,6 +146,8 @@ FAISS_REQUIRED = ["faiss-cpu>=1.7.0,<2"] +QDRANT_REQUIRED = ["qdrant-client>=1.12.0"] + CI_REQUIRED = ( [ "build", @@ -214,6 +216,7 @@ + SINGLESTORE_REQUIRED + OPENTELEMETRY + FAISS_REQUIRED + + QDRANT_REQUIRED ) DOCS_REQUIRED = CI_REQUIRED @@ -284,6 +287,7 @@ "singlestore": SINGLESTORE_REQUIRED, "opentelemetry": OPENTELEMETRY, "faiss": FAISS_REQUIRED, + "qdrant": QDRANT_REQUIRED }, include_package_data=True, license="Apache", From 237c453c2da7d549b9bdb2c044ba284fbb9d9ba7 Mon Sep 17 00:00:00 2001 From: Theodor Mihalache <84387487+tmihalac@users.noreply.github.com> Date: Mon, 28 Oct 2024 10:08:22 -0400 Subject: [PATCH 02/65] fix: Feast create empty online table when FeatureView attribute online=False (#4666) Signed-off-by: Theodor Mihalache --- sdk/python/feast/infra/passthrough_provider.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/sdk/python/feast/infra/passthrough_provider.py b/sdk/python/feast/infra/passthrough_provider.py index a1e9ef82ad7..215b175eb2e 100644 --- a/sdk/python/feast/infra/passthrough_provider.py +++ b/sdk/python/feast/infra/passthrough_provider.py @@ -148,10 +148,16 @@ def update_infra( ): # Call update only if there is an online store if self.online_store: + tables_to_keep_online = [ + fv + for fv in tables_to_keep + if not hasattr(fv, "online") or (hasattr(fv, "online") and fv.online) + ] + self.online_store.update( config=self.repo_config, tables_to_delete=tables_to_delete, - tables_to_keep=tables_to_keep, + tables_to_keep=tables_to_keep_online, entities_to_keep=entities_to_keep, entities_to_delete=entities_to_delete, partial=partial, From 70c6dc1ef82bc829aeaf30c167611fb0f7567cec Mon Sep 17 00:00:00 2001 From: Daniel Dowler <12484302+dandawg@users.noreply.github.com> Date: Mon, 28 Oct 2024 09:21:40 -0600 Subject: [PATCH 03/65] docs: Fix typo feature_tore.yaml -> feature_store.yaml (#4707) --- docs/reference/alpha-vector-database.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/reference/alpha-vector-database.md b/docs/reference/alpha-vector-database.md index fca31ee4780..ae6b47f0422 100644 --- a/docs/reference/alpha-vector-database.md +++ b/docs/reference/alpha-vector-database.md @@ -31,7 +31,7 @@ python batch_score_documents.py The output will be stored in `data/city_wikipedia_summaries.csv.` ### **Initialize Feast feature store and materialize the data to the online store** -Use the feature_tore.yaml file to initialize the feature store. This will use the data as offline store, and Pgvector as online store. +Use the feature_store.yaml file to initialize the feature store. This will use the data as offline store, and Pgvector as online store. ```yaml project: feast_demo_local From 764a8a657c045e99575bb8cfdc51afd9c61fa8e2 Mon Sep 17 00:00:00 2001 From: Tommy Hughes IV Date: Mon, 28 Oct 2024 11:03:02 -0500 Subject: [PATCH 04/65] fix: Update release version in a pertinent Operator file (#4708) --- infra/feast-operator/dist/install.yaml | 2 +- infra/scripts/release/files_to_bump.txt | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/infra/feast-operator/dist/install.yaml b/infra/feast-operator/dist/install.yaml index 63b3a742b16..519649ffba6 100644 --- a/infra/feast-operator/dist/install.yaml +++ b/infra/feast-operator/dist/install.yaml @@ -349,7 +349,7 @@ spec: - --leader-elect command: - /manager - image: feastdev/feast-operator:0.40.0 + image: feastdev/feast-operator:0.41.0 livenessProbe: httpGet: path: /healthz diff --git a/infra/scripts/release/files_to_bump.txt b/infra/scripts/release/files_to_bump.txt index d71f8da37d8..4e708aa8f93 100644 --- a/infra/scripts/release/files_to_bump.txt +++ b/infra/scripts/release/files_to_bump.txt @@ -14,5 +14,6 @@ infra/feast-helm-operator/Makefile 6 infra/feast-helm-operator/config/manager/kustomization.yaml 8 infra/feast-operator/Makefile 6 infra/feast-operator/config/manager/kustomization.yaml 8 +infra/feast-operator/dist/install.yaml 352 java/pom.xml 38 ui/package.json 3 From 9d8d3d88a0ecccef4d610baf84f1b409276044dd Mon Sep 17 00:00:00 2001 From: lokeshrangineni <19699092+lokeshrangineni@users.noreply.github.com> Date: Mon, 28 Oct 2024 14:48:31 -0400 Subject: [PATCH 05/65] feat: Printing more verbose logs when we start the offline server (#4660) Printing more verbose logs when we start the offline server to address the github issue https://github.com/feast-dev/feast/issues/4639 Signed-off-by: lrangine <19699092+lokeshrangineni@users.noreply.github.com> --- sdk/python/feast/offline_server.py | 32 ++++++++++++++++++++++++++++-- 1 file changed, 30 insertions(+), 2 deletions(-) diff --git a/sdk/python/feast/offline_server.py b/sdk/python/feast/offline_server.py index 0cb40ad934c..cec043129e7 100644 --- a/sdk/python/feast/offline_server.py +++ b/sdk/python/feast/offline_server.py @@ -1,10 +1,13 @@ import ast import json import logging +import os +import sys import traceback from datetime import datetime from typing import Any, Dict, List, cast +import click import pyarrow as pa import pyarrow.flight as fl from google.protobuf.json_format import Parse @@ -503,6 +506,24 @@ def get_table_column_names_and_types_from_data_source(self, command: dict): ) return pa.table({"name": column_names, "type": types}) + def serve(self): + message = "offline server starting with pid: " + logger.info( + message + "[%d]", + os.getpid(), + extra={"color_message": message + "[" + click.style("%d", fg="cyan") + "]"}, + ) + super().serve() + + def shutdown(self): + message = "Sending a shutdown signal to the offline server running with pid:: " + logger.info( + message + "[%d]", + os.getpid(), + extra={"color_message": message + "[" + click.style("%d", fg="cyan") + "]"}, + ) + super().shutdown() + def remove_dummies(fv: FeatureView) -> FeatureView: """ @@ -533,5 +554,12 @@ def start_server( location = "grpc+tcp://{}:{}".format(host, port) server = OfflineServer(store, location) - logger.info(f"Offline store server serving on {location}") - server.serve() + try: + logger.info(f"Offline store server serving at: {location}") + server.serve() + except KeyboardInterrupt: + logger.info("KeyboardInterrupt received, stopping the offline server.") + finally: + server.shutdown() + logger.info("offline server stopped.") + sys.exit(0) From c98c806c8d8ffd4c7d2d6a3b3c1e1fb8142e7ab6 Mon Sep 17 00:00:00 2001 From: Francisco Arceo Date: Mon, 28 Oct 2024 23:08:53 -0400 Subject: [PATCH 06/65] chore: Update release process docs (#4706) --- docs/project/release-process.md | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/docs/project/release-process.md b/docs/project/release-process.md index e6f75ffd413..251b9338f0a 100644 --- a/docs/project/release-process.md +++ b/docs/project/release-process.md @@ -4,9 +4,12 @@ For Feast maintainers, these are the concrete steps for making a new release. +Note: Make sure you have a [Personal Access Token](https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/creating-a-personal-access-token) or retrieve your saved personal access token. + +If something goes wrong, investigate the workflow and try to rerun different pieces locally. + ### 0. Cutting a minor release -You only need to hit the `release` workflow using [the GitHub action](https://github.com/feast-dev/feast/blob/master/.github/workflows/release.yml). -First test with a `dry-run` then run it live. This is all you need to do. All deployments to dockerhub, PyPI, and npm are handled by the workflows. +You only need to hit the `release` workflow using [the GitHub action](https://github.com/feast-dev/feast/blob/master/.github/workflows/release.yml). This is all you need to do. All deployments to dockerhub, PyPI, and npm are handled by the workflows. Also note that as a part of the workflow, the [infra/scripts/release/bump_file_versions.py](https://github.com/feast-dev/feast/blob/master/infra/scripts/release/bump_file_versions.py) file will increment the Feast versions in the appropriate files. @@ -98,4 +101,4 @@ In the Feast Gitbook: ![](new_branch_part_5.png) 6. Verify on [docs.feast.dev](http://docs.feast.dev) that this new space is the default (this may take a few minutes to - propagate, and your browser cache may be caching the old branch as the default) \ No newline at end of file + propagate, and your browser cache may be caching the old branch as the default) From a61b93c666a79ec72b48d0927b2a4e1598f6650b Mon Sep 17 00:00:00 2001 From: Bhargav Dodla <13788369+EXPEbdodla@users.noreply.github.com> Date: Mon, 28 Oct 2024 20:52:28 -0700 Subject: [PATCH 07/65] =?UTF-8?q?fix:=20Populates=20project=20created=5Fti?= =?UTF-8?q?me=20correctly=20according=20to=20created=20ti=E2=80=A6=20(#468?= =?UTF-8?q?6)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit fix: Populates project created_time correctly according to created time in feast_metadata table Signed-off-by: Bhargav Dodla Co-authored-by: Bhargav Dodla --- sdk/python/feast/infra/registry/sql.py | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/sdk/python/feast/infra/registry/sql.py b/sdk/python/feast/infra/registry/sql.py index 6ae27acf4e4..c42e6e8b82b 100644 --- a/sdk/python/feast/infra/registry/sql.py +++ b/sdk/python/feast/infra/registry/sql.py @@ -281,7 +281,7 @@ def __init__( ) def _sync_feast_metadata_to_projects_table(self): - feast_metadata_projects: set = [] + feast_metadata_projects: dict = {} projects_set: set = [] with self.read_engine.begin() as conn: stmt = select(feast_metadata).where( @@ -289,7 +289,9 @@ def _sync_feast_metadata_to_projects_table(self): ) rows = conn.execute(stmt).all() for row in rows: - feast_metadata_projects.append(row._mapping["project_id"]) + feast_metadata_projects[row._mapping["project_id"]] = int( + row._mapping["last_updated_timestamp"] + ) if len(feast_metadata_projects) > 0: with self.read_engine.begin() as conn: @@ -299,9 +301,17 @@ def _sync_feast_metadata_to_projects_table(self): projects_set.append(row._mapping["project_id"]) # Find object in feast_metadata_projects but not in projects - projects_to_sync = set(feast_metadata_projects) - set(projects_set) + projects_to_sync = set(feast_metadata_projects.keys()) - set(projects_set) for project_name in projects_to_sync: - self.apply_project(Project(name=project_name), commit=True) + self.apply_project( + Project( + name=project_name, + created_timestamp=datetime.fromtimestamp( + feast_metadata_projects[project_name], tz=timezone.utc + ), + ), + commit=True, + ) if self.purge_feast_metadata: with self.write_engine.begin() as conn: @@ -976,7 +986,8 @@ def _apply_object( if hasattr(obj_proto, "meta") and hasattr( obj_proto.meta, "created_timestamp" ): - obj_proto.meta.created_timestamp.FromDatetime(update_datetime) + if not obj_proto.meta.HasField("created_timestamp"): + obj_proto.meta.created_timestamp.FromDatetime(update_datetime) values = { id_field_name: name, From 430ac535a5bd8311a485e51011a9602ca441d2d3 Mon Sep 17 00:00:00 2001 From: Tommy Hughes IV Date: Tue, 29 Oct 2024 06:39:56 -0500 Subject: [PATCH 08/65] feat: Add Operator support for spec.feastProject & status.applied fields (#4656) spec.feastProject & status.applied logic Signed-off-by: Tommy Hughes --- .../api/feastversion/version.go | 20 ++ .../api/v1alpha1/featurestore_types.go | 45 ++- .../api/v1alpha1/zz_generated.deepcopy.go | 27 +- .../feast-operator.clusterserviceversion.yaml | 35 ++- .../manifests/feast.dev_featurestores.yaml | 107 ++++++- infra/feast-operator/cmd/main.go | 9 + .../crd/bases/feast.dev_featurestores.yaml | 107 ++++++- infra/feast-operator/config/rbac/role.yaml | 23 ++ .../config/samples/v1alpha1_featurestore.yaml | 7 +- infra/feast-operator/dist/install.yaml | 130 ++++++++- .../controller/featurestore_controller.go | 105 ++++++- .../featurestore_controller_test.go | 266 +++++++++++++++++- .../internal/controller/services/client.go | 79 ++++++ .../internal/controller/services/registry.go | 245 ++++++++++++++++ .../controller/services/services_types.go | 72 +++++ infra/scripts/release/files_to_bump.txt | 2 +- 16 files changed, 1237 insertions(+), 42 deletions(-) create mode 100644 infra/feast-operator/api/feastversion/version.go create mode 100644 infra/feast-operator/internal/controller/services/client.go create mode 100644 infra/feast-operator/internal/controller/services/registry.go create mode 100644 infra/feast-operator/internal/controller/services/services_types.go diff --git a/infra/feast-operator/api/feastversion/version.go b/infra/feast-operator/api/feastversion/version.go new file mode 100644 index 00000000000..ac97cd03266 --- /dev/null +++ b/infra/feast-operator/api/feastversion/version.go @@ -0,0 +1,20 @@ +/* +Copyright 2024 Feast Community. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package feastversion + +// Feast release version +const FeastVersion = "0.40.0" diff --git a/infra/feast-operator/api/v1alpha1/featurestore_types.go b/infra/feast-operator/api/v1alpha1/featurestore_types.go index 030ff408b48..1afd7069f0b 100644 --- a/infra/feast-operator/api/v1alpha1/featurestore_types.go +++ b/infra/feast-operator/api/v1alpha1/featurestore_types.go @@ -20,24 +20,59 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) -// EDIT THIS FILE! THIS IS SCAFFOLDING FOR YOU TO OWN! -// NOTE: json tags are required. Any new fields you add must have json tags for the fields to be serialized. +const ( + // Feast phases: + ReadyPhase = "Ready" + PendingPhase = "Pending" + FailedPhase = "Failed" + + // Feast condition types: + ClientReadyType = "Client" + RegistryReadyType = "Registry" + ReadyType = "FeatureStore" + + // Feast condition reasons: + ReadyReason = "Ready" + FailedReason = "FeatureStoreFailed" + RegistryFailedReason = "RegistryDeploymentFailed" + ClientFailedReason = "ClientDeploymentFailed" + + // Feast condition messages: + ReadyMessage = "FeatureStore installation complete" + RegistryReadyMessage = "Registry installation complete" + ClientReadyMessage = "Client installation complete" + + // entity_key_serialization_version + SerializationVersion = 3 +) // FeatureStoreSpec defines the desired state of FeatureStore type FeatureStoreSpec struct { // +kubebuilder:validation:Pattern="^[A-Za-z0-9][A-Za-z0-9_]*$" - // FeastProject is the Feast project id. This can be any alphanumeric string with underscores, but it cannot start with an underscore. + // FeastProject is the Feast project id. This can be any alphanumeric string with underscores, but it cannot start with an underscore. Required. FeastProject string `json:"feastProject"` } // FeatureStoreStatus defines the observed state of FeatureStore type FeatureStoreStatus struct { - // INSERT ADDITIONAL STATUS FIELD - define observed state of cluster - // Important: Run "make" to regenerate code after modifying this file + Applied FeatureStoreSpec `json:"applied,omitempty"` + ClientConfigMap string `json:"clientConfigMap,omitempty"` + Conditions []metav1.Condition `json:"conditions,omitempty"` + FeastVersion string `json:"feastVersion,omitempty"` + Phase string `json:"phase,omitempty"` + ServiceUrls ServiceUrls `json:"serviceUrls,omitempty"` +} + +// ServiceUrls +type ServiceUrls struct { + Registry string `json:"registry,omitempty"` } //+kubebuilder:object:root=true //+kubebuilder:subresource:status +//+kubebuilder:resource:shortName=feast +//+kubebuilder:printcolumn:name="Status",type=string,JSONPath=`.status.phase` +//+kubebuilder:printcolumn:name="Age",type=date,JSONPath=`.metadata.creationTimestamp` // FeatureStore is the Schema for the featurestores API type FeatureStore struct { diff --git a/infra/feast-operator/api/v1alpha1/zz_generated.deepcopy.go b/infra/feast-operator/api/v1alpha1/zz_generated.deepcopy.go index 3f664edded7..b8e410a616e 100644 --- a/infra/feast-operator/api/v1alpha1/zz_generated.deepcopy.go +++ b/infra/feast-operator/api/v1alpha1/zz_generated.deepcopy.go @@ -21,6 +21,7 @@ limitations under the License. package v1alpha1 import ( + "k8s.io/apimachinery/pkg/apis/meta/v1" runtime "k8s.io/apimachinery/pkg/runtime" ) @@ -30,7 +31,7 @@ func (in *FeatureStore) DeepCopyInto(out *FeatureStore) { out.TypeMeta = in.TypeMeta in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) out.Spec = in.Spec - out.Status = in.Status + in.Status.DeepCopyInto(&out.Status) } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new FeatureStore. @@ -101,6 +102,15 @@ func (in *FeatureStoreSpec) DeepCopy() *FeatureStoreSpec { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *FeatureStoreStatus) DeepCopyInto(out *FeatureStoreStatus) { *out = *in + out.Applied = in.Applied + if in.Conditions != nil { + in, out := &in.Conditions, &out.Conditions + *out = make([]v1.Condition, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + out.ServiceUrls = in.ServiceUrls } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new FeatureStoreStatus. @@ -112,3 +122,18 @@ func (in *FeatureStoreStatus) DeepCopy() *FeatureStoreStatus { in.DeepCopyInto(out) return out } + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ServiceUrls) DeepCopyInto(out *ServiceUrls) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ServiceUrls. +func (in *ServiceUrls) DeepCopy() *ServiceUrls { + if in == nil { + return nil + } + out := new(ServiceUrls) + in.DeepCopyInto(out) + return out +} diff --git a/infra/feast-operator/bundle/manifests/feast-operator.clusterserviceversion.yaml b/infra/feast-operator/bundle/manifests/feast-operator.clusterserviceversion.yaml index 8c403f78885..5b989cc2a7f 100644 --- a/infra/feast-operator/bundle/manifests/feast-operator.clusterserviceversion.yaml +++ b/infra/feast-operator/bundle/manifests/feast-operator.clusterserviceversion.yaml @@ -8,17 +8,15 @@ metadata: "apiVersion": "feast.dev/v1alpha1", "kind": "FeatureStore", "metadata": { - "labels": { - "app.kubernetes.io/managed-by": "kustomize", - "app.kubernetes.io/name": "feast-operator" - }, - "name": "featurestore-sample" + "name": "sample" }, - "spec": null + "spec": { + "feastProject": "my_project" + } } ] capabilities: Basic Install - createdAt: "2024-10-09T16:16:53Z" + createdAt: "2024-10-21T16:35:01Z" operators.operatorframework.io/builder: operator-sdk-v1.37.0 operators.operatorframework.io/project_layout: go.kubebuilder.io/v4 name: feast-operator.v0.40.0 @@ -41,6 +39,29 @@ spec: spec: clusterPermissions: - rules: + - apiGroups: + - apps + resources: + - deployments + verbs: + - create + - delete + - get + - list + - update + - watch + - apiGroups: + - "" + resources: + - configmaps + - services + verbs: + - create + - delete + - get + - list + - update + - watch - apiGroups: - feast.dev resources: diff --git a/infra/feast-operator/bundle/manifests/feast.dev_featurestores.yaml b/infra/feast-operator/bundle/manifests/feast.dev_featurestores.yaml index 43df8e3f845..2fb15b432a7 100644 --- a/infra/feast-operator/bundle/manifests/feast.dev_featurestores.yaml +++ b/infra/feast-operator/bundle/manifests/feast.dev_featurestores.yaml @@ -11,10 +11,19 @@ spec: kind: FeatureStore listKind: FeatureStoreList plural: featurestores + shortNames: + - feast singular: featurestore scope: Namespaced versions: - - name: v1alpha1 + - additionalPrinterColumns: + - jsonPath: .status.phase + name: Status + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1alpha1 schema: openAPIV3Schema: description: FeatureStore is the Schema for the featurestores API @@ -42,7 +51,7 @@ spec: feastProject: description: FeastProject is the Feast project id. This can be any alphanumeric string with underscores, but it cannot start with an - underscore. + underscore. Required. pattern: ^[A-Za-z0-9][A-Za-z0-9_]*$ type: string required: @@ -50,6 +59,100 @@ spec: type: object status: description: FeatureStoreStatus defines the observed state of FeatureStore + properties: + applied: + description: FeatureStoreSpec defines the desired state of FeatureStore + properties: + feastProject: + description: FeastProject is the Feast project id. This can be + any alphanumeric string with underscores, but it cannot start + with an underscore. Required. + pattern: ^[A-Za-z0-9][A-Za-z0-9_]*$ + type: string + required: + - feastProject + type: object + clientConfigMap: + type: string + conditions: + items: + description: "Condition contains details for one aspect of the current + state of this API Resource.\n---\nThis struct is intended for + direct use as an array at the field path .status.conditions. For + example,\n\n\n\ttype FooStatus struct{\n\t // Represents the + observations of a foo's current state.\n\t // Known .status.conditions.type + are: \"Available\", \"Progressing\", and \"Degraded\"\n\t // + +patchMergeKey=type\n\t // +patchStrategy=merge\n\t // +listType=map\n\t + \ // +listMapKey=type\n\t Conditions []metav1.Condition `json:\"conditions,omitempty\" + patchStrategy:\"merge\" patchMergeKey:\"type\" protobuf:\"bytes,1,rep,name=conditions\"`\n\n\n\t + \ // other fields\n\t}" + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: |- + type of condition in CamelCase or in foo.example.com/CamelCase. + --- + Many .condition.type values are consistent across resources like Available, but because arbitrary conditions can be + useful (see .node.status.conditions), the ability to deconflict is important. + The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt) + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + feastVersion: + type: string + phase: + type: string + serviceUrls: + description: ServiceUrls + properties: + registry: + type: string + type: object type: object type: object served: true diff --git a/infra/feast-operator/cmd/main.go b/infra/feast-operator/cmd/main.go index 3ca6c895088..eae4f8b1214 100644 --- a/infra/feast-operator/cmd/main.go +++ b/infra/feast-operator/cmd/main.go @@ -25,10 +25,12 @@ import ( // to ensure that exec-entrypoint and run can make use of them. _ "k8s.io/client-go/plugin/pkg/client/auth" + corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime" utilruntime "k8s.io/apimachinery/pkg/util/runtime" clientgoscheme "k8s.io/client-go/kubernetes/scheme" ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/healthz" "sigs.k8s.io/controller-runtime/pkg/log/zap" metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" @@ -116,6 +118,13 @@ func main() { // if you are doing or is intended to do any operation such as perform cleanups // after the manager stops then its usage might be unsafe. // LeaderElectionReleaseOnCancel: true, + Client: client.Options{ + Cache: &client.CacheOptions{ + DisableFor: []client.Object{ + &corev1.ConfigMap{}, + }, + }, + }, }) if err != nil { setupLog.Error(err, "unable to start manager") diff --git a/infra/feast-operator/config/crd/bases/feast.dev_featurestores.yaml b/infra/feast-operator/config/crd/bases/feast.dev_featurestores.yaml index d6bd0536922..8c5c2e62a6d 100644 --- a/infra/feast-operator/config/crd/bases/feast.dev_featurestores.yaml +++ b/infra/feast-operator/config/crd/bases/feast.dev_featurestores.yaml @@ -11,10 +11,19 @@ spec: kind: FeatureStore listKind: FeatureStoreList plural: featurestores + shortNames: + - feast singular: featurestore scope: Namespaced versions: - - name: v1alpha1 + - additionalPrinterColumns: + - jsonPath: .status.phase + name: Status + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1alpha1 schema: openAPIV3Schema: description: FeatureStore is the Schema for the featurestores API @@ -42,7 +51,7 @@ spec: feastProject: description: FeastProject is the Feast project id. This can be any alphanumeric string with underscores, but it cannot start with an - underscore. + underscore. Required. pattern: ^[A-Za-z0-9][A-Za-z0-9_]*$ type: string required: @@ -50,6 +59,100 @@ spec: type: object status: description: FeatureStoreStatus defines the observed state of FeatureStore + properties: + applied: + description: FeatureStoreSpec defines the desired state of FeatureStore + properties: + feastProject: + description: FeastProject is the Feast project id. This can be + any alphanumeric string with underscores, but it cannot start + with an underscore. Required. + pattern: ^[A-Za-z0-9][A-Za-z0-9_]*$ + type: string + required: + - feastProject + type: object + clientConfigMap: + type: string + conditions: + items: + description: "Condition contains details for one aspect of the current + state of this API Resource.\n---\nThis struct is intended for + direct use as an array at the field path .status.conditions. For + example,\n\n\n\ttype FooStatus struct{\n\t // Represents the + observations of a foo's current state.\n\t // Known .status.conditions.type + are: \"Available\", \"Progressing\", and \"Degraded\"\n\t // + +patchMergeKey=type\n\t // +patchStrategy=merge\n\t // +listType=map\n\t + \ // +listMapKey=type\n\t Conditions []metav1.Condition `json:\"conditions,omitempty\" + patchStrategy:\"merge\" patchMergeKey:\"type\" protobuf:\"bytes,1,rep,name=conditions\"`\n\n\n\t + \ // other fields\n\t}" + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: |- + type of condition in CamelCase or in foo.example.com/CamelCase. + --- + Many .condition.type values are consistent across resources like Available, but because arbitrary conditions can be + useful (see .node.status.conditions), the ability to deconflict is important. + The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt) + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + feastVersion: + type: string + phase: + type: string + serviceUrls: + description: ServiceUrls + properties: + registry: + type: string + type: object type: object type: object served: true diff --git a/infra/feast-operator/config/rbac/role.yaml b/infra/feast-operator/config/rbac/role.yaml index f0bb2016af1..5ee64d47051 100644 --- a/infra/feast-operator/config/rbac/role.yaml +++ b/infra/feast-operator/config/rbac/role.yaml @@ -4,6 +4,29 @@ kind: ClusterRole metadata: name: manager-role rules: +- apiGroups: + - apps + resources: + - deployments + verbs: + - create + - delete + - get + - list + - update + - watch +- apiGroups: + - "" + resources: + - configmaps + - services + verbs: + - create + - delete + - get + - list + - update + - watch - apiGroups: - feast.dev resources: diff --git a/infra/feast-operator/config/samples/v1alpha1_featurestore.yaml b/infra/feast-operator/config/samples/v1alpha1_featurestore.yaml index 2800d87e358..3eb62850435 100644 --- a/infra/feast-operator/config/samples/v1alpha1_featurestore.yaml +++ b/infra/feast-operator/config/samples/v1alpha1_featurestore.yaml @@ -1,9 +1,6 @@ apiVersion: feast.dev/v1alpha1 kind: FeatureStore metadata: - labels: - app.kubernetes.io/name: feast-operator - app.kubernetes.io/managed-by: kustomize - name: featurestore-sample + name: sample spec: - # TODO(user): Add fields here + feastProject: my_project diff --git a/infra/feast-operator/dist/install.yaml b/infra/feast-operator/dist/install.yaml index 519649ffba6..77de70ebcb8 100644 --- a/infra/feast-operator/dist/install.yaml +++ b/infra/feast-operator/dist/install.yaml @@ -19,10 +19,19 @@ spec: kind: FeatureStore listKind: FeatureStoreList plural: featurestores + shortNames: + - feast singular: featurestore scope: Namespaced versions: - - name: v1alpha1 + - additionalPrinterColumns: + - jsonPath: .status.phase + name: Status + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1alpha1 schema: openAPIV3Schema: description: FeatureStore is the Schema for the featurestores API @@ -50,7 +59,7 @@ spec: feastProject: description: FeastProject is the Feast project id. This can be any alphanumeric string with underscores, but it cannot start with an - underscore. + underscore. Required. pattern: ^[A-Za-z0-9][A-Za-z0-9_]*$ type: string required: @@ -58,6 +67,100 @@ spec: type: object status: description: FeatureStoreStatus defines the observed state of FeatureStore + properties: + applied: + description: FeatureStoreSpec defines the desired state of FeatureStore + properties: + feastProject: + description: FeastProject is the Feast project id. This can be + any alphanumeric string with underscores, but it cannot start + with an underscore. Required. + pattern: ^[A-Za-z0-9][A-Za-z0-9_]*$ + type: string + required: + - feastProject + type: object + clientConfigMap: + type: string + conditions: + items: + description: "Condition contains details for one aspect of the current + state of this API Resource.\n---\nThis struct is intended for + direct use as an array at the field path .status.conditions. For + example,\n\n\n\ttype FooStatus struct{\n\t // Represents the + observations of a foo's current state.\n\t // Known .status.conditions.type + are: \"Available\", \"Progressing\", and \"Degraded\"\n\t // + +patchMergeKey=type\n\t // +patchStrategy=merge\n\t // +listType=map\n\t + \ // +listMapKey=type\n\t Conditions []metav1.Condition `json:\"conditions,omitempty\" + patchStrategy:\"merge\" patchMergeKey:\"type\" protobuf:\"bytes,1,rep,name=conditions\"`\n\n\n\t + \ // other fields\n\t}" + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: |- + type of condition in CamelCase or in foo.example.com/CamelCase. + --- + Many .condition.type values are consistent across resources like Available, but because arbitrary conditions can be + useful (see .node.status.conditions), the ability to deconflict is important. + The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt) + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + feastVersion: + type: string + phase: + type: string + serviceUrls: + description: ServiceUrls + properties: + registry: + type: string + type: object type: object type: object served: true @@ -170,6 +273,29 @@ kind: ClusterRole metadata: name: feast-operator-manager-role rules: +- apiGroups: + - apps + resources: + - deployments + verbs: + - create + - delete + - get + - list + - update + - watch +- apiGroups: + - "" + resources: + - configmaps + - services + verbs: + - create + - delete + - get + - list + - update + - watch - apiGroups: - feast.dev resources: diff --git a/infra/feast-operator/internal/controller/featurestore_controller.go b/infra/feast-operator/internal/controller/featurestore_controller.go index d56a7ff024d..17227293d80 100644 --- a/infra/feast-operator/internal/controller/featurestore_controller.go +++ b/infra/feast-operator/internal/controller/featurestore_controller.go @@ -18,13 +18,28 @@ package controller import ( "context" + "reflect" + "time" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" + apierrors "k8s.io/apimachinery/pkg/api/errors" + apimeta "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/log" + "github.com/feast-dev/feast/infra/feast-operator/api/feastversion" feastdevv1alpha1 "github.com/feast-dev/feast/infra/feast-operator/api/v1alpha1" + "github.com/feast-dev/feast/infra/feast-operator/internal/controller/services" +) + +// Constants for requeue +const ( + RequeueDelayError = 5 * time.Second ) // FeatureStoreReconciler reconciles a FeatureStore object @@ -36,27 +51,99 @@ type FeatureStoreReconciler struct { //+kubebuilder:rbac:groups=feast.dev,resources=featurestores,verbs=get;list;watch;create;update;patch;delete //+kubebuilder:rbac:groups=feast.dev,resources=featurestores/status,verbs=get;update;patch //+kubebuilder:rbac:groups=feast.dev,resources=featurestores/finalizers,verbs=update +//+kubebuilder:rbac:groups=apps,resources=deployments,verbs=get;list;create;update;watch;delete +//+kubebuilder:rbac:groups=core,resources=services;configmaps,verbs=get;list;create;update;watch;delete // Reconcile is part of the main kubernetes reconciliation loop which aims to // move the current state of the cluster closer to the desired state. -// TODO(user): Modify the Reconcile function to compare the state specified by -// the FeatureStore object against the actual cluster state, and then -// perform operations to make the cluster state reflect the state specified by -// the user. -// // For more details, check Reconcile and its Result here: // - https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.17.3/pkg/reconcile -func (r *FeatureStoreReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { - _ = log.FromContext(ctx) +func (r *FeatureStoreReconciler) Reconcile(ctx context.Context, req ctrl.Request) (result ctrl.Result, recErr error) { + logger := log.FromContext(ctx) + + cr := &feastdevv1alpha1.FeatureStore{} + err := r.Get(ctx, req.NamespacedName, cr) + if err != nil { + if apierrors.IsNotFound(err) { + // CR deleted since request queued, child objects getting GC'd, no requeue + logger.V(1).Info("FeatureStore CR not found, has been deleted") + return ctrl.Result{}, nil + } + // error fetching FeatureStore instance, requeue and try again + logger.Error(err, "Unable to get FeatureStore CR") + return ctrl.Result{}, err + } + currentStatus := cr.Status.DeepCopy() - // TODO(user): your logic here + applyDefaultsToStatus(cr) + result, recErr = r.deployFeast(ctx, cr) + if cr.DeletionTimestamp == nil && !reflect.DeepEqual(currentStatus, cr.Status) { + if err := r.Client.Status().Update(ctx, cr); err != nil { + if errors.IsConflict(err) { + logger.Info("FeatureStore object modified, retry syncing status") + // Re-queue and preserve existing recErr + result = ctrl.Result{Requeue: true, RequeueAfter: RequeueDelayError} + } + logger.Error(err, "Error updating the FeatureStore status") + if recErr == nil { + // There is no existing recErr. Set it to the status update error + recErr = err + } + } + } - return ctrl.Result{}, nil + return result, recErr +} + +func (r *FeatureStoreReconciler) deployFeast(ctx context.Context, cr *feastdevv1alpha1.FeatureStore) (result ctrl.Result, err error) { + logger := log.FromContext(ctx) + condition := metav1.Condition{ + Type: feastdevv1alpha1.ReadyType, + Status: metav1.ConditionTrue, + Reason: feastdevv1alpha1.ReadyReason, + Message: feastdevv1alpha1.ReadyMessage, + } + feast := services.FeastServices{ + Client: r.Client, + Context: ctx, + FeatureStore: cr, + Scheme: r.Scheme, + } + err = feast.Deploy() + if err != nil { + condition = metav1.Condition{ + Type: feastdevv1alpha1.ReadyType, + Status: metav1.ConditionFalse, + Reason: feastdevv1alpha1.FailedReason, + Message: "Error: " + err.Error(), + } + result = ctrl.Result{Requeue: true} + } + logger.Info(condition.Message) + apimeta.SetStatusCondition(&cr.Status.Conditions, condition) + + if apimeta.IsStatusConditionTrue(cr.Status.Conditions, feastdevv1alpha1.ReadyType) { + cr.Status.Phase = feastdevv1alpha1.ReadyPhase + } else if apimeta.IsStatusConditionFalse(cr.Status.Conditions, feastdevv1alpha1.ReadyType) { + cr.Status.Phase = feastdevv1alpha1.FailedPhase + } else { + cr.Status.Phase = feastdevv1alpha1.PendingPhase + } + + return result, err } // SetupWithManager sets up the controller with the Manager. func (r *FeatureStoreReconciler) SetupWithManager(mgr ctrl.Manager) error { return ctrl.NewControllerManagedBy(mgr). For(&feastdevv1alpha1.FeatureStore{}). + Owns(&corev1.ConfigMap{}). + Owns(&appsv1.Deployment{}). + Owns(&corev1.Service{}). Complete(r) } + +func applyDefaultsToStatus(cr *feastdevv1alpha1.FeatureStore) { + cr.Status.Applied.FeastProject = cr.Spec.FeastProject + cr.Status.FeastVersion = feastversion.FeastVersion +} diff --git a/infra/feast-operator/internal/controller/featurestore_controller_test.go b/infra/feast-operator/internal/controller/featurestore_controller_test.go index d4caf254977..5b1e41bb309 100644 --- a/infra/feast-operator/internal/controller/featurestore_controller_test.go +++ b/infra/feast-operator/internal/controller/featurestore_controller_test.go @@ -18,18 +18,28 @@ package controller import ( "context" + "encoding/base64" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + "gopkg.in/yaml.v3" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" + apimeta "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/intstr" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" "sigs.k8s.io/controller-runtime/pkg/reconcile" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - + "github.com/feast-dev/feast/infra/feast-operator/api/feastversion" feastdevv1alpha1 "github.com/feast-dev/feast/infra/feast-operator/api/v1alpha1" + "github.com/feast-dev/feast/infra/feast-operator/internal/controller/services" ) +const feastProject = "test_project" + var _ = Describe("FeatureStore Controller", func() { Context("When reconciling a resource", func() { const resourceName = "test-resource" @@ -38,7 +48,7 @@ var _ = Describe("FeatureStore Controller", func() { typeNamespacedName := types.NamespacedName{ Name: resourceName, - Namespace: "default", // TODO(user):Modify as needed + Namespace: "default", } featurestore := &feastdevv1alpha1.FeatureStore{} @@ -51,14 +61,12 @@ var _ = Describe("FeatureStore Controller", func() { Name: resourceName, Namespace: "default", }, - Spec: feastdevv1alpha1.FeatureStoreSpec{FeastProject: "my_project"}, + Spec: feastdevv1alpha1.FeatureStoreSpec{FeastProject: feastProject}, } Expect(k8sClient.Create(ctx, resource)).To(Succeed()) } }) - AfterEach(func() { - // TODO(user): Cleanup logic after each test, like removing the resource instance. resource := &feastdevv1alpha1.FeatureStore{} err := k8sClient.Get(ctx, typeNamespacedName, resource) Expect(err).NotTo(HaveOccurred()) @@ -66,6 +74,7 @@ var _ = Describe("FeatureStore Controller", func() { By("Cleanup the specific resource instance FeatureStore") Expect(k8sClient.Delete(ctx, resource)).To(Succeed()) }) + It("should successfully reconcile the resource", func() { By("Reconciling the created resource") controllerReconciler := &FeatureStoreReconciler{ @@ -77,8 +86,249 @@ var _ = Describe("FeatureStore Controller", func() { NamespacedName: typeNamespacedName, }) Expect(err).NotTo(HaveOccurred()) - // TODO(user): Add more specific assertions depending on your controller's reconciliation logic. - // Example: If you expect a certain status condition after reconciliation, verify it here. + + resource := &feastdevv1alpha1.FeatureStore{} + err = k8sClient.Get(ctx, typeNamespacedName, resource) + Expect(err).NotTo(HaveOccurred()) + + feast := services.FeastServices{ + Client: controllerReconciler.Client, + Context: ctx, + Scheme: controllerReconciler.Scheme, + FeatureStore: resource, + } + Expect(resource.Status).NotTo(BeNil()) + Expect(resource.Status.FeastVersion).To(Equal(feastversion.FeastVersion)) + Expect(resource.Status.ClientConfigMap).To(Equal(feast.GetFeastServiceName(services.ClientFeastType))) + Expect(resource.Status.ServiceUrls.Registry).To(Equal(feast.GetFeastServiceName(services.RegistryFeastType) + "." + resource.Namespace + ".svc.cluster.local:80")) + Expect(resource.Status.Applied.FeastProject).To(Equal(resource.Spec.FeastProject)) + Expect(resource.Status.Conditions).NotTo(BeEmpty()) + + cond := apimeta.FindStatusCondition(resource.Status.Conditions, feastdevv1alpha1.ReadyType) + Expect(cond).ToNot(BeNil()) + Expect(cond.Status).To(Equal(metav1.ConditionTrue)) + Expect(cond.Reason).To(Equal(feastdevv1alpha1.ReadyReason)) + Expect(cond.Type).To(Equal(feastdevv1alpha1.ReadyType)) + Expect(cond.Message).To(Equal(feastdevv1alpha1.ReadyMessage)) + + cond = apimeta.FindStatusCondition(resource.Status.Conditions, feastdevv1alpha1.RegistryReadyType) + Expect(cond).ToNot(BeNil()) + Expect(cond.Status).To(Equal(metav1.ConditionTrue)) + Expect(cond.Reason).To(Equal(feastdevv1alpha1.ReadyReason)) + Expect(cond.Type).To(Equal(feastdevv1alpha1.RegistryReadyType)) + Expect(cond.Message).To(Equal(feastdevv1alpha1.RegistryReadyMessage)) + + cond = apimeta.FindStatusCondition(resource.Status.Conditions, feastdevv1alpha1.ClientReadyType) + Expect(cond).ToNot(BeNil()) + Expect(cond.Status).To(Equal(metav1.ConditionTrue)) + Expect(cond.Reason).To(Equal(feastdevv1alpha1.ReadyReason)) + Expect(cond.Type).To(Equal(feastdevv1alpha1.ClientReadyType)) + Expect(cond.Message).To(Equal(feastdevv1alpha1.ClientReadyMessage)) + + Expect(resource.Status.Phase).To(Equal(feastdevv1alpha1.ReadyPhase)) + + deploy := &appsv1.Deployment{} + err = k8sClient.Get(ctx, types.NamespacedName{ + Name: feast.GetFeastServiceName(services.RegistryFeastType), + Namespace: resource.Namespace, + }, + deploy) + Expect(err).NotTo(HaveOccurred()) + Expect(controllerutil.HasControllerReference(deploy)).To(BeTrue()) + Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(1)) + + svc := &corev1.Service{} + err = k8sClient.Get(ctx, types.NamespacedName{ + Name: feast.GetFeastServiceName(services.RegistryFeastType), + Namespace: resource.Namespace, + }, + svc) + Expect(err).NotTo(HaveOccurred()) + Expect(controllerutil.HasControllerReference(svc)).To(BeTrue()) + Expect(svc.Spec.Ports[0].TargetPort).To(Equal(intstr.FromInt(int(services.RegistryPort)))) + + }) + + It("should properly encode a feature_store.yaml config", func() { + By("Reconciling the created resource") + controllerReconciler := &FeatureStoreReconciler{ + Client: k8sClient, + Scheme: k8sClient.Scheme(), + } + + _, err := controllerReconciler.Reconcile(ctx, reconcile.Request{ + NamespacedName: typeNamespacedName, + }) + Expect(err).NotTo(HaveOccurred()) + + resource := &feastdevv1alpha1.FeatureStore{} + err = k8sClient.Get(ctx, typeNamespacedName, resource) + Expect(err).NotTo(HaveOccurred()) + + feast := services.FeastServices{ + Client: controllerReconciler.Client, + Context: ctx, + Scheme: controllerReconciler.Scheme, + FeatureStore: resource, + } + + deploy := &appsv1.Deployment{} + err = k8sClient.Get(ctx, types.NamespacedName{ + Name: feast.GetFeastServiceName(services.RegistryFeastType), + Namespace: resource.Namespace, + }, + deploy) + Expect(err).NotTo(HaveOccurred()) + Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(1)) + Expect(deploy.Spec.Template.Spec.Containers[0].Env).To(HaveLen(1)) + env := getEnvVar(services.FeatureStoreYamlEnvVar, deploy.Spec.Template.Spec.Containers[0].Env) + Expect(env).NotTo(BeNil()) + + fsYamlStr, err := feast.GetServiceFeatureStoreYamlBase64() + Expect(err).NotTo(HaveOccurred()) + Expect(fsYamlStr).To(Equal(env.Value)) + + envByte, err := base64.StdEncoding.DecodeString(env.Value) + Expect(err).NotTo(HaveOccurred()) + repoConfig := &services.RepoConfig{} + err = yaml.Unmarshal(envByte, repoConfig) + Expect(err).NotTo(HaveOccurred()) + testConfig := &services.RepoConfig{ + Project: feastProject, + Provider: services.LocalProviderType, + EntityKeySerializationVersion: feastdevv1alpha1.SerializationVersion, + Registry: services.RegistryConfig{ + RegistryType: services.RegistryFileConfigType, + Path: services.LocalRegistryPath, + }, + } + Expect(repoConfig).To(Equal(testConfig)) + + // change feast project and reconcile + resourceNew := resource.DeepCopy() + resourceNew.Spec.FeastProject = "changed" + err = k8sClient.Update(ctx, resourceNew) + Expect(err).NotTo(HaveOccurred()) + _, err = controllerReconciler.Reconcile(ctx, reconcile.Request{ + NamespacedName: typeNamespacedName, + }) + Expect(err).NotTo(HaveOccurred()) + + err = k8sClient.Get(ctx, typeNamespacedName, resource) + Expect(err).NotTo(HaveOccurred()) + Expect(resource.Spec.FeastProject).To(Equal(resourceNew.Spec.FeastProject)) + err = k8sClient.Get(ctx, types.NamespacedName{ + Name: feast.GetFeastServiceName(services.RegistryFeastType), + Namespace: resource.Namespace, + }, + deploy) + Expect(err).NotTo(HaveOccurred()) + + testConfig.Project = resourceNew.Spec.FeastProject + Expect(deploy.Spec.Template.Spec.Containers[0].Env).To(HaveLen(1)) + env = getEnvVar(services.FeatureStoreYamlEnvVar, deploy.Spec.Template.Spec.Containers[0].Env) + Expect(env).NotTo(BeNil()) + + fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64() + Expect(err).NotTo(HaveOccurred()) + Expect(fsYamlStr).To(Equal(env.Value)) + + envByte, err = base64.StdEncoding.DecodeString(env.Value) + Expect(err).NotTo(HaveOccurred()) + err = yaml.Unmarshal(envByte, repoConfig) + Expect(err).NotTo(HaveOccurred()) + Expect(repoConfig).To(Equal(testConfig)) + }) + + It("should error on reconcile", func() { + By("Trying to set the controller OwnerRef of a Deployment that already has a controller") + controllerReconciler := &FeatureStoreReconciler{ + Client: k8sClient, + Scheme: k8sClient.Scheme(), + } + + _, err := controllerReconciler.Reconcile(ctx, reconcile.Request{ + NamespacedName: typeNamespacedName, + }) + Expect(err).NotTo(HaveOccurred()) + + resource := &feastdevv1alpha1.FeatureStore{} + err = k8sClient.Get(ctx, typeNamespacedName, resource) + Expect(err).NotTo(HaveOccurred()) + + feast := services.FeastServices{ + Client: controllerReconciler.Client, + Context: ctx, + Scheme: controllerReconciler.Scheme, + FeatureStore: resource, + } + + deploy := &appsv1.Deployment{} + err = k8sClient.Get(ctx, types.NamespacedName{ + Name: feast.GetFeastServiceName(services.RegistryFeastType), + Namespace: resource.Namespace, + }, + deploy) + Expect(err).NotTo(HaveOccurred()) + + err = controllerutil.RemoveControllerReference(resource, deploy, controllerReconciler.Scheme) + Expect(err).NotTo(HaveOccurred()) + Expect(controllerutil.HasControllerReference(deploy)).To(BeFalse()) + + svc := &corev1.Service{} + name := feast.GetFeastServiceName(services.RegistryFeastType) + err = k8sClient.Get(ctx, types.NamespacedName{ + Name: name, + Namespace: resource.Namespace, + }, + svc) + Expect(err).NotTo(HaveOccurred()) + err = controllerutil.SetControllerReference(svc, deploy, controllerReconciler.Scheme) + Expect(err).NotTo(HaveOccurred()) + Expect(controllerutil.HasControllerReference(deploy)).To(BeTrue()) + err = k8sClient.Update(ctx, deploy) + Expect(err).NotTo(HaveOccurred()) + + _, err = controllerReconciler.Reconcile(ctx, reconcile.Request{ + NamespacedName: typeNamespacedName, + }) + Expect(err).To(HaveOccurred()) + + err = k8sClient.Get(ctx, typeNamespacedName, resource) + Expect(err).NotTo(HaveOccurred()) + Expect(resource.Status.Conditions).To(HaveLen(3)) + + cond := apimeta.FindStatusCondition(resource.Status.Conditions, feastdevv1alpha1.ReadyType) + Expect(cond).ToNot(BeNil()) + Expect(cond.Type).To(Equal(feastdevv1alpha1.ReadyType)) + Expect(cond.Status).To(Equal(metav1.ConditionFalse)) + Expect(cond.Reason).To(Equal(feastdevv1alpha1.FailedReason)) + Expect(cond.Message).To(Equal("Error: Object " + resource.Namespace + "/" + name + " is already owned by another Service controller " + name)) + + cond = apimeta.FindStatusCondition(resource.Status.Conditions, feastdevv1alpha1.RegistryReadyType) + Expect(cond).ToNot(BeNil()) + Expect(cond.Status).To(Equal(metav1.ConditionFalse)) + Expect(cond.Reason).To(Equal(feastdevv1alpha1.RegistryFailedReason)) + Expect(cond.Type).To(Equal(feastdevv1alpha1.RegistryReadyType)) + Expect(cond.Message).To(Equal("Error: Object " + resource.Namespace + "/" + name + " is already owned by another Service controller " + name)) + + cond = apimeta.FindStatusCondition(resource.Status.Conditions, feastdevv1alpha1.ClientReadyType) + Expect(cond).ToNot(BeNil()) + Expect(cond.Status).To(Equal(metav1.ConditionTrue)) + Expect(cond.Reason).To(Equal(feastdevv1alpha1.ReadyReason)) + Expect(cond.Type).To(Equal(feastdevv1alpha1.ClientReadyType)) + Expect(cond.Message).To(Equal(feastdevv1alpha1.ClientReadyMessage)) + + Expect(resource.Status.Phase).To(Equal(feastdevv1alpha1.FailedPhase)) }) }) }) + +func getEnvVar(name string, envs []corev1.EnvVar) *corev1.EnvVar { + for _, e := range envs { + if e.Name == name { + return &e + } + } + return nil +} diff --git a/infra/feast-operator/internal/controller/services/client.go b/infra/feast-operator/internal/controller/services/client.go new file mode 100644 index 00000000000..a46a7c63390 --- /dev/null +++ b/infra/feast-operator/internal/controller/services/client.go @@ -0,0 +1,79 @@ +/* +Copyright 2024 Feast Community. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package services + +import ( + feastdevv1alpha1 "github.com/feast-dev/feast/infra/feast-operator/api/v1alpha1" + "gopkg.in/yaml.v3" + corev1 "k8s.io/api/core/v1" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/log" +) + +func (feast *FeastServices) deployClient() error { + if err := feast.createClientConfigMap(); err != nil { + return err + } + return nil +} + +func (feast *FeastServices) createClientConfigMap() error { + logger := log.FromContext(feast.Context) + cm := &corev1.ConfigMap{ + ObjectMeta: feast.GetObjectMeta(ClientFeastType), + } + cm.SetGroupVersionKind(corev1.SchemeGroupVersion.WithKind("ConfigMap")) + if op, err := controllerutil.CreateOrUpdate(feast.Context, feast.Client, cm, controllerutil.MutateFn(func() error { + return feast.setClientConfigMap(cm) + })); err != nil { + return err + } else if op == controllerutil.OperationResultCreated || op == controllerutil.OperationResultUpdated { + logger.Info("Successfully reconciled", "ConfigMap", cm.Name, "operation", op) + } + return nil +} + +func (feast *FeastServices) setClientConfigMap(cm *corev1.ConfigMap) error { + cm.Labels = feast.getLabels(ClientFeastType) + clientYaml, err := feast.getClientFeatureStoreYaml() + if err != nil { + return err + } + cm.Data = map[string]string{"feature_store.yaml": string(clientYaml)} + feast.FeatureStore.Status.ClientConfigMap = cm.Name + return controllerutil.SetControllerReference(feast.FeatureStore, cm, feast.Scheme) +} + +func (feast *FeastServices) getClientFeatureStoreYaml() ([]byte, error) { + return yaml.Marshal(feast.getClientRepoConfig()) +} + +func (feast *FeastServices) getClientRepoConfig() RepoConfig { + status := feast.FeatureStore.Status + clientRepoConfig := RepoConfig{ + Project: status.Applied.FeastProject, + Provider: LocalProviderType, + EntityKeySerializationVersion: feastdevv1alpha1.SerializationVersion, + } + if len(status.ServiceUrls.Registry) > 0 { + clientRepoConfig.Registry = RegistryConfig{ + RegistryType: RegistryRemoteConfigType, + Path: status.ServiceUrls.Registry, + } + } + return clientRepoConfig +} diff --git a/infra/feast-operator/internal/controller/services/registry.go b/infra/feast-operator/internal/controller/services/registry.go new file mode 100644 index 00000000000..76e01e67982 --- /dev/null +++ b/infra/feast-operator/internal/controller/services/registry.go @@ -0,0 +1,245 @@ +/* +Copyright 2024 Feast Community. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package services + +import ( + "encoding/base64" + + feastdevv1alpha1 "github.com/feast-dev/feast/infra/feast-operator/api/v1alpha1" + "gopkg.in/yaml.v3" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + apimeta "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/log" +) + +// Deploy the feast services +func (feast *FeastServices) Deploy() error { + logger := log.FromContext(feast.Context) + cr := feast.FeatureStore + + if err := feast.deployRegistry(); err != nil { + apimeta.SetStatusCondition(&cr.Status.Conditions, metav1.Condition{ + Type: feastdevv1alpha1.RegistryReadyType, + Status: metav1.ConditionFalse, + Reason: feastdevv1alpha1.RegistryFailedReason, + Message: "Error: " + err.Error(), + }) + logger.Error(err, "Error deploying the FeatureStore "+string(RegistryFeastType)+" service") + return err + } else { + apimeta.SetStatusCondition(&cr.Status.Conditions, metav1.Condition{ + Type: feastdevv1alpha1.RegistryReadyType, + Status: metav1.ConditionTrue, + Reason: feastdevv1alpha1.ReadyReason, + Message: feastdevv1alpha1.RegistryReadyMessage, + }) + } + + if err := feast.deployClient(); err != nil { + apimeta.SetStatusCondition(&cr.Status.Conditions, metav1.Condition{ + Type: feastdevv1alpha1.ClientReadyType, + Status: metav1.ConditionFalse, + Reason: feastdevv1alpha1.ClientFailedReason, + Message: "Error: " + err.Error(), + }) + logger.Error(err, "Error deploying the FeatureStore "+string(ClientFeastType)+" service") + return err + } else { + apimeta.SetStatusCondition(&cr.Status.Conditions, metav1.Condition{ + Type: feastdevv1alpha1.ClientReadyType, + Status: metav1.ConditionTrue, + Reason: feastdevv1alpha1.ReadyReason, + Message: feastdevv1alpha1.ClientReadyMessage, + }) + } + return nil +} + +func (feast *FeastServices) deployRegistry() error { + if err := feast.createRegistryDeployment(); err != nil { + return err + } + if err := feast.createRegistryService(); err != nil { + return err + } + return nil +} + +func (feast *FeastServices) createRegistryDeployment() error { + logger := log.FromContext(feast.Context) + deploy := &appsv1.Deployment{ + ObjectMeta: feast.GetObjectMeta(RegistryFeastType), + } + deploy.SetGroupVersionKind(appsv1.SchemeGroupVersion.WithKind("Deployment")) + if op, err := controllerutil.CreateOrUpdate(feast.Context, feast.Client, deploy, controllerutil.MutateFn(func() error { + return feast.setDeployment(deploy, RegistryFeastType) + })); err != nil { + return err + } else if op == controllerutil.OperationResultCreated || op == controllerutil.OperationResultUpdated { + logger.Info("Successfully reconciled", "Deployment", deploy.Name, "operation", op) + } + + return nil +} + +func (feast *FeastServices) createRegistryService() error { + logger := log.FromContext(feast.Context) + svc := &corev1.Service{ + ObjectMeta: feast.GetObjectMeta(RegistryFeastType), + } + svc.SetGroupVersionKind(corev1.SchemeGroupVersion.WithKind("Service")) + if op, err := controllerutil.CreateOrUpdate(feast.Context, feast.Client, svc, controllerutil.MutateFn(func() error { + return feast.setService(svc, RegistryFeastType) + })); err != nil { + return err + } else if op == controllerutil.OperationResultCreated || op == controllerutil.OperationResultUpdated { + logger.Info("Successfully reconciled", "Service", svc.Name, "operation", op) + } + return nil +} + +func (feast *FeastServices) setDeployment(deploy *appsv1.Deployment, feastType FeastServiceType) error { + fsYamlB64, err := feast.GetServiceFeatureStoreYamlBase64() + if err != nil { + return err + } + replicas := int32(1) + deploy.Labels = feast.getLabels(feastType) + deploy.Spec = appsv1.DeploymentSpec{ + Replicas: &replicas, + Selector: metav1.SetAsLabelSelector(deploy.GetLabels()), + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: deploy.GetLabels(), + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: string(feastType), + Image: "feastdev/feature-server:" + feast.FeatureStore.Status.FeastVersion, + ImagePullPolicy: corev1.PullIfNotPresent, + Env: []corev1.EnvVar{ + { + Name: FeatureStoreYamlEnvVar, + Value: fsYamlB64, + }, + }, + }, + }, + }, + }, + } + if feastType == RegistryFeastType { + deploy.Spec.Template.Spec.Containers[0].Command = []string{ + "feast", "serve_registry", + } + deploy.Spec.Template.Spec.Containers[0].Ports = []corev1.ContainerPort{ + { + Name: string(feastType), + ContainerPort: RegistryPort, + Protocol: corev1.ProtocolTCP, + }, + } + probeHandler := corev1.ProbeHandler{ + TCPSocket: &corev1.TCPSocketAction{ + Port: intstr.FromInt(int(RegistryPort)), + }, + } + deploy.Spec.Template.Spec.Containers[0].LivenessProbe = &corev1.Probe{ + ProbeHandler: probeHandler, + InitialDelaySeconds: 30, + PeriodSeconds: 30, + } + deploy.Spec.Template.Spec.Containers[0].ReadinessProbe = &corev1.Probe{ + ProbeHandler: probeHandler, + InitialDelaySeconds: 20, + PeriodSeconds: 10, + } + } + return controllerutil.SetControllerReference(feast.FeatureStore, deploy, feast.Scheme) +} + +func (feast *FeastServices) setService(svc *corev1.Service, feastType FeastServiceType) error { + svc.Labels = feast.getLabels(feastType) + svc.Spec = corev1.ServiceSpec{ + Selector: svc.GetLabels(), + Type: corev1.ServiceTypeClusterIP, + } + if feastType == RegistryFeastType { + svc.Spec.Ports = []corev1.ServicePort{ + { + Name: "http", + Port: int32(80), + Protocol: corev1.ProtocolTCP, + TargetPort: intstr.FromInt(int(RegistryPort)), + }, + } + feast.FeatureStore.Status.ServiceUrls.Registry = svc.Name + "." + svc.Namespace + ".svc.cluster.local:80" + } + return controllerutil.SetControllerReference(feast.FeatureStore, svc, feast.Scheme) +} + +// GetObjectMeta returns the feast k8s object metadata +func (feast *FeastServices) GetObjectMeta(feastType FeastServiceType) metav1.ObjectMeta { + return metav1.ObjectMeta{Name: feast.GetFeastServiceName(feastType), Namespace: feast.FeatureStore.Namespace} +} + +func (feast *FeastServices) getLabels(feastType FeastServiceType) map[string]string { + return map[string]string{ + feastdevv1alpha1.GroupVersion.Group + "/name": feast.FeatureStore.Name, + feastdevv1alpha1.GroupVersion.Group + "/service-type": string(feastType), + } +} + +func (feast *FeastServices) getFeastName() string { + return FeastPrefix + feast.FeatureStore.Name +} + +// GetFeastServiceName returns the feast service object name based on service type +func (feast *FeastServices) GetFeastServiceName(feastType FeastServiceType) string { + return feast.getFeastName() + "-" + string(feastType) +} + +// GetServiceFeatureStoreYamlBase64 returns a base64 encoded feature_store.yaml config for the feast service +func (feast *FeastServices) GetServiceFeatureStoreYamlBase64() (string, error) { + fsYaml, err := feast.getServiceFeatureStoreYaml() + if err != nil { + return "", err + } + return base64.StdEncoding.EncodeToString(fsYaml), nil +} + +func (feast *FeastServices) getServiceFeatureStoreYaml() ([]byte, error) { + return yaml.Marshal(feast.getServiceRepoConfig()) +} + +func (feast *FeastServices) getServiceRepoConfig() RepoConfig { + appliedSpec := feast.FeatureStore.Status.Applied + return RepoConfig{ + Project: appliedSpec.FeastProject, + Provider: LocalProviderType, + EntityKeySerializationVersion: feastdevv1alpha1.SerializationVersion, + Registry: RegistryConfig{ + RegistryType: RegistryFileConfigType, + Path: LocalRegistryPath, + }, + } +} diff --git a/infra/feast-operator/internal/controller/services/services_types.go b/infra/feast-operator/internal/controller/services/services_types.go new file mode 100644 index 00000000000..e1a318a3943 --- /dev/null +++ b/infra/feast-operator/internal/controller/services/services_types.go @@ -0,0 +1,72 @@ +/* +Copyright 2024 Feast Community. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package services + +import ( + "context" + + feastdevv1alpha1 "github.com/feast-dev/feast/infra/feast-operator/api/v1alpha1" + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +const ( + FeastPrefix = "feast-" + FeatureStoreYamlEnvVar = "FEATURE_STORE_YAML_BASE64" + RegistryPort = int32(6570) + LocalRegistryPath = "/tmp/registry.db" + + RegistryFeastType FeastServiceType = "registry" + ClientFeastType FeastServiceType = "client" + + RegistryRemoteConfigType RegistryConfigType = "remote" + RegistryFileConfigType RegistryConfigType = "file" + + LocalProviderType FeastProviderType = "local" +) + +// FeastServiceType is the type of feast service +type FeastServiceType string + +// RegistryConfigType provider name or a class name that implements Registry +type RegistryConfigType string + +// FeastProviderType defines an implementation of a feature store object +type FeastProviderType string + +// FeastServices is an interface for configuring and deploying feast services +type FeastServices struct { + client.Client + Context context.Context + Scheme *runtime.Scheme + FeatureStore *feastdevv1alpha1.FeatureStore +} + +// RepoConfig is the Repo config. Typically loaded from feature_store.yaml. +// https://rtd.feast.dev/en/stable/#feast.repo_config.RepoConfig +type RepoConfig struct { + Project string `yaml:"project,omitempty"` + Provider FeastProviderType `yaml:"provider,omitempty"` + Registry RegistryConfig `yaml:"registry,omitempty"` + EntityKeySerializationVersion int `yaml:"entity_key_serialization_version,omitempty"` +} + +// RegistryConfig is the configuration that relates to reading from and writing to the Feast registry. +type RegistryConfig struct { + Path string `yaml:"path,omitempty"` + RegistryType RegistryConfigType `yaml:"registry_type,omitempty"` +} diff --git a/infra/scripts/release/files_to_bump.txt b/infra/scripts/release/files_to_bump.txt index 4e708aa8f93..e1731ae8770 100644 --- a/infra/scripts/release/files_to_bump.txt +++ b/infra/scripts/release/files_to_bump.txt @@ -14,6 +14,6 @@ infra/feast-helm-operator/Makefile 6 infra/feast-helm-operator/config/manager/kustomization.yaml 8 infra/feast-operator/Makefile 6 infra/feast-operator/config/manager/kustomization.yaml 8 -infra/feast-operator/dist/install.yaml 352 +infra/feast-operator/api/feastversion/feastversion.go 20 java/pom.xml 38 ui/package.json 3 From 138176e117edc8d8e327cc19a716aba44d9a475a Mon Sep 17 00:00:00 2001 From: Tommy Hughes IV Date: Tue, 29 Oct 2024 08:43:26 -0500 Subject: [PATCH 09/65] chore: Operator-specific release CI (#4709) Add Operator to release CI Signed-off-by: Tommy Hughes --- .github/workflows/release.yml | 9 ++++++++- .../manifests/feast-operator.clusterserviceversion.yaml | 8 ++++---- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index aaa3f48b512..8b0eccd9a92 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -88,7 +88,14 @@ jobs: run: ./infra/scripts/helm/validate-helm-chart-publish.sh - name: Validate all version consistency run: ./infra/scripts/helm/validate-helm-chart-versions.sh $NEXT_VERSION - + - name: Install Go + uses: actions/setup-go@v2 + with: + go-version: 1.21.x + - name: Build & version operator-specific release files + run: | + cd infra/feast-operator/ + make build-installer bundle publish-web-ui-npm: needs: [validate_version_bumps, get_dry_release_versions] diff --git a/infra/feast-operator/bundle/manifests/feast-operator.clusterserviceversion.yaml b/infra/feast-operator/bundle/manifests/feast-operator.clusterserviceversion.yaml index 5b989cc2a7f..2a272660ec2 100644 --- a/infra/feast-operator/bundle/manifests/feast-operator.clusterserviceversion.yaml +++ b/infra/feast-operator/bundle/manifests/feast-operator.clusterserviceversion.yaml @@ -16,10 +16,10 @@ metadata: } ] capabilities: Basic Install - createdAt: "2024-10-21T16:35:01Z" + createdAt: "2024-10-28T15:51:17Z" operators.operatorframework.io/builder: operator-sdk-v1.37.0 operators.operatorframework.io/project_layout: go.kubebuilder.io/v4 - name: feast-operator.v0.40.0 + name: feast-operator.v0.41.0 namespace: placeholder spec: apiservicedefinitions: {} @@ -150,7 +150,7 @@ spec: - --leader-elect command: - /manager - image: feastdev/feast-operator:0.40.0 + image: feastdev/feast-operator:0.41.0 livenessProbe: httpGet: path: /healthz @@ -239,4 +239,4 @@ spec: provider: name: Feast Community url: https://lf-aidata.atlassian.net/wiki/spaces/FEAST/ - version: 0.40.0 + version: 0.41.0 From 53730ef2f6b76e47f18fdcf609dccf08c5c8a1f1 Mon Sep 17 00:00:00 2001 From: Harri Lehtola <1781172+peruukki@users.noreply.github.com> Date: Tue, 29 Oct 2024 20:09:22 -0700 Subject: [PATCH 10/65] chore: Eject Create React App dependency from Feast UI (#4681) * chore: Eject create-react-app with `yarn eject` in /ui Signed-off-by: Harri Lehtola * chore: Move most eject dependencies to devDependencies in /ui Signed-off-by: Harri Lehtola * chore: Remove duplicate babel config added when ejecting in /ui The presets are set in .babelrc.js. Signed-off-by: Harri Lehtola * chore: Upgrade dependencies with vulnerabilities in /ui These two dependencies can now be upgraded after ejecting. Signed-off-by: Harri Lehtola * chore: Extract Jest configuration from package.json in /ui This keeps package.json cleaner. Signed-off-by: Harri Lehtola * chore: Extract ESLint configuration from package.json in /ui Signed-off-by: Harri Lehtola * chore: Remove unused tailwindcss added when ejecting in /ui Signed-off-by: Harri Lehtola * chore: Disable strict mode warnings in scripts and config in /ui Strict mode is implicit in modules, so ESLint gave warnings about using them in the scripts. However, the scripts are run directly with `node`, so they are not used as modules. But setting their sourceType to "script" results in ESLint errors saying "Strict mode is not permitted." I couldn't figure out why, so let's just disable the rule for the scripts. Later I noticed the same warning in the config directory, so let's include that too. Signed-off-by: Harri Lehtola --------- Signed-off-by: Harri Lehtola --- ui/.eslintrc.js | 10 + ui/config/env.js | 104 + ui/config/getHttpsConfig.js | 66 + ui/config/jest/babelTransform.js | 29 + ui/config/jest/cssTransform.js | 14 + ui/config/jest/fileTransform.js | 40 + ui/config/modules.js | 134 ++ ui/config/paths.js | 77 + ui/config/webpack.config.js | 755 ++++++++ .../persistentCache/createEnvironmentHash.js | 9 + ui/config/webpackDevServer.config.js | 127 ++ ui/jest.config.js | 44 + ui/package.json | 66 +- ui/scripts/build.js | 217 +++ ui/scripts/start.js | 154 ++ ui/scripts/test.js | 52 + ui/src/react-app-env.d.ts | 72 +- ui/yarn.lock | 1696 ++++++++++------- 18 files changed, 3002 insertions(+), 664 deletions(-) create mode 100644 ui/.eslintrc.js create mode 100644 ui/config/env.js create mode 100644 ui/config/getHttpsConfig.js create mode 100644 ui/config/jest/babelTransform.js create mode 100644 ui/config/jest/cssTransform.js create mode 100644 ui/config/jest/fileTransform.js create mode 100644 ui/config/modules.js create mode 100644 ui/config/paths.js create mode 100644 ui/config/webpack.config.js create mode 100644 ui/config/webpack/persistentCache/createEnvironmentHash.js create mode 100644 ui/config/webpackDevServer.config.js create mode 100644 ui/jest.config.js create mode 100644 ui/scripts/build.js create mode 100644 ui/scripts/start.js create mode 100644 ui/scripts/test.js diff --git a/ui/.eslintrc.js b/ui/.eslintrc.js new file mode 100644 index 00000000000..b96552de9a6 --- /dev/null +++ b/ui/.eslintrc.js @@ -0,0 +1,10 @@ +module.exports = { + extends: ["react-app", "react-app/jest"], + overrides: [ + { + files: ["./scripts/**", "./config/**"], + parserOptions: { sourceType: "script" }, + rules: { strict: "off" }, + }, + ], +}; diff --git a/ui/config/env.js b/ui/config/env.js new file mode 100644 index 00000000000..ffa7e496aac --- /dev/null +++ b/ui/config/env.js @@ -0,0 +1,104 @@ +'use strict'; + +const fs = require('fs'); +const path = require('path'); +const paths = require('./paths'); + +// Make sure that including paths.js after env.js will read .env variables. +delete require.cache[require.resolve('./paths')]; + +const NODE_ENV = process.env.NODE_ENV; +if (!NODE_ENV) { + throw new Error( + 'The NODE_ENV environment variable is required but was not specified.' + ); +} + +// https://github.com/bkeepers/dotenv#what-other-env-files-can-i-use +const dotenvFiles = [ + `${paths.dotenv}.${NODE_ENV}.local`, + // Don't include `.env.local` for `test` environment + // since normally you expect tests to produce the same + // results for everyone + NODE_ENV !== 'test' && `${paths.dotenv}.local`, + `${paths.dotenv}.${NODE_ENV}`, + paths.dotenv, +].filter(Boolean); + +// Load environment variables from .env* files. Suppress warnings using silent +// if this file is missing. dotenv will never modify any environment variables +// that have already been set. Variable expansion is supported in .env files. +// https://github.com/motdotla/dotenv +// https://github.com/motdotla/dotenv-expand +dotenvFiles.forEach(dotenvFile => { + if (fs.existsSync(dotenvFile)) { + require('dotenv-expand')( + require('dotenv').config({ + path: dotenvFile, + }) + ); + } +}); + +// We support resolving modules according to `NODE_PATH`. +// This lets you use absolute paths in imports inside large monorepos: +// https://github.com/facebook/create-react-app/issues/253. +// It works similar to `NODE_PATH` in Node itself: +// https://nodejs.org/api/modules.html#modules_loading_from_the_global_folders +// Note that unlike in Node, only *relative* paths from `NODE_PATH` are honored. +// Otherwise, we risk importing Node.js core modules into an app instead of webpack shims. +// https://github.com/facebook/create-react-app/issues/1023#issuecomment-265344421 +// We also resolve them to make sure all tools using them work consistently. +const appDirectory = fs.realpathSync(process.cwd()); +process.env.NODE_PATH = (process.env.NODE_PATH || '') + .split(path.delimiter) + .filter(folder => folder && !path.isAbsolute(folder)) + .map(folder => path.resolve(appDirectory, folder)) + .join(path.delimiter); + +// Grab NODE_ENV and REACT_APP_* environment variables and prepare them to be +// injected into the application via DefinePlugin in webpack configuration. +const REACT_APP = /^REACT_APP_/i; + +function getClientEnvironment(publicUrl) { + const raw = Object.keys(process.env) + .filter(key => REACT_APP.test(key)) + .reduce( + (env, key) => { + env[key] = process.env[key]; + return env; + }, + { + // Useful for determining whether we’re running in production mode. + // Most importantly, it switches React into the correct mode. + NODE_ENV: process.env.NODE_ENV || 'development', + // Useful for resolving the correct path to static assets in `public`. + // For example, . + // This should only be used as an escape hatch. Normally you would put + // images into the `src` and `import` them in code to get their paths. + PUBLIC_URL: publicUrl, + // We support configuring the sockjs pathname during development. + // These settings let a developer run multiple simultaneous projects. + // They are used as the connection `hostname`, `pathname` and `port` + // in webpackHotDevClient. They are used as the `sockHost`, `sockPath` + // and `sockPort` options in webpack-dev-server. + WDS_SOCKET_HOST: process.env.WDS_SOCKET_HOST, + WDS_SOCKET_PATH: process.env.WDS_SOCKET_PATH, + WDS_SOCKET_PORT: process.env.WDS_SOCKET_PORT, + // Whether or not react-refresh is enabled. + // It is defined here so it is available in the webpackHotDevClient. + FAST_REFRESH: process.env.FAST_REFRESH !== 'false', + } + ); + // Stringify all values so we can feed into webpack DefinePlugin + const stringified = { + 'process.env': Object.keys(raw).reduce((env, key) => { + env[key] = JSON.stringify(raw[key]); + return env; + }, {}), + }; + + return { raw, stringified }; +} + +module.exports = getClientEnvironment; diff --git a/ui/config/getHttpsConfig.js b/ui/config/getHttpsConfig.js new file mode 100644 index 00000000000..013d493c1bb --- /dev/null +++ b/ui/config/getHttpsConfig.js @@ -0,0 +1,66 @@ +'use strict'; + +const fs = require('fs'); +const path = require('path'); +const crypto = require('crypto'); +const chalk = require('react-dev-utils/chalk'); +const paths = require('./paths'); + +// Ensure the certificate and key provided are valid and if not +// throw an easy to debug error +function validateKeyAndCerts({ cert, key, keyFile, crtFile }) { + let encrypted; + try { + // publicEncrypt will throw an error with an invalid cert + encrypted = crypto.publicEncrypt(cert, Buffer.from('test')); + } catch (err) { + throw new Error( + `The certificate "${chalk.yellow(crtFile)}" is invalid.\n${err.message}` + ); + } + + try { + // privateDecrypt will throw an error with an invalid key + crypto.privateDecrypt(key, encrypted); + } catch (err) { + throw new Error( + `The certificate key "${chalk.yellow(keyFile)}" is invalid.\n${ + err.message + }` + ); + } +} + +// Read file and throw an error if it doesn't exist +function readEnvFile(file, type) { + if (!fs.existsSync(file)) { + throw new Error( + `You specified ${chalk.cyan( + type + )} in your env, but the file "${chalk.yellow(file)}" can't be found.` + ); + } + return fs.readFileSync(file); +} + +// Get the https config +// Return cert files if provided in env, otherwise just true or false +function getHttpsConfig() { + const { SSL_CRT_FILE, SSL_KEY_FILE, HTTPS } = process.env; + const isHttps = HTTPS === 'true'; + + if (isHttps && SSL_CRT_FILE && SSL_KEY_FILE) { + const crtFile = path.resolve(paths.appPath, SSL_CRT_FILE); + const keyFile = path.resolve(paths.appPath, SSL_KEY_FILE); + const config = { + cert: readEnvFile(crtFile, 'SSL_CRT_FILE'), + key: readEnvFile(keyFile, 'SSL_KEY_FILE'), + }; + + validateKeyAndCerts({ ...config, keyFile, crtFile }); + return config; + } + return isHttps; +} + +module.exports = getHttpsConfig; diff --git a/ui/config/jest/babelTransform.js b/ui/config/jest/babelTransform.js new file mode 100644 index 00000000000..5b391e40556 --- /dev/null +++ b/ui/config/jest/babelTransform.js @@ -0,0 +1,29 @@ +'use strict'; + +const babelJest = require('babel-jest').default; + +const hasJsxRuntime = (() => { + if (process.env.DISABLE_NEW_JSX_TRANSFORM === 'true') { + return false; + } + + try { + require.resolve('react/jsx-runtime'); + return true; + } catch (e) { + return false; + } +})(); + +module.exports = babelJest.createTransformer({ + presets: [ + [ + require.resolve('babel-preset-react-app'), + { + runtime: hasJsxRuntime ? 'automatic' : 'classic', + }, + ], + ], + babelrc: false, + configFile: false, +}); diff --git a/ui/config/jest/cssTransform.js b/ui/config/jest/cssTransform.js new file mode 100644 index 00000000000..8f65114812a --- /dev/null +++ b/ui/config/jest/cssTransform.js @@ -0,0 +1,14 @@ +'use strict'; + +// This is a custom Jest transformer turning style imports into empty objects. +// http://facebook.github.io/jest/docs/en/webpack.html + +module.exports = { + process() { + return 'module.exports = {};'; + }, + getCacheKey() { + // The output is always the same. + return 'cssTransform'; + }, +}; diff --git a/ui/config/jest/fileTransform.js b/ui/config/jest/fileTransform.js new file mode 100644 index 00000000000..aab67618c38 --- /dev/null +++ b/ui/config/jest/fileTransform.js @@ -0,0 +1,40 @@ +'use strict'; + +const path = require('path'); +const camelcase = require('camelcase'); + +// This is a custom Jest transformer turning file imports into filenames. +// http://facebook.github.io/jest/docs/en/webpack.html + +module.exports = { + process(src, filename) { + const assetFilename = JSON.stringify(path.basename(filename)); + + if (filename.match(/\.svg$/)) { + // Based on how SVGR generates a component name: + // https://github.com/smooth-code/svgr/blob/01b194cf967347d43d4cbe6b434404731b87cf27/packages/core/src/state.js#L6 + const pascalCaseFilename = camelcase(path.parse(filename).name, { + pascalCase: true, + }); + const componentName = `Svg${pascalCaseFilename}`; + return `const React = require('react'); + module.exports = { + __esModule: true, + default: ${assetFilename}, + ReactComponent: React.forwardRef(function ${componentName}(props, ref) { + return { + $$typeof: Symbol.for('react.element'), + type: 'svg', + ref: ref, + key: null, + props: Object.assign({}, props, { + children: ${assetFilename} + }) + }; + }), + };`; + } + + return `module.exports = ${assetFilename};`; + }, +}; diff --git a/ui/config/modules.js b/ui/config/modules.js new file mode 100644 index 00000000000..d63e41d78dc --- /dev/null +++ b/ui/config/modules.js @@ -0,0 +1,134 @@ +'use strict'; + +const fs = require('fs'); +const path = require('path'); +const paths = require('./paths'); +const chalk = require('react-dev-utils/chalk'); +const resolve = require('resolve'); + +/** + * Get additional module paths based on the baseUrl of a compilerOptions object. + * + * @param {Object} options + */ +function getAdditionalModulePaths(options = {}) { + const baseUrl = options.baseUrl; + + if (!baseUrl) { + return ''; + } + + const baseUrlResolved = path.resolve(paths.appPath, baseUrl); + + // We don't need to do anything if `baseUrl` is set to `node_modules`. This is + // the default behavior. + if (path.relative(paths.appNodeModules, baseUrlResolved) === '') { + return null; + } + + // Allow the user set the `baseUrl` to `appSrc`. + if (path.relative(paths.appSrc, baseUrlResolved) === '') { + return [paths.appSrc]; + } + + // If the path is equal to the root directory we ignore it here. + // We don't want to allow importing from the root directly as source files are + // not transpiled outside of `src`. We do allow importing them with the + // absolute path (e.g. `src/Components/Button.js`) but we set that up with + // an alias. + if (path.relative(paths.appPath, baseUrlResolved) === '') { + return null; + } + + // Otherwise, throw an error. + throw new Error( + chalk.red.bold( + "Your project's `baseUrl` can only be set to `src` or `node_modules`." + + ' Create React App does not support other values at this time.' + ) + ); +} + +/** + * Get webpack aliases based on the baseUrl of a compilerOptions object. + * + * @param {*} options + */ +function getWebpackAliases(options = {}) { + const baseUrl = options.baseUrl; + + if (!baseUrl) { + return {}; + } + + const baseUrlResolved = path.resolve(paths.appPath, baseUrl); + + if (path.relative(paths.appPath, baseUrlResolved) === '') { + return { + src: paths.appSrc, + }; + } +} + +/** + * Get jest aliases based on the baseUrl of a compilerOptions object. + * + * @param {*} options + */ +function getJestAliases(options = {}) { + const baseUrl = options.baseUrl; + + if (!baseUrl) { + return {}; + } + + const baseUrlResolved = path.resolve(paths.appPath, baseUrl); + + if (path.relative(paths.appPath, baseUrlResolved) === '') { + return { + '^src/(.*)$': '/src/$1', + }; + } +} + +function getModules() { + // Check if TypeScript is setup + const hasTsConfig = fs.existsSync(paths.appTsConfig); + const hasJsConfig = fs.existsSync(paths.appJsConfig); + + if (hasTsConfig && hasJsConfig) { + throw new Error( + 'You have both a tsconfig.json and a jsconfig.json. If you are using TypeScript please remove your jsconfig.json file.' + ); + } + + let config; + + // If there's a tsconfig.json we assume it's a + // TypeScript project and set up the config + // based on tsconfig.json + if (hasTsConfig) { + const ts = require(resolve.sync('typescript', { + basedir: paths.appNodeModules, + })); + config = ts.readConfigFile(paths.appTsConfig, ts.sys.readFile).config; + // Otherwise we'll check if there is jsconfig.json + // for non TS projects. + } else if (hasJsConfig) { + config = require(paths.appJsConfig); + } + + config = config || {}; + const options = config.compilerOptions || {}; + + const additionalModulePaths = getAdditionalModulePaths(options); + + return { + additionalModulePaths: additionalModulePaths, + webpackAliases: getWebpackAliases(options), + jestAliases: getJestAliases(options), + hasTsConfig, + }; +} + +module.exports = getModules(); diff --git a/ui/config/paths.js b/ui/config/paths.js new file mode 100644 index 00000000000..f0a6cd9c986 --- /dev/null +++ b/ui/config/paths.js @@ -0,0 +1,77 @@ +'use strict'; + +const path = require('path'); +const fs = require('fs'); +const getPublicUrlOrPath = require('react-dev-utils/getPublicUrlOrPath'); + +// Make sure any symlinks in the project folder are resolved: +// https://github.com/facebook/create-react-app/issues/637 +const appDirectory = fs.realpathSync(process.cwd()); +const resolveApp = relativePath => path.resolve(appDirectory, relativePath); + +// We use `PUBLIC_URL` environment variable or "homepage" field to infer +// "public path" at which the app is served. +// webpack needs to know it to put the right