diff --git a/docs/getting-started/quickstart.md b/docs/getting-started/quickstart.md index a83897005fd..184167ebe29 100644 --- a/docs/getting-started/quickstart.md +++ b/docs/getting-started/quickstart.md @@ -113,7 +113,7 @@ provider: local online_store: type: sqlite path: data/online_store.db -entity_key_serialization_version: 2 +entity_key_serialization_version: 3 ``` {% endtab %} diff --git a/docs/how-to-guides/entity-reserialization-of-from-v2-to-v3.md b/docs/how-to-guides/entity-reserialization-of-from-v2-to-v3.md index dc43e481a96..8135c350fd7 100644 --- a/docs/how-to-guides/entity-reserialization-of-from-v2-to-v3.md +++ b/docs/how-to-guides/entity-reserialization-of-from-v2-to-v3.md @@ -1,8 +1,8 @@ -# Entity Key Re-Serialization from Version 2 to 3 +# Entity Key Serialization from Version 2 to Version 3 -Entity Key Serialization version 2 will soon be deprecated, hence we need to shift the serialization and deserilization to version 3. +Entity Key Serialization version 2 is now deprecated. All new and existing Feast deployments should shift to using serialization version 3. -But here comes the challegnge where existing FeatuteViews on stores has written features with version 2. A version 2 serialized entity key cant be retrived using version 3 deserilization algorithm. +However, a challenge arises when existing FeatureViews in online or offline stores have written features with version 2. A version 2 serialized entity key cannot be retrieved using the version 3 deserialization algorithm. ## Reserialize the Feature Views entity Keys to version 3 @@ -10,7 +10,7 @@ The solution is to reserialize the entity keys from version 2 to version 3. Follow the following procedures to reserialize the entity key to version 3 in feature View in an offline / online store. -In hosrt, you need to iterate through all the feature views in your Feast repository, retrieve their serialized entity keys (if stored in version 2), reserialize them to version 3, and then update the online/offline store or wherever the serialized keys are stored. +In short, you need to iterate through all the feature views in your Feast repository, retrieve their serialized entity keys (if stored in version 2), reserialize them to version 3, and then update the online/offline store or wherever the serialized keys are stored. ### 1. Initialize the Feature Store diff --git a/docs/how-to-guides/running-feast-in-production.md b/docs/how-to-guides/running-feast-in-production.md index 7aeb9e96650..65ab2c82d9d 100644 --- a/docs/how-to-guides/running-feast-in-production.md +++ b/docs/how-to-guides/running-feast-in-production.md @@ -88,7 +88,7 @@ def materialize(data_interval_start=None, data_interval_end=None): provider="aws", offline_store="file", online_store=DynamoDBOnlineStoreConfig(region="us-west-2"), - entity_key_serialization_version=2 + entity_key_serialization_version=3 ) store = FeatureStore(config=repo_config) # Option 1: materialize just one feature view diff --git a/docs/how-to-guides/starting-feast-servers-tls-mode.md b/docs/how-to-guides/starting-feast-servers-tls-mode.md index a868e17cf96..ffc7e5d9e90 100644 --- a/docs/how-to-guides/starting-feast-servers-tls-mode.md +++ b/docs/how-to-guides/starting-feast-servers-tls-mode.md @@ -83,7 +83,7 @@ online_store: path: http://localhost:6566 type: remote cert: /path/to/cert.pem -entity_key_serialization_version: 2 +entity_key_serialization_version: 3 auth: type: no_auth ``` @@ -121,7 +121,7 @@ online_store: path: http://localhost:6566 type: remote cert: /path/to/cert.pem -entity_key_serialization_version: 2 +entity_key_serialization_version: 3 auth: type: no_auth ``` @@ -161,7 +161,7 @@ online_store: path: http://localhost:6566 type: remote cert: /path/to/cert.pem -entity_key_serialization_version: 2 +entity_key_serialization_version: 3 offline_store: type: remote host: localhost diff --git a/docs/reference/batch-materialization/spark.md b/docs/reference/batch-materialization/spark.md index 27a1388c48e..7a61c0ccb6d 100644 --- a/docs/reference/batch-materialization/spark.md +++ b/docs/reference/batch-materialization/spark.md @@ -47,7 +47,7 @@ repo_config = RepoConfig( "partitions": 10 }, online_store=DynamoDBOnlineStoreConfig(region="us-west-1"), - entity_key_serialization_version=2 + entity_key_serialization_version=3 ) store = FeatureStore(config=repo_config) diff --git a/docs/reference/feast-cli-commands.md b/docs/reference/feast-cli-commands.md index 712df18a6b6..71e0ab3c76d 100644 --- a/docs/reference/feast-cli-commands.md +++ b/docs/reference/feast-cli-commands.md @@ -79,7 +79,7 @@ online_store: path: data/online_store.db offline_store: type: dask -entity_key_serialization_version: 2 +entity_key_serialization_version: 3 auth: type: no_auth ``` diff --git a/docs/reference/online-stores/couchbase.md b/docs/reference/online-stores/couchbase.md index 2878deb97ee..99bdd710793 100644 --- a/docs/reference/online-stores/couchbase.md +++ b/docs/reference/online-stores/couchbase.md @@ -44,7 +44,7 @@ online_store: password: password # Couchbase password from access credentials bucket_name: feast # Couchbase bucket name, defaults to feast kv_port: 11210 # Couchbase key-value port, defaults to 11210. Required if custom ports are used. -entity_key_serialization_version: 2 +entity_key_serialization_version: 3 ``` {% endcode %} diff --git a/docs/reference/online-stores/remote.md b/docs/reference/online-stores/remote.md index aa97a495baa..b6734ccc1ec 100644 --- a/docs/reference/online-stores/remote.md +++ b/docs/reference/online-stores/remote.md @@ -17,7 +17,7 @@ online_store: path: http://localhost:6566 type: remote cert: /path/to/cert.pem -entity_key_serialization_version: 2 +entity_key_serialization_version: 3 auth: type: no_auth ``` diff --git a/examples/credit-risk-end-to-end/02_Deploying_the_Feature_Store.ipynb b/examples/credit-risk-end-to-end/02_Deploying_the_Feature_Store.ipynb index f736cdaed93..6379ba46421 100644 --- a/examples/credit-risk-end-to-end/02_Deploying_the_Feature_Store.ipynb +++ b/examples/credit-risk-end-to-end/02_Deploying_the_Feature_Store.ipynb @@ -62,7 +62,7 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": 2, "id": "8bd21689-4a8e-4b0c-937d-0911df9db1d3", "metadata": {}, "outputs": [], @@ -102,7 +102,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 6, "id": "b3757221-2037-49eb-867f-b9529fec06e2", "metadata": {}, "outputs": [ @@ -110,7 +110,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "Writing Feature_Store/feature_store.yaml\n" + "Overwriting Feature_Store/feature_store.yaml\n" ] } ], @@ -125,7 +125,7 @@ "online_store:\n", " type: sqlite\n", " path: data/online_store.db\n", - "entity_key_serialization_version: 2" + "entity_key_serialization_version: 3" ] }, { @@ -383,10 +383,18 @@ "id": "c764a60a-b911-41a8-ba8f-7ef0a0bc7257", "metadata": {}, "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/Users/jyejare/gitrepos/feast/.venv/lib/python3.11/site-packages/pydantic/_internal/_fields.py:192: UserWarning: Field name \"vector_enabled\" in \"SqliteOnlineStoreConfig\" shadows an attribute in parent \"VectorStoreConfig\"\n", + " warnings.warn(\n" + ] + }, { "data": { "text/plain": [ - "RepoConfig(project='loan_applications', provider='local', registry_config='data/registry.db', online_config={'type': 'sqlite', 'path': 'data/online_store.db'}, offline_config={'type': 'dask'}, batch_engine_config='local', feature_server=None, flags=None, repo_path=PosixPath('Feature_Store'), entity_key_serialization_version=2, coerce_tz_aware=True)" + "RepoConfig(project='loan_applications', provider='local', registry_config='data/registry.db', online_config={'type': 'sqlite', 'path': 'data/online_store.db'}, auth={'type': 'no_auth'}, offline_config={'type': 'dask'}, batch_engine_config='local', feature_server=None, flags=None, repo_path=PosixPath('Feature_Store'), entity_key_serialization_version=3, coerce_tz_aware=True)" ] }, "execution_count": 7, @@ -779,7 +787,7 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3 (ipykernel)", + "display_name": ".venv", "language": "python", "name": "python3" }, @@ -793,7 +801,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.11.9" + "version": "3.11.13" } }, "nbformat": 4, diff --git a/examples/credit-risk-end-to-end/03_Credit_Risk_Model_Training.ipynb b/examples/credit-risk-end-to-end/03_Credit_Risk_Model_Training.ipynb index ca0d0e29d95..c7148b89c30 100644 --- a/examples/credit-risk-end-to-end/03_Credit_Risk_Model_Training.ipynb +++ b/examples/credit-risk-end-to-end/03_Credit_Risk_Model_Training.ipynb @@ -256,7 +256,7 @@ " \"host\": \"localhost\",\n", " \"port\": 8815\n", " },\n", - " entity_key_serialization_version=2\n", + " entity_key_serialization_version=3\n", "))" ] }, diff --git a/examples/java-demo/README.md b/examples/java-demo/README.md index 0ae085e0a7a..a0bde89f2ba 100644 --- a/examples/java-demo/README.md +++ b/examples/java-demo/README.md @@ -1,4 +1,3 @@ - # Running Feast Java Server with Redis & calling with python (with registry in GCP) For this tutorial, we setup Feast with Redis, using the Feast CLI to register and materialize features, and then retrieving via a Feast Java server deployed in Kubernetes via a gRPC call. @@ -40,7 +39,7 @@ For this tutorial, we setup Feast with Redis, using the Feast CLI to register an connection_string: localhost:6379,password=[YOUR PASSWORD] offline_store: type: file - entity_key_serialization_version: 2 + entity_key_serialization_version: 3 ``` 4. Run `feast apply` to apply your local features to the remote registry - Note: you may need to authenticate to gcloud first with `gcloud auth login` @@ -86,7 +85,7 @@ For this tutorial, we setup Feast with Redis, using the Feast CLI to register an >2. `make build-java-docker-dev` >3. In the `application-override.yaml`, uncomment the two `image: tag: dev` blocks >4. `helm install feast-release ../../../infra/charts/feast --values application-override.yaml` -5. (Optional): check logs of the server to make sure it’s working +5. (Optional): check logs of the server to make sure it's working ```bash kubectl logs svc/feast-release-feature-server ``` diff --git a/examples/java-demo/feature_repo/application-override.yaml b/examples/java-demo/feature_repo/application-override.yaml index caaa5411e2f..0db26a9dd39 100644 --- a/examples/java-demo/feature_repo/application-override.yaml +++ b/examples/java-demo/feature_repo/application-override.yaml @@ -10,7 +10,7 @@ feature-server: host: my-redis-master port: 6379 password: [YOUR PASSWORD] - entityKeySerializationVersion: 2 + entityKeySerializationVersion: 3 # Uncomment below for dev # image: # tag: dev diff --git a/examples/java-demo/feature_repo/feature_store.yaml b/examples/java-demo/feature_repo/feature_store.yaml index 16d426fc5a8..0928f41e07a 100644 --- a/examples/java-demo/feature_repo/feature_store.yaml +++ b/examples/java-demo/feature_repo/feature_store.yaml @@ -7,4 +7,4 @@ online_store: connection_string: localhost:6379,password=[YOUR PASSWORD] offline_store: type: file -entity_key_serialization_version: 2 \ No newline at end of file +entity_key_serialization_version: 3 \ No newline at end of file diff --git a/examples/kind-quickstart/client/feature_store.yaml b/examples/kind-quickstart/client/feature_store.yaml index 62acd3ead66..2671f525e4f 100644 --- a/examples/kind-quickstart/client/feature_store.yaml +++ b/examples/kind-quickstart/client/feature_store.yaml @@ -9,6 +9,6 @@ offline_store: online_store: path: http://localhost:8003 type: remote -entity_key_serialization_version: 2 +entity_key_serialization_version: 3 auth: type: no_auth diff --git a/examples/podman_local/feature_repo/feature_store.yaml b/examples/podman_local/feature_repo/feature_store.yaml index 3e6a3603162..5ad0c0b77e8 100644 --- a/examples/podman_local/feature_repo/feature_store.yaml +++ b/examples/podman_local/feature_repo/feature_store.yaml @@ -6,4 +6,4 @@ provider: local online_store: type: sqlite path: data/online_store.db -entity_key_serialization_version: 2 +entity_key_serialization_version: 3 diff --git a/examples/python-helm-demo/README.md b/examples/python-helm-demo/README.md index 078550ae392..031355f3f49 100644 --- a/examples/python-helm-demo/README.md +++ b/examples/python-helm-demo/README.md @@ -1,4 +1,3 @@ - # Running Feast Python / Go Feature Server with Redis on Kubernetes For this tutorial, we set up Feast with Redis. @@ -64,7 +63,7 @@ Manifests have been taken from [Deploy Minio in your project](https://ai-on-open connection_string: localhost:6379,password=**** offline_store: type: file - entity_key_serialization_version: 2 + entity_key_serialization_version: 3 ``` 1. To run `feast apply` from the current machine we need to define the AWS credentials to connect the MinIO S3 store, which are defined in [minio.env](./minio.env): @@ -142,7 +141,7 @@ are defined in [minio.env](./minio.env): ]' kubectl wait --for=condition=available deployment/online-server --timeout=2m ``` -1. (Optional): check logs of the server to make sure it’s working +1. (Optional): check logs of the server to make sure it's working ```bash kubectl logs svc/online-server ``` diff --git a/examples/python-helm-demo/feature_repo/feature_store.yaml.template b/examples/python-helm-demo/feature_repo/feature_store.yaml.template index 585ba23e63b..81f0fa4083d 100644 --- a/examples/python-helm-demo/feature_repo/feature_store.yaml.template +++ b/examples/python-helm-demo/feature_repo/feature_store.yaml.template @@ -6,4 +6,4 @@ online_store: connection_string: localhost:6379,password=_REDIS_PASSWORD_ offline_store: type: file -entity_key_serialization_version: 2 \ No newline at end of file +entity_key_serialization_version: 3 \ No newline at end of file diff --git a/examples/python-helm-demo/test/feature_store.yaml b/examples/python-helm-demo/test/feature_store.yaml index 13e99873ee7..a81c66f1f7f 100644 --- a/examples/python-helm-demo/test/feature_store.yaml +++ b/examples/python-helm-demo/test/feature_store.yaml @@ -4,4 +4,4 @@ provider: local online_store: path: http://localhost:6566 type: remote -entity_key_serialization_version: 2 \ No newline at end of file +entity_key_serialization_version: 3 \ No newline at end of file diff --git a/examples/rbac-local/client/feature_store.yaml b/examples/rbac-local/client/feature_store.yaml index d428adf6712..971b7098af9 100644 --- a/examples/rbac-local/client/feature_store.yaml +++ b/examples/rbac-local/client/feature_store.yaml @@ -1,4 +1,3 @@ -entity_key_serialization_version: 2 offline_store: host: localhost port: 8815 @@ -10,3 +9,4 @@ project: rbac registry: path: localhost:6570 registry_type: remote +entity_key_serialization_version: 3 \ No newline at end of file diff --git a/examples/rbac-remote/client/oidc/feature_repo/feature_store.yaml b/examples/rbac-remote/client/oidc/feature_repo/feature_store.yaml index 1454e16df92..d36762ce95e 100644 --- a/examples/rbac-remote/client/oidc/feature_repo/feature_store.yaml +++ b/examples/rbac-remote/client/oidc/feature_repo/feature_store.yaml @@ -16,4 +16,4 @@ auth: client_secret: update-this-value username: ${FEAST_USERNAME} password: ${FEAST_PASSWORD} -entity_key_serialization_version: 2 +entity_key_serialization_version: 3 diff --git a/examples/rbac-remote/server/feature_repo/feature_store.yaml b/examples/rbac-remote/server/feature_repo/feature_store.yaml index 78b13c660bf..25267f1bece 100644 --- a/examples/rbac-remote/server/feature_repo/feature_store.yaml +++ b/examples/rbac-remote/server/feature_repo/feature_store.yaml @@ -23,4 +23,4 @@ offline_store: db_schema: public user: feast password: feast -entity_key_serialization_version: 2 +entity_key_serialization_version: 3 diff --git a/examples/rbac-remote/server/k8s/feature_store_offline.yaml b/examples/rbac-remote/server/k8s/feature_store_offline.yaml index 4fc01508bd1..1e68df239f7 100644 --- a/examples/rbac-remote/server/k8s/feature_store_offline.yaml +++ b/examples/rbac-remote/server/k8s/feature_store_offline.yaml @@ -13,4 +13,4 @@ offline_store: password: feast auth: type: kubernetes -entity_key_serialization_version: 2 +entity_key_serialization_version: 3 diff --git a/examples/rbac-remote/server/k8s/feature_store_online.yaml b/examples/rbac-remote/server/k8s/feature_store_online.yaml index aa167731b2e..bd5c8cf85fe 100644 --- a/examples/rbac-remote/server/k8s/feature_store_online.yaml +++ b/examples/rbac-remote/server/k8s/feature_store_online.yaml @@ -17,4 +17,4 @@ offline_store: port: 80 auth: type: kubernetes -entity_key_serialization_version: 2 +entity_key_serialization_version: 3 diff --git a/examples/rbac-remote/server/k8s/feature_store_registry.yaml b/examples/rbac-remote/server/k8s/feature_store_registry.yaml index 579141fb010..6da7ef5d573 100644 --- a/examples/rbac-remote/server/k8s/feature_store_registry.yaml +++ b/examples/rbac-remote/server/k8s/feature_store_registry.yaml @@ -9,4 +9,4 @@ registry: pool_pre_ping: true auth: type: kubernetes -entity_key_serialization_version: 2 +entity_key_serialization_version: 3 diff --git a/examples/rbac-remote/server/oidc/feature_store_offline.yaml b/examples/rbac-remote/server/oidc/feature_store_offline.yaml index 8ed4cc1ff3c..b6fba1a7d49 100644 --- a/examples/rbac-remote/server/oidc/feature_store_offline.yaml +++ b/examples/rbac-remote/server/oidc/feature_store_offline.yaml @@ -15,4 +15,4 @@ auth: type: oidc auth_discovery_url: https://keycloak-feast-dev.apps.com/realms/feast-rbac/.well-known/openid-configuration client_id: feast-client -entity_key_serialization_version: 2 +entity_key_serialization_version: 3 diff --git a/examples/rbac-remote/server/oidc/feature_store_online.yaml b/examples/rbac-remote/server/oidc/feature_store_online.yaml index c47c3a0662c..e9e051d7e7b 100644 --- a/examples/rbac-remote/server/oidc/feature_store_online.yaml +++ b/examples/rbac-remote/server/oidc/feature_store_online.yaml @@ -19,4 +19,4 @@ auth: type: oidc auth_discovery_url: https://keycloak-feast-dev.apps.com/realms/feast-rbac/.well-known/openid-configuration client_id: feast-client -entity_key_serialization_version: 2 +entity_key_serialization_version: 3 diff --git a/examples/rbac-remote/server/oidc/feature_store_registry.yaml b/examples/rbac-remote/server/oidc/feature_store_registry.yaml index a661d9dc566..1ffe9a09225 100644 --- a/examples/rbac-remote/server/oidc/feature_store_registry.yaml +++ b/examples/rbac-remote/server/oidc/feature_store_registry.yaml @@ -11,4 +11,4 @@ auth: type: oidc auth_discovery_url: https://keycloak-feast-dev.apps.com/realms/feast-rbac/.well-known/openid-configuration client_id: feast-client -entity_key_serialization_version: 2 +entity_key_serialization_version: 3 diff --git a/examples/remote-offline-store/offline_client/feature_store.yaml b/examples/remote-offline-store/offline_client/feature_store.yaml index 24ee5d70426..5e916432b7f 100644 --- a/examples/remote-offline-store/offline_client/feature_store.yaml +++ b/examples/remote-offline-store/offline_client/feature_store.yaml @@ -7,4 +7,4 @@ offline_store: type: remote host: localhost port: 8815 -entity_key_serialization_version: 2 +entity_key_serialization_version: 3 diff --git a/examples/remote-offline-store/offline_server/feature_repo/feature_store.yaml b/examples/remote-offline-store/offline_server/feature_repo/feature_store.yaml index a751706d07a..758cf6f460f 100644 --- a/examples/remote-offline-store/offline_server/feature_repo/feature_store.yaml +++ b/examples/remote-offline-store/offline_server/feature_repo/feature_store.yaml @@ -6,4 +6,4 @@ provider: local online_store: type: sqlite path: data/online_store.db -entity_key_serialization_version: 2 +entity_key_serialization_version: 3 diff --git a/examples/rhoai-quickstart/feast-demo-quickstart.ipynb b/examples/rhoai-quickstart/feast-demo-quickstart.ipynb index 18874e462ec..8777798e49f 100644 --- a/examples/rhoai-quickstart/feast-demo-quickstart.ipynb +++ b/examples/rhoai-quickstart/feast-demo-quickstart.ipynb @@ -22,8 +22,8 @@ "output_type": "stream", "text": [ "\n", - "\u001B[1m[\u001B[0m\u001B[34;49mnotice\u001B[0m\u001B[1;39;49m]\u001B[0m\u001B[39;49m A new release of pip available: \u001B[0m\u001B[31;49m22.2.2\u001B[0m\u001B[39;49m -> \u001B[0m\u001B[32;49m24.2\u001B[0m\n", - "\u001B[1m[\u001B[0m\u001B[34;49mnotice\u001B[0m\u001B[1;39;49m]\u001B[0m\u001B[39;49m To update, run: \u001B[0m\u001B[32;49mpip install --upgrade pip\u001B[0m\n", + "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip available: \u001b[0m\u001b[31;49m22.2.2\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m24.2\u001b[0m\n", + "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpip install --upgrade pip\u001b[0m\n", "Note: you may need to restart the kernel to use updated packages.\n", "feast package is installed\n" ] @@ -82,7 +82,7 @@ "output_type": "stream", "text": [ "\n", - "Creating a new Feast repository in \u001B[1m\u001B[32m/opt/app-root/src/feast/examples/rhoai-quickstart/my_feast_project\u001B[0m.\n", + "Creating a new Feast repository in \u001b[1m\u001b[32m/opt/app-root/src/feast/examples/rhoai-quickstart/my_feast_project\u001b[0m.\n", "\n" ] } @@ -463,19 +463,19 @@ "09/24/2024 06:01:41 PM root WARNING: Cannot use sqlite_vec for vector search\n", "09/24/2024 06:01:41 PM root WARNING: Cannot use sqlite_vec for vector search\n", "09/24/2024 06:01:41 PM root WARNING: Cannot use sqlite_vec for vector search\n", - "Created entity \u001B[1m\u001B[32mdriver\u001B[0m\n", - "Created feature view \u001B[1m\u001B[32mdriver_hourly_stats\u001B[0m\n", - "Created feature view \u001B[1m\u001B[32mdriver_hourly_stats_fresh\u001B[0m\n", - "Created on demand feature view \u001B[1m\u001B[32mtransformed_conv_rate\u001B[0m\n", - "Created on demand feature view \u001B[1m\u001B[32mtransformed_conv_rate_fresh\u001B[0m\n", - "Created feature service \u001B[1m\u001B[32mdriver_activity_v2\u001B[0m\n", - "Created feature service \u001B[1m\u001B[32mdriver_activity_v1\u001B[0m\n", - "Created feature service \u001B[1m\u001B[32mdriver_activity_v3\u001B[0m\n", + "Created entity \u001b[1m\u001b[32mdriver\u001b[0m\n", + "Created feature view \u001b[1m\u001b[32mdriver_hourly_stats\u001b[0m\n", + "Created feature view \u001b[1m\u001b[32mdriver_hourly_stats_fresh\u001b[0m\n", + "Created on demand feature view \u001b[1m\u001b[32mtransformed_conv_rate\u001b[0m\n", + "Created on demand feature view \u001b[1m\u001b[32mtransformed_conv_rate_fresh\u001b[0m\n", + "Created feature service \u001b[1m\u001b[32mdriver_activity_v2\u001b[0m\n", + "Created feature service \u001b[1m\u001b[32mdriver_activity_v1\u001b[0m\n", + "Created feature service \u001b[1m\u001b[32mdriver_activity_v3\u001b[0m\n", "\n", "09/24/2024 06:01:41 PM root WARNING: Cannot use sqlite_vec for vector search\n", "09/24/2024 06:01:41 PM root WARNING: Cannot use sqlite_vec for vector search\n", - "Created sqlite table \u001B[1m\u001B[32mmy_feast_project_driver_hourly_stats_fresh\u001B[0m\n", - "Created sqlite table \u001B[1m\u001B[32mmy_feast_project_driver_hourly_stats\u001B[0m\n", + "Created sqlite table \u001b[1m\u001b[32mmy_feast_project_driver_hourly_stats_fresh\u001b[0m\n", + "Created sqlite table \u001b[1m\u001b[32mmy_feast_project_driver_hourly_stats\u001b[0m\n", "\n" ] } @@ -706,12 +706,12 @@ "output_type": "stream", "text": [ "09/24/2024 06:02:09 PM root WARNING: _list_feature_views will make breaking changes. Please use _list_batch_feature_views instead. _list_feature_views will behave like _list_all_feature_views in the future.\n", - "Materializing \u001B[1m\u001B[32m2\u001B[0m feature views to \u001B[1m\u001B[32m2024-09-24 18:02:06+00:00\u001B[0m into the \u001B[1m\u001B[32msqlite\u001B[0m online store.\n", + "Materializing \u001b[1m\u001b[32m2\u001b[0m feature views to \u001b[1m\u001b[32m2024-09-24 18:02:06+00:00\u001b[0m into the \u001b[1m\u001b[32msqlite\u001b[0m online store.\n", "\n", - "\u001B[1m\u001B[32mdriver_hourly_stats\u001B[0m from \u001B[1m\u001B[32m2024-09-23 18:02:09+00:00\u001B[0m to \u001B[1m\u001B[32m2024-09-24 18:02:06+00:00\u001B[0m:\n", + "\u001b[1m\u001b[32mdriver_hourly_stats\u001b[0m from \u001b[1m\u001b[32m2024-09-23 18:02:09+00:00\u001b[0m to \u001b[1m\u001b[32m2024-09-24 18:02:06+00:00\u001b[0m:\n", " 0%| | 0/5 [00:00= 3 { + byteBuffer := make([]byte, 4) + binary.LittleEndian.PutUint32(byteBuffer, uint32(len(keys))) + bufferList = append([][]byte{byteBuffer}, bufferList...) + } + for i := 0; i < len(keys); i++ { offset := i * 2 byteBuffer := make([]byte, 4) @@ -415,16 +428,9 @@ func serializeValue(value interface{}, entityKeySerializationVersion int64) (*[] binary.LittleEndian.PutUint32(valueBuffer, uint32(x.Int32Val)) return &valueBuffer, types.ValueType_INT32, nil case *types.Value_Int64Val: - if entityKeySerializationVersion <= 1 { - // We unfortunately have to use 32 bit here for backward compatibility :( - valueBuffer := make([]byte, 4) - binary.LittleEndian.PutUint32(valueBuffer, uint32(x.Int64Val)) - return &valueBuffer, types.ValueType_INT64, nil - } else { - valueBuffer := make([]byte, 8) - binary.LittleEndian.PutUint64(valueBuffer, uint64(x.Int64Val)) - return &valueBuffer, types.ValueType_INT64, nil - } + valueBuffer := make([]byte, 8) + binary.LittleEndian.PutUint64(valueBuffer, uint64(x.Int64Val)) + return &valueBuffer, types.ValueType_INT64, nil case nil: return nil, types.ValueType_INVALID, fmt.Errorf("could not detect type for %v", x) default: diff --git a/go/internal/feast/onlinestore/redisonlinestore_test.go b/go/internal/feast/onlinestore/redisonlinestore_test.go index 34adee191e8..466f8a763f1 100644 --- a/go/internal/feast/onlinestore/redisonlinestore_test.go +++ b/go/internal/feast/onlinestore/redisonlinestore_test.go @@ -15,7 +15,7 @@ func TestNewRedisOnlineStore(t *testing.T) { } rc := ®istry.RepoConfig{ OnlineStore: config, - EntityKeySerializationVersion: 2, + EntityKeySerializationVersion: 3, } store, err := NewRedisOnlineStore("test", rc, config) assert.Nil(t, err) @@ -32,7 +32,7 @@ func TestNewRedisOnlineStoreWithPassword(t *testing.T) { } rc := ®istry.RepoConfig{ OnlineStore: config, - EntityKeySerializationVersion: 2, + EntityKeySerializationVersion: 3, } store, err := NewRedisOnlineStore("test", rc, config) assert.Nil(t, err) @@ -47,7 +47,7 @@ func TestNewRedisOnlineStoreWithDB(t *testing.T) { } rc := ®istry.RepoConfig{ OnlineStore: config, - EntityKeySerializationVersion: 2, + EntityKeySerializationVersion: 3, } store, err := NewRedisOnlineStore("test", rc, config) assert.Nil(t, err) @@ -62,7 +62,7 @@ func TestNewRedisOnlineStoreWithSsl(t *testing.T) { } rc := ®istry.RepoConfig{ OnlineStore: config, - EntityKeySerializationVersion: 2, + EntityKeySerializationVersion: 3, } store, err := NewRedisOnlineStore("test", rc, config) assert.Nil(t, err) @@ -142,7 +142,7 @@ func TestBuildRedisKeys(t *testing.T) { r := &RedisOnlineStore{ project: "test_project", config: ®istry.RepoConfig{ - EntityKeySerializationVersion: 2, + EntityKeySerializationVersion: 3, }, } diff --git a/infra/charts/feast/README.md b/infra/charts/feast/README.md index 2753d330de0..bb28647a396 100644 --- a/infra/charts/feast/README.md +++ b/infra/charts/feast/README.md @@ -43,7 +43,7 @@ feature-server: config: host: localhost port: 6379 - entityKeySerializationVersion: 2 + entityKeySerializationVersion: 3 global: registry: diff --git a/infra/charts/feast/README.md.gotmpl b/infra/charts/feast/README.md.gotmpl index 7a32d22c7f1..81352b90e0d 100644 --- a/infra/charts/feast/README.md.gotmpl +++ b/infra/charts/feast/README.md.gotmpl @@ -43,7 +43,7 @@ feature-server: config: host: localhost port: 6379 - entityKeySerializationVersion: 2 + entityKeySerializationVersion: 3 global: registry: diff --git a/java/serving/README.md b/java/serving/README.md index 5cf5ec65354..24c1db344cc 100644 --- a/java/serving/README.md +++ b/java/serving/README.md @@ -30,14 +30,14 @@ From the Feast GitHub root, run: feast: project: feast_demo registry: /Users/[your username]/GitHub/feast-demo/feature_repo/data/registry.db - entityKeySerializationVersion: 2 + entityKeySerializationVersion: 3 ``` 2. An example of if you're using Redis with a remote registry: ```yaml feast: project: feast_java_demo registry: gs://[YOUR BUCKET]/demo-repo/registry.db - entityKeySerializationVersion: 2 + entityKeySerializationVersion: 3 activeStore: online stores: - name: online diff --git a/java/serving/src/main/java/feast/serving/connectors/redis/retriever/EntityKeySerializerV2.java b/java/serving/src/main/java/feast/serving/connectors/redis/retriever/EntityKeySerializerV2.java index 672f4d7c318..8b2991385b2 100644 --- a/java/serving/src/main/java/feast/serving/connectors/redis/retriever/EntityKeySerializerV2.java +++ b/java/serving/src/main/java/feast/serving/connectors/redis/retriever/EntityKeySerializerV2.java @@ -91,9 +91,8 @@ public byte[] serialize(RedisProto.RedisKeyV2 entityKey) { we use `struct.pack(" Tuple[bytes, int]: if value_type == "string_val": return v.string_val.encode("utf8"), ValueType.STRING @@ -41,7 +42,9 @@ def _deserialize_value(value_type, value_bytes) -> ValueProto: raise ValueError(f"Unsupported value type: {value_type}") -def serialize_entity_key_prefix(entity_keys: List[str]) -> bytes: +def serialize_entity_key_prefix( + entity_keys: List[str], entity_key_serialization_version: int = 3 +) -> bytes: """ Serialize keys to a bytestring, so it can be used to prefix-scan through items stored in the online store using serialize_entity_key. @@ -51,8 +54,12 @@ def serialize_entity_key_prefix(entity_keys: List[str]) -> bytes: """ sorted_keys = sorted(entity_keys) output: List[bytes] = [] + if entity_key_serialization_version > 2: + output.append(struct.pack(" 2: + output.append(struct.pack(" bytes: """ Serialize entity key to a bytestring so it can be used as a lookup key in a hash table. @@ -117,13 +124,19 @@ def serialize_entity_key( Args: entity_key_serialization_version: version of the entity key serialization - version 1: int64 values are serialized as 4 bytes - version 2: int64 values are serialized as 8 bytes + Versions: version 3: entity_key size is added to the serialization for deserialization purposes entity_key: EntityKeyProto Returns: bytes of the serialized entity key """ + if entity_key_serialization_version < 3: + # Not raising the error, keeping it in warning state for reserialization purpose + # We should remove this after few releases + warnings.warn( + "Serialization of entity key with version < 3 is removed. Please use version 3 by setting entity_key_serialization_version=3." + "To reserializa your online store featrues refer - https://github.com/feast-dev/feast/blob/master/docs/how-to-guides/entity-reserialization-of-from-v2-to-v3.md" + ) sorted_keys, sorted_values = zip( *sorted(zip(entity_key.join_keys, entity_key.entity_values)) ) @@ -163,9 +176,12 @@ def deserialize_entity_key( Returns: EntityKeyProto """ - if entity_key_serialization_version <= 2: - raise ValueError( - "Deserialization of entity key with version <= 2 is not supported. Please use version > 2 by setting entity_key_serialization_version=3" + if entity_key_serialization_version < 3: + # Not raising the error, keeping it in warning state for reserialization purpose + # We should remove this after few releases + warnings.warn( + "Deserialization of entity key with version < 3 is removed. Please use version 3 by setting entity_key_serialization_version=3." + "To reserializa your online store featrues refer - https://github.com/feast-dev/feast/blob/master/docs/how-to-guides/entity-reserialization-of-from-v2-to-v3.md" ) offset = 0 keys = [] diff --git a/sdk/python/feast/infra/online_stores/couchbase_online_store/README.md b/sdk/python/feast/infra/online_stores/couchbase_online_store/README.md index 8f95884fe03..885250a66af 100644 --- a/sdk/python/feast/infra/online_stores/couchbase_online_store/README.md +++ b/sdk/python/feast/infra/online_stores/couchbase_online_store/README.md @@ -41,7 +41,7 @@ online_store: password: password # Couchbase password from access credentials bucket_name: feast # Couchbase bucket name, defaults to feast kv_port: 11210 # Couchbase key-value port, defaults to 11210. Required if custom ports are used. -entity_key_serialization_version: 2 +entity_key_serialization_version: 3 ``` #### Apply the feature definitions in [`example.py`](https://github.com/feast-dev/feast/blob/master/go/internal/test/feature_repo/example.py) diff --git a/sdk/python/feast/infra/online_stores/hazelcast_online_store/hazelcast_online_store.py b/sdk/python/feast/infra/online_stores/hazelcast_online_store/hazelcast_online_store.py index c56d394c21a..21359b45bca 100644 --- a/sdk/python/feast/infra/online_stores/hazelcast_online_store/hazelcast_online_store.py +++ b/sdk/python/feast/infra/online_stores/hazelcast_online_store/hazelcast_online_store.py @@ -163,7 +163,7 @@ def online_write_batch( entity_key_str = base64.b64encode( serialize_entity_key( entity_key, - entity_key_serialization_version=2, + entity_key_serialization_version=3, ) ).decode("utf-8") event_ts_utc = event_ts.astimezone(tz=timezone.utc).timestamp() @@ -214,7 +214,7 @@ def online_read( entity_key_str = base64.b64encode( serialize_entity_key( entity_key, - entity_key_serialization_version=2, + entity_key_serialization_version=3, ) ).decode("utf-8") if requested_features: diff --git a/sdk/python/feast/infra/online_stores/helpers.py b/sdk/python/feast/infra/online_stores/helpers.py index 409a1eb2126..b657bd44d00 100644 --- a/sdk/python/feast/infra/online_stores/helpers.py +++ b/sdk/python/feast/infra/online_stores/helpers.py @@ -23,7 +23,7 @@ def get_online_store_from_config(online_store_config: Any) -> OnlineStore: def _redis_key( - project: str, entity_key: EntityKeyProto, entity_key_serialization_version=1 + project: str, entity_key: EntityKeyProto, entity_key_serialization_version=3 ) -> bytes: key: List[bytes] = [ serialize_entity_key( @@ -50,7 +50,7 @@ def _mmh3(key: str): def compute_entity_id( - entity_key: EntityKeyProto, entity_key_serialization_version=1 + entity_key: EntityKeyProto, entity_key_serialization_version=3 ) -> str: """ Compute Entity id given Feast Entity Key for online stores. diff --git a/sdk/python/feast/infra/online_stores/mysql_online_store/mysql.py b/sdk/python/feast/infra/online_stores/mysql_online_store/mysql.py index 64111ca42c1..d44eddfbd0b 100644 --- a/sdk/python/feast/infra/online_stores/mysql_online_store/mysql.py +++ b/sdk/python/feast/infra/online_stores/mysql_online_store/mysql.py @@ -72,7 +72,7 @@ def online_write_batch( for entity_key, values, timestamp, created_ts in data: entity_key_bin = serialize_entity_key( entity_key, - entity_key_serialization_version=2, + entity_key_serialization_version=3, ).hex() timestamp = to_naive_utc(timestamp) if created_ts is not None: @@ -137,7 +137,7 @@ def online_read( for entity_key in entity_keys: entity_key_bin = serialize_entity_key( entity_key, - entity_key_serialization_version=2, + entity_key_serialization_version=3, ).hex() cur.execute( diff --git a/sdk/python/feast/infra/online_stores/singlestore_online_store/singlestore.py b/sdk/python/feast/infra/online_stores/singlestore_online_store/singlestore.py index a1535589542..eb598ec5e7a 100644 --- a/sdk/python/feast/infra/online_stores/singlestore_online_store/singlestore.py +++ b/sdk/python/feast/infra/online_stores/singlestore_online_store/singlestore.py @@ -80,7 +80,7 @@ def online_write_batch( for entity_key, values, timestamp, created_ts in data: entity_key_bin = serialize_entity_key( entity_key, - entity_key_serialization_version=2, + entity_key_serialization_version=3, ).hex() timestamp = _to_naive_utc(timestamp) if created_ts is not None: @@ -130,7 +130,7 @@ def online_read( keys.append( serialize_entity_key( entity_key, - entity_key_serialization_version=2, + entity_key_serialization_version=3, ).hex() ) diff --git a/sdk/python/feast/infra/online_stores/sqlite.py b/sdk/python/feast/infra/online_stores/sqlite.py index 07180fe75ed..461277631f5 100644 --- a/sdk/python/feast/infra/online_stores/sqlite.py +++ b/sdk/python/feast/infra/online_stores/sqlite.py @@ -111,7 +111,6 @@ class SqliteOnlineStoreConfig(FeastConfigBaseModel, VectorStoreConfig): path: StrictStr = "data/online.db" """ (optional) Path to sqlite db """ - vector_enabled: bool = False text_search_enabled: bool = False diff --git a/sdk/python/feast/infra/utils/hbase_utils.py b/sdk/python/feast/infra/utils/hbase_utils.py index 72afda2ef3d..d26587cc6a8 100644 --- a/sdk/python/feast/infra/utils/hbase_utils.py +++ b/sdk/python/feast/infra/utils/hbase_utils.py @@ -193,19 +193,19 @@ def main(): EntityKey( join_keys=["driver_id"], entity_values=[Value(int64_val=1004)] ), - entity_key_serialization_version=2, + entity_key_serialization_version=3, ).hex(), serialize_entity_key( EntityKey( join_keys=["driver_id"], entity_values=[Value(int64_val=1005)] ), - entity_key_serialization_version=2, + entity_key_serialization_version=3, ).hex(), serialize_entity_key( EntityKey( join_keys=["driver_id"], entity_values=[Value(int64_val=1024)] ), - entity_key_serialization_version=2, + entity_key_serialization_version=3, ).hex(), ] rows = table.rows(row_keys) diff --git a/sdk/python/feast/infra/utils/snowflake/snowpark/snowflake_udfs.py b/sdk/python/feast/infra/utils/snowflake/snowpark/snowflake_udfs.py index ebba3e9b84e..277d8e18946 100644 --- a/sdk/python/feast/infra/utils/snowflake/snowpark/snowflake_udfs.py +++ b/sdk/python/feast/infra/utils/snowflake/snowpark/snowflake_udfs.py @@ -394,7 +394,7 @@ def feast_serialize_entity_keys(df): join_keys=join_keys, entity_values=[proto_values_by_column[k][idx] for k in join_keys], ), - entity_key_serialization_version=2, + entity_key_serialization_version=3, ) for idx in range(df.shape[0]) ] diff --git a/sdk/python/feast/repo_config.py b/sdk/python/feast/repo_config.py index d172b80f076..41d4971ea4e 100644 --- a/sdk/python/feast/repo_config.py +++ b/sdk/python/feast/repo_config.py @@ -215,16 +215,12 @@ class RepoConfig(FeastBaseModel): repo_path: Optional[Path] = None """When using relative path in FileSource path, this parameter is mandatory""" - entity_key_serialization_version: StrictInt = 1 + entity_key_serialization_version: StrictInt = 3 """ Entity key serialization version: This version is used to control what serialization scheme is used when writing data to the online store. - A value <= 1 uses the serialization scheme used by feast up to Feast 0.22. - A value of 2 uses a newer serialization scheme, supported as of Feast 0.23. A value of 3 uses the latest serialization scheme, supported as of Feast 0.38. - The main difference between the three schema is that - v1: the serialization scheme v1 stored `long` values as `int`s, which would result in errors trying to serialize a range of values. - v2: fixes this error, but v1 is kept around to ensure backwards compatibility - specifically the ability to read - feature values for entities that have already been written into the online store. + + Version Schemas: v3: add entity_key value length to serialized bytes to enable deserialization, which can be used in retrieval of entity_key in document retrieval. """ @@ -266,9 +262,9 @@ def __init__(self, **data: Any): self.feature_server["type"] )(**self.feature_server) - if self.entity_key_serialization_version <= 2: + if self.entity_key_serialization_version < 3: warnings.warn( - "The serialization version 2 and below will be deprecated in the next release. " + "The serialization version below 3 are deprecated. " "Specifying `entity_key_serialization_version` to 3 is recommended.", DeprecationWarning, ) diff --git a/sdk/python/feast/templates/athena/feature_repo/feature_store.yaml b/sdk/python/feast/templates/athena/feature_repo/feature_store.yaml index bd12e906d1f..a3e74083f5e 100644 --- a/sdk/python/feast/templates/athena/feature_repo/feature_store.yaml +++ b/sdk/python/feast/templates/athena/feature_repo/feature_store.yaml @@ -11,4 +11,4 @@ offline_store: data_source: AwsDataCatalog s3_staging_location: s3://{S3 bucket to be used by Feast} workgroup: {Workgroup for Athena} -entity_key_serialization_version: 2 \ No newline at end of file +entity_key_serialization_version: 3 \ No newline at end of file diff --git a/sdk/python/feast/templates/aws/feature_repo/feature_store.yaml b/sdk/python/feast/templates/aws/feature_repo/feature_store.yaml index c29496711bb..27fb0d6dfbe 100644 --- a/sdk/python/feast/templates/aws/feature_repo/feature_store.yaml +++ b/sdk/python/feast/templates/aws/feature_repo/feature_store.yaml @@ -25,4 +25,4 @@ offline_store: user: %REDSHIFT_USER% s3_staging_location: %REDSHIFT_S3_STAGING_LOCATION% iam_role: %REDSHIFT_IAM_ROLE% -entity_key_serialization_version: 2 +entity_key_serialization_version: 3 diff --git a/sdk/python/feast/templates/cassandra/feature_repo/feature_store.yaml b/sdk/python/feast/templates/cassandra/feature_repo/feature_store.yaml index ce50275554b..9c6263c7e45 100644 --- a/sdk/python/feast/templates/cassandra/feature_repo/feature_store.yaml +++ b/sdk/python/feast/templates/cassandra/feature_repo/feature_store.yaml @@ -18,4 +18,4 @@ online_store: load_balancing_policy: c_load_balancing_policy read_concurrency: c_r_concurrency write_concurrency: c_w_concurrency -entity_key_serialization_version: 2 +entity_key_serialization_version: 3 diff --git a/sdk/python/feast/templates/couchbase/feature_repo/feature_store.yaml b/sdk/python/feast/templates/couchbase/feature_repo/feature_store.yaml index 96f45934eb5..69a2577951f 100644 --- a/sdk/python/feast/templates/couchbase/feature_repo/feature_store.yaml +++ b/sdk/python/feast/templates/couchbase/feature_repo/feature_store.yaml @@ -14,4 +14,4 @@ offline_store: user: COUCHBASE_COLUMNAR_USER # Couchbase cluster access name from Settings > Access Control page in Capella Columnar console password: COUCHBASE_COLUMNAR_PASSWORD # Couchbase password from Settings > Access Control page in Capella Columnar console timeout: COUCHBASE_COLUMNAR_TIMEOUT # Timeout in seconds for Columnar operations, optional -entity_key_serialization_version: 2 +entity_key_serialization_version: 3 diff --git a/sdk/python/feast/templates/gcp/feature_repo/feature_store.yaml b/sdk/python/feast/templates/gcp/feature_repo/feature_store.yaml index e3d9d1d1e61..e24093c9745 100644 --- a/sdk/python/feast/templates/gcp/feature_repo/feature_store.yaml +++ b/sdk/python/feast/templates/gcp/feature_repo/feature_store.yaml @@ -22,4 +22,4 @@ online_store: #online_store: # type: redis # connection_string: "localhost:6379" -entity_key_serialization_version: 2 +entity_key_serialization_version: 3 diff --git a/sdk/python/feast/templates/hazelcast/feature_repo/feature_store.yaml b/sdk/python/feast/templates/hazelcast/feature_repo/feature_store.yaml index e26d1bf7506..a7c687f6455 100644 --- a/sdk/python/feast/templates/hazelcast/feature_repo/feature_store.yaml +++ b/sdk/python/feast/templates/hazelcast/feature_repo/feature_store.yaml @@ -11,4 +11,4 @@ online_store: ssl_keyfile_path: c_key_path ssl_password: ${SSL_PASSWORD} # This value will be read form the `SSL_PASSWORD` environment variable. key_ttl_seconds: c_ttl_seconds -entity_key_serialization_version: 2 +entity_key_serialization_version: 3 diff --git a/sdk/python/feast/templates/hbase/feature_repo/feature_store.yaml b/sdk/python/feast/templates/hbase/feature_repo/feature_store.yaml index bde49486ad3..028c48f029b 100644 --- a/sdk/python/feast/templates/hbase/feature_repo/feature_store.yaml +++ b/sdk/python/feast/templates/hbase/feature_repo/feature_store.yaml @@ -7,4 +7,4 @@ online_store: type: hbase host: 127.0.0.1 port: 9090 -entity_key_serialization_version: 2 +entity_key_serialization_version: 3 diff --git a/sdk/python/feast/templates/local/feature_repo/feature_store.yaml b/sdk/python/feast/templates/local/feature_repo/feature_store.yaml index 11b339583e0..356e706c638 100644 --- a/sdk/python/feast/templates/local/feature_repo/feature_store.yaml +++ b/sdk/python/feast/templates/local/feature_repo/feature_store.yaml @@ -6,7 +6,7 @@ provider: local online_store: type: sqlite path: data/online_store.db -entity_key_serialization_version: 2 +entity_key_serialization_version: 3 # By default, no_auth for authentication and authorization, other possible values kubernetes and oidc. Refer the documentation for more details. auth: type: no_auth diff --git a/sdk/python/feast/templates/minimal/feature_repo/feature_store.yaml b/sdk/python/feast/templates/minimal/feature_repo/feature_store.yaml index 45a0ce77186..244a3b0f453 100644 --- a/sdk/python/feast/templates/minimal/feature_repo/feature_store.yaml +++ b/sdk/python/feast/templates/minimal/feature_repo/feature_store.yaml @@ -3,4 +3,4 @@ registry: /path/to/registry.db provider: local online_store: path: /path/to/online_store.db -entity_key_serialization_version: 2 \ No newline at end of file +entity_key_serialization_version: 3 \ No newline at end of file diff --git a/sdk/python/feast/templates/postgres/feature_repo/feature_store.yaml b/sdk/python/feast/templates/postgres/feature_repo/feature_store.yaml index f14510f820e..0663ff0ad97 100644 --- a/sdk/python/feast/templates/postgres/feature_repo/feature_store.yaml +++ b/sdk/python/feast/templates/postgres/feature_repo/feature_store.yaml @@ -23,4 +23,4 @@ offline_store: db_schema: DB_SCHEMA user: DB_USERNAME password: DB_PASSWORD -entity_key_serialization_version: 2 +entity_key_serialization_version: 3 diff --git a/sdk/python/feast/templates/snowflake/feature_repo/feature_store.yaml b/sdk/python/feast/templates/snowflake/feature_repo/feature_store.yaml index 104e6394c6b..a7de457a2f2 100644 --- a/sdk/python/feast/templates/snowflake/feature_repo/feature_store.yaml +++ b/sdk/python/feast/templates/snowflake/feature_repo/feature_store.yaml @@ -25,4 +25,4 @@ online_store: role: SNOWFLAKE_ROLE warehouse: SNOWFLAKE_WAREHOUSE database: SNOWFLAKE_DATABASE -entity_key_serialization_version: 2 +entity_key_serialization_version: 3 diff --git a/sdk/python/feast/templates/spark/feature_repo/feature_store.yaml b/sdk/python/feast/templates/spark/feature_repo/feature_store.yaml index 08383a29e13..873ce113f71 100644 --- a/sdk/python/feast/templates/spark/feature_repo/feature_store.yaml +++ b/sdk/python/feast/templates/spark/feature_repo/feature_store.yaml @@ -16,4 +16,4 @@ offline_store: spark.sql.execution.arrow.pyspark.enabled: "true" online_store: path: data/online_store.db -entity_key_serialization_version: 2 +entity_key_serialization_version: 3 diff --git a/sdk/python/feast/templates/spark/feature_store.yaml b/sdk/python/feast/templates/spark/feature_store.yaml index 91e3ecf4724..230d2c23d9f 100644 --- a/sdk/python/feast/templates/spark/feature_store.yaml +++ b/sdk/python/feast/templates/spark/feature_store.yaml @@ -12,4 +12,4 @@ offline_store: spark.sql.session.timeZone: "UTC" online_store: path: data/online_store.db -entity_key_serialization_version: 2 +entity_key_serialization_version: 3 diff --git a/sdk/python/tests/integration/compute_engines/spark/test_compute.py b/sdk/python/tests/integration/compute_engines/spark/test_compute.py index 5254db1e690..3d44a130d64 100644 --- a/sdk/python/tests/integration/compute_engines/spark/test_compute.py +++ b/sdk/python/tests/integration/compute_engines/spark/test_compute.py @@ -109,7 +109,7 @@ def create_spark_environment(): batch_engine={"type": "spark.engine", "partitions": 10}, ) spark_environment = construct_test_environment( - spark_config, None, entity_key_serialization_version=2 + spark_config, None, entity_key_serialization_version=3 ) spark_environment.setup() return spark_environment diff --git a/sdk/python/tests/integration/feature_repos/repo_configuration.py b/sdk/python/tests/integration/feature_repos/repo_configuration.py index b851261e819..24e611c4f33 100644 --- a/sdk/python/tests/integration/feature_repos/repo_configuration.py +++ b/sdk/python/tests/integration/feature_repos/repo_configuration.py @@ -528,7 +528,7 @@ def construct_test_environment( fixture_request: Optional[pytest.FixtureRequest], test_suite_name: str = "integration_test", worker_id: str = "worker_id", - entity_key_serialization_version: int = 2, + entity_key_serialization_version: int = 3, ) -> Environment: _uuid = str(uuid.uuid4()).replace("-", "")[:6] @@ -565,14 +565,6 @@ def construct_test_environment( cache_ttl_seconds=1, ) - online_store = ( - test_repo_config.online_store.get("type") - if isinstance(test_repo_config.online_store, dict) - else test_repo_config.online_store - ) - if online_store in ["milvus", "pgvector", "qdrant", "elasticsearch"]: - entity_key_serialization_version = 3 - environment_params = { "name": project, "provider": test_repo_config.provider, diff --git a/sdk/python/tests/integration/feature_repos/universal/data_sources/file.py b/sdk/python/tests/integration/feature_repos/universal/data_sources/file.py index 6f6e5d68133..7e6334b1b88 100644 --- a/sdk/python/tests/integration/feature_repos/universal/data_sources/file.py +++ b/sdk/python/tests/integration/feature_repos/universal/data_sources/file.py @@ -385,7 +385,7 @@ def setup(self, registry: RegistryConfig): provider="local", offline_store=parent_offline_config, registry=registry.path, - entity_key_serialization_version=2, + entity_key_serialization_version=3, ) repo_path = Path(tempfile.mkdtemp()) @@ -435,7 +435,7 @@ def setup(self, registry: RegistryConfig): provider="local", offline_store=parent_offline_config, registry=registry.path, - entity_key_serialization_version=2, + entity_key_serialization_version=3, ) certificates_path = tempfile.mkdtemp() @@ -536,7 +536,7 @@ def setup(self, registry: RegistryConfig): provider="local", offline_store=parent_offline_config, registry=registry.path, - entity_key_serialization_version=2, + entity_key_serialization_version=3, ) repo_base_path = Path(tempfile.mkdtemp()) diff --git a/sdk/python/tests/integration/materialization/contrib/spark/test_spark_materialization_engine.py b/sdk/python/tests/integration/materialization/contrib/spark/test_spark_materialization_engine.py index ae0e03c9441..03f942c2f96 100644 --- a/sdk/python/tests/integration/materialization/contrib/spark/test_spark_materialization_engine.py +++ b/sdk/python/tests/integration/materialization/contrib/spark/test_spark_materialization_engine.py @@ -31,7 +31,7 @@ def test_spark_materialization_consistency(): batch_engine={"type": "spark.engine", "partitions": 10}, ) spark_environment = construct_test_environment( - spark_config, None, entity_key_serialization_version=2 + spark_config, None, entity_key_serialization_version=3 ) spark_environment.setup() diff --git a/sdk/python/tests/integration/materialization/test_lambda.py b/sdk/python/tests/integration/materialization/test_lambda.py index 07ab93e2e13..f0c1e108694 100644 --- a/sdk/python/tests/integration/materialization/test_lambda.py +++ b/sdk/python/tests/integration/materialization/test_lambda.py @@ -38,9 +38,8 @@ def test_lambda_materialization_consistency(): }, registry_location=RegistryLocation.S3, ) - # TODO(adchia): figure out why entity_key_serialization_version 2 breaks with this test lambda_environment = construct_test_environment( - lambda_config, None, entity_key_serialization_version=1 + lambda_config, None, entity_key_serialization_version=3 ) df = create_basic_driver_dataset() diff --git a/sdk/python/tests/integration/online_store/test_remote_online_store.py b/sdk/python/tests/integration/online_store/test_remote_online_store.py index 3b5b707dcb7..80166abf431 100644 --- a/sdk/python/tests/integration/online_store/test_remote_online_store.py +++ b/sdk/python/tests/integration/online_store/test_remote_online_store.py @@ -260,7 +260,7 @@ def _overwrite_remote_client_feature_store_yaml( ): repo_config = os.path.join(repo_path, "feature_store.yaml") - config_content = "entity_key_serialization_version: 2\n" + auth_config + config_content = "entity_key_serialization_version: 3\n" + auth_config config_content += dedent( f""" project: {PROJECT_NAME} diff --git a/sdk/python/tests/integration/registration/test_feature_store.py b/sdk/python/tests/integration/registration/test_feature_store.py index d7ffb83059b..b59af900190 100644 --- a/sdk/python/tests/integration/registration/test_feature_store.py +++ b/sdk/python/tests/integration/registration/test_feature_store.py @@ -72,6 +72,6 @@ def feature_store_with_local_registry(): project="default", provider="local", online_store=SqliteOnlineStoreConfig(path=online_store_path), - entity_key_serialization_version=2, + entity_key_serialization_version=3, ) ) diff --git a/sdk/python/tests/integration/registration/test_inference.py b/sdk/python/tests/integration/registration/test_inference.py index 9f490d7f4e2..7f053c46224 100644 --- a/sdk/python/tests/integration/registration/test_inference.py +++ b/sdk/python/tests/integration/registration/test_inference.py @@ -23,7 +23,7 @@ def test_update_file_data_source_with_inferred_event_timestamp_col(simple_datase provider="local", project="test", registry="test.pb", - entity_key_serialization_version=2, + entity_key_serialization_version=3, ), ) actual_event_timestamp_cols = [ @@ -41,7 +41,7 @@ def test_update_file_data_source_with_inferred_event_timestamp_col(simple_datase provider="local", project="test", registry="test.pb", - entity_key_serialization_version=2, + entity_key_serialization_version=3, ), ) @@ -62,7 +62,7 @@ def test_update_data_sources_with_inferred_event_timestamp_col(universal_data_so provider="local", project="test", registry="test.pb", - entity_key_serialization_version=2, + entity_key_serialization_version=3, ), ) actual_event_timestamp_cols = [ diff --git a/sdk/python/tests/unit/cli/test_cli.py b/sdk/python/tests/unit/cli/test_cli.py index 0dd51e9ce68..c8649f5cfb5 100644 --- a/sdk/python/tests/unit/cli/test_cli.py +++ b/sdk/python/tests/unit/cli/test_cli.py @@ -121,7 +121,7 @@ def setup_third_party_provider_repo(provider_name: str): type: sqlite offline_store: type: file - entity_key_serialization_version: 2 + entity_key_serialization_version: 3 """ ) ) @@ -158,7 +158,7 @@ def setup_third_party_registry_store_repo( type: sqlite offline_store: type: file - entity_key_serialization_version: 2 + entity_key_serialization_version: 3 """ ) ) @@ -189,4 +189,4 @@ def test_cli_configuration(): assertpy.assert_that(output).contains(b"type: sqlite") assertpy.assert_that(output).contains(b"path: data/online_store.db") assertpy.assert_that(output).contains(b"type: file") - assertpy.assert_that(output).contains(b"entity_key_serialization_version: 2") + assertpy.assert_that(output).contains(b"entity_key_serialization_version: 3") diff --git a/sdk/python/tests/unit/infra/offline_stores/test_redshift.py b/sdk/python/tests/unit/infra/offline_stores/test_redshift.py index a9ed4c2b59f..dc4caccc8f9 100644 --- a/sdk/python/tests/unit/infra/offline_stores/test_redshift.py +++ b/sdk/python/tests/unit/infra/offline_stores/test_redshift.py @@ -33,7 +33,7 @@ def test_offline_write_batch( s3_staging_location="s3://bucket/path", workgroup="", ), - entity_key_serialization_version=2, + entity_key_serialization_version=3, ) batch_source = RedshiftSource( diff --git a/sdk/python/tests/unit/infra/offline_stores/test_snowflake.py b/sdk/python/tests/unit/infra/offline_stores/test_snowflake.py index d692d0f957a..0a4c1df9b1c 100644 --- a/sdk/python/tests/unit/infra/offline_stores/test_snowflake.py +++ b/sdk/python/tests/unit/infra/offline_stores/test_snowflake.py @@ -38,7 +38,7 @@ def retrieval_job(request): provider="snowflake.offline", online_store=SqliteOnlineStoreConfig(type="sqlite"), offline_store=offline_store_config, - entity_key_serialization_version=2, + entity_key_serialization_version=3, ), full_feature_names=True, on_demand_feature_views=[], diff --git a/sdk/python/tests/unit/infra/online_store/test_dynamodb_online_store.py b/sdk/python/tests/unit/infra/online_store/test_dynamodb_online_store.py index 527d3f16181..1fd032bbd54 100644 --- a/sdk/python/tests/unit/infra/online_store/test_dynamodb_online_store.py +++ b/sdk/python/tests/unit/infra/online_store/test_dynamodb_online_store.py @@ -49,7 +49,7 @@ def repo_config(): online_store=DynamoDBOnlineStoreConfig(region=REGION), # online_store={"type": "dynamodb", "region": REGION}, offline_store=DaskOfflineStoreConfig(), - entity_key_serialization_version=2, + entity_key_serialization_version=3, ) diff --git a/sdk/python/tests/unit/infra/online_store/test_redis.py b/sdk/python/tests/unit/infra/online_store/test_redis.py index c26c2f25c5f..83c8d3d61e4 100644 --- a/sdk/python/tests/unit/infra/online_store/test_redis.py +++ b/sdk/python/tests/unit/infra/online_store/test_redis.py @@ -18,7 +18,7 @@ def repo_config(): return RepoConfig( provider="local", project="test", - entity_key_serialization_version=2, + entity_key_serialization_version=3, registry="dummy_registry.db", ) @@ -49,7 +49,7 @@ def test_generate_entity_redis_keys(redis_online_store: RedisOnlineStore, repo_c repo_config, entity_keys ) expected = [ - b"\x02\x00\x00\x00entity\x03\x00\x00\x00\x04\x00\x00\x00\x01\x00\x00\x00test" + b"\x01\x00\x00\x00\x02\x00\x00\x00\x06\x00\x00\x00entity\x03\x00\x00\x00\x04\x00\x00\x00\x01\x00\x00\x00test" ] assert actual == expected diff --git a/sdk/python/tests/unit/infra/scaffolding/test_repo_config.py b/sdk/python/tests/unit/infra/scaffolding/test_repo_config.py index 9dcf7e4caf6..bd6283e2d7b 100644 --- a/sdk/python/tests/unit/infra/scaffolding/test_repo_config.py +++ b/sdk/python/tests/unit/infra/scaffolding/test_repo_config.py @@ -48,7 +48,7 @@ def test_nullable_online_store_local(): registry: "registry.db" provider: local online_store: null - entity_key_serialization_version: 2 + entity_key_serialization_version: 3 """ ), expect_error=None, @@ -62,7 +62,7 @@ def test_local_config(): project: foo registry: "registry.db" provider: local - entity_key_serialization_version: 2 + entity_key_serialization_version: 3 """ ), expect_error=None, @@ -78,7 +78,7 @@ def test_local_config_with_full_online_class(): provider: local online_store: type: feast.infra.online_stores.sqlite.SqliteOnlineStore - entity_key_serialization_version: 2 + entity_key_serialization_version: 3 """ ), expect_error=None, @@ -94,7 +94,7 @@ def test_local_config_with_full_online_class_directly(): registry: "registry.db" provider: local online_store: feast.infra.online_stores.sqlite.SqliteOnlineStore - entity_key_serialization_version: 2 + entity_key_serialization_version: 3 """ ), expect_error=None, @@ -128,7 +128,7 @@ def test_no_online_store_type(): provider: local online_store: path: "blah" - entity_key_serialization_version: 2 + entity_key_serialization_version: 3 """ ), expect_error=None, @@ -158,7 +158,7 @@ def test_no_project(): provider: local online_store: path: foo - entity_key_serialization_version: 2 + entity_key_serialization_version: 3 """ ), expect_error="1 validation error for RepoConfig\nproject\n Field required", @@ -197,7 +197,7 @@ def test_no_provider(): registry: "registry.db" online_store: path: "blah" - entity_key_serialization_version: 2 + entity_key_serialization_version: 3 """ ), expect_error=None, @@ -219,7 +219,7 @@ def test_auth_config(): provider: local online_store: path: foo - entity_key_serialization_version: 2 + entity_key_serialization_version: 3 """ ), expect_error="missing authentication type", @@ -240,7 +240,7 @@ def test_auth_config(): provider: local online_store: path: foo - entity_key_serialization_version: 2 + entity_key_serialization_version: 3 """ ), expect_error="invalid authentication type=not_valid_auth_type", @@ -258,7 +258,7 @@ def test_auth_config(): provider: local online_store: path: foo - entity_key_serialization_version: 2 + entity_key_serialization_version: 3 """ ), expect_error=None, @@ -286,7 +286,7 @@ def test_auth_config(): provider: local online_store: path: foo - entity_key_serialization_version: 2 + entity_key_serialization_version: 3 """ ), expect_error=None, @@ -310,7 +310,7 @@ def test_auth_config(): provider: local online_store: path: foo - entity_key_serialization_version: 2 + entity_key_serialization_version: 3 """ ), expect_error=None, @@ -328,7 +328,7 @@ def test_auth_config(): provider: local online_store: path: foo - entity_key_serialization_version: 2 + entity_key_serialization_version: 3 """ ), expect_error=None, diff --git a/sdk/python/tests/unit/infra/test_inference_unit_tests.py b/sdk/python/tests/unit/infra/test_inference_unit_tests.py index 951f7033d23..f1aef20d113 100644 --- a/sdk/python/tests/unit/infra/test_inference_unit_tests.py +++ b/sdk/python/tests/unit/infra/test_inference_unit_tests.py @@ -243,7 +243,7 @@ def test_feature_view_inference_respects_basic_inference(): config = RepoConfig( provider="local", project="test", - entity_key_serialization_version=2, + entity_key_serialization_version=3, registry="dummy_registry.pb", ) provider = get_provider(config) @@ -264,7 +264,7 @@ def test_feature_view_inference_respects_basic_inference(): config = RepoConfig( provider="local", project="test", - entity_key_serialization_version=2, + entity_key_serialization_version=3, registry="dummy_registry.pb", ) provider = get_provider(config) @@ -301,7 +301,7 @@ def test_feature_view_inference_on_entity_value_types(): config = RepoConfig( provider="local", project="test", - entity_key_serialization_version=2, + entity_key_serialization_version=3, registry="dummy_registry.pb", ) provider = get_provider(config) @@ -377,7 +377,7 @@ def test_feature_view_inference_on_entity_columns(simple_dataset_1): config = RepoConfig( provider="local", project="test", - entity_key_serialization_version=2, + entity_key_serialization_version=3, registry="dummy_registry.pb", ) provider = get_provider(config) @@ -418,7 +418,7 @@ def test_feature_view_inference_on_feature_columns(simple_dataset_1): config = RepoConfig( provider="local", project="test", - entity_key_serialization_version=2, + entity_key_serialization_version=3, registry="dummy_registry.pb", ) provider = get_provider(config) @@ -476,7 +476,7 @@ def test_update_feature_services_with_inferred_features(simple_dataset_1): config = RepoConfig( provider="local", project="test", - entity_key_serialization_version=2, + entity_key_serialization_version=3, registry="dummy_registry.pb", ) provider = get_provider(config) @@ -539,7 +539,7 @@ def test_update_feature_services_with_specified_features(simple_dataset_1): config = RepoConfig( provider="local", project="test", - entity_key_serialization_version=2, + entity_key_serialization_version=3, registry="dummy_registry.pb", ) provider = get_provider(config) diff --git a/sdk/python/tests/unit/infra/test_key_encoding_utils.py b/sdk/python/tests/unit/infra/test_key_encoding_utils.py index bad94534bdb..4c82945fe88 100644 --- a/sdk/python/tests/unit/infra/test_key_encoding_utils.py +++ b/sdk/python/tests/unit/infra/test_key_encoding_utils.py @@ -1,5 +1,3 @@ -import pytest - from feast.infra.key_encoding_utils import ( _deserialize_value, _serialize_val, @@ -18,24 +16,16 @@ def test_serialize_entity_key(): EntityKeyProto( join_keys=["user"], entity_values=[ValueProto(int64_val=int(2**15))] ), - entity_key_serialization_version=2, + entity_key_serialization_version=3, ) # True int64, but should also be fine. serialize_entity_key( EntityKeyProto( join_keys=["user"], entity_values=[ValueProto(int64_val=int(2**31))] ), - entity_key_serialization_version=2, + entity_key_serialization_version=3, ) - # Old serialization scheme, should fail. - with pytest.raises(BaseException): - serialize_entity_key( - EntityKeyProto( - join_keys=["user"], entity_values=[ValueProto(int64_val=int(2**31))] - ), - ) - def test_deserialize_entity_key(): serialized_entity_key = serialize_entity_key( @@ -84,23 +74,9 @@ def test_serialize_value(): assert t == ValueType.INT32 assert v == b"\x01\x00\x00\x00" - # default entity_key_serialization_version is 1, so the result should be 4 bytes + # Default entity_key_serialization_version is 3, so result is of 8 bytes v, t = _serialize_val("int64_val", ValueProto(int64_val=1)) assert t == ValueType.INT64 - assert v == b"\x01\x00\x00\x00" - - # current entity_key_serialization_version is 2, so the result should be 8 bytes - v, t = _serialize_val( - "int64_val", ValueProto(int64_val=1), entity_key_serialization_version=2 - ) - assert t == ValueType.INT64 - assert v == b"\x01\x00\x00\x00\x00\x00\x00\x00" - - # new entity_key_serialization_version is 3, the result should be same as version 2 - v, t = _serialize_val( - "int64_val", ValueProto(int64_val=1), entity_key_serialization_version=3 - ) - assert t == ValueType.INT64 assert v == b"\x01\x00\x00\x00\x00\x00\x00\x00" diff --git a/sdk/python/tests/unit/local_feast_tests/test_local_feature_store.py b/sdk/python/tests/unit/local_feast_tests/test_local_feature_store.py index 931acfb3919..07b6fe093de 100644 --- a/sdk/python/tests/unit/local_feast_tests/test_local_feature_store.py +++ b/sdk/python/tests/unit/local_feast_tests/test_local_feature_store.py @@ -742,6 +742,6 @@ def feature_store_with_local_registry(): project="default", provider="local", online_store=SqliteOnlineStoreConfig(path=online_store_path), - entity_key_serialization_version=2, + entity_key_serialization_version=3, ) ) diff --git a/sdk/python/tests/unit/online_store/test_online_retrieval.py b/sdk/python/tests/unit/online_store/test_online_retrieval.py index 0615a4ca9d1..35f1e0a3238 100644 --- a/sdk/python/tests/unit/online_store/test_online_retrieval.py +++ b/sdk/python/tests/unit/online_store/test_online_retrieval.py @@ -224,7 +224,7 @@ def test_get_online_features() -> None: online_store=store.config.online_store, project=store.project, provider=store.config.provider, - entity_key_serialization_version=2, + entity_key_serialization_version=3, ) ) @@ -287,7 +287,7 @@ def test_get_online_features() -> None: online_store=store.config.online_store, project=store.project, provider=store.config.provider, - entity_key_serialization_version=2, + entity_key_serialization_version=3, ) ) diff --git a/sdk/python/tests/unit/online_store/test_online_writes.py b/sdk/python/tests/unit/online_store/test_online_writes.py index 5d573bf6b92..9c6f5c8af10 100644 --- a/sdk/python/tests/unit/online_store/test_online_writes.py +++ b/sdk/python/tests/unit/online_store/test_online_writes.py @@ -43,7 +43,7 @@ def setUp(self): project="test_write_to_online_store", registry=os.path.join(data_dir, "registry.db"), provider="local", - entity_key_serialization_version=2, + entity_key_serialization_version=3, online_store=SqliteOnlineStoreConfig( path=os.path.join(data_dir, "online.db") ), @@ -163,7 +163,7 @@ def test_transform_on_write_pdf(self): project="test_write_to_online_store_with_transform", registry=os.path.join(data_dir, "registry.db"), provider="local", - entity_key_serialization_version=2, + entity_key_serialization_version=3, online_store=SqliteOnlineStoreConfig( path=os.path.join(data_dir, "online.db") ), diff --git a/sdk/python/tests/unit/test_offline_server.py b/sdk/python/tests/unit/test_offline_server.py index 74752f6952d..c5f309eb323 100644 --- a/sdk/python/tests/unit/test_offline_server.py +++ b/sdk/python/tests/unit/test_offline_server.py @@ -95,7 +95,7 @@ def remote_feature_store(offline_server): registry=registry_path, provider="local", offline_store=offline_config, - entity_key_serialization_version=2, + entity_key_serialization_version=3, # repo_config = ) ) diff --git a/sdk/python/tests/unit/test_on_demand_pandas_transformation.py b/sdk/python/tests/unit/test_on_demand_pandas_transformation.py index 1a04a466fbc..c5ab657becc 100644 --- a/sdk/python/tests/unit/test_on_demand_pandas_transformation.py +++ b/sdk/python/tests/unit/test_on_demand_pandas_transformation.py @@ -35,7 +35,7 @@ def test_pandas_transformation(): project="test_on_demand_python_transformation", registry=os.path.join(data_dir, "registry.db"), provider="local", - entity_key_serialization_version=2, + entity_key_serialization_version=3, online_store=SqliteOnlineStoreConfig( path=os.path.join(data_dir, "online.db") ), @@ -116,7 +116,7 @@ def test_pandas_transformation_returning_all_data_types(): project="test_on_demand_python_transformation", registry=os.path.join(data_dir, "registry.db"), provider="local", - entity_key_serialization_version=2, + entity_key_serialization_version=3, online_store=SqliteOnlineStoreConfig( path=os.path.join(data_dir, "online.db") ), @@ -312,7 +312,7 @@ def test_invalid_pandas_transformation_raises_type_error_on_apply(): project="test_on_demand_python_transformation", registry=os.path.join(data_dir, "registry.db"), provider="local", - entity_key_serialization_version=2, + entity_key_serialization_version=3, online_store=SqliteOnlineStoreConfig( path=os.path.join(data_dir, "online.db") ), diff --git a/sdk/python/tests/unit/test_on_demand_python_transformation.py b/sdk/python/tests/unit/test_on_demand_python_transformation.py index eb29c645e53..f53e100cf22 100644 --- a/sdk/python/tests/unit/test_on_demand_python_transformation.py +++ b/sdk/python/tests/unit/test_on_demand_python_transformation.py @@ -51,7 +51,7 @@ def setUp(self): project="test_on_demand_python_transformation", registry=os.path.join(data_dir, "registry.db"), provider="local", - entity_key_serialization_version=2, + entity_key_serialization_version=3, online_store=SqliteOnlineStoreConfig( path=os.path.join(data_dir, "online.db") ), @@ -407,7 +407,7 @@ def setUp(self): project="test_on_demand_python_transformation", registry=os.path.join(data_dir, "registry.db"), provider="local", - entity_key_serialization_version=2, + entity_key_serialization_version=3, online_store=SqliteOnlineStoreConfig( path=os.path.join(data_dir, "online.db") ), @@ -775,7 +775,7 @@ def test_invalid_python_transformation_raises_type_error_on_apply(): project="test_on_demand_python_transformation", registry=os.path.join(data_dir, "registry.db"), provider="local", - entity_key_serialization_version=2, + entity_key_serialization_version=3, online_store=SqliteOnlineStoreConfig( path=os.path.join(data_dir, "online.db") ), @@ -822,7 +822,7 @@ def test_stored_writes(self): project="test_on_demand_python_transformation", registry=os.path.join(data_dir, "registry.db"), provider="local", - entity_key_serialization_version=2, + entity_key_serialization_version=3, online_store=SqliteOnlineStoreConfig( path=os.path.join(data_dir, "online.db") ), diff --git a/sdk/python/tests/unit/test_serialization_version.py b/sdk/python/tests/unit/test_serialization_version.py index 00562e4000a..474d3de9ebe 100644 --- a/sdk/python/tests/unit/test_serialization_version.py +++ b/sdk/python/tests/unit/test_serialization_version.py @@ -12,6 +12,6 @@ def test_registry_entity_serialization_version(): provider="local", online_store="redis", registry=f"{tmpdir}/registry.db", - entity_key_serialization_version=2, + entity_key_serialization_version=3, ) - assertpy.assert_that(r.entity_key_serialization_version).is_equal_to(2) + assertpy.assert_that(r.entity_key_serialization_version).is_equal_to(3) diff --git a/sdk/python/tests/unit/test_substrait_transformation.py b/sdk/python/tests/unit/test_substrait_transformation.py index 351651cfda7..1e5cd0889ae 100644 --- a/sdk/python/tests/unit/test_substrait_transformation.py +++ b/sdk/python/tests/unit/test_substrait_transformation.py @@ -19,7 +19,7 @@ def test_ibis_pandas_parity(): project="test_on_demand_substrait_transformation", registry=os.path.join(data_dir, "registry.db"), provider="local", - entity_key_serialization_version=2, + entity_key_serialization_version=3, online_store=SqliteOnlineStoreConfig( path=os.path.join(data_dir, "online.db") ), diff --git a/sdk/python/tests/utils/auth_permissions_util.py b/sdk/python/tests/utils/auth_permissions_util.py index dcc456e1d82..c332a5ab8d3 100644 --- a/sdk/python/tests/utils/auth_permissions_util.py +++ b/sdk/python/tests/utils/auth_permissions_util.py @@ -158,7 +158,7 @@ def get_remote_registry_store(server_port, feature_store, tls_mode): auth=feature_store.config.auth, registry=registry_config, provider="local", - entity_key_serialization_version=2, + entity_key_serialization_version=3, repo_path=feature_store.repo_path, ) ) diff --git a/sdk/python/tests/utils/cli_repo_creator.py b/sdk/python/tests/utils/cli_repo_creator.py index 4b8f9aad04b..ea1d7fcf10b 100644 --- a/sdk/python/tests/utils/cli_repo_creator.py +++ b/sdk/python/tests/utils/cli_repo_creator.py @@ -84,7 +84,7 @@ def local_repo( path: {data_path / "online_store.db"} offline_store: type: {offline_store} - entity_key_serialization_version: 2 + entity_key_serialization_version: 3 """ ) elif online_store == "milvus": diff --git a/sdk/python/tests/utils/dynamo_table_creator.py b/sdk/python/tests/utils/dynamo_table_creator.py index 0ebc939dc11..ad547358a90 100644 --- a/sdk/python/tests/utils/dynamo_table_creator.py +++ b/sdk/python/tests/utils/dynamo_table_creator.py @@ -44,7 +44,7 @@ def insert_data_test_table(data, project, tbl_name, region): dynamodb_resource = boto3.resource("dynamodb", region_name=region) table_instance = dynamodb_resource.Table(f"{project}.{tbl_name}") for entity_key, features, timestamp, created_ts in data: - entity_id = compute_entity_id(entity_key, entity_key_serialization_version=2) + entity_id = compute_entity_id(entity_key, entity_key_serialization_version=3) with table_instance.batch_writer() as batch: batch.put_item( Item={ diff --git a/sdk/python/tests/utils/e2e_test_validation.py b/sdk/python/tests/utils/e2e_test_validation.py index ed66aead87d..6d887b8bb07 100644 --- a/sdk/python/tests/utils/e2e_test_validation.py +++ b/sdk/python/tests/utils/e2e_test_validation.py @@ -195,7 +195,7 @@ def make_feature_store_yaml( offline_store=offline_store_config, online_store=online_store, repo_path=str(Path(repo_dir_name)), - entity_key_serialization_version=2, + entity_key_serialization_version=3, ) config_dict = config.model_dump(by_alias=True) if ( diff --git a/ui/feature_repo/feature_store.yaml b/ui/feature_repo/feature_store.yaml index 60342c96bdb..80fcb7354fa 100644 --- a/ui/feature_repo/feature_store.yaml +++ b/ui/feature_repo/feature_store.yaml @@ -5,4 +5,4 @@ online_store: type: sqlite offline_store: type: file -entity_key_serialization_version: 2 +entity_key_serialization_version: 3