|
-| apigateway:*
| arn:aws:apigateway:*::/apis/*/routes/*/routeresponses
arn:aws:apigateway:*::/apis/*/routes/*/routeresponses/*
arn:aws:apigateway:*::/apis/*/routes/*
arn:aws:apigateway:*::/apis/*/routes
arn:aws:apigateway:*::/apis/*/integrations
arn:aws:apigateway:*::/apis/*/stages/*/routesettings/*
arn:aws:apigateway:*::/apis/*
arn:aws:apigateway:*::/apis
|
+| Permissions | Resources |
+| --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ |
+| lambda:CreateFunction
lambda:GetFunction
lambda:DeleteFunction
lambda:AddPermission
lambda:UpdateFunctionConfiguration
| arn:aws:lambda:\:\:function:feast-\* |
+| ecr:CreateRepository
ecr:DescribeRepositories
ecr:DeleteRepository
ecr:PutImage
ecr:DescribeImages
ecr:BatchDeleteImage
ecr:CompleteLayerUpload
ecr:UploadLayerPart
ecr:InitiateLayerUpload
ecr:BatchCheckLayerAvailability
ecr:GetDownloadUrlForLayer
ecr:GetRepositoryPolicy
ecr:SetRepositoryPolicy
ecr:GetAuthorizationToken
| \* |
+| iam:PassRole
| arn:aws:iam::\:role/ |
+| apigateway:*
| arn:aws:apigateway:*::/apis/*/routes/*/routeresponses
arn:aws:apigateway:*::/apis/*/routes/*/routeresponses/*
arn:aws:apigateway:*::/apis/*/routes/*
arn:aws:apigateway:*::/apis/*/routes
arn:aws:apigateway:*::/apis/*/integrations
arn:aws:apigateway:*::/apis/*/stages/*/routesettings/*
arn:aws:apigateway:*::/apis/*
arn:aws:apigateway:*::/apis
|
The following inline policy can be used to grant Feast the necessary permissions:
@@ -202,4 +195,3 @@ $ curl -X POST \
]
}
```
-
diff --git a/docs/reference/feature-servers/python-feature-server.md b/docs/reference/feature-servers/python-feature-server.md
index 2646c28ef4..7588435576 100644
--- a/docs/reference/feature-servers/python-feature-server.md
+++ b/docs/reference/feature-servers/python-feature-server.md
@@ -10,13 +10,14 @@ There is a CLI command that starts the server: `feast serve`. By default, Feast
## Deploying as a service
-One can deploy a feature server by building a docker image that bundles in the project's `feature_store.yaml`. See this [helm chart](https://github.com/feast-dev/feast/blob/master/infra/charts/feast-python-server) for an example.
+One can deploy a feature server by building a docker image that bundles in the project's `feature_store.yaml`. See this [helm chart](https://github.com/feast-dev/feast/blob/master/infra/charts/feast-python-server) for an example on how to run Feast on Kubernetes.
-A [remote feature server](../alpha-aws-lambda-feature-server.md) on AWS Lambda is also available.
+A [remote feature server](alpha-aws-lambda-feature-server.md) on AWS Lambda is also available.
## Example
### Initializing a feature server
+
Here's an example of how to start the Python feature server with a local feature repo:
```bash
@@ -49,6 +50,7 @@ INFO: Uvicorn running on http://127.0.0.1:6566 (Press CTRL+C to quit)
```
### Retrieving features
+
After the server starts, we can execute cURL commands from another terminal tab:
```bash
@@ -140,7 +142,7 @@ $ curl -X POST \
It's also possible to specify a feature service name instead of the list of features:
-```text
+```
curl -X POST \
"http://localhost:6566/get-online-features" \
-d '{
@@ -152,10 +154,12 @@ curl -X POST \
```
### Pushing features to the online and offline stores
+
The Python feature server also exposes an endpoint for [push sources](../../data-sources/push.md). This endpoint allows you to push data to the online and/or offline store.
-The request definition for pushmode is a string parameter `to` where the options are: ["online", "offline", "online_and_offline"]. Note that timestamps need to be strings.
-```text
+The request definition for pushmode is a string parameter `to` where the options are: \["online", "offline", "online\_and\_offline"]. Note that timestamps need to be strings.
+
+```
curl -X POST "http://localhost:6566/push" -d '{
"push_source_name": "driver_hourly_stats_push_source",
"df": {
@@ -171,6 +175,7 @@ curl -X POST "http://localhost:6566/push" -d '{
```
or equivalently from Python:
+
```python
import json
import requests
diff --git a/docs/reference/offline-stores/README.md b/docs/reference/offline-stores/README.md
index 57d7f35dea..08a28f9e7e 100644
--- a/docs/reference/offline-stores/README.md
+++ b/docs/reference/offline-stores/README.md
@@ -1,16 +1,37 @@
# Offline stores
-Please see [Offline Store](../../getting-started/architecture-and-components/offline-store.md) for an explanation of offline stores.
+Please see [Offline Store](../../getting-started/architecture-and-components/offline-store.md) for a conceptual explanation of offline stores.
-{% page-ref page="file.md" %}
+## Reference
-{% page-ref page="snowflake.md" %}
+{% content-ref url="overview.md" %}
+[overview.md](overview.md)
+{% endcontent-ref %}
-{% page-ref page="bigquery.md" %}
+{% content-ref url="file.md" %}
+[file.md](file.md)
+{% endcontent-ref %}
-{% page-ref page="redshift.md" %}
+{% content-ref url="snowflake.md" %}
+[snowflake.md](snowflake.md)
+{% endcontent-ref %}
-{% page-ref page="spark.md" %}
+{% content-ref url="bigquery.md" %}
+[bigquery.md](bigquery.md)
+{% endcontent-ref %}
-{% page-ref page="postgres.md" %}
+{% content-ref url="redshift.md" %}
+[redshift.md](redshift.md)
+{% endcontent-ref %}
+{% content-ref url="spark.md" %}
+[spark.md](spark.md)
+{% endcontent-ref %}
+
+{% content-ref url="postgres.md" %}
+[postgres.md](postgres.md)
+{% endcontent-ref %}
+
+{% content-ref url="trino.md" %}
+[trino.md](trino.md)
+{% endcontent-ref %}
diff --git a/docs/reference/offline-stores/bigquery.md b/docs/reference/offline-stores/bigquery.md
index 255c587d6b..0e286d78c4 100644
--- a/docs/reference/offline-stores/bigquery.md
+++ b/docs/reference/offline-stores/bigquery.md
@@ -1,13 +1,11 @@
-# BigQuery
+# BigQuery offline store
## Description
The BigQuery offline store provides support for reading [BigQuerySources](../data-sources/bigquery.md).
-* BigQuery tables and views are allowed as sources.
* All joins happen within BigQuery.
-* Entity dataframes can be provided as a SQL query or can be provided as a Pandas dataframe. Pandas dataframes will be uploaded to BigQuery in order to complete join operations.
-* A [BigQueryRetrievalJob](https://github.com/feast-dev/feast/blob/c50a36ec1ad5b8d81c6f773c23204db7c7a7d218/sdk/python/feast/infra/offline_stores/bigquery.py#L210) is returned when calling `get_historical_features()`.
+* Entity dataframes can be provided as a SQL query or can be provided as a Pandas dataframe. A Pandas dataframes will be uploaded to BigQuery as a table (marked for expiration) in order to complete join operations.
## Example
@@ -22,4 +20,38 @@ offline_store:
```
{% endcode %}
-Configuration options are available [here](https://rtd.feast.dev/en/latest/#feast.repo_config.BigQueryOfflineStoreConfig).
+The full set of configuration options is available in [BigQueryOfflineStoreConfig](https://rtd.feast.dev/en/latest/index.html#feast.infra.offline_stores.bigquery.BigQueryOfflineStoreConfig).
+
+## Functionality Matrix
+
+The set of functionality supported by offline stores is described in detail [here](overview.md#functionality).
+Below is a matrix indicating which functionality is supported by the BigQuery offline store.
+
+| | BigQuery |
+| :----------------------------------------------------------------- | :------- |
+| `get_historical_features` (point-in-time correct join) | yes |
+| `pull_latest_from_table_or_query` (retrieve latest feature values) | yes |
+| `pull_all_from_table_or_query` (retrieve a saved dataset) | yes |
+| `offline_write_batch` (persist dataframes to offline store) | yes |
+| `write_logged_features` (persist logged features to offline store) | yes |
+
+Below is a matrix indicating which functionality is supported by `BigQueryRetrievalJob`.
+
+| | BigQuery |
+| ----------------------------------------------------- | -------- |
+| export to dataframe | yes |
+| export to arrow table | yes |
+| export to arrow batches | no |
+| export to SQL | yes |
+| export to data lake (S3, GCS, etc.) | no |
+| export to data warehouse | yes |
+| export as Spark dataframe | no |
+| local execution of Python-based on-demand transforms | yes |
+| remote execution of Python-based on-demand transforms | no |
+| persist results in the offline store | yes |
+| preview the query plan before execution | yes |
+| read partitioned data* | partial |
+
+*See [GitHub issue](https://github.com/feast-dev/feast/issues/2530) for details on proposed solutions for enabling the BigQuery offline store to understand tables that use `_PARTITIONTIME` as the partition column.
+
+To compare this set of functionality against other offline stores, please see the full [functionality matrix](overview.md#functionality-matrix).
diff --git a/docs/reference/offline-stores/file.md b/docs/reference/offline-stores/file.md
index 42ac821691..4b76d9af90 100644
--- a/docs/reference/offline-stores/file.md
+++ b/docs/reference/offline-stores/file.md
@@ -1,11 +1,13 @@
-# File
+# File offline store
## Description
-The File offline store provides support for reading [FileSources](../data-sources/file.md).
+The file offline store provides support for reading [FileSources](../data-sources/file.md).
+It uses Dask as the compute engine.
-* Only Parquet files are currently supported.
-* All data is downloaded and joined using Python and may not scale to production workloads.
+{% hint style="warning" %}
+All data is downloaded and joined using Python and therefore may not scale to production workloads.
+{% endhint %}
## Example
@@ -19,4 +21,36 @@ offline_store:
```
{% endcode %}
-Configuration options are available [here](https://rtd.feast.dev/en/latest/#feast.repo_config.FileOfflineStoreConfig).
+The full set of configuration options is available in [FileOfflineStoreConfig](https://rtd.feast.dev/en/latest/#feast.infra.offline_stores.file.FileOfflineStoreConfig).
+
+## Functionality Matrix
+
+The set of functionality supported by offline stores is described in detail [here](overview.md#functionality).
+Below is a matrix indicating which functionality is supported by the file offline store.
+
+| | File |
+| :-------------------------------- | :-- |
+| `get_historical_features` (point-in-time correct join) | yes |
+| `pull_latest_from_table_or_query` (retrieve latest feature values) | yes |
+| `pull_all_from_table_or_query` (retrieve a saved dataset) | yes |
+| `offline_write_batch` (persist dataframes to offline store) | yes |
+| `write_logged_features` (persist logged features to offline store) | yes |
+
+Below is a matrix indicating which functionality is supported by `FileRetrievalJob`.
+
+| | File |
+| --------------------------------- | --- |
+| export to dataframe | yes |
+| export to arrow table | yes |
+| export to arrow batches | no |
+| export to SQL | no |
+| export to data lake (S3, GCS, etc.) | no |
+| export to data warehouse | no |
+| export as Spark dataframe | no |
+| local execution of Python-based on-demand transforms | yes |
+| remote execution of Python-based on-demand transforms | no |
+| persist results in the offline store | yes |
+| preview the query plan before execution | yes |
+| read partitioned data | yes |
+
+To compare this set of functionality against other offline stores, please see the full [functionality matrix](overview.md#functionality-matrix).
diff --git a/docs/reference/offline-stores/overview.md b/docs/reference/offline-stores/overview.md
new file mode 100644
index 0000000000..10f99813ba
--- /dev/null
+++ b/docs/reference/offline-stores/overview.md
@@ -0,0 +1,58 @@
+# Overview
+
+## Functionality
+
+Here are the methods exposed by the `OfflineStore` interface, along with the core functionality supported by the method:
+* `get_historical_features`: point-in-time correct join to retrieve historical features
+* `pull_latest_from_table_or_query`: retrieve latest feature values for materialization into the online store
+* `pull_all_from_table_or_query`: retrieve a saved dataset
+* `offline_write_batch`: persist dataframes to the offline store, primarily for push sources
+* `write_logged_features`: persist logged features to the offline store, for feature logging
+
+The first three of these methods all return a `RetrievalJob` specific to an offline store, such as a `SnowflakeRetrievalJob`. Here is a list of functionality supported by `RetrievalJob`s:
+* export to dataframe
+* export to arrow table
+* export to arrow batches (to handle large datasets in memory)
+* export to SQL
+* export to data lake (S3, GCS, etc.)
+* export to data warehouse
+* export as Spark dataframe
+* local execution of Python-based on-demand transforms
+* remote execution of Python-based on-demand transforms
+* persist results in the offline store
+* preview the query plan before execution (`RetrievalJob`s are lazily executed)
+* read partitioned data
+
+## Functionality Matrix
+
+There are currently four core offline store implementations: `FileOfflineStore`, `BigQueryOfflineStore`, `SnowflakeOfflineStore`, and `RedshiftOfflineStore`.
+There are several additional implementations contributed by the Feast community (`PostgreSQLOfflineStore`, `SparkOfflineStore`, and `TrinoOfflineStore`), which are not guaranteed to be stable or to match the functionality of the core implementations.
+Details for each specific offline store, such as how to configure it in a `feature_store.yaml`, can be found [here](README.md).
+
+Below is a matrix indicating which offline stores support which methods.
+
+| | File | BigQuery | Snowflake | Redshift | Postgres | Spark | Trino |
+| :-------------------------------- | :-- | :-- | :-- | :-- | :-- | :-- | :-- |
+| `get_historical_features` | yes | yes | yes | yes | yes | yes | yes |
+| `pull_latest_from_table_or_query` | yes | yes | yes | yes | yes | yes | yes |
+| `pull_all_from_table_or_query` | yes | yes | yes | yes | yes | yes | yes |
+| `offline_write_batch` | yes | yes | yes | yes | no | no | no |
+| `write_logged_features` | yes | yes | yes | yes | no | no | no |
+
+
+Below is a matrix indicating which `RetrievalJob`s support what functionality.
+
+| | File | BigQuery | Snowflake | Redshift | Postgres | Spark | Trino |
+| --------------------------------- | --- | --- | --- | --- | --- | --- | --- |
+| export to dataframe | yes | yes | yes | yes | yes | yes | yes |
+| export to arrow table | yes | yes | yes | yes | yes | yes | yes |
+| export to arrow batches | no | no | no | yes | no | no | no |
+| export to SQL | no | yes | no | yes | yes | no | yes |
+| export to data lake (S3, GCS, etc.) | no | no | yes | no | yes | no | no |
+| export to data warehouse | no | yes | yes | yes | yes | no | no |
+| export as Spark dataframe | no | no | no | no | no | yes | no |
+| local execution of Python-based on-demand transforms | yes | yes | yes | yes | yes | no | yes |
+| remote execution of Python-based on-demand transforms | no | no | no | no | no | no | no |
+| persist results in the offline store | yes | yes | yes | yes | yes | yes | no |
+| preview the query plan before execution | yes | yes | yes | yes | yes | yes | yes |
+| read partitioned data | yes | yes | yes | yes | yes | yes | yes |
diff --git a/docs/reference/offline-stores/postgres.md b/docs/reference/offline-stores/postgres.md
index 9bd472673a..506666fc37 100644
--- a/docs/reference/offline-stores/postgres.md
+++ b/docs/reference/offline-stores/postgres.md
@@ -1,20 +1,14 @@
-# PostgreSQL (contrib)
+# PostgreSQL offline store (contrib)
## Description
-The PostgreSQL offline store is an offline store that provides support for reading [PostgreSQL](../data-sources/postgres.md) data sources.
+The PostgreSQL offline store provides support for reading [PostgreSQLSources](../data-sources/postgres.md).
+* Entity dataframes can be provided as a SQL query or can be provided as a Pandas dataframe. A Pandas dataframes will be uploaded to Postgres as a table in order to complete join operations.
+## Disclaimer
-**DISCLAIMER**: This PostgreSQL offline store still does not achieve full test coverage.
-
-* Entity dataframes can be provided as a SQL query or can be provided as a Pandas dataframe. Pandas dataframes will be converted to a Spark dataframe and processed as a temporary view.
-* A `PostgreSQLRetrievalJob` is returned when calling `get_historical_features()`.
- * This allows you to call
- * `to_df` to retrieve the pandas dataframe.
- * `to_arrow` to retrieve the dataframe as a PyArrow table.
- * `to_sql` to get the SQL query used to pull the features.
-
-* sslmode, sslkey_path, sslcert_path, and sslrootcert_path are optional
+The PostgreSQL offline store does not achieve full test coverage.
+Please do not assume complete stability.
## Example
@@ -39,3 +33,38 @@ online_store:
path: data/online_store.db
```
{% endcode %}
+
+Note that `sslmode`, `sslkey_path`, `sslcert_path`, and `sslrootcert_path` are optional parameters.
+The full set of configuration options is available in [PostgreSQLOfflineStoreConfig](https://rtd.feast.dev/en/master/#feast.infra.offline_stores.contrib.postgres_offline_store.postgres.PostgreSQLOfflineStoreConfig).
+
+## Functionality Matrix
+
+The set of functionality supported by offline stores is described in detail [here](overview.md#functionality).
+Below is a matrix indicating which functionality is supported by the PostgreSQL offline store.
+
+| | Postgres |
+| :-------------------------------- | :-- |
+| `get_historical_features` (point-in-time correct join) | yes |
+| `pull_latest_from_table_or_query` (retrieve latest feature values) | yes |
+| `pull_all_from_table_or_query` (retrieve a saved dataset) | yes |
+| `offline_write_batch` (persist dataframes to offline store) | no |
+| `write_logged_features` (persist logged features to offline store) | no |
+
+Below is a matrix indicating which functionality is supported by `PostgreSQLRetrievalJob`.
+
+| | Postgres |
+| --------------------------------- | --- |
+| export to dataframe | yes |
+| export to arrow table | yes |
+| export to arrow batches | no |
+| export to SQL | yes |
+| export to data lake (S3, GCS, etc.) | yes |
+| export to data warehouse | yes |
+| export as Spark dataframe | no |
+| local execution of Python-based on-demand transforms | yes |
+| remote execution of Python-based on-demand transforms | no |
+| persist results in the offline store | yes |
+| preview the query plan before execution | yes |
+| read partitioned data | yes |
+
+To compare this set of functionality against other offline stores, please see the full [functionality matrix](overview.md#functionality-matrix).
diff --git a/docs/reference/offline-stores/redshift.md b/docs/reference/offline-stores/redshift.md
index 73148730c5..2cdf49bdb9 100644
--- a/docs/reference/offline-stores/redshift.md
+++ b/docs/reference/offline-stores/redshift.md
@@ -1,13 +1,11 @@
-# Redshift
+# Redshift offline store
## Description
The Redshift offline store provides support for reading [RedshiftSources](../data-sources/redshift.md).
-* Redshift tables and views are allowed as sources.
* All joins happen within Redshift.
-* Entity dataframes can be provided as a SQL query or can be provided as a Pandas dataframe. Pandas dataframes will be uploaded to Redshift in order to complete join operations.
-* A [RedshiftRetrievalJob](https://github.com/feast-dev/feast/blob/bf557bcb72c7878a16dccb48443bbbe9dc3efa49/sdk/python/feast/infra/offline_stores/redshift.py#L161) is returned when calling `get_historical_features()`.
+* Entity dataframes can be provided as a SQL query or can be provided as a Pandas dataframe. A Pandas dataframes will be uploaded to Redshift temporarily in order to complete join operations.
## Example
@@ -27,7 +25,39 @@ offline_store:
```
{% endcode %}
-Configuration options are available [here](https://github.com/feast-dev/feast/blob/bf557bcb72c7878a16dccb48443bbbe9dc3efa49/sdk/python/feast/infra/offline_stores/redshift.py#L22).
+The full set of configuration options is available in [RedshiftOfflineStoreConfig](https://rtd.feast.dev/en/master/#feast.infra.offline_stores.redshift.RedshiftOfflineStoreConfig).
+
+## Functionality Matrix
+
+The set of functionality supported by offline stores is described in detail [here](overview.md#functionality).
+Below is a matrix indicating which functionality is supported by the Redshift offline store.
+
+| | Redshift |
+| :-------------------------------- | :-- |
+| `get_historical_features` (point-in-time correct join) | yes |
+| `pull_latest_from_table_or_query` (retrieve latest feature values) | yes |
+| `pull_all_from_table_or_query` (retrieve a saved dataset) | yes |
+| `offline_write_batch` (persist dataframes to offline store) | yes |
+| `write_logged_features` (persist logged features to offline store) | yes |
+
+Below is a matrix indicating which functionality is supported by `RedshiftRetrievalJob`.
+
+| | Redshift |
+| --------------------------------- | --- |
+| export to dataframe | yes |
+| export to arrow table | yes |
+| export to arrow batches | yes |
+| export to SQL | yes |
+| export to data lake (S3, GCS, etc.) | no |
+| export to data warehouse | yes |
+| export as Spark dataframe | no |
+| local execution of Python-based on-demand transforms | yes |
+| remote execution of Python-based on-demand transforms | no |
+| persist results in the offline store | yes |
+| preview the query plan before execution | yes |
+| read partitioned data | yes |
+
+To compare this set of functionality against other offline stores, please see the full [functionality matrix](overview.md#functionality-matrix).
## Permissions
diff --git a/docs/reference/offline-stores/snowflake.md b/docs/reference/offline-stores/snowflake.md
index e2afaef90d..b3b58fe786 100644
--- a/docs/reference/offline-stores/snowflake.md
+++ b/docs/reference/offline-stores/snowflake.md
@@ -1,17 +1,10 @@
-# Snowflake
+# Snowflake offline store
## Description
The [Snowflake](https://trial.snowflake.com) offline store provides support for reading [SnowflakeSources](../data-sources/snowflake.md).
-
-* Snowflake tables and views are allowed as sources.
* All joins happen within Snowflake.
-* Entity dataframes can be provided as a SQL query or can be provided as a Pandas dataframe. Pandas dataframes will be uploaded to Snowflake in order to complete join operations.
-* A `SnowflakeRetrievalJob` is returned when calling `get_historical_features()`.
- * This allows you to call
- * `to_snowflake` to save the dataset into Snowflake
- * `to_sql` to get the SQL query that would execute on `to_df`
- * `to_arrow_chunks` to get the result in batches ([Snowflake python connector docs](https://docs.snowflake.com/en/user-guide/python-connector-api.html#get_result_batches))
+* Entity dataframes can be provided as a SQL query or can be provided as a Pandas dataframe. A Pandas dataframes will be uploaded to Snowflake as a temporary table in order to complete join operations.
## Example
@@ -31,4 +24,36 @@ offline_store:
```
{% endcode %}
-Configuration options are available in [SnowflakeOfflineStoreConfig](https://github.com/feast-dev/feast/blob/master/sdk/python/feast/infra/offline_stores/snowflake.py#L56).
+The full set of configuration options is available in [SnowflakeOfflineStoreConfig](https://rtd.feast.dev/en/latest/#feast.infra.offline_stores.snowflake.SnowflakeOfflineStoreConfig).
+
+## Functionality Matrix
+
+The set of functionality supported by offline stores is described in detail [here](overview.md#functionality).
+Below is a matrix indicating which functionality is supported by the Snowflake offline store.
+
+| | Snowflake |
+| :-------------------------------- | :-- |
+| `get_historical_features` (point-in-time correct join) | yes |
+| `pull_latest_from_table_or_query` (retrieve latest feature values) | yes |
+| `pull_all_from_table_or_query` (retrieve a saved dataset) | yes |
+| `offline_write_batch` (persist dataframes to offline store) | yes |
+| `write_logged_features` (persist logged features to offline store) | yes |
+
+Below is a matrix indicating which functionality is supported by `SnowflakeRetrievalJob`.
+
+| | Snowflake |
+| --------------------------------- | --- |
+| export to dataframe | yes |
+| export to arrow table | yes |
+| export to arrow batches | no |
+| export to SQL | no |
+| export to data lake (S3, GCS, etc.) | yes |
+| export to data warehouse | yes |
+| export as Spark dataframe | no |
+| local execution of Python-based on-demand transforms | yes |
+| remote execution of Python-based on-demand transforms | no |
+| persist results in the offline store | yes |
+| preview the query plan before execution | yes |
+| read partitioned data | yes |
+
+To compare this set of functionality against other offline stores, please see the full [functionality matrix](overview.md#functionality-matrix).
diff --git a/docs/reference/offline-stores/spark.md b/docs/reference/offline-stores/spark.md
index 7eec8d7b73..f1ef1300bd 100644
--- a/docs/reference/offline-stores/spark.md
+++ b/docs/reference/offline-stores/spark.md
@@ -1,20 +1,15 @@
-# Spark (contrib)
+# Spark offline store (contrib)
## Description
-The Spark offline store is an offline store currently in alpha development that provides support for reading [SparkSources](../data-sources/spark.md).
+The Spark offline store provides support for reading [SparkSources](../data-sources/spark.md).
-## Disclaimer
+* Entity dataframes can be provided as a SQL query or can be provided as a Pandas dataframe. A Pandas dataframes will be converted to a Spark dataframe and processed as a temporary view.
-This Spark offline store still does not achieve full test coverage and continues to fail some integration tests when integrating with the feast universal test suite. Please do NOT assume complete stability of the API.
+## Disclaimer
-* Spark tables and views are allowed as sources that are loaded in from some Spark store(e.g in Hive or in memory).
-* Entity dataframes can be provided as a SQL query or can be provided as a Pandas dataframe. Pandas dataframes will be converted to a Spark dataframe and processed as a temporary view.
-* A `SparkRetrievalJob` is returned when calling `get_historical_features()`.
- * This allows you to call
- * `to_df` to retrieve the pandas dataframe.
- * `to_arrow` to retrieve the dataframe as a pyarrow Table.
- * `to_spark_df` to retrieve the dataframe the spark.
+The Spark offline store does not achieve full test coverage.
+Please do not assume complete stability.
## Example
@@ -36,3 +31,37 @@ online_store:
path: data/online_store.db
```
{% endcode %}
+
+The full set of configuration options is available in [SparkOfflineStoreConfig](https://rtd.feast.dev/en/master/#feast.infra.offline_stores.contrib.spark_offline_store.spark.SparkOfflineStoreConfig).
+
+## Functionality Matrix
+
+The set of functionality supported by offline stores is described in detail [here](overview.md#functionality).
+Below is a matrix indicating which functionality is supported by the Spark offline store.
+
+| | Spark |
+| :-------------------------------- | :-- |
+| `get_historical_features` (point-in-time correct join) | yes |
+| `pull_latest_from_table_or_query` (retrieve latest feature values) | yes |
+| `pull_all_from_table_or_query` (retrieve a saved dataset) | yes |
+| `offline_write_batch` (persist dataframes to offline store) | no |
+| `write_logged_features` (persist logged features to offline store) | no |
+
+Below is a matrix indicating which functionality is supported by `SparkRetrievalJob`.
+
+| | Spark |
+| --------------------------------- | --- |
+| export to dataframe | yes |
+| export to arrow table | yes |
+| export to arrow batches | no |
+| export to SQL | no |
+| export to data lake (S3, GCS, etc.) | no |
+| export to data warehouse | no |
+| export as Spark dataframe | yes |
+| local execution of Python-based on-demand transforms | no |
+| remote execution of Python-based on-demand transforms | no |
+| persist results in the offline store | yes |
+| preview the query plan before execution | yes |
+| read partitioned data | yes |
+
+To compare this set of functionality against other offline stores, please see the full [functionality matrix](overview.md#functionality-matrix).
diff --git a/docs/reference/offline-stores/trino.md b/docs/reference/offline-stores/trino.md
new file mode 100644
index 0000000000..8cc604248f
--- /dev/null
+++ b/docs/reference/offline-stores/trino.md
@@ -0,0 +1,64 @@
+# Trino offline store (contrib)
+
+## Description
+
+The Trino offline store provides support for reading [TrinoSources](../data-sources/trino.md).
+* Entity dataframes can be provided as a SQL query or can be provided as a Pandas dataframe. A Pandas dataframes will be uploaded to Trino as a table in order to complete join operations.
+
+## Disclaimer
+
+The Trino offline store does not achieve full test coverage.
+Please do not assume complete stability.
+
+## Example
+
+{% code title="feature_store.yaml" %}
+```yaml
+project: feature_repo
+registry: data/registry.db
+provider: local
+offline_store:
+ type: feast_trino.trino.TrinoOfflineStore
+ host: localhost
+ port: 8080
+ catalog: memory
+ connector:
+ type: memory
+online_store:
+ path: data/online_store.db
+```
+{% endcode %}
+
+The full set of configuration options is available in [TrinoOfflineStoreConfig](https://rtd.feast.dev/en/master/#trino-offline-store).
+
+## Functionality Matrix
+
+The set of functionality supported by offline stores is described in detail [here](overview.md#functionality).
+Below is a matrix indicating which functionality is supported by the Trino offline store.
+
+| | Trino |
+| :-------------------------------- | :-- |
+| `get_historical_features` (point-in-time correct join) | yes |
+| `pull_latest_from_table_or_query` (retrieve latest feature values) | yes |
+| `pull_all_from_table_or_query` (retrieve a saved dataset) | yes |
+| `offline_write_batch` (persist dataframes to offline store) | no |
+| `write_logged_features` (persist logged features to offline store) | no |
+
+Below is a matrix indicating which functionality is supported by `TrinoRetrievalJob`.
+
+| | Trino |
+| --------------------------------- | --- |
+| export to dataframe | yes |
+| export to arrow table | yes |
+| export to arrow batches | no |
+| export to SQL | yes |
+| export to data lake (S3, GCS, etc.) | no |
+| export to data warehouse | no |
+| export as Spark dataframe | no |
+| local execution of Python-based on-demand transforms | yes |
+| remote execution of Python-based on-demand transforms | no |
+| persist results in the offline store | no |
+| preview the query plan before execution | yes |
+| read partitioned data | yes |
+
+To compare this set of functionality against other offline stores, please see the full [functionality matrix](overview.md#functionality-matrix).
diff --git a/docs/reference/online-stores/README.md b/docs/reference/online-stores/README.md
index 5eb566af3c..8367e2ce74 100644
--- a/docs/reference/online-stores/README.md
+++ b/docs/reference/online-stores/README.md
@@ -2,14 +2,26 @@
Please see [Online Store](../../getting-started/architecture-and-components/online-store.md) for an explanation of online stores.
-{% page-ref page="sqlite.md" %}
+{% content-ref url="sqlite.md" %}
+[sqlite.md](sqlite.md)
+{% endcontent-ref %}
-{% page-ref page="snowflake.md" %}
+{% content-ref url="snowflake.md" %}
+[snowflake.md](snowflake.md)
+{% endcontent-ref %}
-{% page-ref page="redis.md" %}
+{% content-ref url="redis.md" %}
+[redis.md](redis.md)
+{% endcontent-ref %}
-{% page-ref page="datastore.md" %}
+{% content-ref url="datastore.md" %}
+[datastore.md](datastore.md)
+{% endcontent-ref %}
-{% page-ref page="dynamodb.md" %}
+{% content-ref url="dynamodb.md" %}
+[dynamodb.md](dynamodb.md)
+{% endcontent-ref %}
-{% page-ref page="postgres.md" %}
+{% content-ref url="postgres.md" %}
+[postgres.md](postgres.md)
+{% endcontent-ref %}
diff --git a/docs/reference/online-stores/datastore.md b/docs/reference/online-stores/datastore.md
index 012d497f30..ed1425abb6 100644
--- a/docs/reference/online-stores/datastore.md
+++ b/docs/reference/online-stores/datastore.md
@@ -1,4 +1,4 @@
-# Datastore
+# Datastore online store
## Description
diff --git a/docs/reference/online-stores/dynamodb.md b/docs/reference/online-stores/dynamodb.md
index 2af7e422d6..f9f8b4339d 100644
--- a/docs/reference/online-stores/dynamodb.md
+++ b/docs/reference/online-stores/dynamodb.md
@@ -1,4 +1,4 @@
-# DynamoDB
+# DynamoDB online store
## Description
diff --git a/docs/reference/online-stores/postgres.md b/docs/reference/online-stores/postgres.md
index 7d24079da9..4f51dff617 100644
--- a/docs/reference/online-stores/postgres.md
+++ b/docs/reference/online-stores/postgres.md
@@ -1,4 +1,4 @@
-# PostgreSQL (contrib)
+# PostgreSQL online store (contrib)
## Description
diff --git a/docs/reference/online-stores/redis.md b/docs/reference/online-stores/redis.md
index ce1de2ad54..4388ccfa0a 100644
--- a/docs/reference/online-stores/redis.md
+++ b/docs/reference/online-stores/redis.md
@@ -1,4 +1,4 @@
-# Redis
+# Redis online store
## Description
diff --git a/docs/reference/online-stores/snowflake.md b/docs/reference/online-stores/snowflake.md
index ccf3d526da..bf975fa7ea 100644
--- a/docs/reference/online-stores/snowflake.md
+++ b/docs/reference/online-stores/snowflake.md
@@ -1,4 +1,4 @@
-# Snowflake
+# Snowflake online store
## Description
diff --git a/docs/reference/online-stores/sqlite.md b/docs/reference/online-stores/sqlite.md
index fd11e3439c..668e6024e3 100644
--- a/docs/reference/online-stores/sqlite.md
+++ b/docs/reference/online-stores/sqlite.md
@@ -1,4 +1,4 @@
-# SQLite
+# SQLite online store
## Description
diff --git a/docs/tutorials/driver-ranking-with-feast.md b/docs/tutorials/driver-ranking-with-feast.md
deleted file mode 100644
index 4ad34cd9c0..0000000000
--- a/docs/tutorials/driver-ranking-with-feast.md
+++ /dev/null
@@ -1,25 +0,0 @@
----
-description: >-
- Making a prediction using a linear regression model is a common use case in
- ML. This model predicts if a driver will complete a trip based on features
- ingested into Feast.
----
-
-# Driver ranking
-
-In this example, you'll learn how to use some of the key functionality in Feast. The tutorial runs in both local mode and on the Google Cloud Platform \(GCP\). For GCP, you must have access to a GCP project already, including read and write permissions to BigQuery.
-
-## [Driver Ranking Example](https://github.com/feast-dev/feast-driver-ranking-tutorial)
-
-This tutorial guides you on how to use Feast with [Scikit-learn](https://scikit-learn.org/stable/). You will learn how to:
-
-* Train a model locally \(on your laptop\) using data from [BigQuery](https://cloud.google.com/bigquery/)
-* Test the model for online inference using [SQLite](https://www.sqlite.org/index.html) \(for fast iteration\)
-* Test the model for online inference using [Firestore](https://firebase.google.com/products/firestore) \(for production use\)
-
-Try it and let us know what you think!
-
-| [ Run in Google Colab ](https://colab.research.google.com/github/feast-dev/feast-driver-ranking-tutorial/blob/master/notebooks/Driver_Ranking_Tutorial.ipynb) | [ View Source in Github](https://github.com/feast-dev/feast-driver-ranking-tutorial/blob/master/notebooks/Driver_Ranking_Tutorial.ipynb) |
-| :--- | :--- |
-
-
diff --git a/docs/tutorials/tutorials-overview.md b/docs/tutorials/tutorials-overview.md
deleted file mode 100644
index 9432783a69..0000000000
--- a/docs/tutorials/tutorials-overview.md
+++ /dev/null
@@ -1,15 +0,0 @@
-# Overview
-
-These Feast tutorials showcase how to use Feast to simplify end to end model training / serving.
-
-{% page-ref page="fraud-detection.md" %}
-
-{% page-ref page="driver-ranking-with-feast.md" %}
-
-{% page-ref page="real-time-credit-scoring-on-aws.md" %}
-
-{% page-ref page="driver-stats-on-snowflake.md" %}
-
-{% page-ref page="validating-historical-features.md" %}
-
-{% page-ref page="using-scalable-registry.md" %}
diff --git a/docs/tutorials/tutorials-overview/README.md b/docs/tutorials/tutorials-overview/README.md
new file mode 100644
index 0000000000..76cb2bea6b
--- /dev/null
+++ b/docs/tutorials/tutorials-overview/README.md
@@ -0,0 +1,19 @@
+# Sample use-case tutorials
+
+These Feast tutorials showcase how to use Feast to simplify end to end model training / serving.
+
+{% content-ref url="driver-ranking-with-feast.md" %}
+[driver-ranking-with-feast.md](driver-ranking-with-feast.md)
+{% endcontent-ref %}
+
+{% content-ref url="fraud-detection.md" %}
+[fraud-detection.md](fraud-detection.md)
+{% endcontent-ref %}
+
+{% content-ref url="real-time-credit-scoring-on-aws.md" %}
+[real-time-credit-scoring-on-aws.md](real-time-credit-scoring-on-aws.md)
+{% endcontent-ref %}
+
+{% content-ref url="driver-stats-on-snowflake.md" %}
+[driver-stats-on-snowflake.md](driver-stats-on-snowflake.md)
+{% endcontent-ref %}
diff --git a/docs/tutorials/tutorials-overview/driver-ranking-with-feast.md b/docs/tutorials/tutorials-overview/driver-ranking-with-feast.md
new file mode 100644
index 0000000000..54f3035319
--- /dev/null
+++ b/docs/tutorials/tutorials-overview/driver-ranking-with-feast.md
@@ -0,0 +1,23 @@
+---
+description: >-
+ Making a prediction using a linear regression model is a common use case in
+ ML. This model predicts if a driver will complete a trip based on features
+ ingested into Feast.
+---
+
+# Driver ranking
+
+In this example, you'll learn how to use some of the key functionality in Feast. The tutorial runs in both local mode and on the Google Cloud Platform (GCP). For GCP, you must have access to a GCP project already, including read and write permissions to BigQuery.
+
+## [Driver Ranking Example](https://github.com/feast-dev/feast-driver-ranking-tutorial)
+
+This tutorial guides you on how to use Feast with [Scikit-learn](https://scikit-learn.org/stable/). You will learn how to:
+
+* Train a model locally (on your laptop) using data from [BigQuery](https://cloud.google.com/bigquery/)
+* Test the model for online inference using [SQLite](https://www.sqlite.org/index.html) (for fast iteration)
+* Test the model for online inference using [Firestore](https://firebase.google.com/products/firestore) (for production use)
+
+Try it and let us know what you think!
+
+| [ Run in Google Colab](https://colab.research.google.com/github/feast-dev/feast-driver-ranking-tutorial/blob/master/notebooks/Driver\_Ranking\_Tutorial.ipynb) | [ View Source in Github](https://github.com/feast-dev/feast-driver-ranking-tutorial/blob/master/notebooks/Driver\_Ranking\_Tutorial.ipynb) |
+| -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
diff --git a/docs/tutorials/driver-stats-on-snowflake.md b/docs/tutorials/tutorials-overview/driver-stats-on-snowflake.md
similarity index 100%
rename from docs/tutorials/driver-stats-on-snowflake.md
rename to docs/tutorials/tutorials-overview/driver-stats-on-snowflake.md
diff --git a/docs/tutorials/fraud-detection.md b/docs/tutorials/tutorials-overview/fraud-detection.md
similarity index 51%
rename from docs/tutorials/fraud-detection.md
rename to docs/tutorials/tutorials-overview/fraud-detection.md
index 7bdfde760e..30564d0b0c 100644
--- a/docs/tutorials/fraud-detection.md
+++ b/docs/tutorials/tutorials-overview/fraud-detection.md
@@ -17,13 +17,9 @@ Our end-to-end example will perform the following workflows:
* Building point-in-time correct training datasets from feature data and training a model
* Making online predictions from feature data
-Here's a high-level picture of our system architecture on Google Cloud Platform \(GCP\):
-
-
-
-
-
-|  [Run in Google Colab](https://colab.research.google.com/github/feast-dev/feast-fraud-tutorial/blob/master/notebooks/Fraud_Detection_Tutorial.ipynb) | [ View Source on Github](https://github.com/feast-dev/feast-fraud-tutorial/blob/main/notebooks/Fraud_Detection_Tutorial.ipynb) |
-| :--- | :--- |
+Here's a high-level picture of our system architecture on Google Cloud Platform (GCP):
+
+|  [Run in Google Colab](https://colab.research.google.com/github/feast-dev/feast-fraud-tutorial/blob/master/notebooks/Fraud\_Detection\_Tutorial.ipynb) | [ View Source on Github](https://github.com/feast-dev/feast-fraud-tutorial/blob/main/notebooks/Fraud\_Detection\_Tutorial.ipynb) |
+| ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
diff --git a/docs/tutorials/real-time-credit-scoring-on-aws.md b/docs/tutorials/tutorials-overview/real-time-credit-scoring-on-aws.md
similarity index 74%
rename from docs/tutorials/real-time-credit-scoring-on-aws.md
rename to docs/tutorials/tutorials-overview/real-time-credit-scoring-on-aws.md
index 43f8c98133..6268aba1f1 100644
--- a/docs/tutorials/real-time-credit-scoring-on-aws.md
+++ b/docs/tutorials/tutorials-overview/real-time-credit-scoring-on-aws.md
@@ -10,20 +10,18 @@ When individuals apply for loans from banks and other credit providers, the deci
In this example, we will demonstrate how a real-time credit scoring system can be built using Feast and Scikit-Learn on AWS, using feature data from S3.
-This real-time system accepts a loan request from a customer and responds within 100ms with a decision on whether their loan has been approved or rejected.
+This real-time system accepts a loan request from a customer and responds within 100ms with a decision on whether their loan has been approved or rejected.
## [Real-time Credit Scoring Example](https://github.com/feast-dev/real-time-credit-scoring-on-aws-tutorial)
This end-to-end tutorial will take you through the following steps:
-* Deploying S3 with Parquet as your primary data source, containing both [loan features](https://github.com/feast-dev/real-time-credit-scoring-on-aws-tutorial/blob/22fc6c7272ef033e7ba0afc64ffaa6f6f8fc0277/data/loan_table_sample.csv) and [zip code features](https://github.com/feast-dev/real-time-credit-scoring-on-aws-tutorial/blob/22fc6c7272ef033e7ba0afc64ffaa6f6f8fc0277/data/zipcode_table_sample.csv)
+* Deploying S3 with Parquet as your primary data source, containing both [loan features](https://github.com/feast-dev/real-time-credit-scoring-on-aws-tutorial/blob/22fc6c7272ef033e7ba0afc64ffaa6f6f8fc0277/data/loan\_table\_sample.csv) and [zip code features](https://github.com/feast-dev/real-time-credit-scoring-on-aws-tutorial/blob/22fc6c7272ef033e7ba0afc64ffaa6f6f8fc0277/data/zipcode\_table\_sample.csv)
* Deploying Redshift as the interface Feast uses to build training datasets
* Registering your features with Feast and configuring DynamoDB for online serving
* Building a training dataset with Feast to train your credit scoring model
* Loading feature values from S3 into DynamoDB
* Making online predictions with your credit scoring model using features from DynamoDB
-| [ View Source on Github](https://github.com/feast-dev/real-time-credit-scoring-on-aws-tutorial) |
-| :--- |
-
-
+| [ View Source on Github](https://github.com/feast-dev/real-time-credit-scoring-on-aws-tutorial) |
+| ---------------------------------------------------------------------------------------------------------------------------------------------- |
diff --git a/docs/tutorials/using-scalable-registry.md b/docs/tutorials/using-scalable-registry.md
index 51fa50ff33..0ee02674b1 100644
--- a/docs/tutorials/using-scalable-registry.md
+++ b/docs/tutorials/using-scalable-registry.md
@@ -13,6 +13,11 @@ However, there's inherent limitations with a file-based registry, since changing
An alternative to the file-based registry is the [SQLRegistry](https://rtd.feast.dev/en/latest/feast.infra.registry_stores.html#feast.infra.registry_stores.sql.SqlRegistry) which ships with Feast. This implementation stores the registry in a relational database, and allows for changes to individual objects atomically.
Under the hood, the SQL Registry implementation uses [SQLAlchemy](https://docs.sqlalchemy.org/en/14/) to abstract over the different databases. Consequently, any [database supported](https://docs.sqlalchemy.org/en/14/core/engines.html#supported-databases) by SQLAlchemy can be used by the SQL Registry.
+The following databases are supported and tested out of the box:
+- PostgreSQL
+- MySQL
+- Sqlite
+
Feast can use the SQL Registry via a config change in the feature_store.yaml file. An example of how to configure this would be:
```yaml
diff --git a/examples/java-demo/README.md b/examples/java-demo/README.md
index b908bb7625..2b1d7f75a5 100644
--- a/examples/java-demo/README.md
+++ b/examples/java-demo/README.md
@@ -39,9 +39,6 @@ For this tutorial, we setup Feast with Redis, using the Feast CLI to register an
connection_string: localhost:6379,password=[YOUR PASSWORD]
offline_store:
type: file
- flags:
- alpha_features: true
- on_demand_transforms: true
```
4. Run `feast apply` to apply your local features to the remote registry
5. Materialize features to the online store:
diff --git a/examples/java-demo/feature_repo/application-override.yaml b/examples/java-demo/feature_repo/application-override.yaml
index dbdeda4c04..5a43d886dc 100644
--- a/examples/java-demo/feature_repo/application-override.yaml
+++ b/examples/java-demo/feature_repo/application-override.yaml
@@ -10,6 +10,7 @@ feature-server:
host: my-redis-master
port: 6379
password: [YOUR PASSWORD]
+ entityKeySerializationVersion: 2
global:
registry:
path: gs://[YOUR BUCKET]/demo-repo/registry.db
diff --git a/examples/java-demo/feature_repo/driver_repo.py b/examples/java-demo/feature_repo/driver_repo.py
index e17a5d9cf8..4c4d45c456 100644
--- a/examples/java-demo/feature_repo/driver_repo.py
+++ b/examples/java-demo/feature_repo/driver_repo.py
@@ -1,13 +1,11 @@
import pandas as pd
from feast.data_source import RequestSource
-from feast.field import Field
from feast.on_demand_feature_view import on_demand_feature_view
-from feast.request_feature_view import RequestFeatureView
from feast.types import Float32, Float64, Int64, String
from google.protobuf.duration_pb2 import Duration
from feast.field import Field
-from feast import Entity, Feature, BatchFeatureView, FileSource
+from feast import Entity, FileSource, FeatureView
driver_hourly_stats = FileSource(
path="data/driver_stats_with_string.parquet",
@@ -15,10 +13,10 @@
created_timestamp_column="created",
)
driver = Entity(name="driver_id", description="driver id",)
-driver_hourly_stats_view = BatchFeatureView(
+driver_hourly_stats_view = FeatureView(
name="driver_hourly_stats",
- entities=["driver_id"],
- ttl=Duration(seconds=86400000),
+ entities=[driver],
+ ttl=timedelta(days=365),
schema=[
Field(name="conv_rate", dtype=Float32),
Field(name="acc_rate", dtype=Float32),
@@ -40,6 +38,7 @@
],
)
+
# Define an on demand feature view which can generate new features based on
# existing feature views and RequestSource features
@on_demand_feature_view(
@@ -58,14 +57,3 @@ def transformed_conv_rate(inputs: pd.DataFrame) -> pd.DataFrame:
df["conv_rate_plus_val2"] = inputs["conv_rate"] + inputs["val_to_add_2"]
return df
-
-# Define request feature view
-driver_age_request_fv = RequestFeatureView(
- name="driver_age",
- request_data_source=RequestSource(
- name="driver_age",
- schema=[
- Field(name="driver_age", dtype=Int64),
- ],
- ),
-)
diff --git a/examples/java-demo/feature_repo/feature_store.yaml b/examples/java-demo/feature_repo/feature_store.yaml
index 91c65b512a..cfb0a89e8d 100644
--- a/examples/java-demo/feature_repo/feature_store.yaml
+++ b/examples/java-demo/feature_repo/feature_store.yaml
@@ -6,6 +6,4 @@ online_store:
connection_string: localhost:6379,password=[YOUR PASSWORD]
offline_store:
type: file
-flags:
- alpha_features: true
- on_demand_transforms: true
+entity_key_serialization_version: 2
\ No newline at end of file
diff --git a/examples/java-demo/feature_repo/test_python_fetch.py b/examples/java-demo/feature_repo/test_python_fetch.py
new file mode 100644
index 0000000000..5e2781e150
--- /dev/null
+++ b/examples/java-demo/feature_repo/test_python_fetch.py
@@ -0,0 +1,26 @@
+from feast import FeatureStore
+
+
+def run_demo():
+ store = FeatureStore(repo_path=".")
+
+ print("\n--- Online features ---")
+ features = store.get_online_features(
+ features=[
+ "driver_hourly_stats:conv_rate",
+ ],
+ entity_rows=[
+ {
+ "driver_id": 1001,
+ },
+ {
+ "driver_id": 1002,
+ }
+ ],
+ ).to_dict()
+ for key, value in sorted(features.items()):
+ print(key, " : ", value)
+
+
+if __name__ == "__main__":
+ run_demo()
diff --git a/infra/charts/feast-feature-server/Chart.yaml b/infra/charts/feast-feature-server/Chart.yaml
index 6c1afc9540..aca9574b0c 100644
--- a/infra/charts/feast-feature-server/Chart.yaml
+++ b/infra/charts/feast-feature-server/Chart.yaml
@@ -2,7 +2,7 @@ apiVersion: v2
name: feast-feature-server
description: Feast Feature Server in Go or Python
type: application
-version: 0.22.0
+version: 0.23.1
keywords:
- machine learning
- big data
diff --git a/infra/charts/feast-feature-server/README.md b/infra/charts/feast-feature-server/README.md
index a55451e788..e83f75b39d 100644
--- a/infra/charts/feast-feature-server/README.md
+++ b/infra/charts/feast-feature-server/README.md
@@ -1,6 +1,6 @@
# feast-feature-server
- 
+ 
Feast Feature Server in Go or Python
@@ -12,28 +12,28 @@ Feast Feature Server in Go or Python
## Values
-| Key | Type | Default | Description |
-|-----|------|---------|-------------|
-| affinity | object | `{}` | |
-| fullnameOverride | string | `""` | |
-| image.pullPolicy | string | `"IfNotPresent"` | |
-| image.repository | string | `""` | |
-| image.tag | string | `""` | |
-| imagePullSecrets | list | `[]` | |
-| livenessProbe.initialDelaySeconds | int | `30` | |
-| livenessProbe.periodSeconds | int | `30` | |
-| nameOverride | string | `""` | |
-| nodeSelector | object | `{}` | |
-| podAnnotations | object | `{}` | |
-| podSecurityContext | object | `{}` | |
-| readinessProbe.initialDelaySeconds | int | `20` | |
-| readinessProbe.periodSeconds | int | `10` | |
-| replicaCount | int | `1` | |
-| resources | object | `{}` | |
-| securityContext | object | `{}` | |
-| service.port | int | `80` | |
-| service.type | string | `"ClusterIP"` | |
-| tolerations | list | `[]` | |
+| Key | Type | Default | Description |
+| ---------------------------------- | ------ | ---------------- | ----------- |
+| affinity | object | `{}` | |
+| fullnameOverride | string | `""` | |
+| image.pullPolicy | string | `"IfNotPresent"` | |
+| image.repository | string | `""` | |
+| image.tag | string | `""` | |
+| imagePullSecrets | list | `[]` | |
+| livenessProbe.initialDelaySeconds | int | `30` | |
+| livenessProbe.periodSeconds | int | `30` | |
+| nameOverride | string | `""` | |
+| nodeSelector | object | `{}` | |
+| podAnnotations | object | `{}` | |
+| podSecurityContext | object | `{}` | |
+| readinessProbe.initialDelaySeconds | int | `20` | |
+| readinessProbe.periodSeconds | int | `10` | |
+| replicaCount | int | `1` | |
+| resources | object | `{}` | |
+| securityContext | object | `{}` | |
+| service.port | int | `80` | |
+| service.type | string | `"ClusterIP"` | |
+| tolerations | list | `[]` | |
----------------------------------------------
Autogenerated from chart metadata using [helm-docs v1.11.0](https://github.com/norwoodj/helm-docs/releases/v1.11.0)
diff --git a/infra/charts/feast-python-server/Chart.yaml b/infra/charts/feast-python-server/Chart.yaml
index 6ab82b7a65..b8b229fd59 100644
--- a/infra/charts/feast-python-server/Chart.yaml
+++ b/infra/charts/feast-python-server/Chart.yaml
@@ -2,7 +2,7 @@ apiVersion: v2
name: feast-python-server
description: Feast Feature Server in Python
type: application
-version: 0.23.0
+version: 0.23.1
keywords:
- machine learning
- big data
diff --git a/infra/charts/feast-python-server/README.md b/infra/charts/feast-python-server/README.md
index e3da9b1d29..5f25b7d8fe 100644
--- a/infra/charts/feast-python-server/README.md
+++ b/infra/charts/feast-python-server/README.md
@@ -1,6 +1,6 @@
# feast-python-server
- 
+ 
Feast Feature Server in Python
diff --git a/infra/charts/feast/Chart.yaml b/infra/charts/feast/Chart.yaml
index f4e33de7f3..e7bc00d8a6 100644
--- a/infra/charts/feast/Chart.yaml
+++ b/infra/charts/feast/Chart.yaml
@@ -1,7 +1,7 @@
apiVersion: v1
description: Feature store for machine learning
name: feast
-version: 0.23.0
+version: 0.23.1
keywords:
- machine learning
- big data
diff --git a/infra/charts/feast/README.md b/infra/charts/feast/README.md
index f71dcf6124..b7741c4b45 100644
--- a/infra/charts/feast/README.md
+++ b/infra/charts/feast/README.md
@@ -8,7 +8,7 @@ This repo contains Helm charts for Feast components that are being installed on
## Chart: Feast
-Feature store for machine learning Current chart version is `0.23.0`
+Feature store for machine learning Current chart version is `0.23.1`
## Installation
@@ -55,8 +55,8 @@ For more details, please see: https://docs.feast.dev/how-to-guides/running-feast
| Repository | Name | Version |
|------------|------|---------|
| https://charts.helm.sh/stable | redis | 10.5.6 |
-| https://feast-helm-charts.storage.googleapis.com | feature-server(feature-server) | 0.23.0 |
-| https://feast-helm-charts.storage.googleapis.com | transformation-service(transformation-service) | 0.23.0 |
+| https://feast-helm-charts.storage.googleapis.com | feature-server(feature-server) | 0.23.1 |
+| https://feast-helm-charts.storage.googleapis.com | transformation-service(transformation-service) | 0.23.1 |
## Values
diff --git a/infra/charts/feast/charts/feature-server/Chart.yaml b/infra/charts/feast/charts/feature-server/Chart.yaml
index ee08b0b0f8..42b366e0e7 100644
--- a/infra/charts/feast/charts/feature-server/Chart.yaml
+++ b/infra/charts/feast/charts/feature-server/Chart.yaml
@@ -1,8 +1,8 @@
apiVersion: v1
description: "Feast Feature Server: Online feature serving service for Feast"
name: feature-server
-version: 0.23.0
-appVersion: v0.23.0
+version: 0.23.1
+appVersion: v0.23.1
keywords:
- machine learning
- big data
diff --git a/infra/charts/feast/charts/feature-server/README.md b/infra/charts/feast/charts/feature-server/README.md
index 4717cfff3a..84ed8abd25 100644
--- a/infra/charts/feast/charts/feature-server/README.md
+++ b/infra/charts/feast/charts/feature-server/README.md
@@ -1,6 +1,6 @@
# feature-server
- 
+ 
Feast Feature Server: Online feature serving service for Feast
@@ -17,7 +17,7 @@ Feast Feature Server: Online feature serving service for Feast
| envOverrides | object | `{}` | Extra environment variables to set |
| image.pullPolicy | string | `"IfNotPresent"` | Image pull policy |
| image.repository | string | `"feastdev/feature-server-java"` | Docker image for Feature Server repository |
-| image.tag | string | `"0.23.0"` | Image tag |
+| image.tag | string | `"0.23.1"` | Image tag |
| ingress.grpc.annotations | object | `{}` | Extra annotations for the ingress |
| ingress.grpc.auth.enabled | bool | `false` | Flag to enable auth |
| ingress.grpc.class | string | `"nginx"` | Which ingress controller to use |
diff --git a/infra/charts/feast/charts/feature-server/values.yaml b/infra/charts/feast/charts/feature-server/values.yaml
index 011ce9dc33..f23c77dd1a 100644
--- a/infra/charts/feast/charts/feature-server/values.yaml
+++ b/infra/charts/feast/charts/feature-server/values.yaml
@@ -5,7 +5,7 @@ image:
# image.repository -- Docker image for Feature Server repository
repository: feastdev/feature-server-java
# image.tag -- Image tag
- tag: 0.23.0
+ tag: 0.23.1
# image.pullPolicy -- Image pull policy
pullPolicy: IfNotPresent
diff --git a/infra/charts/feast/charts/transformation-service/Chart.yaml b/infra/charts/feast/charts/transformation-service/Chart.yaml
index 07055730c5..834ce7fe56 100644
--- a/infra/charts/feast/charts/transformation-service/Chart.yaml
+++ b/infra/charts/feast/charts/transformation-service/Chart.yaml
@@ -1,8 +1,8 @@
apiVersion: v1
description: "Transformation service: to compute on-demand features"
name: transformation-service
-version: 0.23.0
-appVersion: v0.23.0
+version: 0.23.1
+appVersion: v0.23.1
keywords:
- machine learning
- big data
diff --git a/infra/charts/feast/charts/transformation-service/README.md b/infra/charts/feast/charts/transformation-service/README.md
index 9bc7a1e5d6..84525fc0d6 100644
--- a/infra/charts/feast/charts/transformation-service/README.md
+++ b/infra/charts/feast/charts/transformation-service/README.md
@@ -1,6 +1,6 @@
# transformation-service
- 
+ 
Transformation service: to compute on-demand features
@@ -13,7 +13,7 @@ Transformation service: to compute on-demand features
| envOverrides | object | `{}` | Extra environment variables to set |
| image.pullPolicy | string | `"IfNotPresent"` | Image pull policy |
| image.repository | string | `"feastdev/feature-transformation-server"` | Docker image for Transformation Server repository |
-| image.tag | string | `"0.23.0"` | Image tag |
+| image.tag | string | `"0.23.1"` | Image tag |
| nodeSelector | object | `{}` | Node labels for pod assignment |
| podLabels | object | `{}` | Labels to be added to Feast Serving pods |
| replicaCount | int | `1` | Number of pods that will be created |
diff --git a/infra/charts/feast/charts/transformation-service/config/feature_store.yaml b/infra/charts/feast/charts/transformation-service/config/feature_store.yaml
index 555e93a306..c003b87cc2 100644
--- a/infra/charts/feast/charts/transformation-service/config/feature_store.yaml
+++ b/infra/charts/feast/charts/transformation-service/config/feature_store.yaml
@@ -2,7 +2,4 @@ registry:
path: {{ .Values.global.registry.path }}
cache_ttl_seconds: {{ .Values.global.registry.cache_ttl_seconds }}
provider: local
-project: {{ .Values.global.project }}
-flags:
- on_demand_transforms: true
- alpha_features: true
\ No newline at end of file
+project: {{ .Values.global.project }}
\ No newline at end of file
diff --git a/infra/charts/feast/charts/transformation-service/values.yaml b/infra/charts/feast/charts/transformation-service/values.yaml
index c1e506a476..53841df813 100644
--- a/infra/charts/feast/charts/transformation-service/values.yaml
+++ b/infra/charts/feast/charts/transformation-service/values.yaml
@@ -5,7 +5,7 @@ image:
# image.repository -- Docker image for Transformation Server repository
repository: feastdev/feature-transformation-server
# image.tag -- Image tag
- tag: 0.23.0
+ tag: 0.23.1
# image.pullPolicy -- Image pull policy
pullPolicy: IfNotPresent
diff --git a/infra/charts/feast/requirements.yaml b/infra/charts/feast/requirements.yaml
index c88fb7a4fa..8d24fcd9e6 100644
--- a/infra/charts/feast/requirements.yaml
+++ b/infra/charts/feast/requirements.yaml
@@ -1,12 +1,12 @@
dependencies:
- name: feature-server
alias: feature-server
- version: 0.23.0
+ version: 0.23.1
condition: feature-server.enabled
repository: https://feast-helm-charts.storage.googleapis.com
- name: transformation-service
alias: transformation-service
- version: 0.23.0
+ version: 0.23.1
condition: transformation-service.enabled
repository: https://feast-helm-charts.storage.googleapis.com
- name: redis
diff --git a/infra/scripts/helm/push-helm-charts.sh b/infra/scripts/helm/push-helm-charts.sh
index 08753adb3c..1c32ee985b 100755
--- a/infra/scripts/helm/push-helm-charts.sh
+++ b/infra/scripts/helm/push-helm-charts.sh
@@ -17,7 +17,9 @@ helm repo add feast-helm-chart-repo $bucket
cd infra/charts
helm package feast
helm package feast-python-server
+helm package feast-feature-server
helm gcs push --public feast-${1}.tgz feast-helm-chart-repo --force
helm gcs push --public feast-python-server-${1}.tgz feast-helm-chart-repo --force
+helm gcs push --public feast-feature-server-${1}.tgz feast-helm-chart-repo --force
rm -f ./*.tgz
\ No newline at end of file
diff --git a/infra/scripts/helm/validate-helm-chart-versions.sh b/infra/scripts/helm/validate-helm-chart-versions.sh
index 0ba75bd744..aac79d9315 100755
--- a/infra/scripts/helm/validate-helm-chart-versions.sh
+++ b/infra/scripts/helm/validate-helm-chart-versions.sh
@@ -3,7 +3,7 @@
set -e
# Amount of file locations that need to be bumped in unison when versions increment
-UNIQUE_VERSIONS_COUNT=18
+UNIQUE_VERSIONS_COUNT=20
if [ $# -ne 1 ]; then
echo "Please provide a single semver version (without a \"v\" prefix) to test the repository against, e.g 0.99.0"
diff --git a/infra/scripts/publish-java-sdk.sh b/infra/scripts/publish-java-sdk.sh
index 68174db17a..0e8b62478f 100755
--- a/infra/scripts/publish-java-sdk.sh
+++ b/infra/scripts/publish-java-sdk.sh
@@ -69,4 +69,4 @@ gpg --import --batch --yes $GPG_KEY_IMPORT_DIR/private-key
echo "============================================================"
echo "Deploying Java SDK with revision: $REVISION"
echo "============================================================"
-mvn -f java/pom.xml --projects .,datatypes,sdk -Drevision=$REVISION --batch-mode clean deploy
+mvn -f java/pom.xml --projects .,datatypes,serving-client -Drevision=$REVISION --batch-mode clean deploy
diff --git a/infra/scripts/create-cluster.sh b/infra/scripts/redis-cluster.sh
similarity index 100%
rename from infra/scripts/create-cluster.sh
rename to infra/scripts/redis-cluster.sh
diff --git a/infra/scripts/release/files_to_bump.txt b/infra/scripts/release/files_to_bump.txt
index a1e2d29623..6a558e04f0 100644
--- a/infra/scripts/release/files_to_bump.txt
+++ b/infra/scripts/release/files_to_bump.txt
@@ -9,4 +9,7 @@ infra/charts/feast/charts/feature-server/values.yaml 8
infra/charts/feast/README.md 11 58 59
infra/charts/feast-python-server/Chart.yaml 5
infra/charts/feast-python-server/README.md 3
-java/pom.xml 41
+infra/charts/feast-feature-server/Chart.yaml 5
+infra/charts/feast-feature-server/README.md 3
+java/pom.xml 38
+ui/package.json 3
diff --git a/infra/templates/README.md.jinja2 b/infra/templates/README.md.jinja2
index 6a8ebdbab7..e59a364d81 100644
--- a/infra/templates/README.md.jinja2
+++ b/infra/templates/README.md.jinja2
@@ -21,7 +21,7 @@ Feast (**Fea**ture **St**ore) is an open source feature store for machine learni
Feast allows ML platform teams to:
-* **Make features consistently available for training and serving** by managing an _offline store_ (to process historical data for scale-out batch scoring or model training), a low-latency _online store_ (to power real-time prediction)_,_ and a battle-tested _feature server_ (for serving pre-computed features online).
+* **Make features consistently available for training and serving** by managing an _offline store_ (to process historical data for scale-out batch scoring or model training), a low-latency _online store_ (to power real-time prediction)_,_ and a battle-tested _feature server_ (to serve pre-computed features online).
* **Avoid data leakage** by generating point-in-time correct feature sets so data scientists can focus on feature engineering rather than debugging error-prone dataset joining logic. This ensure that future feature values do not leak to models during training.
* **Decouple ML from data infrastructure** by providing a single data access layer that abstracts feature storage from feature retrieval, ensuring models remain portable as you move from training models to serving models, from batch models to realtime models, and from one data infra system to another.
diff --git a/java/CONTRIBUTING.md b/java/CONTRIBUTING.md
index f6c789d984..74549034b9 100644
--- a/java/CONTRIBUTING.md
+++ b/java/CONTRIBUTING.md
@@ -2,17 +2,40 @@
> The higher level [Development Guide](https://docs.feast.dev/v/master/project/development-guide)
> gives contributing to Feast codebase as a whole.
-### Overview
+## Overview
This guide is targeted at developers looking to contribute to Feast components in
the feast-java Repository:
- [Feast Serving](#feast-serving)
-- [Feast Java Client](#feast-java-client)
+- [Feast Serving Client](#feast-serving-client)
> Don't see the Feast component that you want to contribute to here?
> Check out the [Development Guide](https://docs.feast.dev/v/master/project/development-guide)
> to learn how Feast components are distributed over multiple repositories.
-#### Common Setup
+### Repository structure
+There are four key top level packages:
+- `serving`: Feast Serving (a gRPC service to serve features)
+- `serving-client`: Feast Serving Client (a thin Java client to communicate with Feast serving via gRPC )
+- `datatypes`: A symlink to the overall project protos. These include the core serving gRPC protos, proto representations of all objects in the Feast registry.
+- `coverage`: Generates JaCoCo coverage reports
+
+#### Feast Serving
+> **Note:** there are references to metrics collection in the code. These are unused and exist for legacy reasons (from when this used Spring Boot), but remain in the code until published to StatsD / Prometheus Pushgateway.
+
+The primary entrypoint into the Feast Serving server is `ServingGuiceApplication`, which connects to the rest of the packages:
+- `connectors`: Contains online store connectors (e.g. Redis)
+- `exception`: Contains user-facing exceptions thrown by Feast Serving
+- `registry`: Logic to parse a Feast file-based registry (in GCS, S3, or local) into the `Registry` proto object, and automatically re-sync the registry.
+- `service`: Core logic that exposes and backs the serving APIs. This includes communication with a feature transformation server to execute on demand transformations
+ - The root code in this package creates the main entrypoint (`ServingServiceV2`) which is injected into `OnlineServingGrpcServiceV2` in `grpc/` implement the gRPC service.
+ - `config`: Guice modules to power the server and config
+ - Includes server config / guice modules in `ServerModule`
+ - Maps overall Feast Serving user configuration from Java to YAML in `ApplicationPropertiesModule` and `ApplicationProperties`
+ - `controller`: server controllers (right now, only a gRPC health check)
+ - `grpc`: Implementation of the gRPC serving service
+ - `interceptors`: gRPC interceptors (currently used to produce metrics around each gRPC request)
+
+### Common Setup
Common Environment Setup for all feast-java Feast components:
Ensure following development tools are installed:
@@ -20,7 +43,7 @@ Ensure following development tools are installed:
- Maven 3.6
- `make`
-#### Code Style
+### Code Style
Feast's Java codebase conforms to the [Google Java Style Guide](https://google.github.io/styleguide/javaguide.html).
Automatically format the code to conform the style guide by:
@@ -33,7 +56,7 @@ mvn spotless:apply
> If you're using IntelliJ, you can import these [code style settings](https://github.com/google/styleguide/blob/gh-pages/intellij-java-google-style.xml)
> if you'd like to use the IDE's reformat function.
-#### Project Makefile
+### Project Makefile
The Project Makefile provides useful shorthands for common development tasks:
@@ -42,18 +65,18 @@ Run all Unit tests:
make test-java
```
-Run all Integration tests:
+Run all Integration tests (note: this also runs GCS + S3 based tests which should fail):
```
make test-java-integration
```
-Building Docker images for Feast Core & Feast Serving:
+Building Docker images for Feast Serving:
```
make build-docker REGISTRY=gcr.io/kf-feast VERSION=develop
```
-#### IDE Setup
+### IDE Setup
If you're using IntelliJ, some additional steps may be needed to make sure IntelliJ autocomplete works as expected.
Specifically, proto-generated code is not indexed by IntelliJ. To fix this, navigate to the following window in IntelliJ:
`Project Structure > Modules > datatypes-java`, and mark the following folders as `Source` directorys:
@@ -64,12 +87,12 @@ Specifically, proto-generated code is not indexed by IntelliJ. To fix this, navi
## Feast Serving
See instructions [here](serving/README.md) for developing.
-## Feast Java Client
+## Feast Serving Client
### Environment Setup
-Setting up your development environment for Feast Java SDK:
+Setting up your development environment:
1. Complete the feast-java [Common Setup](#common-setup)
-> Feast Java Client is a Java Client for retrieving Features from a running Feast Serving instance.
+> Feast Serving Client is a Serving Client for retrieving Features from a running Feast Serving instance.
> See the [Feast Serving Section](#feast-serving) section for how to get a Feast Serving instance running.
### Building
diff --git a/java/README.md b/java/README.md
index 8c3d93628e..53573a6fed 100644
--- a/java/README.md
+++ b/java/README.md
@@ -3,8 +3,8 @@
### Overview
This repository contains the following Feast components.
-* Feast Serving: A service used to serve the latest feature values to models.
-* Feast Java SDK: A client used to retrieve features from Feast Serving.
+* Feast Serving: A gRPC service used to serve the latest feature values to models.
+* Feast Serving Client: A client used to retrieve features from Feast Serving.
### Architecture
@@ -16,6 +16,7 @@ Guides on Contributing:
- [Contribution Process for Feast](https://docs.feast.dev/v/master/project/contributing)
- [Development Guide for Feast](https://docs.feast.dev/v/master/project/development-guide)
- [Development Guide for feast-java (this repository)](CONTRIBUTING.md)
+ - **Note**: includes installing without using Helm
### Installing using Helm
Please see the Helm charts in [infra/charts/feast](../infra/charts/feast).
diff --git a/java/common/pom.xml b/java/common/pom.xml
deleted file mode 100644
index 6b580880f1..0000000000
--- a/java/common/pom.xml
+++ /dev/null
@@ -1,162 +0,0 @@
-
-
-
- 4.0.0
-
-
- feast-parent
- dev.feast
- ${revision}
-
-
- Feast Common
- Feast common module with functionality that can be reused
- feast-common
-
-
-
- dev.feast
- feast-datatypes
- ${project.version}
- compile
-
-
- com.google.protobuf
- protobuf-java-util
- ${protobuf.version}
-
-
-
- org.apache.commons
- commons-lang3
- 3.6
-
-
-
-
- org.projectlombok
- lombok
- ${lombok.version}
-
-
- com.google.auto.value
- auto-value-annotations
- ${auto.value.version}
-
-
-
-
- com.google.code.gson
- gson
- ${gson.version}
-
-
- io.gsonfire
- gson-fire
- ${gson.fire.version}
-
-
- com.fasterxml.jackson.core
- jackson-databind
- 2.12.6.1
-
-
- com.fasterxml.jackson.datatype
- jackson-datatype-jsr310
- ${jackson.version}
-
-
-
-
- org.slf4j
- slf4j-api
-
-
- org.fluentd
- fluent-logger
- 0.3.1
-
-
-
- javax.xml.bind
- jaxb-api
-
-
- javax.validation
- validation-api
-
-
-
-
- com.google.code.findbugs
- jsr305
- 3.0.2
-
-
-
-
- org.hamcrest
- hamcrest-library
- test
- ${hamcrest.version}
-
-
-
- junit
- junit
- 4.13.2
-
-
- org.mockito
- mockito-core
- ${mockito.version}
- test
-
-
-
-
-
- org.apache.maven.plugins
- maven-javadoc-plugin
-
-
-
-
- org.jacoco
- jacoco-maven-plugin
-
-
- org.apache.maven.plugins
- maven-surefire-plugin
- 3.0.0-M4
-
- -Xms2048m -Xmx2048m -Djdk.net.URLClassPath.disableClassPathURLCheck=true
-
-
-
- org.sonatype.plugins
- nexus-staging-maven-plugin
-
- true
-
-
-
-
-
diff --git a/java/common/src/main/java/feast/common/logging/AuditLogger.java b/java/common/src/main/java/feast/common/logging/AuditLogger.java
deleted file mode 100644
index f3538a794b..0000000000
--- a/java/common/src/main/java/feast/common/logging/AuditLogger.java
+++ /dev/null
@@ -1,168 +0,0 @@
-/*
- * SPDX-License-Identifier: Apache-2.0
- * Copyright 2018-2020 The Feast Authors
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * https://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package feast.common.logging;
-
-import com.google.protobuf.InvalidProtocolBufferException;
-import com.google.protobuf.util.JsonFormat;
-import feast.common.logging.config.LoggingProperties;
-import feast.common.logging.config.LoggingProperties.AuditLogProperties;
-import feast.common.logging.entry.*;
-import feast.common.logging.entry.LogResource.ResourceType;
-import java.net.InetAddress;
-import java.net.UnknownHostException;
-import java.util.HashMap;
-import java.util.Map;
-import lombok.extern.slf4j.Slf4j;
-import org.apache.commons.lang3.StringUtils;
-import org.fluentd.logger.FluentLogger;
-import org.slf4j.Marker;
-import org.slf4j.MarkerFactory;
-import org.slf4j.event.Level;
-
-@Slf4j
-public class AuditLogger {
- private static final String FLUENTD_DESTINATION = "fluentd";
- private static final Marker AUDIT_MARKER = MarkerFactory.getMarker("AUDIT_MARK");
- private static FluentLogger fluentLogger;
- private static AuditLogProperties properties;
- private static String artifact;
- private static String version;
-
- public AuditLogger(LoggingProperties loggingProperties, String artifact, String version) {
- // Spring runs this constructor when creating the AuditLogger bean,
- // which allows us to populate the AuditLogger class with dependencies.
- // This allows us to use the dependencies in the AuditLogger's static methods
- AuditLogger.properties = loggingProperties.getAudit();
- AuditLogger.artifact = artifact;
- AuditLogger.version = version;
- if (AuditLogger.properties.getMessageLogging() != null
- && AuditLogger.properties.getMessageLogging().isEnabled()) {
- AuditLogger.fluentLogger =
- FluentLogger.getLogger(
- "feast",
- AuditLogger.properties.getMessageLogging().getFluentdHost(),
- AuditLogger.properties.getMessageLogging().getFluentdPort());
- }
- }
-
- /**
- * Log the handling of a Protobuf message by a service call.
- *
- * @param level log level
- * @param entryBuilder with all fields set except instance.
- */
- public static void logMessage(Level level, MessageAuditLogEntry.Builder entryBuilder) {
- log(level, entryBuilder.setComponent(artifact).setVersion(version).build());
- }
-
- /**
- * Log an action being taken on a specific resource
- *
- * @param level describing the severity of the log.
- * @param action name of the action being taken on specific resource.
- * @param resourceType the type of resource being logged.
- * @param resourceId resource specific identifier identifing the instance of the resource.
- */
- public static void logAction(
- Level level, String action, ResourceType resourceType, String resourceId) {
- log(
- level,
- ActionAuditLogEntry.of(
- artifact, version, LogResource.of(resourceType, resourceId), action));
- }
-
- /**
- * Log a transition in state/status in a specific resource.
- *
- * @param level describing the severity of the log.
- * @param status name of end status which the resource transition to.
- * @param resourceType the type of resource being logged.
- * @param resourceId resource specific identifier identifing the instance of the resource.
- */
- public static void logTransition(
- Level level, String status, ResourceType resourceType, String resourceId) {
- log(
- level,
- TransitionAuditLogEntry.of(
- artifact, version, LogResource.of(resourceType, resourceId), status));
- }
-
- /**
- * Log given {@link AuditLogEntry} at the given logging {@link Level} to the Audit log.
- *
- * @param level describing the severity of the log.
- * @param entry the {@link AuditLogEntry} to push to the audit log.
- */
- private static void log(Level level, AuditLogEntry entry) {
- // Check if audit logging is of this specific log entry enabled.
- if (!properties.isEnabled()) {
- return;
- }
-
- // Either forward log to logging layer or log to console
- String destination = properties.getMessageLogging().getDestination();
- if (destination.equals(FLUENTD_DESTINATION)) {
- if (entry.getKind() == AuditLogEntryKind.MESSAGE) {
- Map fluentdLogs = new HashMap<>();
- MessageAuditLogEntry messageAuditLogEntry = (MessageAuditLogEntry) entry;
- String releaseName;
-
- try {
- releaseName =
- StringUtils.defaultIfEmpty(
- System.getenv("RELEASE_NAME"), InetAddress.getLocalHost().getHostAddress());
- } catch (UnknownHostException e) {
- releaseName = StringUtils.defaultIfEmpty(System.getenv("RELEASE_NAME"), "");
- }
-
- fluentdLogs.put("id", messageAuditLogEntry.getId());
- fluentdLogs.put("identity", messageAuditLogEntry.getIdentity());
- fluentdLogs.put("service", messageAuditLogEntry.getService());
- fluentdLogs.put("status_code", messageAuditLogEntry.getStatusCode());
- fluentdLogs.put("method", messageAuditLogEntry.getMethod());
- fluentdLogs.put("release_name", releaseName);
- try {
- fluentdLogs.put("request", JsonFormat.printer().print(messageAuditLogEntry.getRequest()));
- fluentdLogs.put(
- "response", JsonFormat.printer().print(messageAuditLogEntry.getResponse()));
- } catch (InvalidProtocolBufferException e) {
- }
- fluentLogger.log("fluentd", fluentdLogs);
- }
- } else {
- // Log event to audit log through enabled formats
- String entryJSON = entry.toJSON();
- switch (level) {
- case TRACE:
- log.trace(AUDIT_MARKER, entryJSON);
- break;
- case DEBUG:
- log.debug(AUDIT_MARKER, entryJSON);
- break;
- case INFO:
- log.info(AUDIT_MARKER, entryJSON);
- break;
- case WARN:
- log.warn(AUDIT_MARKER, entryJSON);
- break;
- case ERROR:
- log.error(AUDIT_MARKER, entryJSON);
- break;
- }
- }
- }
-}
diff --git a/java/common/src/main/java/feast/common/logging/config/LoggingProperties.java b/java/common/src/main/java/feast/common/logging/config/LoggingProperties.java
deleted file mode 100644
index 06e62f71af..0000000000
--- a/java/common/src/main/java/feast/common/logging/config/LoggingProperties.java
+++ /dev/null
@@ -1,54 +0,0 @@
-/*
- * SPDX-License-Identifier: Apache-2.0
- * Copyright 2018-2019 The Feast Authors
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * https://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package feast.common.logging.config;
-
-import feast.common.validators.OneOfStrings;
-import javax.validation.constraints.NotNull;
-import lombok.Getter;
-import lombok.Setter;
-
-@Getter
-@Setter
-public class LoggingProperties {
- @NotNull private AuditLogProperties audit;
-
- @Getter
- @Setter
- public static class AuditLogProperties {
- // Whether to enable/disable audit logging entirely.
- private boolean enabled;
-
- private MessageLogging messageLogging;
-
- @Getter
- @Setter
- public static class MessageLogging {
- // Whether to enable/disable message level (ie request/response) audit logging.
- private boolean enabled;
-
- // Whether to log to console or fluentd
- @OneOfStrings({"console", "fluentd"})
- private String destination;
-
- // fluentD service host for external (request/response) logging.
- private String fluentdHost;
-
- // fluentD service port for external (request/response) logging.
- private Integer fluentdPort;
- }
- }
-}
diff --git a/java/common/src/main/java/feast/common/logging/entry/ActionAuditLogEntry.java b/java/common/src/main/java/feast/common/logging/entry/ActionAuditLogEntry.java
deleted file mode 100644
index 4fdeaee32a..0000000000
--- a/java/common/src/main/java/feast/common/logging/entry/ActionAuditLogEntry.java
+++ /dev/null
@@ -1,44 +0,0 @@
-/*
- * SPDX-License-Identifier: Apache-2.0
- * Copyright 2018-2020 The Feast Authors
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * https://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package feast.common.logging.entry;
-
-import com.google.auto.value.AutoValue;
-
-/** ActionAuditLogEntry records an action being taken on a specific resource */
-@AutoValue
-public abstract class ActionAuditLogEntry extends AuditLogEntry {
- /** @return The name of the action taken on the resource. */
- public abstract String getAction();
-
- /** @return The target resource of which the action was taken on. */
- public abstract LogResource getResource();
-
- /**
- * Create an {@link AuditLogEntry} that records an action being taken on a specific resource.
- *
- * @param component The name of th Feast component producing this {@link AuditLogEntry}.
- * @param version The version of Feast producing this {@link AuditLogEntry}.
- * @param resource The target resource of which the action was taken on.
- * @param action The name of the action being taken on the given resource.
- * @return log entry that records an action being taken on a specific resource
- */
- public static ActionAuditLogEntry of(
- String component, String version, LogResource resource, String action) {
- return new AutoValue_ActionAuditLogEntry(
- component, version, AuditLogEntryKind.ACTION, action, resource);
- }
-}
diff --git a/java/common/src/main/java/feast/common/logging/entry/AuditLogEntry.java b/java/common/src/main/java/feast/common/logging/entry/AuditLogEntry.java
deleted file mode 100644
index 8148c474b0..0000000000
--- a/java/common/src/main/java/feast/common/logging/entry/AuditLogEntry.java
+++ /dev/null
@@ -1,57 +0,0 @@
-/*
- * SPDX-License-Identifier: Apache-2.0
- * Copyright 2018-2019 The Feast Authors
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * https://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package feast.common.logging.entry;
-
-import com.google.gson.Gson;
-
-/**
- * AuditLogEntry represents a single audit Log Entry. Audit log entry can converted into string with
- * {{@link #toString()} for human readable representation. Or structured JSON with {{@link
- * #toJSON()} for a machine parsable representation.
- */
-public abstract class AuditLogEntry {
- /** Declare Log Type to allow external Logging systems to filter out {@link AuditLogEntry} */
- public final String logType = "FeastAuditLogEntry";
-
- public final String application = "Feast";
-
- /**
- * The name of the Feast component producing this {@link AuditLogEntry}
- *
- * @return the component
- */
- public abstract String getComponent();
-
- /**
- * The version of Feast producing this {@link AuditLogEntry}
- *
- * @return version
- */
- public abstract String getVersion();
-
- public abstract AuditLogEntryKind getKind();
-
- /**
- * Return a structured JSON representation of this {@link AuditLogEntry}
- *
- * @return structured JSON representation
- */
- public String toJSON() {
- Gson gson = new Gson();
- return gson.toJson(this);
- }
-}
diff --git a/java/common/src/main/java/feast/common/logging/entry/AuditLogEntryKind.java b/java/common/src/main/java/feast/common/logging/entry/AuditLogEntryKind.java
deleted file mode 100644
index d673f6bdb3..0000000000
--- a/java/common/src/main/java/feast/common/logging/entry/AuditLogEntryKind.java
+++ /dev/null
@@ -1,24 +0,0 @@
-/*
- * SPDX-License-Identifier: Apache-2.0
- * Copyright 2018-2019 The Feast Authors
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * https://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package feast.common.logging.entry;
-
-/** AuditLogEntryKind lists the various kinds of {@link AuditLogEntry} */
-public enum AuditLogEntryKind {
- MESSAGE,
- ACTION,
- TRANSITION,
-}
diff --git a/java/common/src/main/java/feast/common/logging/entry/LogResource.java b/java/common/src/main/java/feast/common/logging/entry/LogResource.java
deleted file mode 100644
index 1d0345a404..0000000000
--- a/java/common/src/main/java/feast/common/logging/entry/LogResource.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/*
- * SPDX-License-Identifier: Apache-2.0
- * Copyright 2018-2019 The Feast Authors
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * https://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package feast.common.logging.entry;
-
-import com.google.auto.value.AutoValue;
-
-@AutoValue
-/**
- * LogResource is used in {@link AuditLogEntry} to reference a specific resource as the subject of
- * the log
- */
-public abstract class LogResource {
- public enum ResourceType {
- JOB,
- FEATURE_TABLE
- }
-
- public abstract ResourceType getType();
-
- public abstract String getId();
-
- public static LogResource of(ResourceType type, String id) {
- return new AutoValue_LogResource(type, id);
- }
-}
diff --git a/java/common/src/main/java/feast/common/logging/entry/MessageAuditLogEntry.java b/java/common/src/main/java/feast/common/logging/entry/MessageAuditLogEntry.java
deleted file mode 100644
index 8ad428a3a3..0000000000
--- a/java/common/src/main/java/feast/common/logging/entry/MessageAuditLogEntry.java
+++ /dev/null
@@ -1,117 +0,0 @@
-/*
- * SPDX-License-Identifier: Apache-2.0
- * Copyright 2018-2020 The Feast Authors
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * https://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package feast.common.logging.entry;
-
-import com.google.auto.value.AutoValue;
-import com.google.gson.Gson;
-import com.google.gson.GsonBuilder;
-import com.google.gson.JsonParser;
-import com.google.gson.JsonSerializer;
-import com.google.protobuf.Empty;
-import com.google.protobuf.InvalidProtocolBufferException;
-import com.google.protobuf.Message;
-import com.google.protobuf.util.JsonFormat;
-import io.grpc.Status.Code;
-import java.util.UUID;
-
-/** MessageAuditLogEntry records the handling of a Protobuf message by a service call. */
-@AutoValue
-public abstract class MessageAuditLogEntry extends AuditLogEntry {
- /** @return Id used to identify the service call that the log entry is recording */
- public abstract UUID getId();
-
- /** @return The name of the service that was used to handle the service call. */
- public abstract String getService();
-
- /** @return The name of the method that was used to handle the service call. */
- public abstract String getMethod();
-
- /**
- * @return The request Protobuf {@link Message} that was passed to the Service in the service
- * call.
- */
- public abstract Message getRequest();
-
- /**
- * @return The response Protobuf {@link Message} that was passed to the Service in the service
- * call. May be an {@link Empty} protobuf no request could be collected due to an error.
- */
- public abstract Message getResponse();
-
- /**
- * @return The authenticated identity that was assumed during the handling of the service call.
- * For example, the user id or email that identifies the user making the call. Empty if the
- * service call is not authenticated.
- */
- public abstract String getIdentity();
-
- /** @return The result status code of the service call. */
- public abstract Code getStatusCode();
-
- @AutoValue.Builder
- public abstract static class Builder {
- public abstract Builder setId(UUID id);
-
- public abstract Builder setComponent(String component);
-
- public abstract Builder setVersion(String component);
-
- public abstract Builder setKind(AuditLogEntryKind kind);
-
- public abstract Builder setService(String name);
-
- public abstract Builder setMethod(String name);
-
- public abstract Builder setRequest(Message request);
-
- public abstract Builder setResponse(Message response);
-
- public abstract Builder setIdentity(String identity);
-
- public abstract Builder setStatusCode(Code statusCode);
-
- public abstract MessageAuditLogEntry build();
- }
-
- public static MessageAuditLogEntry.Builder newBuilder() {
- return new AutoValue_MessageAuditLogEntry.Builder()
- .setKind(AuditLogEntryKind.MESSAGE)
- .setId(UUID.randomUUID());
- }
-
- @Override
- public String toJSON() {
- // GSON requires custom typeadapter (serializer) to convert Protobuf messages to JSON properly
- Gson gson =
- new GsonBuilder()
- .registerTypeAdapter(
- Message.class,
- (JsonSerializer)
- (message, type, context) -> {
- try {
- String messageJSON = JsonFormat.printer().print(message);
- return new JsonParser().parse(messageJSON);
- } catch (InvalidProtocolBufferException e) {
-
- throw new RuntimeException(
- "Unexpected exception converting Protobuf to JSON", e);
- }
- })
- .create();
- return gson.toJson(this);
- }
-}
diff --git a/java/common/src/main/java/feast/common/logging/entry/TransitionAuditLogEntry.java b/java/common/src/main/java/feast/common/logging/entry/TransitionAuditLogEntry.java
deleted file mode 100644
index 224f10e0b5..0000000000
--- a/java/common/src/main/java/feast/common/logging/entry/TransitionAuditLogEntry.java
+++ /dev/null
@@ -1,45 +0,0 @@
-/*
- * SPDX-License-Identifier: Apache-2.0
- * Copyright 2018-2020 The Feast Authors
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * https://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package feast.common.logging.entry;
-
-import com.google.auto.value.AutoValue;
-
-/** TransitionAuditLogEntry records a transition in state/status in a specific resource. */
-@AutoValue
-public abstract class TransitionAuditLogEntry extends AuditLogEntry {
- /** @return The resource which the state/status transition occured. */
- public abstract LogResource getResource();
-
- /** @return The end status with the resource transition to. */
- public abstract String getStatus();
-
- /**
- * Construct a new {@link AuditLogEntry} to record a transition in state/status in a specific
- * resource.
- *
- * @param component The name of th Feast component producing this {@link AuditLogEntry}.
- * @param version The version of Feast producing this {@link AuditLogEntry}.
- * @param resource the resource which the transtion occured
- * @param status the end status which the resource transitioned to.
- * @return log entry to record a transition in state/status in a specific resource
- */
- public static TransitionAuditLogEntry of(
- String component, String version, LogResource resource, String status) {
- return new AutoValue_TransitionAuditLogEntry(
- component, version, AuditLogEntryKind.TRANSITION, resource, status);
- }
-}
diff --git a/java/common/src/main/java/feast/common/logging/interceptors/GrpcMessageInterceptor.java b/java/common/src/main/java/feast/common/logging/interceptors/GrpcMessageInterceptor.java
deleted file mode 100644
index e34fefd115..0000000000
--- a/java/common/src/main/java/feast/common/logging/interceptors/GrpcMessageInterceptor.java
+++ /dev/null
@@ -1,110 +0,0 @@
-/*
- * SPDX-License-Identifier: Apache-2.0
- * Copyright 2018-2019 The Feast Authors
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * https://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package feast.common.logging.interceptors;
-
-import com.google.protobuf.Empty;
-import com.google.protobuf.Message;
-import feast.common.logging.AuditLogger;
-import feast.common.logging.config.LoggingProperties;
-import feast.common.logging.entry.MessageAuditLogEntry;
-import io.grpc.ForwardingServerCall.SimpleForwardingServerCall;
-import io.grpc.ForwardingServerCallListener.SimpleForwardingServerCallListener;
-import io.grpc.Metadata;
-import io.grpc.ServerCall;
-import io.grpc.ServerCall.Listener;
-import io.grpc.ServerCallHandler;
-import io.grpc.ServerInterceptor;
-import io.grpc.Status;
-import org.slf4j.event.Level;
-
-/**
- * GrpcMessageInterceptor intercepts a GRPC calls to log handling of GRPC messages to the Audit Log.
- * Intercepts the incoming and outgoing messages logs them to the audit log, together with method
- * name and assumed authenticated identity (if authentication is enabled). NOTE:
- * GrpcMessageInterceptor assumes that all service calls are unary (ie single request/response).
- */
-public class GrpcMessageInterceptor implements ServerInterceptor {
- private final LoggingProperties loggingProperties;
-
- /**
- * Construct GrpcMessageIntercetor.
- *
- * @param loggingProperties properties used to configure logging interceptor.
- */
- public GrpcMessageInterceptor(LoggingProperties loggingProperties) {
- this.loggingProperties = loggingProperties;
- }
-
- @Override
- public Listener interceptCall(
- ServerCall call, Metadata headers, ServerCallHandler next) {
- // Disable the message logging interceptor entirely if message logging is disabled.
- if (!loggingProperties.getAudit().getMessageLogging().isEnabled()) {
- return next.startCall(call, headers);
- }
-
- MessageAuditLogEntry.Builder entryBuilder = MessageAuditLogEntry.newBuilder();
- // default response/request message to empty proto in log entry.
- // request could be empty when the client closes the connection before sending a request
- // message.
- // response could be unset when the service encounters an error when processsing the service
- // call.
- entryBuilder.setRequest(Empty.newBuilder().build());
- entryBuilder.setResponse(Empty.newBuilder().build());
-
- // Unpack service & method name from call
- // full method name is in format ./
- String fullMethodName = call.getMethodDescriptor().getFullMethodName();
- entryBuilder.setService(
- fullMethodName.substring(fullMethodName.lastIndexOf(".") + 1, fullMethodName.indexOf("/")));
- entryBuilder.setMethod(fullMethodName.substring(fullMethodName.indexOf("/") + 1));
-
- // Attempt Extract current authenticated identity.
- entryBuilder.setIdentity("");
-
- // Register forwarding call to intercept outgoing response and log to audit log
- call =
- new SimpleForwardingServerCall<>(call) {
- @Override
- public void sendMessage(RespT message) {
- // 2. Track the response & Log entry to audit logger
- super.sendMessage(message);
- entryBuilder.setResponse((Message) message);
- }
-
- @Override
- public void close(Status status, Metadata trailers) {
- super.close(status, trailers);
- // 3. Log the message log entry to the audit log
- Level logLevel = (status.isOk()) ? Level.INFO : Level.ERROR;
- entryBuilder.setStatusCode(status.getCode());
- AuditLogger.logMessage(logLevel, entryBuilder);
- }
- };
-
- ServerCall.Listener listener = next.startCall(call, headers);
- return new SimpleForwardingServerCallListener<>(listener) {
- @Override
- // Register listener to intercept incoming request messages and log to audit log
- public void onMessage(ReqT message) {
- super.onMessage(message);
- // 1. Track the request.
- entryBuilder.setRequest((Message) message);
- }
- };
- }
-}
diff --git a/java/common/src/main/java/feast/common/validators/OneOfStringValidator.java b/java/common/src/main/java/feast/common/validators/OneOfStringValidator.java
deleted file mode 100644
index 924953a2c4..0000000000
--- a/java/common/src/main/java/feast/common/validators/OneOfStringValidator.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * SPDX-License-Identifier: Apache-2.0
- * Copyright 2018-2020 The Feast Authors
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * https://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package feast.common.validators;
-
-import java.util.Arrays;
-import javax.validation.ConstraintValidator;
-import javax.validation.ConstraintValidatorContext;
-
-/** Validates whether a string value is found within a collection. */
-public class OneOfStringValidator implements ConstraintValidator {
-
- /** Values that are permitted for a specific instance of this validator */
- String[] allowedValues;
-
- /**
- * Initialize the OneOfStringValidator with a collection of allowed String values.
- *
- * @param constraintAnnotation constraint annotation
- */
- @Override
- public void initialize(OneOfStrings constraintAnnotation) {
- allowedValues = constraintAnnotation.value();
- }
-
- /**
- * Validates whether a string value is found within the collection defined in the annotation.
- *
- * @param value String value that should be validated
- * @param context Provides contextual data and operation when applying a given constraint
- * validator
- * @return Boolean value indicating whether the string is found within the allowed values.
- */
- @Override
- public boolean isValid(String value, ConstraintValidatorContext context) {
- return Arrays.asList(allowedValues).contains(value);
- }
-}
diff --git a/java/common/src/main/java/feast/common/validators/OneOfStrings.java b/java/common/src/main/java/feast/common/validators/OneOfStrings.java
deleted file mode 100644
index b236f6f1af..0000000000
--- a/java/common/src/main/java/feast/common/validators/OneOfStrings.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * SPDX-License-Identifier: Apache-2.0
- * Copyright 2018-2020 The Feast Authors
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * https://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package feast.common.validators;
-
-import java.lang.annotation.*;
-import javax.validation.Constraint;
-import javax.validation.Payload;
-
-/**
- * Annotation for String "one of" validation. Allows for the definition of a collection through an
- * annotation. The collection is used to test values defined in the object.
- */
-@Target({
- ElementType.METHOD,
- ElementType.FIELD,
- ElementType.ANNOTATION_TYPE,
- ElementType.CONSTRUCTOR,
- ElementType.PARAMETER
-})
-@Retention(RetentionPolicy.RUNTIME)
-@Documented
-@Constraint(validatedBy = OneOfStringValidator.class)
-public @interface OneOfStrings {
- /** @return Default error message that is returned if the incorrect value is set */
- String message() default "Field value must be one of the following: {value}";
-
- /** @return Allows for the specification of validation groups to which this constraint belongs. */
- Class[] groups() default {};
-
- /**
- * @return An attribute payload that can be used to assign custom payload objects to a constraint.
- */
- Class[] payload() default {};
-
- /** @return Default value that is returned if no allowed values are configured */
- String[] value() default {};
-}
diff --git a/java/common/src/main/resources/log4j2.xml b/java/common/src/main/resources/log4j2.xml
deleted file mode 100644
index c75c2db13c..0000000000
--- a/java/common/src/main/resources/log4j2.xml
+++ /dev/null
@@ -1,48 +0,0 @@
-
-
-
-
-
-
- %d{yyyy-MM-dd HH:mm:ss.SSS} %5p ${hostName} --- [%15.15t] %-40.40c{1.} : %m%n%ex
-
-
- {"time":"%d{yyyy-MM-dd'T'HH:mm:ssXXX}","hostname":"${hostName}","severity":"%p","message":%m}%n%ex
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
diff --git a/java/common/src/test/java/feast/common/logging/entry/AuditLogEntryTest.java b/java/common/src/test/java/feast/common/logging/entry/AuditLogEntryTest.java
deleted file mode 100644
index bc3dcbcf74..0000000000
--- a/java/common/src/test/java/feast/common/logging/entry/AuditLogEntryTest.java
+++ /dev/null
@@ -1,107 +0,0 @@
-/*
- * SPDX-License-Identifier: Apache-2.0
- * Copyright 2018-2020 The Feast Authors
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * https://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package feast.common.logging.entry;
-
-import static org.hamcrest.MatcherAssert.assertThat;
-import static org.hamcrest.Matchers.equalTo;
-
-import com.google.gson.JsonObject;
-import com.google.gson.JsonParser;
-import com.google.protobuf.Timestamp;
-import feast.common.logging.entry.LogResource.ResourceType;
-import feast.proto.serving.ServingAPIProto;
-import feast.proto.serving.ServingAPIProto.FeatureReferenceV2;
-import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesRequestV2;
-import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesResponse;
-import feast.proto.types.ValueProto.Value;
-import io.grpc.Status;
-import java.util.Arrays;
-import java.util.List;
-import org.junit.Test;
-
-public class AuditLogEntryTest {
- public List getTestAuditLogs() {
- GetOnlineFeaturesRequestV2 requestSpec =
- GetOnlineFeaturesRequestV2.newBuilder()
- .addAllFeatures(
- Arrays.asList(
- FeatureReferenceV2.newBuilder()
- .setFeatureViewName("featuretable_1")
- .setFeatureName("feature1")
- .build(),
- FeatureReferenceV2.newBuilder()
- .setFeatureViewName("featuretable_1")
- .setFeatureName("feature2")
- .build()))
- .build();
-
- GetOnlineFeaturesResponse responseSpec =
- GetOnlineFeaturesResponse.newBuilder()
- .setMetadata(
- ServingAPIProto.GetOnlineFeaturesResponseMetadata.newBuilder()
- .setFeatureNames(
- ServingAPIProto.FeatureList.newBuilder()
- .addAllVal(
- Arrays.asList(
- "featuretable_1:feature_1", "featuretable_1:feature2"))))
- .addAllResults(
- Arrays.asList(
- GetOnlineFeaturesResponse.FeatureVector.newBuilder()
- .addValues(Value.newBuilder().setInt32Val(32).build())
- .addStatuses(ServingAPIProto.FieldStatus.PRESENT)
- .addEventTimestamps(Timestamp.newBuilder().build())
- .build(),
- GetOnlineFeaturesResponse.FeatureVector.newBuilder()
- .addValues(Value.newBuilder().setInt32Val(64).build())
- .addStatuses(ServingAPIProto.FieldStatus.PRESENT)
- .addEventTimestamps(Timestamp.newBuilder().build())
- .build()))
- .build();
-
- return Arrays.asList(
- MessageAuditLogEntry.newBuilder()
- .setComponent("feast-serving")
- .setVersion("0.9")
- .setService("ServingService")
- .setMethod("getOnlineFeatures")
- .setRequest(requestSpec)
- .setResponse(responseSpec)
- .setStatusCode(Status.OK.getCode())
- .setIdentity("adam@no.such.email")
- .build(),
- ActionAuditLogEntry.of(
- "core", "0.9", LogResource.of(ResourceType.JOB, "kafka-to-redis"), "CREATE"),
- TransitionAuditLogEntry.of(
- "core", "0.9", LogResource.of(ResourceType.FEATURE_TABLE, "featuretable_1"), "READY"));
- }
-
- @Test
- public void shouldReturnJSONRepresentationOfAuditLog() {
- for (AuditLogEntry auditLog : getTestAuditLogs()) {
- // Check that auditLog's toJSON() returns valid JSON
- String logJSON = auditLog.toJSON();
- System.out.println(logJSON);
- JsonParser parser = new JsonParser();
-
- // check basic fields are present in JSON representation.
- JsonObject logObject = parser.parse(logJSON).getAsJsonObject();
- assertThat(logObject.getAsJsonPrimitive("logType").getAsString(), equalTo(auditLog.logType));
- assertThat(
- logObject.getAsJsonPrimitive("kind").getAsString(), equalTo(auditLog.getKind().name()));
- }
- }
-}
diff --git a/java/docs/coverage/pom.xml b/java/coverage/pom.xml
similarity index 85%
rename from java/docs/coverage/pom.xml
rename to java/coverage/pom.xml
index f6e08909ee..a604135c79 100644
--- a/java/docs/coverage/pom.xml
+++ b/java/coverage/pom.xml
@@ -30,7 +30,7 @@
dev.feast
feast-parent
${revision}
- ../..
+ ..
Feast Coverage Java
@@ -41,18 +41,6 @@
-
- dev.feast
- feast-storage-api
- ${project.version}
-
-
-
- dev.feast
- feast-storage-connector-redis
- ${project.version}
-
-
dev.feast
feast-serving
diff --git a/java/infra/docker/feature-server/Dockerfile b/java/infra/docker/feature-server/Dockerfile
index dbd8c91472..5cd0e6e37b 100644
--- a/java/infra/docker/feature-server/Dockerfile
+++ b/java/infra/docker/feature-server/Dockerfile
@@ -8,13 +8,9 @@ WORKDIR /build
COPY java/pom.xml .
COPY java/datatypes/pom.xml datatypes/pom.xml
-COPY java/common/pom.xml common/pom.xml
COPY java/serving/pom.xml serving/pom.xml
-COPY java/storage/api/pom.xml storage/api/pom.xml
-COPY java/storage/connectors/pom.xml storage/connectors/pom.xml
-COPY java/storage/connectors/redis/pom.xml storage/connectors/redis/pom.xml
-COPY java/sdk/pom.xml sdk/pom.xml
-COPY java/docs/coverage/pom.xml docs/coverage/pom.xml
+COPY java/serving-client/pom.xml serving-client/pom.xml
+COPY java/coverage/pom.xml coverage/pom.xml
# Setting Maven repository .m2 directory relative to /build folder gives the
# user to optionally use cached repository when building the image by copying
diff --git a/java/pom.xml b/java/pom.xml
index 0bf92ee244..cac85cdd38 100644
--- a/java/pom.xml
+++ b/java/pom.xml
@@ -29,16 +29,13 @@
datatypes
- storage/api
- storage/connectors
serving
- sdk
- docs/coverage
- common
+ serving-client
+ coverage
- 0.23.0
+ 0.23.1
https://github.com/feast-dev/feast
UTF-8
@@ -91,6 +88,7 @@
*/
]]>
+
${maven.multiModuleProjectDirectory}
false
diff --git a/java/sdk/pom.xml b/java/serving-client/pom.xml
similarity index 97%
rename from java/sdk/pom.xml
rename to java/serving-client/pom.xml
index 5896214b27..7b8838a009 100644
--- a/java/sdk/pom.xml
+++ b/java/serving-client/pom.xml
@@ -4,8 +4,8 @@
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
4.0.0
- Feast SDK for Java
- SDK for registering, storing, and retrieving features
+ Feast Serving Client
+ Client for retrieving features from a Feast feature server
feast-serving-client
diff --git a/java/sdk/src/main/java/dev/feast/FeastClient.java b/java/serving-client/src/main/java/dev/feast/FeastClient.java
similarity index 100%
rename from java/sdk/src/main/java/dev/feast/FeastClient.java
rename to java/serving-client/src/main/java/dev/feast/FeastClient.java
diff --git a/java/sdk/src/main/java/dev/feast/RequestUtil.java b/java/serving-client/src/main/java/dev/feast/RequestUtil.java
similarity index 100%
rename from java/sdk/src/main/java/dev/feast/RequestUtil.java
rename to java/serving-client/src/main/java/dev/feast/RequestUtil.java
diff --git a/java/sdk/src/main/java/dev/feast/Row.java b/java/serving-client/src/main/java/dev/feast/Row.java
similarity index 100%
rename from java/sdk/src/main/java/dev/feast/Row.java
rename to java/serving-client/src/main/java/dev/feast/Row.java
diff --git a/java/sdk/src/main/java/dev/feast/SecurityConfig.java b/java/serving-client/src/main/java/dev/feast/SecurityConfig.java
similarity index 100%
rename from java/sdk/src/main/java/dev/feast/SecurityConfig.java
rename to java/serving-client/src/main/java/dev/feast/SecurityConfig.java
diff --git a/java/sdk/src/test/java/dev/feast/FeastClientTest.java b/java/serving-client/src/test/java/dev/feast/FeastClientTest.java
similarity index 100%
rename from java/sdk/src/test/java/dev/feast/FeastClientTest.java
rename to java/serving-client/src/test/java/dev/feast/FeastClientTest.java
diff --git a/java/sdk/src/test/java/dev/feast/RequestUtilTest.java b/java/serving-client/src/test/java/dev/feast/RequestUtilTest.java
similarity index 100%
rename from java/sdk/src/test/java/dev/feast/RequestUtilTest.java
rename to java/serving-client/src/test/java/dev/feast/RequestUtilTest.java
diff --git a/java/serving/README.md b/java/serving/README.md
index 5ac7194924..a0d87563a9 100644
--- a/java/serving/README.md
+++ b/java/serving/README.md
@@ -41,7 +41,7 @@ From the Feast GitHub root, run:
java \
-Xms1g \
-Xmx4g \
- -jar java/serving/target/feast-serving-0.17.1-SNAPSHOT-jar-with-dependencies.jar \
+ -jar java/serving/target/feast-serving-[YOUR VERSION]-jar-with-dependencies.jar \
classpath:/application.yml,file:./application-override.yaml
```
5. Now you have a Feast Serving gRPC service running on port 6566 locally!
@@ -124,7 +124,7 @@ You can debug this like any other Java executable. Swap the java command above w
-Xrunjdwp:transport=dt_socket,address=5005,server=y,suspend=y \
-Xms1g \
-Xmx4g \
- -jar java/serving/target/feast-serving-0.17.1-SNAPSHOT-jar-with-dependencies.jar \
+ -jar java/serving/target/feast-serving-[YOUR VERSION]-jar-with-dependencies.jar \
classpath:/application.yml,file:./application-override.yaml
```
Now you can attach e.g. a Remote debugger in IntelliJ to port 5005 to debug / make breakpoints.
diff --git a/java/serving/pom.xml b/java/serving/pom.xml
index f173cdd5fe..e597775f9b 100644
--- a/java/serving/pom.xml
+++ b/java/serving/pom.xml
@@ -92,24 +92,6 @@
${project.version}
-
- dev.feast
- feast-common
- ${project.version}
-
-
-
- dev.feast
- feast-storage-api
- ${project.version}
-
-
-
- dev.feast
- feast-storage-connector-redis
- ${project.version}
-
-
com.google.inject
guice
@@ -345,6 +327,16 @@
2.7.4
test
+
+ io.lettuce
+ lettuce-core
+ 6.0.2.RELEASE
+
+
+ org.apache.commons
+ commons-lang3
+ 3.10
+
diff --git a/java/serving/src/main/java/feast/serving/ServingGuiceApplication.java b/java/serving/src/main/java/feast/serving/ServingGuiceApplication.java
index 664d6dd4ec..d91af8abb1 100644
--- a/java/serving/src/main/java/feast/serving/ServingGuiceApplication.java
+++ b/java/serving/src/main/java/feast/serving/ServingGuiceApplication.java
@@ -18,7 +18,7 @@
import com.google.inject.Guice;
import com.google.inject.Injector;
-import feast.serving.config.*;
+import feast.serving.service.config.*;
import io.grpc.Server;
import java.io.IOException;
@@ -32,9 +32,9 @@ public static void main(String[] args) throws InterruptedException, IOException
final Injector i =
Guice.createInjector(
- new ServingServiceConfigV2(),
- new RegistryConfig(),
- new InstrumentationConfig(),
+ new ServingServiceV2Module(),
+ new RegistryConfigModule(),
+ new InstrumentationConfigModule(),
new ServerModule(),
new ApplicationPropertiesModule(args));
diff --git a/java/storage/api/src/main/java/feast/storage/api/retriever/Feature.java b/java/serving/src/main/java/feast/serving/connectors/Feature.java
similarity index 94%
rename from java/storage/api/src/main/java/feast/storage/api/retriever/Feature.java
rename to java/serving/src/main/java/feast/serving/connectors/Feature.java
index 92ae1f31fb..af96a90866 100644
--- a/java/storage/api/src/main/java/feast/storage/api/retriever/Feature.java
+++ b/java/serving/src/main/java/feast/serving/connectors/Feature.java
@@ -14,18 +14,19 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package feast.storage.api.retriever;
+package feast.serving.connectors;
import com.google.protobuf.Timestamp;
import feast.proto.serving.ServingAPIProto.FeatureReferenceV2;
import feast.proto.types.ValueProto;
import feast.proto.types.ValueProto.Value;
import java.util.HashMap;
+import java.util.Map;
public interface Feature {
- HashMap TYPE_TO_VAL_CASE =
- new HashMap() {
+ Map TYPE_TO_VAL_CASE =
+ new HashMap<>() {
{
put(ValueProto.ValueType.Enum.BYTES, ValueProto.Value.ValCase.BYTES_VAL);
put(ValueProto.ValueType.Enum.STRING, ValueProto.Value.ValCase.STRING_VAL);
diff --git a/java/storage/api/src/main/java/feast/storage/api/retriever/OnlineRetrieverV2.java b/java/serving/src/main/java/feast/serving/connectors/OnlineRetriever.java
similarity index 96%
rename from java/storage/api/src/main/java/feast/storage/api/retriever/OnlineRetrieverV2.java
rename to java/serving/src/main/java/feast/serving/connectors/OnlineRetriever.java
index fde8ba7396..79c062814b 100644
--- a/java/storage/api/src/main/java/feast/storage/api/retriever/OnlineRetrieverV2.java
+++ b/java/serving/src/main/java/feast/serving/connectors/OnlineRetriever.java
@@ -14,14 +14,14 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package feast.storage.api.retriever;
+package feast.serving.connectors;
import feast.proto.serving.ServingAPIProto;
import feast.proto.types.ValueProto;
import java.util.List;
import java.util.Map;
-public interface OnlineRetrieverV2 {
+public interface OnlineRetriever {
/**
* Get online features for the given entity rows using data retrieved from the Feature references
* specified in FeatureTable request.
diff --git a/java/storage/api/src/main/java/feast/storage/api/retriever/ProtoFeature.java b/java/serving/src/main/java/feast/serving/connectors/ProtoFeature.java
similarity index 98%
rename from java/storage/api/src/main/java/feast/storage/api/retriever/ProtoFeature.java
rename to java/serving/src/main/java/feast/serving/connectors/ProtoFeature.java
index 09f6b75f49..9820898d00 100644
--- a/java/storage/api/src/main/java/feast/storage/api/retriever/ProtoFeature.java
+++ b/java/serving/src/main/java/feast/serving/connectors/ProtoFeature.java
@@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package feast.storage.api.retriever;
+package feast.serving.connectors;
import com.google.protobuf.Timestamp;
import feast.proto.serving.ServingAPIProto;
diff --git a/java/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/common/RedisHashDecoder.java b/java/serving/src/main/java/feast/serving/connectors/redis/common/RedisHashDecoder.java
similarity index 96%
rename from java/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/common/RedisHashDecoder.java
rename to java/serving/src/main/java/feast/serving/connectors/redis/common/RedisHashDecoder.java
index 78b64fd141..9f5c94924d 100644
--- a/java/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/common/RedisHashDecoder.java
+++ b/java/serving/src/main/java/feast/serving/connectors/redis/common/RedisHashDecoder.java
@@ -14,15 +14,15 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package feast.storage.connectors.redis.common;
+package feast.serving.connectors.redis.common;
import com.google.common.hash.Hashing;
import com.google.protobuf.InvalidProtocolBufferException;
import com.google.protobuf.Timestamp;
import feast.proto.serving.ServingAPIProto;
import feast.proto.types.ValueProto;
-import feast.storage.api.retriever.Feature;
-import feast.storage.api.retriever.ProtoFeature;
+import feast.serving.connectors.Feature;
+import feast.serving.connectors.ProtoFeature;
import java.nio.ByteBuffer;
import java.nio.charset.StandardCharsets;
import java.util.*;
diff --git a/java/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/common/RedisKeyGenerator.java b/java/serving/src/main/java/feast/serving/connectors/redis/common/RedisKeyGenerator.java
similarity index 97%
rename from java/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/common/RedisKeyGenerator.java
rename to java/serving/src/main/java/feast/serving/connectors/redis/common/RedisKeyGenerator.java
index 389ca0abfd..defb337a82 100644
--- a/java/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/common/RedisKeyGenerator.java
+++ b/java/serving/src/main/java/feast/serving/connectors/redis/common/RedisKeyGenerator.java
@@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package feast.storage.connectors.redis.common;
+package feast.serving.connectors.redis.common;
import feast.proto.serving.ServingAPIProto;
import feast.proto.storage.RedisProto;
diff --git a/java/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/retriever/EntityKeySerializer.java b/java/serving/src/main/java/feast/serving/connectors/redis/retriever/EntityKeySerializer.java
similarity index 94%
rename from java/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/retriever/EntityKeySerializer.java
rename to java/serving/src/main/java/feast/serving/connectors/redis/retriever/EntityKeySerializer.java
index 6220dd29d4..d25f0da4f9 100644
--- a/java/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/retriever/EntityKeySerializer.java
+++ b/java/serving/src/main/java/feast/serving/connectors/redis/retriever/EntityKeySerializer.java
@@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package feast.storage.connectors.redis.retriever;
+package feast.serving.connectors.redis.retriever;
import feast.proto.storage.RedisProto;
diff --git a/java/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/retriever/EntityKeySerializerV2.java b/java/serving/src/main/java/feast/serving/connectors/redis/retriever/EntityKeySerializerV2.java
similarity index 96%
rename from java/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/retriever/EntityKeySerializerV2.java
rename to java/serving/src/main/java/feast/serving/connectors/redis/retriever/EntityKeySerializerV2.java
index f99e5cbdb1..672f4d7c31 100644
--- a/java/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/retriever/EntityKeySerializerV2.java
+++ b/java/serving/src/main/java/feast/serving/connectors/redis/retriever/EntityKeySerializerV2.java
@@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package feast.storage.connectors.redis.retriever;
+package feast.serving.connectors.redis.retriever;
import com.google.protobuf.ProtocolStringList;
import feast.proto.storage.RedisProto;
@@ -87,14 +87,15 @@ public byte[] serialize(RedisProto.RedisKeyV2 entityKey) {
break;
case INT64_VAL:
buffer.addAll(encodeInteger(ValueProto.ValueType.Enum.INT64.getNumber()));
- buffer.addAll(encodeInteger(Integer.BYTES));
/* This is super dumb - but in https://github.com/feast-dev/feast/blob/dcae1606f53028ce5413567fb8b66f92cfef0f8e/sdk/python/feast/infra/key_encoding_utils.py#L9
we use `struct.pack(" tracerOptional;
+ private final OnlineRetriever retriever;
private final RegistryRepository registryRepository;
private final OnlineTransformationService onlineTransformationService;
private final String project;
@@ -56,16 +62,16 @@ public class OnlineServingServiceV2 implements ServingServiceV2 {
ValueProto.Value.newBuilder().setStringVal(DUMMY_ENTITY_VAL).build();
public OnlineServingServiceV2(
- OnlineRetrieverV2 retriever,
- Tracer tracer,
+ OnlineRetriever retriever,
RegistryRepository registryRepository,
OnlineTransformationService onlineTransformationService,
- String project) {
+ String project,
+ Optional tracerOptional) {
this.retriever = retriever;
- this.tracer = tracer;
this.registryRepository = registryRepository;
this.onlineTransformationService = onlineTransformationService;
this.project = project;
+ this.tracerOptional = tracerOptional;
}
/** {@inheritDoc} */
@@ -107,20 +113,21 @@ public ServingAPIProto.GetOnlineFeaturesResponse getOnlineFeatures(
List> entityRows = getEntityRows(request);
- Span storageRetrievalSpan = tracer.buildSpan("storageRetrieval").start();
+ Span storageRetrievalSpan =
+ tracerOptional.map(tracer -> tracer.buildSpan("storageRetrieval").start()).orElse(null);
if (storageRetrievalSpan != null) {
storageRetrievalSpan.setTag("entities", entityRows.size());
storageRetrievalSpan.setTag("features", retrievedFeatureReferences.size());
}
- List> features =
- retrieveFeatures(retrievedFeatureReferences, entityRows);
+ List> features = retrieveFeatures(retrievedFeatureReferences, entityRows);
if (storageRetrievalSpan != null) {
storageRetrievalSpan.finish();
}
- Span postProcessingSpan = tracer.buildSpan("postProcessing").start();
+ Span postProcessingSpan =
+ tracerOptional.map(tracer -> tracer.buildSpan("postProcessing").start()).orElse(null);
ServingAPIProto.GetOnlineFeaturesResponse.Builder responseBuilder =
ServingAPIProto.GetOnlineFeaturesResponse.newBuilder();
@@ -141,7 +148,7 @@ public ServingAPIProto.GetOnlineFeaturesResponse getOnlineFeatures(
responseBuilder.addResultsBuilder();
for (int rowIdx = 0; rowIdx < features.size(); rowIdx++) {
- feast.storage.api.retriever.Feature feature = features.get(rowIdx).get(featureIdx);
+ Feature feature = features.get(rowIdx).get(featureIdx);
if (feature == null) {
vectorBuilder.addValues(nullValue);
vectorBuilder.addStatuses(FieldStatus.NOT_FOUND);
@@ -172,7 +179,7 @@ public ServingAPIProto.GetOnlineFeaturesResponse getOnlineFeatures(
ServingAPIProto.FeatureList.newBuilder()
.addAllVal(
retrievedFeatureReferences.stream()
- .map(Feature::getFeatureReference)
+ .map(FeatureUtil::getFeatureReference)
.collect(Collectors.toList()))));
if (postProcessingSpan != null) {
@@ -202,7 +209,7 @@ private List getFeaturesList(
ServingAPIProto.GetOnlineFeaturesRequest request) {
if (request.getFeatures().getValCount() > 0) {
return request.getFeatures().getValList().stream()
- .map(Feature::parseFeatureReference)
+ .map(FeatureUtil::parseFeatureReference)
.collect(Collectors.toList());
}
@@ -246,7 +253,7 @@ private List> getEntityRows(
return entityRows;
}
- private List> retrieveFeatures(
+ private List> retrieveFeatures(
List featureReferences, List> entityRows) {
// Prepare feature reference to index mapping. This mapping will be used to arrange the
// retrieved features to the same order as in the input.
@@ -267,10 +274,9 @@ private List> retrieveFeatures(
}
// Create placeholders for retrieved features.
- List> features = new ArrayList<>(entityRows.size());
+ List> features = new ArrayList<>(entityRows.size());
for (int i = 0; i < entityRows.size(); i++) {
- List featuresPerEntity =
- new ArrayList<>(featureReferences.size());
+ List featuresPerEntity = new ArrayList<>(featureReferences.size());
for (int j = 0; j < featureReferences.size(); j++) {
featuresPerEntity.add(null);
}
@@ -311,7 +317,7 @@ private List> retrieveFeatures(
});
entityRowsPerGroup.add(entityRowPerGroup);
}
- List> featuresPerGroup =
+ List> featuresPerGroup =
retriever.getOnlineFeatures(entityRowsPerGroup, featureReferencesPerGroup, entityNames);
for (int i = 0; i < featuresPerGroup.size(); i++) {
for (int j = 0; j < featureReferencesPerGroup.size(); j++) {
@@ -329,7 +335,7 @@ private void populateOnDemandFeatures(
List onDemandFeatureSources,
List retrievedFeatureReferences,
ServingAPIProto.GetOnlineFeaturesRequest request,
- List> features,
+ List> features,
ServingAPIProto.GetOnlineFeaturesResponse.Builder responseBuilder) {
List>> onDemandContext =
@@ -366,7 +372,7 @@ private void populateOnDemandFeatures(
// Send out requests to the FTS and process the responses.
Set onDemandFeatureStringReferences =
onDemandFeatureReferences.stream()
- .map(r -> Feature.getFeatureReference(r))
+ .map(r -> FeatureUtil.getFeatureReference(r))
.collect(Collectors.toSet());
for (FeatureReferenceV2 featureReference : onDemandFeatureReferences) {
@@ -418,7 +424,7 @@ private static FieldStatus getFeatureStatus(ValueProto.Value value, boolean isOu
* @param maxAge feature's max age.
*/
private static boolean checkOutsideMaxAge(
- feast.storage.api.retriever.Feature feature, Timestamp entityTimestamp, Duration maxAge) {
+ Feature feature, Timestamp entityTimestamp, Duration maxAge) {
if (maxAge.equals(Duration.getDefaultInstance())) { // max age is not set
return false;
@@ -457,7 +463,7 @@ private void populateHistogramMetrics(
private void populateCountMetrics(
FeatureReferenceV2 featureRef,
ServingAPIProto.GetOnlineFeaturesResponse.FeatureVectorOrBuilder featureVector) {
- String featureRefString = Feature.getFeatureReference(featureRef);
+ String featureRefString = FeatureUtil.getFeatureReference(featureRef);
featureVector
.getStatusesList()
.forEach(
@@ -475,7 +481,7 @@ private void populateFeatureCountMetrics(List featureReferen
featureReferences.forEach(
featureReference ->
Metrics.requestFeatureCount
- .labels(project, Feature.getFeatureReference(featureReference))
+ .labels(project, FeatureUtil.getFeatureReference(featureReference))
.inc());
}
}
diff --git a/java/serving/src/main/java/feast/serving/config/ApplicationProperties.java b/java/serving/src/main/java/feast/serving/service/config/ApplicationProperties.java
similarity index 94%
rename from java/serving/src/main/java/feast/serving/config/ApplicationProperties.java
rename to java/serving/src/main/java/feast/serving/service/config/ApplicationProperties.java
index 5850eb6483..e4c33434a1 100644
--- a/java/serving/src/main/java/feast/serving/config/ApplicationProperties.java
+++ b/java/serving/src/main/java/feast/serving/service/config/ApplicationProperties.java
@@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package feast.serving.config;
+package feast.serving.service.config;
// Feast configuration properties that maps Feast configuration from default application.yml file to
// a Java object.
@@ -23,16 +23,14 @@
import com.fasterxml.jackson.annotation.JsonMerge;
import com.fasterxml.jackson.annotation.OptBoolean;
-import feast.common.logging.config.LoggingProperties;
-import feast.storage.connectors.redis.retriever.RedisClusterStoreConfig;
-import feast.storage.connectors.redis.retriever.RedisStoreConfig;
+import feast.serving.connectors.redis.retriever.RedisClusterStoreConfig;
+import feast.serving.connectors.redis.retriever.RedisStoreConfig;
import io.lettuce.core.ReadFrom;
import java.time.Duration;
import java.util.*;
import javax.annotation.PostConstruct;
import javax.validation.*;
import javax.validation.constraints.NotBlank;
-import javax.validation.constraints.NotNull;
import org.slf4j.Logger;
/** Feast Serving properties. */
@@ -58,14 +56,6 @@ public void setGrpc(GrpcServer grpc) {
this.grpc = grpc;
}
- public RestServer getRest() {
- return rest;
- }
-
- public void setRest(RestServer rest) {
- this.rest = rest;
- }
-
/**
* Validates all FeastProperties. This method runs after properties have been initialized and
* individually and conditionally validates each class.
@@ -103,8 +93,6 @@ public static class FeastProperties {
private List stores = new ArrayList<>();
/* Metric tracing properties. */
private TracingProperties tracing;
- /* Feast Audit Logging properties */
- @NotNull private LoggingProperties logging;
private String gcpProject;
private String awsRegion;
private String transformationServiceEndpoint;
@@ -195,15 +183,6 @@ public void setTracing(TracingProperties tracing) {
this.tracing = tracing;
}
- /**
- * Gets logging properties
- *
- * @return logging properties
- */
- public LoggingProperties getLogging() {
- return logging;
- }
-
public String getGcpProject() {
return gcpProject;
}
diff --git a/java/serving/src/main/java/feast/serving/config/ApplicationPropertiesModule.java b/java/serving/src/main/java/feast/serving/service/config/ApplicationPropertiesModule.java
similarity index 98%
rename from java/serving/src/main/java/feast/serving/config/ApplicationPropertiesModule.java
rename to java/serving/src/main/java/feast/serving/service/config/ApplicationPropertiesModule.java
index 07183fc710..3575733073 100644
--- a/java/serving/src/main/java/feast/serving/config/ApplicationPropertiesModule.java
+++ b/java/serving/src/main/java/feast/serving/service/config/ApplicationPropertiesModule.java
@@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package feast.serving.config;
+package feast.serving.service.config;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.ObjectReader;
diff --git a/java/serving/src/main/java/feast/serving/config/InstrumentationConfig.java b/java/serving/src/main/java/feast/serving/service/config/InstrumentationConfigModule.java
similarity index 93%
rename from java/serving/src/main/java/feast/serving/config/InstrumentationConfig.java
rename to java/serving/src/main/java/feast/serving/service/config/InstrumentationConfigModule.java
index 7f8590bb84..3eb748e451 100644
--- a/java/serving/src/main/java/feast/serving/config/InstrumentationConfig.java
+++ b/java/serving/src/main/java/feast/serving/service/config/InstrumentationConfigModule.java
@@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package feast.serving.config;
+package feast.serving.service.config;
import com.google.inject.AbstractModule;
import com.google.inject.Provides;
@@ -22,7 +22,7 @@
import io.opentracing.contrib.grpc.TracingServerInterceptor;
import io.opentracing.noop.NoopTracerFactory;
-public class InstrumentationConfig extends AbstractModule {
+public class InstrumentationConfigModule extends AbstractModule {
@Provides
public Tracer tracer(ApplicationProperties applicationProperties) {
diff --git a/java/serving/src/main/java/feast/serving/config/RegistryConfig.java b/java/serving/src/main/java/feast/serving/service/config/RegistryConfigModule.java
similarity index 96%
rename from java/serving/src/main/java/feast/serving/config/RegistryConfig.java
rename to java/serving/src/main/java/feast/serving/service/config/RegistryConfigModule.java
index 49e08e03b6..cfb4666f07 100644
--- a/java/serving/src/main/java/feast/serving/config/RegistryConfig.java
+++ b/java/serving/src/main/java/feast/serving/service/config/RegistryConfigModule.java
@@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package feast.serving.config;
+package feast.serving.service.config;
import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.AmazonS3ClientBuilder;
@@ -27,7 +27,7 @@
import java.net.URI;
import java.util.Optional;
-public class RegistryConfig extends AbstractModule {
+public class RegistryConfigModule extends AbstractModule {
@Provides
Storage googleStorage(ApplicationProperties applicationProperties) {
return StorageOptions.newBuilder()
diff --git a/java/serving/src/main/java/feast/serving/config/ServerModule.java b/java/serving/src/main/java/feast/serving/service/config/ServerModule.java
similarity index 92%
rename from java/serving/src/main/java/feast/serving/config/ServerModule.java
rename to java/serving/src/main/java/feast/serving/service/config/ServerModule.java
index 5428306f2b..a5d902b17b 100644
--- a/java/serving/src/main/java/feast/serving/config/ServerModule.java
+++ b/java/serving/src/main/java/feast/serving/service/config/ServerModule.java
@@ -14,13 +14,13 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package feast.serving.config;
+package feast.serving.service.config;
import com.google.inject.AbstractModule;
import com.google.inject.Provides;
-import feast.serving.controller.HealthServiceController;
-import feast.serving.grpc.OnlineServingGrpcServiceV2;
import feast.serving.service.ServingServiceV2;
+import feast.serving.service.controller.HealthServiceController;
+import feast.serving.service.grpc.OnlineServingGrpcServiceV2;
import io.grpc.Server;
import io.grpc.ServerBuilder;
import io.grpc.health.v1.HealthGrpc;
diff --git a/java/serving/src/main/java/feast/serving/config/ServingServiceConfigV2.java b/java/serving/src/main/java/feast/serving/service/config/ServingServiceV2Module.java
similarity index 83%
rename from java/serving/src/main/java/feast/serving/config/ServingServiceConfigV2.java
rename to java/serving/src/main/java/feast/serving/service/config/ServingServiceV2Module.java
index 868e3b83d1..564159ceed 100644
--- a/java/serving/src/main/java/feast/serving/config/ServingServiceConfigV2.java
+++ b/java/serving/src/main/java/feast/serving/service/config/ServingServiceV2Module.java
@@ -14,21 +14,22 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package feast.serving.config;
+package feast.serving.service.config;
import com.google.inject.AbstractModule;
import com.google.inject.Provides;
-import feast.serving.registry.*;
+import feast.serving.connectors.OnlineRetriever;
+import feast.serving.connectors.redis.retriever.*;
+import feast.serving.registry.RegistryRepository;
import feast.serving.service.OnlineServingServiceV2;
import feast.serving.service.OnlineTransformationService;
import feast.serving.service.ServingServiceV2;
-import feast.storage.api.retriever.OnlineRetrieverV2;
-import feast.storage.connectors.redis.retriever.*;
import io.opentracing.Tracer;
+import java.util.Optional;
import org.slf4j.Logger;
-public class ServingServiceConfigV2 extends AbstractModule {
- private static final Logger log = org.slf4j.LoggerFactory.getLogger(ServingServiceConfigV2.class);
+public class ServingServiceV2Module extends AbstractModule {
+ private static final Logger log = org.slf4j.LoggerFactory.getLogger(ServingServiceV2Module.class);
@Provides
public ServingServiceV2 registryBasedServingServiceV2(
@@ -38,14 +39,14 @@ public ServingServiceV2 registryBasedServingServiceV2(
final ServingServiceV2 servingService;
final ApplicationProperties.Store store = applicationProperties.getFeast().getActiveStore();
- OnlineRetrieverV2 retrieverV2;
+ OnlineRetriever retriever;
// TODO: Support more store types, and potentially use a plugin model here.
switch (store.getType()) {
case REDIS_CLUSTER:
RedisClientAdapter redisClusterClient =
RedisClusterClient.create(store.getRedisClusterConfig());
- retrieverV2 =
- new OnlineRetriever(
+ retriever =
+ new RedisOnlineRetriever(
applicationProperties.getFeast().getProject(),
redisClusterClient,
new EntityKeySerializerV2(
@@ -54,8 +55,8 @@ public ServingServiceV2 registryBasedServingServiceV2(
case REDIS:
RedisClientAdapter redisClient = RedisClient.create(store.getRedisConfig());
log.info("Created EntityKeySerializerV2");
- retrieverV2 =
- new OnlineRetriever(
+ retriever =
+ new RedisOnlineRetriever(
applicationProperties.getFeast().getProject(),
redisClient,
new EntityKeySerializerV2(
@@ -77,11 +78,11 @@ public ServingServiceV2 registryBasedServingServiceV2(
servingService =
new OnlineServingServiceV2(
- retrieverV2,
- tracer,
+ retriever,
registryRepository,
onlineTransformationService,
- applicationProperties.getFeast().getProject());
+ applicationProperties.getFeast().getProject(),
+ Optional.of(tracer));
return servingService;
}
diff --git a/java/serving/src/main/java/feast/serving/controller/HealthServiceController.java b/java/serving/src/main/java/feast/serving/service/controller/HealthServiceController.java
similarity index 98%
rename from java/serving/src/main/java/feast/serving/controller/HealthServiceController.java
rename to java/serving/src/main/java/feast/serving/service/controller/HealthServiceController.java
index 2f98ae032f..2ce17a5751 100644
--- a/java/serving/src/main/java/feast/serving/controller/HealthServiceController.java
+++ b/java/serving/src/main/java/feast/serving/service/controller/HealthServiceController.java
@@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package feast.serving.controller;
+package feast.serving.service.controller;
import com.google.inject.Inject;
import feast.proto.serving.ServingAPIProto.GetFeastServingInfoRequest;
diff --git a/java/serving/src/main/java/feast/serving/grpc/OnlineServingGrpcServiceV2.java b/java/serving/src/main/java/feast/serving/service/grpc/OnlineServingGrpcServiceV2.java
similarity index 98%
rename from java/serving/src/main/java/feast/serving/grpc/OnlineServingGrpcServiceV2.java
rename to java/serving/src/main/java/feast/serving/service/grpc/OnlineServingGrpcServiceV2.java
index fe024404f3..bc155a7fa2 100644
--- a/java/serving/src/main/java/feast/serving/grpc/OnlineServingGrpcServiceV2.java
+++ b/java/serving/src/main/java/feast/serving/service/grpc/OnlineServingGrpcServiceV2.java
@@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package feast.serving.grpc;
+package feast.serving.service.grpc;
import feast.proto.serving.ServingAPIProto;
import feast.proto.serving.ServingServiceGrpc;
diff --git a/java/serving/src/main/java/feast/serving/interceptors/GrpcMonitoringContext.java b/java/serving/src/main/java/feast/serving/service/interceptors/GrpcMonitoringContext.java
similarity index 96%
rename from java/serving/src/main/java/feast/serving/interceptors/GrpcMonitoringContext.java
rename to java/serving/src/main/java/feast/serving/service/interceptors/GrpcMonitoringContext.java
index 48d8d76a91..c1803e99fc 100644
--- a/java/serving/src/main/java/feast/serving/interceptors/GrpcMonitoringContext.java
+++ b/java/serving/src/main/java/feast/serving/service/interceptors/GrpcMonitoringContext.java
@@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package feast.serving.interceptors;
+package feast.serving.service.interceptors;
import java.util.Optional;
diff --git a/java/serving/src/main/java/feast/serving/interceptors/GrpcMonitoringInterceptor.java b/java/serving/src/main/java/feast/serving/service/interceptors/GrpcMonitoringInterceptor.java
similarity index 98%
rename from java/serving/src/main/java/feast/serving/interceptors/GrpcMonitoringInterceptor.java
rename to java/serving/src/main/java/feast/serving/service/interceptors/GrpcMonitoringInterceptor.java
index 735f8c556d..8777e8f844 100644
--- a/java/serving/src/main/java/feast/serving/interceptors/GrpcMonitoringInterceptor.java
+++ b/java/serving/src/main/java/feast/serving/service/interceptors/GrpcMonitoringInterceptor.java
@@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package feast.serving.interceptors;
+package feast.serving.service.interceptors;
import feast.serving.util.Metrics;
import io.grpc.ForwardingServerCall.SimpleForwardingServerCall;
diff --git a/java/serving/src/main/java/feast/serving/util/Metrics.java b/java/serving/src/main/java/feast/serving/util/Metrics.java
index dca2b5e82a..669f79ddff 100644
--- a/java/serving/src/main/java/feast/serving/util/Metrics.java
+++ b/java/serving/src/main/java/feast/serving/util/Metrics.java
@@ -19,6 +19,7 @@
import io.prometheus.client.Counter;
import io.prometheus.client.Histogram;
+// TODO: send these metrics either via Prometheus push gateway or StatsD
public class Metrics {
public static final Histogram requestLatency =
diff --git a/java/serving/src/main/java/feast/serving/util/RequestHelper.java b/java/serving/src/main/java/feast/serving/util/RequestHelper.java
index f730e01982..0f66e806ef 100644
--- a/java/serving/src/main/java/feast/serving/util/RequestHelper.java
+++ b/java/serving/src/main/java/feast/serving/util/RequestHelper.java
@@ -16,9 +16,9 @@
*/
package feast.serving.util;
-import feast.common.models.Feature;
import feast.proto.serving.ServingAPIProto;
import feast.proto.serving.ServingAPIProto.FeatureReferenceV2;
+import feast.serving.service.FeatureUtil;
public class RequestHelper {
@@ -29,7 +29,7 @@ public static void validateOnlineRequest(ServingAPIProto.GetOnlineFeaturesReques
}
// All FeatureReferences should have FeatureTable name and Feature name
for (String featureReference : request.getFeatures().getValList()) {
- validateOnlineRequestFeatureReference(Feature.parseFeatureReference(featureReference));
+ validateOnlineRequestFeatureReference(FeatureUtil.parseFeatureReference(featureReference));
}
}
diff --git a/java/serving/src/main/java/feast/serving/util/mappers/ResponseJSONMapper.java b/java/serving/src/main/java/feast/serving/util/mappers/ResponseJSONMapper.java
deleted file mode 100644
index 3ab9f43c34..0000000000
--- a/java/serving/src/main/java/feast/serving/util/mappers/ResponseJSONMapper.java
+++ /dev/null
@@ -1,82 +0,0 @@
-/*
- * SPDX-License-Identifier: Apache-2.0
- * Copyright 2018-2019 The Feast Authors
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * https://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package feast.serving.util.mappers;
-
-import feast.proto.serving.ServingAPIProto;
-import feast.proto.types.ValueProto.Value;
-import java.util.List;
-import java.util.Map;
-import java.util.stream.Collectors;
-
-// ResponseJSONMapper maps GRPC Response types to more human readable JSON responses
-public class ResponseJSONMapper {
-
- public static List> mapGetOnlineFeaturesResponse(
- ServingAPIProto.GetOnlineFeaturesResponse response) {
- return response.getResultsList().stream()
- .map(fieldValues -> convertFieldValuesToMap(fieldValues))
- .collect(Collectors.toList());
- }
-
- private static Map convertFieldValuesToMap(
- ServingAPIProto.GetOnlineFeaturesResponse.FeatureVector vec) {
- return Map.of(
- "values",
- vec.getValuesList().stream()
- .map(ResponseJSONMapper::extractValue)
- .collect(Collectors.toList()),
- "statuses",
- vec.getStatusesList(),
- "event_timestamp",
- vec.getEventTimestampsList());
- }
-
- private static Object extractValue(Value value) {
- switch (value.getValCase().getNumber()) {
- case 1:
- return value.getBytesVal();
- case 2:
- return value.getStringVal();
- case 3:
- return value.getInt32Val();
- case 4:
- return value.getInt64Val();
- case 5:
- return value.getDoubleVal();
- case 6:
- return value.getFloatVal();
- case 7:
- return value.getBoolVal();
- case 11:
- return value.getBytesListVal();
- case 12:
- return value.getStringListVal();
- case 13:
- return value.getInt32ListVal();
- case 14:
- return value.getInt64ListVal();
- case 15:
- return value.getDoubleListVal();
- case 16:
- return value.getFloatListVal();
- case 17:
- return value.getBoolListVal();
- default:
- return null;
- }
- }
-}
diff --git a/java/serving/src/main/resources/application.yml b/java/serving/src/main/resources/application.yml
index 1f6d5b34c4..b0c3bf7636 100644
--- a/java/serving/src/main/resources/application.yml
+++ b/java/serving/src/main/resources/application.yml
@@ -34,22 +34,6 @@ feast:
# The service name identifier for the tracing data
serviceName: feast_serving
- logging:
- # Audit logging provides a machine readable structured JSON log that can give better
- # insight into what is happening in Feast.
- audit:
- # Whether audit logging is enabled.
- enabled: true
- # Whether to enable message level (ie request/response) audit logging
- messageLogging:
- enabled: false
- # Logging forwarder currently provides a machine readable structured JSON log to an
- # external fluentd service that can give better insight into what is happening in Feast.
- # Accepts console / fluentd as destination
- destination: console
- fluentdHost: localhost
- fluentdPort: 24224
-
grpc:
server:
# The port number Feast Serving GRPC service should listen on
diff --git a/java/serving/src/test/java/feast/serving/it/ServingBenchmarkIT.java b/java/serving/src/test/java/feast/serving/it/ServingBenchmarkIT.java
index 1d77c2e4f7..4254a80e17 100644
--- a/java/serving/src/test/java/feast/serving/it/ServingBenchmarkIT.java
+++ b/java/serving/src/test/java/feast/serving/it/ServingBenchmarkIT.java
@@ -22,7 +22,7 @@
import com.google.common.math.Quantiles;
import feast.proto.serving.ServingAPIProto;
import feast.proto.types.ValueProto;
-import feast.serving.config.ApplicationProperties;
+import feast.serving.service.config.ApplicationProperties;
import feast.serving.util.DataGenerator;
import java.util.List;
import java.util.LongSummaryStatistics;
diff --git a/java/serving/src/test/java/feast/serving/it/ServingEnvironment.java b/java/serving/src/test/java/feast/serving/it/ServingEnvironment.java
index ebbfa89d37..43b82345c6 100644
--- a/java/serving/src/test/java/feast/serving/it/ServingEnvironment.java
+++ b/java/serving/src/test/java/feast/serving/it/ServingEnvironment.java
@@ -24,8 +24,8 @@
import com.google.inject.Module;
import com.google.inject.util.Modules;
import feast.proto.serving.ServingServiceGrpc;
-import feast.serving.config.*;
-import feast.serving.grpc.OnlineServingGrpcServiceV2;
+import feast.serving.service.config.*;
+import feast.serving.service.grpc.OnlineServingGrpcServiceV2;
import io.grpc.ManagedChannel;
import io.grpc.ManagedChannelBuilder;
import io.grpc.Server;
@@ -120,16 +120,16 @@ ApplicationProperties applicationProperties() {
Module overrideConfig = registryConfig();
Module registryConfig;
if (overrideConfig != null) {
- registryConfig = Modules.override(new RegistryConfig()).with(registryConfig());
+ registryConfig = Modules.override(new RegistryConfigModule()).with(registryConfig());
} else {
- registryConfig = new RegistryConfig();
+ registryConfig = new RegistryConfigModule();
}
injector =
Guice.createInjector(
- new ServingServiceConfigV2(),
+ new ServingServiceV2Module(),
registryConfig,
- new InstrumentationConfig(),
+ new InstrumentationConfigModule(),
appPropertiesModule,
new ServerModule());
diff --git a/java/serving/src/test/java/feast/serving/it/ServingRedisGSRegistryIT.java b/java/serving/src/test/java/feast/serving/it/ServingRedisGSRegistryIT.java
index 78871cd45c..925f1887d2 100644
--- a/java/serving/src/test/java/feast/serving/it/ServingRedisGSRegistryIT.java
+++ b/java/serving/src/test/java/feast/serving/it/ServingRedisGSRegistryIT.java
@@ -16,12 +16,12 @@
*/
package feast.serving.it;
-import static org.junit.jupiter.api.Assertions.*;
+import static org.junit.jupiter.api.Assertions.assertArrayEquals;
import com.google.cloud.storage.*;
import com.google.cloud.storage.testing.RemoteStorageHelper;
import feast.proto.core.RegistryProto;
-import feast.serving.config.ApplicationProperties;
+import feast.serving.service.config.ApplicationProperties;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit;
import org.junit.jupiter.api.AfterAll;
diff --git a/java/serving/src/test/java/feast/serving/it/ServingRedisLocalRegistryIT.java b/java/serving/src/test/java/feast/serving/it/ServingRedisLocalRegistryIT.java
index c83d8dbbf1..91fc0ed5e1 100644
--- a/java/serving/src/test/java/feast/serving/it/ServingRedisLocalRegistryIT.java
+++ b/java/serving/src/test/java/feast/serving/it/ServingRedisLocalRegistryIT.java
@@ -17,7 +17,7 @@
package feast.serving.it;
import feast.proto.core.RegistryProto;
-import feast.serving.config.ApplicationProperties;
+import feast.serving.service.config.ApplicationProperties;
public class ServingRedisLocalRegistryIT extends ServingBaseTests {
@Override
diff --git a/java/serving/src/test/java/feast/serving/it/ServingRedisS3RegistryIT.java b/java/serving/src/test/java/feast/serving/it/ServingRedisS3RegistryIT.java
index d67fbf2621..12315c9e48 100644
--- a/java/serving/src/test/java/feast/serving/it/ServingRedisS3RegistryIT.java
+++ b/java/serving/src/test/java/feast/serving/it/ServingRedisS3RegistryIT.java
@@ -24,7 +24,7 @@
import com.google.inject.AbstractModule;
import com.google.inject.Provides;
import feast.proto.core.RegistryProto;
-import feast.serving.config.ApplicationProperties;
+import feast.serving.service.config.ApplicationProperties;
import java.io.ByteArrayInputStream;
import org.junit.jupiter.api.BeforeAll;
import org.testcontainers.junit.jupiter.Container;
diff --git a/java/serving/src/test/java/feast/serving/it/TestUtils.java b/java/serving/src/test/java/feast/serving/it/TestUtils.java
index 9bca14db4e..180b26fd02 100644
--- a/java/serving/src/test/java/feast/serving/it/TestUtils.java
+++ b/java/serving/src/test/java/feast/serving/it/TestUtils.java
@@ -22,7 +22,7 @@
import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesRequest;
import feast.proto.serving.ServingServiceGrpc;
import feast.proto.types.ValueProto;
-import feast.serving.config.ApplicationProperties;
+import feast.serving.service.config.ApplicationProperties;
import io.grpc.Channel;
import io.grpc.ManagedChannelBuilder;
import java.util.*;
@@ -77,7 +77,7 @@ public static ApplicationProperties.FeastProperties createBasicFeastProperties(
feastProperties.setActiveStore("online");
feastProperties.setProject("feast_project");
-
+ feastProperties.setEntityKeySerializationVersion(2);
feastProperties.setStores(
ImmutableList.of(
new ApplicationProperties.Store(
diff --git a/java/serving/src/test/java/feast/serving/it/TransformationServiceIT.java b/java/serving/src/test/java/feast/serving/it/TransformationServiceIT.java
index 102d851528..81147d4268 100644
--- a/java/serving/src/test/java/feast/serving/it/TransformationServiceIT.java
+++ b/java/serving/src/test/java/feast/serving/it/TransformationServiceIT.java
@@ -22,7 +22,7 @@
import com.google.common.collect.Lists;
import feast.proto.serving.ServingAPIProto;
import feast.proto.types.ValueProto;
-import feast.serving.config.ApplicationProperties;
+import feast.serving.service.config.ApplicationProperties;
import feast.serving.util.DataGenerator;
import java.util.List;
import java.util.Map;
diff --git a/java/common/src/test/java/feast/common/models/FeaturesTest.java b/java/serving/src/test/java/feast/serving/service/FeaturesTest.java
similarity index 91%
rename from java/common/src/test/java/feast/common/models/FeaturesTest.java
rename to java/serving/src/test/java/feast/serving/service/FeaturesTest.java
index 953da61afe..27c0b21d2c 100644
--- a/java/common/src/test/java/feast/common/models/FeaturesTest.java
+++ b/java/serving/src/test/java/feast/serving/service/FeaturesTest.java
@@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package feast.common.models;
+package feast.serving.service;
import static org.hamcrest.MatcherAssert.assertThat;
import static org.hamcrest.core.IsEqual.equalTo;
@@ -38,7 +38,7 @@ public void setUp() {
@Test
public void shouldReturnFeatureStringRef() {
- String actualFeatureStringRef = Feature.getFeatureReference(featureReference);
+ String actualFeatureStringRef = FeatureUtil.getFeatureReference(featureReference);
String expectedFeatureStringRef = "featuretable_1:feature1";
assertThat(actualFeatureStringRef, equalTo(expectedFeatureStringRef));
diff --git a/java/serving/src/test/java/feast/serving/service/OnlineServingServiceTest.java b/java/serving/src/test/java/feast/serving/service/OnlineServingServiceTest.java
index 933e38f056..32bf2a4cb8 100644
--- a/java/serving/src/test/java/feast/serving/service/OnlineServingServiceTest.java
+++ b/java/serving/src/test/java/feast/serving/service/OnlineServingServiceTest.java
@@ -32,16 +32,17 @@
import feast.proto.serving.ServingAPIProto.FieldStatus;
import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesResponse;
import feast.proto.types.ValueProto;
+import feast.serving.connectors.Feature;
+import feast.serving.connectors.ProtoFeature;
+import feast.serving.connectors.redis.retriever.RedisOnlineRetriever;
import feast.serving.registry.Registry;
import feast.serving.registry.RegistryRepository;
-import feast.storage.api.retriever.Feature;
-import feast.storage.api.retriever.ProtoFeature;
-import feast.storage.connectors.redis.retriever.OnlineRetriever;
import io.opentracing.Tracer;
import io.opentracing.Tracer.SpanBuilder;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
+import java.util.Optional;
import java.util.stream.Collectors;
import org.junit.Before;
import org.junit.Test;
@@ -53,7 +54,7 @@ public class OnlineServingServiceTest {
@Mock Registry registry;
@Mock Tracer tracer;
- @Mock OnlineRetriever retrieverV2;
+ @Mock RedisOnlineRetriever retrieverV2;
private String transformationServiceEndpoint;
private OnlineServingServiceV2 onlineServingServiceV2;
@@ -73,7 +74,11 @@ public void setUp() {
new OnlineTransformationService(transformationServiceEndpoint, registryRepo);
onlineServingServiceV2 =
new OnlineServingServiceV2(
- retrieverV2, tracer, registryRepo, onlineTransformationService, "feast_project");
+ retrieverV2,
+ registryRepo,
+ onlineTransformationService,
+ "feast_project",
+ Optional.of(tracer));
mockedFeatureRows = new ArrayList<>();
mockedFeatureRows.add(
@@ -378,7 +383,7 @@ private ServingAPIProto.GetOnlineFeaturesRequest getOnlineFeaturesRequest(
ServingAPIProto.FeatureList.newBuilder()
.addAllVal(
featureReferences.stream()
- .map(feast.common.models.Feature::getFeatureReference)
+ .map(FeatureUtil::getFeatureReference)
.collect(Collectors.toList()))
.build())
.putAllEntities(
diff --git a/java/serving/src/test/resources/docker-compose/feast10/feature_store.yaml b/java/serving/src/test/resources/docker-compose/feast10/feature_store.yaml
index 2e6625c025..56e469bbee 100644
--- a/java/serving/src/test/resources/docker-compose/feast10/feature_store.yaml
+++ b/java/serving/src/test/resources/docker-compose/feast10/feature_store.yaml
@@ -5,6 +5,4 @@ online_store:
type: redis
connection_string: "redis:6379,password=testpw"
offline_store: {}
-flags:
- alpha_features: true
- on_demand_transforms: true
+entity_key_serialization_version: 2
diff --git a/java/storage/api/pom.xml b/java/storage/api/pom.xml
deleted file mode 100644
index 90f656e281..0000000000
--- a/java/storage/api/pom.xml
+++ /dev/null
@@ -1,72 +0,0 @@
-
-
-
- dev.feast
- feast-parent
- ${revision}
- ../../pom.xml
-
-
- 4.0.0
- feast-storage-api
-
- Feast Storage API
-
-
-
-
- org.apache.maven.plugins
- maven-dependency-plugin
-
-
-
- javax.annotation
-
-
-
-
-
-
-
-
-
- dev.feast
- feast-datatypes
- ${project.version}
-
-
-
- com.google.auto.value
- auto-value-annotations
- 1.6.6
-
-
-
- com.google.auto.value
- auto-value
- 1.6.6
- provided
-
-
-
- org.apache.commons
- commons-lang3
- 3.9
-
-
-
- org.apache.avro
- avro
- 1.10.2
-
-
-
- junit
- junit
- 4.13.2
- test
-
-
-
-
diff --git a/java/storage/api/src/main/java/feast/storage/api/retriever/AvroFeature.java b/java/storage/api/src/main/java/feast/storage/api/retriever/AvroFeature.java
deleted file mode 100644
index d8f2763010..0000000000
--- a/java/storage/api/src/main/java/feast/storage/api/retriever/AvroFeature.java
+++ /dev/null
@@ -1,172 +0,0 @@
-/*
- * SPDX-License-Identifier: Apache-2.0
- * Copyright 2018-2021 The Feast Authors
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * https://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package feast.storage.api.retriever;
-
-import com.google.protobuf.ByteString;
-import com.google.protobuf.Timestamp;
-import feast.proto.serving.ServingAPIProto;
-import feast.proto.types.ValueProto;
-import java.nio.ByteBuffer;
-import java.util.stream.Collectors;
-import org.apache.avro.generic.GenericData;
-import org.apache.avro.util.Utf8;
-
-public class AvroFeature implements Feature {
- private final ServingAPIProto.FeatureReferenceV2 featureReference;
-
- private final Timestamp eventTimestamp;
-
- private final Object featureValue;
-
- public AvroFeature(
- ServingAPIProto.FeatureReferenceV2 featureReference,
- Timestamp eventTimestamp,
- Object featureValue) {
- this.featureReference = featureReference;
- this.eventTimestamp = eventTimestamp;
- this.featureValue = featureValue;
- }
-
- /**
- * Casts feature value of Object type based on Feast valueType. Empty object i.e new Object() is
- * interpreted as VAL_NOT_SET Feast valueType.
- *
- * @param valueType Feast valueType of feature as specified in FeatureSpec
- * @return ValueProto.Value representation of feature
- */
- @Override
- public ValueProto.Value getFeatureValue(ValueProto.ValueType.Enum valueType) {
- ValueProto.Value finalValue;
-
- try {
- switch (valueType) {
- case STRING:
- finalValue =
- ValueProto.Value.newBuilder().setStringVal(((Utf8) featureValue).toString()).build();
- break;
- case INT32:
- finalValue = ValueProto.Value.newBuilder().setInt32Val((Integer) featureValue).build();
- break;
- case INT64:
- finalValue = ValueProto.Value.newBuilder().setInt64Val((Long) featureValue).build();
- break;
- case DOUBLE:
- finalValue = ValueProto.Value.newBuilder().setDoubleVal((Double) featureValue).build();
- break;
- case FLOAT:
- finalValue = ValueProto.Value.newBuilder().setFloatVal((Float) featureValue).build();
- break;
- case BYTES:
- finalValue =
- ValueProto.Value.newBuilder()
- .setBytesVal(ByteString.copyFrom(((ByteBuffer) featureValue).array()))
- .build();
- break;
- case BOOL:
- finalValue = ValueProto.Value.newBuilder().setBoolVal((Boolean) featureValue).build();
- break;
- case STRING_LIST:
- finalValue =
- ValueProto.Value.newBuilder()
- .setStringListVal(
- ValueProto.StringList.newBuilder()
- .addAllVal(
- ((GenericData.Array) featureValue)
- .stream().map(Utf8::toString).collect(Collectors.toList()))
- .build())
- .build();
- break;
- case INT64_LIST:
- finalValue =
- ValueProto.Value.newBuilder()
- .setInt64ListVal(
- ValueProto.Int64List.newBuilder()
- .addAllVal(((GenericData.Array) featureValue))
- .build())
- .build();
- break;
- case INT32_LIST:
- finalValue =
- ValueProto.Value.newBuilder()
- .setInt32ListVal(
- ValueProto.Int32List.newBuilder()
- .addAllVal(((GenericData.Array) featureValue))
- .build())
- .build();
- break;
- case FLOAT_LIST:
- finalValue =
- ValueProto.Value.newBuilder()
- .setFloatListVal(
- ValueProto.FloatList.newBuilder()
- .addAllVal(((GenericData.Array) featureValue))
- .build())
- .build();
- break;
- case DOUBLE_LIST:
- finalValue =
- ValueProto.Value.newBuilder()
- .setDoubleListVal(
- ValueProto.DoubleList.newBuilder()
- .addAllVal(((GenericData.Array) featureValue))
- .build())
- .build();
- break;
- case BOOL_LIST:
- finalValue =
- ValueProto.Value.newBuilder()
- .setBoolListVal(
- ValueProto.BoolList.newBuilder()
- .addAllVal(((GenericData.Array) featureValue))
- .build())
- .build();
- break;
- case BYTES_LIST:
- finalValue =
- ValueProto.Value.newBuilder()
- .setBytesListVal(
- ValueProto.BytesList.newBuilder()
- .addAllVal(
- ((GenericData.Array) featureValue)
- .stream()
- .map(byteBuffer -> ByteString.copyFrom(byteBuffer.array()))
- .collect(Collectors.toList()))
- .build())
- .build();
- break;
- default:
- throw new RuntimeException(
- String.format("FeatureType %s is not supported", valueType.name()));
- }
- } catch (ClassCastException e) {
- // Feature type has changed
- finalValue = ValueProto.Value.newBuilder().build();
- }
-
- return finalValue;
- }
-
- @Override
- public ServingAPIProto.FeatureReferenceV2 getFeatureReference() {
- return this.featureReference;
- }
-
- @Override
- public Timestamp getEventTimestamp() {
- return this.eventTimestamp;
- }
-}
diff --git a/java/storage/api/src/main/java/feast/storage/api/retriever/FeatureTableRequest.java b/java/storage/api/src/main/java/feast/storage/api/retriever/FeatureTableRequest.java
deleted file mode 100644
index 2f181e6de8..0000000000
--- a/java/storage/api/src/main/java/feast/storage/api/retriever/FeatureTableRequest.java
+++ /dev/null
@@ -1,62 +0,0 @@
-/*
- * SPDX-License-Identifier: Apache-2.0
- * Copyright 2018-2020 The Feast Authors
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * https://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package feast.storage.api.retriever;
-
-import com.google.auto.value.AutoValue;
-import com.google.common.collect.ImmutableSet;
-import feast.proto.core.FeatureTableProto.FeatureTableSpec;
-import feast.proto.serving.ServingAPIProto.FeatureReferenceV2;
-import java.util.List;
-import java.util.Map;
-import java.util.stream.Collectors;
-
-@AutoValue
-public abstract class FeatureTableRequest {
- public abstract FeatureTableSpec getSpec();
-
- public abstract ImmutableSet getFeatureReferences();
-
- public static Builder newBuilder() {
- return new AutoValue_FeatureTableRequest.Builder();
- }
-
- @AutoValue.Builder
- public abstract static class Builder {
- public abstract Builder setSpec(FeatureTableSpec spec);
-
- abstract ImmutableSet.Builder featureReferencesBuilder();
-
- public Builder addAllFeatureReferences(List featureReferenceList) {
- featureReferencesBuilder().addAll(featureReferenceList);
- return this;
- }
-
- public Builder addFeatureReference(FeatureReferenceV2 featureReference) {
- featureReferencesBuilder().add(featureReference);
- return this;
- }
-
- public abstract FeatureTableRequest build();
- }
-
- public Map getFeatureRefsByName() {
- return getFeatureReferences().stream()
- .collect(
- Collectors.toMap(
- FeatureReferenceV2::getFeatureName, featureReference -> featureReference));
- }
-}
diff --git a/java/storage/connectors/pom.xml b/java/storage/connectors/pom.xml
deleted file mode 100644
index 11e32a154c..0000000000
--- a/java/storage/connectors/pom.xml
+++ /dev/null
@@ -1,61 +0,0 @@
-
-
-
- dev.feast
- feast-parent
- ${revision}
- ../../pom.xml
-
-
- 4.0.0
- feast-storage-connectors
- pom
-
- Feast Storage Connectors
-
-
- redis
-
-
-
-
-
- org.apache.maven.plugins
- maven-dependency-plugin
-
-
-
- javax.annotation
-
-
-
-
-
- org.jacoco
- jacoco-maven-plugin
-
-
-
-
-
-
- dev.feast
- feast-datatypes
- ${project.version}
-
-
-
- dev.feast
- feast-common
- ${project.version}
-
-
-
- dev.feast
- feast-storage-api
- ${project.version}
-
-
-
-
diff --git a/java/storage/connectors/redis/pom.xml b/java/storage/connectors/redis/pom.xml
deleted file mode 100644
index ce25f41da6..0000000000
--- a/java/storage/connectors/redis/pom.xml
+++ /dev/null
@@ -1,104 +0,0 @@
-
-
-
- dev.feast
- feast-storage-connectors
- ${revision}
-
-
- 4.0.0
- feast-storage-connector-redis
-
- Feast Storage Connector for Redis
-
-
-
- io.lettuce
- lettuce-core
- 6.0.2.RELEASE
-
-
-
- io.netty
- netty-transport-native-epoll
- 4.1.52.Final
- linux-x86_64
-
-
-
- org.apache.commons
- commons-lang3
- 3.9
-
-
-
- com.google.auto.value
- auto-value-annotations
- 1.6.6
-
-
-
- com.google.auto.value
- auto-value
- 1.6.6
- provided
-
-
-
- com.google.guava
- guava
- ${guava.version}
-
-
-
- org.mockito
- mockito-core
- ${mockito.version}
- test
-
-
-
-
- com.github.kstyrc
- embedded-redis
- 0.6
- test
-
-
-
- org.hamcrest
- hamcrest-core
- test
- ${hamcrest.version}
-
-
-
- org.hamcrest
- hamcrest-library
- test
- ${hamcrest.version}
-
-
-
- net.ishiis.redis
- redis-unit
- 1.0.3
- test
-
-
-
-
- junit
- junit
- 4.13.2
- test
-
-
- org.slf4j
- slf4j-simple
- 1.7.32
- test
-
-
-
diff --git a/java/storage/connectors/redis/src/test/java/feast/storage/connectors/redis/test/TestUtil.java b/java/storage/connectors/redis/src/test/java/feast/storage/connectors/redis/test/TestUtil.java
deleted file mode 100644
index 66aba44bc2..0000000000
--- a/java/storage/connectors/redis/src/test/java/feast/storage/connectors/redis/test/TestUtil.java
+++ /dev/null
@@ -1,44 +0,0 @@
-/*
- * SPDX-License-Identifier: Apache-2.0
- * Copyright 2018-2020 The Feast Authors
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * https://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package feast.storage.connectors.redis.test;
-
-import java.io.IOException;
-import redis.embedded.RedisServer;
-
-public class TestUtil {
- public static class LocalRedis {
-
- private static RedisServer server;
-
- /**
- * Start local Redis for used in testing at "localhost"
- *
- * @param port port number
- * @throws IOException if Redis failed to start
- */
- public static void start(int port) throws IOException {
- server = new RedisServer(port);
- server.start();
- }
-
- public static void stop() {
- if (server != null) {
- server.stop();
- }
- }
- }
-}
diff --git a/sdk/python/docs/source/feast.rst b/sdk/python/docs/source/feast.rst
index c000ac2e2b..b1fb70a362 100644
--- a/sdk/python/docs/source/feast.rst
+++ b/sdk/python/docs/source/feast.rst
@@ -169,14 +169,6 @@ feast.field module
:undoc-members:
:show-inheritance:
-feast.flags module
-------------------
-
-.. automodule:: feast.flags
- :members:
- :undoc-members:
- :show-inheritance:
-
feast.flags\_helper module
--------------------------
diff --git a/sdk/python/feast/cli.py b/sdk/python/feast/cli.py
index 153c1a5ddd..8c1f988e36 100644
--- a/sdk/python/feast/cli.py
+++ b/sdk/python/feast/cli.py
@@ -25,7 +25,7 @@
from dateutil import parser
from pygments import formatters, highlight, lexers
-from feast import flags, flags_helper, utils
+from feast import utils
from feast.constants import DEFAULT_FEATURE_TRANSFORMATION_SERVER_PORT
from feast.errors import FeastObjectNotFoundException, FeastProviderLoginError
from feast.feature_store import FeatureStore
@@ -691,113 +691,6 @@ def serve_transformations_command(ctx: click.Context, port: int):
store.serve_transformations(port)
-@cli.group(name="alpha")
-def alpha_cmd():
- """
- Access alpha features
- """
- pass
-
-
-@alpha_cmd.command("list")
-@click.pass_context
-def list_alpha_features(ctx: click.Context):
- """
- Lists all alpha features
- """
- repo = ctx.obj["CHDIR"]
- cli_check_repo(repo)
- repo_path = str(repo)
- store = FeatureStore(repo_path=repo_path)
-
- flags_to_show = flags.FLAG_NAMES.copy()
- flags_to_show.remove(flags.FLAG_ALPHA_FEATURES_NAME)
- print("Alpha features:")
- for flag in flags_to_show:
- enabled_string = (
- "enabled"
- if flags_helper.feature_flag_enabled(store.config, flag)
- else "disabled"
- )
- print(f"{flag}: {enabled_string}")
-
-
-@alpha_cmd.command("enable-all")
-@click.pass_context
-def enable_alpha_features(ctx: click.Context):
- """
- Enables all alpha features
- """
- repo = ctx.obj["CHDIR"]
- cli_check_repo(repo)
- repo_path = str(repo)
- store = FeatureStore(repo_path=repo_path)
-
- if store.config.flags is None:
- store.config.flags = {}
- for flag_name in flags.FLAG_NAMES:
- store.config.flags[flag_name] = True
- store.config.write_to_path(Path(repo_path))
-
-
-@alpha_cmd.command("enable")
-@click.argument("name", type=click.STRING)
-@click.pass_context
-def enable_alpha_feature(ctx: click.Context, name: str):
- """
- Enables an alpha feature
- """
- if name not in flags.FLAG_NAMES:
- raise ValueError(f"Flag name, {name}, not valid.")
-
- repo = ctx.obj["CHDIR"]
- cli_check_repo(repo)
- repo_path = str(repo)
- store = FeatureStore(repo_path=repo_path)
-
- if store.config.flags is None:
- store.config.flags = {}
- store.config.flags[flags.FLAG_ALPHA_FEATURES_NAME] = True
- store.config.flags[name] = True
- store.config.write_to_path(Path(repo_path))
-
-
-@alpha_cmd.command("disable")
-@click.argument("name", type=click.STRING)
-@click.pass_context
-def disable_alpha_feature(ctx: click.Context, name: str):
- """
- Disables an alpha feature
- """
- if name not in flags.FLAG_NAMES:
- raise ValueError(f"Flag name, {name}, not valid.")
-
- repo = ctx.obj["CHDIR"]
- cli_check_repo(repo)
- repo_path = str(repo)
- store = FeatureStore(repo_path=repo_path)
-
- if store.config.flags is None or name not in store.config.flags:
- return
- store.config.flags[name] = False
- store.config.write_to_path(Path(repo_path))
-
-
-@alpha_cmd.command("disable-all")
-@click.pass_context
-def disable_alpha_features(ctx: click.Context):
- """
- Disables all alpha features
- """
- repo = ctx.obj["CHDIR"]
- cli_check_repo(repo)
- repo_path = str(repo)
- store = FeatureStore(repo_path=repo_path)
-
- store.config.flags = None
- store.config.write_to_path(Path(repo_path))
-
-
@cli.command("validate")
@click.option(
"--feature-service",
diff --git a/sdk/python/feast/errors.py b/sdk/python/feast/errors.py
index 980dfd470f..518d648533 100644
--- a/sdk/python/feast/errors.py
+++ b/sdk/python/feast/errors.py
@@ -197,6 +197,13 @@ def __init__(
)
+class FeastOfflineStoreInvalidName(Exception):
+ def __init__(self, offline_store_class_name: str):
+ super().__init__(
+ f"Offline Store Class '{offline_store_class_name}' should end with the string `OfflineStore`.'"
+ )
+
+
class FeastOnlineStoreInvalidName(Exception):
def __init__(self, online_store_class_name: str):
super().__init__(
@@ -312,14 +319,6 @@ def __init__(self, feature_view_name: str):
)
-class ExperimentalFeatureNotEnabled(Exception):
- def __init__(self, feature_flag_name: str):
- super().__init__(
- f"You are attempting to use an experimental feature that is not enabled. Please run "
- f"`feast alpha enable {feature_flag_name}` "
- )
-
-
class RepoConfigPathDoesNotExist(Exception):
def __init__(self):
super().__init__("The repo_path attribute does not exist for the repo_config.")
diff --git a/sdk/python/feast/feature_store.py b/sdk/python/feast/feature_store.py
index c4ccc9a648..0c93eeb843 100644
--- a/sdk/python/feast/feature_store.py
+++ b/sdk/python/feast/feature_store.py
@@ -40,7 +40,7 @@
from google.protobuf.timestamp_pb2 import Timestamp
from tqdm import tqdm
-from feast import feature_server, flags, flags_helper, ui_server, utils
+from feast import feature_server, ui_server, utils
from feast.base_feature_view import BaseFeatureView
from feast.batch_feature_view import BatchFeatureView
from feast.data_source import DataSource, PushMode
@@ -50,7 +50,6 @@
from feast.entity import Entity
from feast.errors import (
EntityNotFoundException,
- ExperimentalFeatureNotEnabled,
FeatureNameCollisionError,
FeatureViewNotFoundException,
RequestDataNotFoundInEntityDfException,
@@ -502,8 +501,8 @@ def _get_features(
)
if feature_service_from_registry != _features:
warnings.warn(
- "The FeatureService object that has been passed in as an argument is"
- "inconsistent with the version from Registry. Potentially a newer version"
+ "The FeatureService object that has been passed in as an argument is "
+ "inconsistent with the version from the registry. Potentially a newer version "
"of the FeatureService has been applied to the registry."
)
for projection in feature_service_from_registry.feature_view_projections:
@@ -533,11 +532,12 @@ def _validate_all_feature_views(
sfvs_to_update: List[StreamFeatureView],
):
"""Validates all feature views."""
- if (
- not flags_helper.enable_on_demand_feature_views(self.config)
- and len(odfvs_to_update) > 0
- ):
- raise ExperimentalFeatureNotEnabled(flags.FLAG_ON_DEMAND_TRANSFORM_NAME)
+ if len(odfvs_to_update) > 0:
+ warnings.warn(
+ "On demand feature view is an experimental feature. "
+ "This API is stable, but the functionality does not scale well for offline retrieval",
+ RuntimeWarning,
+ )
set_usage_attribute("odfv", bool(odfvs_to_update))
@@ -839,7 +839,7 @@ def apply(
)
for rfv in request_views_to_update:
- data_sources_set_to_update.add(rfv.request_data_source)
+ data_sources_set_to_update.add(rfv.request_source)
for odfv in odfvs_to_update:
for v in odfv.source_request_sources.values():
@@ -2284,8 +2284,11 @@ def serve_ui(
@log_exceptions_and_usage
def serve_transformations(self, port: int) -> None:
"""Start the feature transformation server locally on a given port."""
- if not flags_helper.enable_on_demand_feature_views(self.config):
- raise ExperimentalFeatureNotEnabled(flags.FLAG_ON_DEMAND_TRANSFORM_NAME)
+ warnings.warn(
+ "On demand feature view is an experimental feature. "
+ "This API is stable, but the functionality does not scale well for offline retrieval",
+ RuntimeWarning,
+ )
from feast import transformation_server
diff --git a/sdk/python/feast/flags.py b/sdk/python/feast/flags.py
deleted file mode 100644
index 26e20d81f6..0000000000
--- a/sdk/python/feast/flags.py
+++ /dev/null
@@ -1,10 +0,0 @@
-FLAG_ALPHA_FEATURES_NAME = "alpha_features"
-FLAG_ON_DEMAND_TRANSFORM_NAME = "on_demand_transforms"
-FLAG_AWS_LAMBDA_FEATURE_SERVER_NAME = "aws_lambda_feature_server"
-ENV_FLAG_IS_TEST = "IS_TEST"
-
-FLAG_NAMES = {
- FLAG_ALPHA_FEATURES_NAME,
- FLAG_ON_DEMAND_TRANSFORM_NAME,
- FLAG_AWS_LAMBDA_FEATURE_SERVER_NAME,
-}
diff --git a/sdk/python/feast/flags_helper.py b/sdk/python/feast/flags_helper.py
index 7cf16dbf0b..4763f4a283 100644
--- a/sdk/python/feast/flags_helper.py
+++ b/sdk/python/feast/flags_helper.py
@@ -1,39 +1,11 @@
import os
-from feast import flags
-from feast.repo_config import RepoConfig
+ENV_FLAG_IS_TEST = "IS_TEST"
def _env_flag_enabled(name: str) -> bool:
return os.getenv(name, default="False") == "True"
-def feature_flag_enabled(repo_config: RepoConfig, flag_name: str) -> bool:
- if is_test():
- return True
- return (
- _alpha_feature_flag_enabled(repo_config)
- and repo_config.flags is not None
- and flag_name in repo_config.flags
- and repo_config.flags[flag_name]
- )
-
-
-def _alpha_feature_flag_enabled(repo_config: RepoConfig) -> bool:
- return (
- repo_config.flags is not None
- and flags.FLAG_ALPHA_FEATURES_NAME in repo_config.flags
- and repo_config.flags[flags.FLAG_ALPHA_FEATURES_NAME]
- )
-
-
def is_test() -> bool:
- return _env_flag_enabled(flags.ENV_FLAG_IS_TEST)
-
-
-def enable_on_demand_feature_views(repo_config: RepoConfig) -> bool:
- return feature_flag_enabled(repo_config, flags.FLAG_ON_DEMAND_TRANSFORM_NAME)
-
-
-def enable_aws_lambda_feature_server(repo_config: RepoConfig) -> bool:
- return feature_flag_enabled(repo_config, flags.FLAG_AWS_LAMBDA_FEATURE_SERVER_NAME)
+ return _env_flag_enabled(ENV_FLAG_IS_TEST)
diff --git a/sdk/python/feast/infra/aws.py b/sdk/python/feast/infra/aws.py
index 145c55952e..523848f2f2 100644
--- a/sdk/python/feast/infra/aws.py
+++ b/sdk/python/feast/infra/aws.py
@@ -3,6 +3,7 @@
import logging
import os
import uuid
+import warnings
from datetime import datetime
from pathlib import Path
from tempfile import TemporaryFile
@@ -22,15 +23,12 @@
from feast.errors import (
AwsAPIGatewayDoesNotExist,
AwsLambdaDoesNotExist,
- ExperimentalFeatureNotEnabled,
IncompatibleRegistryStoreClass,
RepoConfigPathDoesNotExist,
S3RegistryBucketForbiddenAccess,
S3RegistryBucketNotExist,
)
from feast.feature_view import FeatureView
-from feast.flags import FLAG_AWS_LAMBDA_FEATURE_SERVER_NAME
-from feast.flags_helper import enable_aws_lambda_feature_server
from feast.infra.feature_servers.aws_lambda.config import AwsLambdaFeatureServerConfig
from feast.infra.passthrough_provider import PassthroughProvider
from feast.infra.utils import aws_utils
@@ -74,8 +72,11 @@ def update_infra(
)
if self.repo_config.feature_server and self.repo_config.feature_server.enabled:
- if not enable_aws_lambda_feature_server(self.repo_config):
- raise ExperimentalFeatureNotEnabled(FLAG_AWS_LAMBDA_FEATURE_SERVER_NAME)
+ warnings.warn(
+ "AWS Lambda based feature serving is an experimental feature. "
+ "We do not guarantee that future changes will maintain backward compatibility.",
+ RuntimeWarning,
+ )
# Since the AWS Lambda feature server will attempt to load the registry, we
# only allow the registry to be in S3.
diff --git a/sdk/python/feast/infra/offline_stores/contrib/contrib_repo_configuration.py b/sdk/python/feast/infra/offline_stores/contrib/spark_repo_configuration.py
similarity index 75%
rename from sdk/python/feast/infra/offline_stores/contrib/contrib_repo_configuration.py
rename to sdk/python/feast/infra/offline_stores/contrib/spark_repo_configuration.py
index 083ec2b210..ec414f202a 100644
--- a/sdk/python/feast/infra/offline_stores/contrib/contrib_repo_configuration.py
+++ b/sdk/python/feast/infra/offline_stores/contrib/spark_repo_configuration.py
@@ -1,9 +1,6 @@
from feast.infra.offline_stores.contrib.spark_offline_store.tests.data_source import (
SparkDataSourceCreator,
)
-from feast.infra.offline_stores.contrib.trino_offline_store.tests.data_source import (
- TrinoSourceCreator,
-)
from tests.integration.feature_repos.repo_configuration import REDIS_CONFIG
from tests.integration.feature_repos.universal.online_store.redis import (
RedisOnlineStoreCreator,
@@ -11,7 +8,6 @@
AVAILABLE_OFFLINE_STORES = [
("local", SparkDataSourceCreator),
- ("local", TrinoSourceCreator),
]
AVAILABLE_ONLINE_STORES = {"redis": (REDIS_CONFIG, RedisOnlineStoreCreator)}
diff --git a/sdk/python/feast/infra/offline_stores/contrib/trino_repo_configuration.py b/sdk/python/feast/infra/offline_stores/contrib/trino_repo_configuration.py
new file mode 100644
index 0000000000..198227095d
--- /dev/null
+++ b/sdk/python/feast/infra/offline_stores/contrib/trino_repo_configuration.py
@@ -0,0 +1,13 @@
+from feast.infra.offline_stores.contrib.trino_offline_store.tests.data_source import (
+ TrinoSourceCreator,
+)
+from tests.integration.feature_repos.repo_configuration import REDIS_CONFIG
+from tests.integration.feature_repos.universal.online_store.redis import (
+ RedisOnlineStoreCreator,
+)
+
+AVAILABLE_OFFLINE_STORES = [
+ ("local", TrinoSourceCreator),
+]
+
+AVAILABLE_ONLINE_STORES = {"redis": (REDIS_CONFIG, RedisOnlineStoreCreator)}
diff --git a/sdk/python/feast/infra/offline_stores/snowflake.py b/sdk/python/feast/infra/offline_stores/snowflake.py
index 0f4c6a7b52..5936151d71 100644
--- a/sdk/python/feast/infra/offline_stores/snowflake.py
+++ b/sdk/python/feast/infra/offline_stores/snowflake.py
@@ -429,7 +429,7 @@ def _to_arrow_internal(self) -> pa.Table:
def to_snowflake(self, table_name: str, temporary=False) -> None:
"""Save dataset as a new Snowflake table"""
- if self.on_demand_feature_views is not None:
+ if self.on_demand_feature_views:
transformed_df = self.to_df()
write_pandas(
diff --git a/sdk/python/feast/infra/registry_stores/sql.py b/sdk/python/feast/infra/registry_stores/sql.py
index 9c6b47a714..39ed2a802e 100644
--- a/sdk/python/feast/infra/registry_stores/sql.py
+++ b/sdk/python/feast/infra/registry_stores/sql.py
@@ -2,7 +2,7 @@
from datetime import datetime
from enum import Enum
from pathlib import Path
-from typing import Any, List, Optional, Set, Union
+from typing import Any, Callable, List, Optional, Set, Union
from sqlalchemy import ( # type: ignore
BigInteger,
@@ -560,7 +560,7 @@ def update_infra(self, infra: Infra, project: str, commit: bool = True):
)
def get_infra(self, project: str, allow_cache: bool = False) -> Infra:
- return self._get_object(
+ infra_object = self._get_object(
managed_infra,
"infra_obj",
project,
@@ -570,6 +570,8 @@ def get_infra(self, project: str, allow_cache: bool = False) -> Infra:
"infra_proto",
None,
)
+ infra_object = infra_object or InfraProto()
+ return Infra.from_proto(infra_object)
def apply_user_metadata(
self,
@@ -683,11 +685,19 @@ def commit(self):
pass
def _apply_object(
- self, table, project: str, id_field_name, obj, proto_field_name, name=None
+ self,
+ table: Table,
+ project: str,
+ id_field_name: str,
+ obj: Any,
+ proto_field_name: str,
+ name: Optional[str] = None,
):
self._maybe_init_project_metadata(project)
- name = name or obj.name
+ name = name or obj.name if hasattr(obj, "name") else None
+ assert name, f"name needs to be provided for {obj}"
+
with self.engine.connect() as conn:
update_datetime = datetime.utcnow()
update_time = int(update_datetime.timestamp())
@@ -712,9 +722,16 @@ def _apply_object(
)
conn.execute(update_stmt)
else:
+ obj_proto = obj.to_proto()
+
+ if hasattr(obj_proto, "meta") and hasattr(
+ obj_proto.meta, "created_timestamp"
+ ):
+ obj_proto.meta.created_timestamp.FromDatetime(update_datetime)
+
values = {
id_field_name: name,
- proto_field_name: obj.to_proto().SerializeToString(),
+ proto_field_name: obj_proto.SerializeToString(),
"last_updated_timestamp": update_time,
"project_id": project,
}
@@ -749,7 +766,14 @@ def _maybe_init_project_metadata(self, project):
conn.execute(insert_stmt)
usage.set_current_project_uuid(new_project_uuid)
- def _delete_object(self, table, name, project, id_field_name, not_found_exception):
+ def _delete_object(
+ self,
+ table: Table,
+ name: str,
+ project: str,
+ id_field_name: str,
+ not_found_exception: Optional[Callable],
+ ):
with self.engine.connect() as conn:
stmt = delete(table).where(
getattr(table.c, id_field_name) == name, table.c.project_id == project
@@ -763,14 +787,14 @@ def _delete_object(self, table, name, project, id_field_name, not_found_exceptio
def _get_object(
self,
- table,
- name,
- project,
- proto_class,
- python_class,
- id_field_name,
- proto_field_name,
- not_found_exception,
+ table: Table,
+ name: str,
+ project: str,
+ proto_class: Any,
+ python_class: Any,
+ id_field_name: str,
+ proto_field_name: str,
+ not_found_exception: Optional[Callable],
):
self._maybe_init_project_metadata(project)
@@ -782,10 +806,18 @@ def _get_object(
if row:
_proto = proto_class.FromString(row[proto_field_name])
return python_class.from_proto(_proto)
- raise not_found_exception(name, project)
+ if not_found_exception:
+ raise not_found_exception(name, project)
+ else:
+ return None
def _list_objects(
- self, table, project, proto_class, python_class, proto_field_name
+ self,
+ table: Table,
+ project: str,
+ proto_class: Any,
+ python_class: Any,
+ proto_field_name: str,
):
self._maybe_init_project_metadata(project)
with self.engine.connect() as conn:
diff --git a/sdk/python/feast/repo_config.py b/sdk/python/feast/repo_config.py
index 587907b284..278bc7da69 100644
--- a/sdk/python/feast/repo_config.py
+++ b/sdk/python/feast/repo_config.py
@@ -17,10 +17,11 @@
from pydantic.error_wrappers import ErrorWrapper
from pydantic.typing import Dict, Optional, Union
-from feast import flags
from feast.errors import (
FeastFeatureServerTypeInvalidError,
FeastFeatureServerTypeSetError,
+ FeastOfflineStoreInvalidName,
+ FeastOnlineStoreInvalidName,
FeastProviderNotSetError,
)
from feast.importer import import_class
@@ -135,7 +136,7 @@ class RepoConfig(FeastBaseModel):
""" FeatureServerConfig: Feature server configuration (optional depending on provider) """
flags: Any
- """ Flags: Feature flags for experimental features (optional) """
+ """ Flags (deprecated field): Feature flags for experimental features """
repo_path: Optional[Path] = None
@@ -278,7 +279,8 @@ def _validate_online_store_config(cls, values):
return values
# Make sure that the provider configuration is set. We need it to set the defaults
- assert "provider" in values
+ if "provider" not in values:
+ raise FeastProviderNotSetError()
# Set the default type
# This is only direct reference to a provider or online store that we should have
@@ -315,7 +317,8 @@ def _validate_offline_store_config(cls, values):
return values
# Make sure that the provider configuration is set. We need it to set the defaults
- assert "provider" in values
+ if "provider" not in values:
+ raise FeastProviderNotSetError()
# Set the default type
if "type" not in values["offline_store"]:
@@ -391,15 +394,9 @@ def _validate_flags(cls, v):
if not isinstance(v, Dict):
return
- for flag_name, val in v.items():
- if flag_name not in flags.FLAG_NAMES:
- _logger.warn(
- "Unrecognized flag: %s. This feature may be invalid, or may refer "
- "to a previously experimental feature which has graduated to production.",
- flag_name,
- )
- if type(val) is not bool:
- raise ValueError(f"Flag value, {val}, not valid.")
+ _logger.warning(
+ "Flags are no longer necessary in Feast. Experimental features will log warnings instead."
+ )
return v
@@ -455,8 +452,8 @@ def get_batch_engine_config_from_type(batch_engine_type: str):
def get_online_config_from_type(online_store_type: str):
if online_store_type in ONLINE_STORE_CLASS_FOR_TYPE:
online_store_type = ONLINE_STORE_CLASS_FOR_TYPE[online_store_type]
- else:
- assert online_store_type.endswith("OnlineStore")
+ elif not online_store_type.endswith("OnlineStore"):
+ raise FeastOnlineStoreInvalidName(online_store_type)
module_name, online_store_class_type = online_store_type.rsplit(".", 1)
config_class_name = f"{online_store_class_type}Config"
@@ -466,8 +463,8 @@ def get_online_config_from_type(online_store_type: str):
def get_offline_config_from_type(offline_store_type: str):
if offline_store_type in OFFLINE_STORE_CLASS_FOR_TYPE:
offline_store_type = OFFLINE_STORE_CLASS_FOR_TYPE[offline_store_type]
- else:
- assert offline_store_type.endswith("OfflineStore")
+ elif not offline_store_type.endswith("OfflineStore"):
+ raise FeastOfflineStoreInvalidName(offline_store_type)
module_name, offline_store_class_type = offline_store_type.rsplit(".", 1)
config_class_name = f"{offline_store_class_type}Config"
diff --git a/sdk/python/feast/ui/package.json b/sdk/python/feast/ui/package.json
index 883c19660b..358aa2cdd2 100644
--- a/sdk/python/feast/ui/package.json
+++ b/sdk/python/feast/ui/package.json
@@ -6,7 +6,7 @@
"@elastic/datemath": "^5.0.3",
"@elastic/eui": "^57.0.0",
"@emotion/react": "^11.9.0",
- "@feast-dev/feast-ui": "^0.20.5",
+ "@feast-dev/feast-ui": "latest",
"@testing-library/jest-dom": "^5.16.4",
"@testing-library/react": "^13.2.0",
"@testing-library/user-event": "^13.5.0",
diff --git a/sdk/python/feast/ui/yarn.lock b/sdk/python/feast/ui/yarn.lock
index b44fc5f51a..dc6c92859c 100644
--- a/sdk/python/feast/ui/yarn.lock
+++ b/sdk/python/feast/ui/yarn.lock
@@ -1345,10 +1345,10 @@
minimatch "^3.1.2"
strip-json-comments "^3.1.1"
-"@feast-dev/feast-ui@^0.20.5":
- version "0.20.5"
- resolved "https://registry.yarnpkg.com/@feast-dev/feast-ui/-/feast-ui-0.20.5.tgz#bb0d6fc81cbd92ca69b779982ab151a8d9cabaee"
- integrity sha512-BwMPJSv1MkylHxPnU/2fZX77AC/G4H2DIf+HAj80ZklwB0zbmeZzhXFrVh4xSheevGZFh0L839JeL14WfXPZsA==
+"@feast-dev/feast-ui@latest":
+ version "0.23.1"
+ resolved "https://registry.yarnpkg.com/@feast-dev/feast-ui/-/feast-ui-0.23.1.tgz#391ba62efb3976b87745fe1a09ead50f3b6bd408"
+ integrity sha512-jMElqDfdNRrC+CxIXJ3Kkxa1CDwLyr4cCyO67e8KVEhiIb9eIYkIrfU8GpGm9sU2dErpIIqziWFvdR4aU/hNJA==
dependencies:
"@elastic/datemath" "^5.0.3"
"@elastic/eui" "^55.0.1"
diff --git a/sdk/python/feast/usage.py b/sdk/python/feast/usage.py
index 5e78aa52d2..9791f86c27 100644
--- a/sdk/python/feast/usage.py
+++ b/sdk/python/feast/usage.py
@@ -29,6 +29,7 @@
import requests
+from feast import flags_helper
from feast.constants import DEFAULT_FEAST_USAGE_VALUE, FEAST_USAGE
from feast.version import get_version
@@ -172,7 +173,8 @@ def _export(event: typing.Dict[str, typing.Any]):
def _produce_event(ctx: UsageContext):
- is_test = bool({"pytest", "unittest"} & sys.modules.keys())
+ # Cannot check for unittest because typeguard pulls in unittest
+ is_test = flags_helper.is_test() or bool({"pytest"} & sys.modules.keys())
event = {
"timestamp": datetime.utcnow().isoformat(),
"is_test": is_test,
diff --git a/sdk/python/tests/README.md b/sdk/python/tests/README.md
new file mode 100644
index 0000000000..0f56e0eee2
--- /dev/null
+++ b/sdk/python/tests/README.md
@@ -0,0 +1,343 @@
+# Testing Suite
+
+## Overview
+
+This guide will go over:
+
+1. how Feast tests are setup
+2. how to extend the test suite to test new functionality
+3. how to use the existing test suite to test a new custom offline / online store.
+
+## Test suite overview
+
+Let's inspect the test setup in `sdk/python/tests/integration`:
+
+```bash
+$ tree
+.
+├── e2e
+│ ├── test_go_feature_server.py
+│ ├── test_python_feature_server.py
+│ ├── test_universal_e2e.py
+│ ├── test_usage_e2e.py
+│ └── test_validation.py
+├── feature_repos
+│ ├── integration_test_repo_config.py
+│ ├── repo_configuration.py
+│ └── universal
+│ ├── catalog
+│ ├── data_source_creator.py
+│ ├── data_sources
+│ │ ├── __init__.py
+│ │ ├── bigquery.py
+│ │ ├── file.py
+│ │ ├── redshift.py
+│ │ └── snowflake.py
+│ ├── entities.py
+│ ├── feature_views.py
+│ ├── online_store
+│ │ ├── __init__.py
+│ │ ├── datastore.py
+│ │ ├── dynamodb.py
+│ │ ├── hbase.py
+│ │ └── redis.py
+│ └── online_store_creator.py
+├── materialization
+│ └── test_lambda.py
+├── offline_store
+│ ├── test_feature_logging.py
+│ ├── test_offline_write.py
+│ ├── test_push_features_to_offline_store.py
+│ ├── test_s3_custom_endpoint.py
+│ └── test_universal_historical_retrieval.py
+├── online_store
+│ ├── test_online_retrieval.py
+│ ├── test_push_features_to_online_store.py
+│ └── test_universal_online.py
+└── registration
+ ├── test_feature_store.py
+ ├── test_inference.py
+ ├── test_registry.py
+ ├── test_sql_registry.py
+ ├── test_universal_cli.py
+ ├── test_universal_odfv_feature_inference.py
+ └── test_universal_types.py
+
+```
+
+* `feature_repos` has setup files for most tests in the test suite.
+* `conftest.py` and some of the individual test files contain fixtures which can be used to on different offline stores, online stores, etc. and thus abstract away store specific implementations so we don't need to rewrite the same test implementation for different stores.
+
+## Structure of the test suite
+
+### What is the universal test suite?
+
+The universal test suite verifies that crucial Feast functions (e.g `get_historical_features`, `get_online_features` etc.) have the correct behavior for each of the different environments that Feast could be used in. These environments are combinations of an offline store, online store, and provider and the universal test suite serves to run basic functional verification against all of these different permutations.
+
+We use pytest [fixtures](https://docs.pytest.org/en/6.2.x/fixture.html) to accomplish this without writing excess code.
+
+Tests in Feast are split into integration and unit tests.
+
+### Is it an integration or unit test?
+
+* Integration tests test non local Feast behavior. Integration tests mainly involve testing of Feast components that connect to services outside of Feast(e.g connecting to gcp or aws clients).
+ * Generally if the test requires the initialization of a feature store in an external environment in order to test (i.e using our universal test fixtures), it is probably an integration test.
+* Unit tests, on the other hand, unit tests primarily test local and class level behavior that does not require spinning up an external service. If your test can be run locally without using any other services besides pytest, it is a unit test.
+
+### Main types of tests
+
+#### Integration tests
+
+1. E2E tests
+ * E2E tests test end-to-end functionality of Feast over the various codepaths (initialize a feature store, apply, and materialize).
+ * The main codepaths include:
+ * basic e2e tests for offline stores
+ * `test_universal_e2e.py`
+ * go feature server
+ * `test_go_feature_server.py`
+ * python http server
+ * `test_python_feature_server.py`
+ * usage tracking
+ * `test_usage_e2e.py`
+ * data quality monitoring feature validation
+ * `test_validation.py`
+2. Offline and Online Store Tests
+ * Offline and online store tests mainly test for the offline and online retrieval functionality.
+ * The various specific functionalities that are tested include:
+ * push API tests
+ * `test_push_features_to_offline_store.py`
+ * `test_push_features_to_online_store.py`
+ * `test_offline_write.py`
+ * historical retrieval tests
+ * `test_universal_historical_retrieval.py`
+ * online retrieval tests
+ * `test_universal_online.py`
+ * data quality monitoring feature logging tests
+ * `test_feature_logging.py`
+ * online store tests
+ * `test_universal_online.py`
+3. Registration Tests
+ * The registration folder contains all of the registry tests and some universal cli tests. This includes:
+ * CLI Apply and Materialize tests tested against on the universal test suite
+ * Data type inference tests
+ * Registry tests
+4. Miscellaneous Tests
+ * AWS Lambda Materialization Tests (Currently do not work)
+ * `test_lambda.py`
+
+#### Unit tests
+
+1. Registry Diff Tests
+ * These are tests for the infrastructure and registry diff functionality that Feast uses to determine if changes to the registry or infrastructure is needed.
+2. Local CLI Tests and Local Feast Tests
+ * These tests test all of the cli commands against the local file offline store.
+3. Infrastructure Unit Tests
+ * DynamoDB tests with dynamo mocked out
+ * Repository configuration tests
+ * Schema inference unit tests
+ * Key serialization tests
+ * Basic provider unit tests
+4. Feature Store Validation Tests
+ * These test mainly contain class level validation like hashing tests, protobuf and class serialization, and error and warning handling.
+ * Data source unit tests
+ * Feature service unit tests
+ * Feature service, feature view, and feature validation tests
+ * Protobuf/json tests for Feast ValueTypes
+ * Serialization tests
+ * Type mapping
+ * Feast types
+ * Serialization tests due to this [issue](https://github.com/feast-dev/feast/issues/2345)
+ * Feast usage tracking unit tests
+
+#### Docstring tests
+
+Docstring tests are primarily smoke tests to make sure imports and setup functions can be executed without errors.
+
+## Understanding the test suite with an example test
+
+### Example test
+
+Let's look at a sample test using the universal repo:
+
+{% tabs %}
+{% tab code="sdk/python/tests/integration/offline_store/test_universal_historical_retrieval.py" %}
+```python
+@pytest.mark.integration
+@pytest.mark.universal_offline_stores
+@pytest.mark.parametrize("full_feature_names", [True, False], ids=lambda v: f"full:{v}")
+def test_historical_features(environment, universal_data_sources, full_feature_names):
+ store = environment.feature_store
+
+ (entities, datasets, data_sources) = universal_data_sources
+
+ feature_views = construct_universal_feature_views(data_sources)
+
+ entity_df_with_request_data = datasets.entity_df.copy(deep=True)
+ entity_df_with_request_data["val_to_add"] = [
+ i for i in range(len(entity_df_with_request_data))
+ ]
+ entity_df_with_request_data["driver_age"] = [
+ i + 100 for i in range(len(entity_df_with_request_data))
+ ]
+
+ feature_service = FeatureService(
+ name="convrate_plus100",
+ features=[feature_views.driver[["conv_rate"]], feature_views.driver_odfv],
+ )
+ feature_service_entity_mapping = FeatureService(
+ name="entity_mapping",
+ features=[
+ feature_views.location.with_name("origin").with_join_key_map(
+ {"location_id": "origin_id"}
+ ),
+ feature_views.location.with_name("destination").with_join_key_map(
+ {"location_id": "destination_id"}
+ ),
+ ],
+ )
+
+ store.apply(
+ [
+ driver(),
+ customer(),
+ location(),
+ feature_service,
+ feature_service_entity_mapping,
+ *feature_views.values(),
+ ]
+ )
+ # ... more test code
+
+ job_from_df = store.get_historical_features(
+ entity_df=entity_df_with_request_data,
+ features=[
+ "driver_stats:conv_rate",
+ "driver_stats:avg_daily_trips",
+ "customer_profile:current_balance",
+ "customer_profile:avg_passenger_count",
+ "customer_profile:lifetime_trip_count",
+ "conv_rate_plus_100:conv_rate_plus_100",
+ "conv_rate_plus_100:conv_rate_plus_100_rounded",
+ "conv_rate_plus_100:conv_rate_plus_val_to_add",
+ "order:order_is_success",
+ "global_stats:num_rides",
+ "global_stats:avg_ride_length",
+ "field_mapping:feature_name",
+ ],
+ full_feature_names=full_feature_names,
+ )
+
+ if job_from_df.supports_remote_storage_export():
+ files = job_from_df.to_remote_storage()
+ print(files)
+ assert len(files) > 0 # This test should be way more detailed
+
+ start_time = datetime.utcnow()
+ actual_df_from_df_entities = job_from_df.to_df()
+ # ... more test code
+
+ validate_dataframes(
+ expected_df,
+ table_from_df_entities,
+ keys=[event_timestamp, "order_id", "driver_id", "customer_id"],
+ )
+ # ... more test code
+```
+{% endtab %}
+{% endtabs %}
+
+* The key fixtures are the `environment` and `universal_data_sources` fixtures, which are defined in the `feature_repos` directories and the `conftest.py` file. This by default pulls in a standard dataset with driver and customer entities (that we have pre-defined), certain feature views, and feature values.
+ * The `environment` fixture sets up a feature store, parametrized by the provider and the online/offline store. It allows the test to query against that feature store without needing to worry about the underlying implementation or any setup that may be involved in creating instances of these datastores.
+ * Each fixture creates a different integration test with its own `IntegrationTestRepoConfig` which is used by pytest to generate a unique test testing one of the different environments that require testing.
+
+* Feast tests also use a variety of markers:
+ * The `@pytest.mark.integration` marker is used to designate integration tests which will cause the test to be run when you call `make test-python-integration`.
+ * The `@pytest.mark.universal_offline_stores` marker will parametrize the test on all of the universal offline stores including file, redshift, bigquery and snowflake.
+ * The `full_feature_names` parametrization defines whether or not the test should reference features as their full feature name (fully qualified path) or just the feature name itself.
+
+
+## Writing a new test or reusing existing tests
+
+### To add a new test to an existing test file
+
+* Use the same function signatures as an existing test (e.g. use `environment` and `universal_data_sources` as an argument) to include the relevant test fixtures.
+* If possible, expand an individual test instead of writing a new test, due to the cost of starting up offline / online stores.
+* Use the `universal_offline_stores` and `universal_online_store` markers to parametrize the test against different offline store and online store combinations. You can also designate specific online and offline stores to test by using the `only` parameter on the marker.
+
+```python
+@pytest.mark.universal_online_stores(only=["redis"])
+```
+### To test a new offline / online store from a plugin repo
+
+* Install Feast in editable mode with `pip install -e`.
+* The core tests for offline / online store behavior are parametrized by the `FULL_REPO_CONFIGS` variable defined in `feature_repos/repo_configuration.py`. To overwrite this variable without modifying the Feast repo, create your own file that contains a `FULL_REPO_CONFIGS` (which will require adding a new `IntegrationTestRepoConfig` or two) and set the environment variable `FULL_REPO_CONFIGS_MODULE` to point to that file. Then the core offline / online store tests can be run with `make test-python-universal`.
+* See the [custom offline store demo](https://github.com/feast-dev/feast-custom-offline-store-demo) and the [custom online store demo](https://github.com/feast-dev/feast-custom-online-store-demo) for examples.
+
+### What are some important things to keep in mind when adding a new offline / online store?
+
+#### Type mapping/Inference
+
+Many problems arise when implementing your data store's type conversion to interface with Feast datatypes.
+1. You will need to correctly update `inference.py` so that Feast can infer your datasource schemas
+2. You also need to update `type_map.py` so that Feast knows how to convert your datastores types to Feast-recognized types in `feast/types.py`.
+
+#### Historical and online retrieval
+
+The most important functionality in Feast is historical and online retrieval. Most of the e2e and universal integration test test this functionality in some way. Making sure this functionality works also indirectly asserts that reading and writing from your datastore works as intended.
+
+
+### To include a new offline / online store in the main Feast repo
+
+* Extend `data_source_creator.py` for your offline store.
+* In `repo_configuration.py` add a new `IntegrationTestRepoConfig` or two (depending on how many online stores you want to test).
+ * Generally, you should only need to test against sqlite. However, if you need to test against a production online store, then you can also test against Redis or dynamodb.
+* Run the full test suite with `make test-python-integration.`
+
+### Including a new offline / online store in the main Feast repo from external plugins with community maintainers.
+
+* This folder is for plugins that are officially maintained with community owners. Place the APIs in `feast/infra/offline_stores/contrib/`.
+* Extend `data_source_creator.py` for your offline store and implement the required APIs.
+* In `contrib_repo_configuration.py` add a new `IntegrationTestRepoConfig` (depending on how many online stores you want to test).
+* Run the test suite on the contrib test suite with `make test-python-contrib-universal`.
+
+### To include a new online store
+
+* In `repo_configuration.py` add a new config that maps to a serialized version of configuration you need in `feature_store.yaml` to setup the online store.
+* In `repo_configuration.py`, add new `IntegrationTestRepoConfig` for online stores you want to test.
+* Run the full test suite with `make test-python-integration`
+
+### To use custom data in a new test
+
+* Check `test_universal_types.py` for an example of how to do this.
+
+```python
+@pytest.mark.integration
+def your_test(environment: Environment):
+ df = #...#
+ data_source = environment.data_source_creator.create_data_source(
+ df,
+ destination_name=environment.feature_store.project
+ )
+ your_fv = driver_feature_view(data_source)
+ entity = driver(value_type=ValueType.UNKNOWN)
+ fs.apply([fv, entity])
+
+ # ... run test
+```
+
+### Running your own Redis cluster for testing
+
+* Install Redis on your computer. If you are a mac user, you should be able to `brew install redis`.
+ * Running `redis-server --help` and `redis-cli --help` should show corresponding help menus.
+* * Run `./infra/scripts/redis-cluster.sh start` then `./infra/scripts/redis-cluster.sh create` to start the Redis cluster locally. You should see output that looks like this:
+~~~~
+Starting 6001
+Starting 6002
+Starting 6003
+Starting 6004
+Starting 6005
+Starting 6006
+~~~~
+* You should be able to run the integration tests and have the Redis cluster tests pass.
+* If you would like to run your own Redis cluster, you can run the above commands with your own specified ports and connect to the newly configured cluster.
+* To stop the cluster, run `./infra/scripts/redis-cluster.sh stop` and then `./infra/scripts/redis-cluster.sh clean`.
diff --git a/sdk/python/tests/integration/feature_repos/repo_configuration.py b/sdk/python/tests/integration/feature_repos/repo_configuration.py
index 776fff3bb9..52675242b7 100644
--- a/sdk/python/tests/integration/feature_repos/repo_configuration.py
+++ b/sdk/python/tests/integration/feature_repos/repo_configuration.py
@@ -75,11 +75,11 @@
SNOWFLAKE_CONFIG = {
"type": "snowflake.online",
- "account": os.environ.get("SNOWFLAKE_CI_DEPLOYMENT", ""),
- "user": os.environ.get("SNOWFLAKE_CI_USER", ""),
- "password": os.environ.get("SNOWFLAKE_CI_PASSWORD", ""),
- "role": os.environ.get("SNOWFLAKE_CI_ROLE", ""),
- "warehouse": os.environ.get("SNOWFLAKE_CI_WAREHOUSE", ""),
+ "account": os.getenv("SNOWFLAKE_CI_DEPLOYMENT", ""),
+ "user": os.getenv("SNOWFLAKE_CI_USER", ""),
+ "password": os.getenv("SNOWFLAKE_CI_PASSWORD", ""),
+ "role": os.getenv("SNOWFLAKE_CI_ROLE", ""),
+ "warehouse": os.getenv("SNOWFLAKE_CI_WAREHOUSE", ""),
"database": "FEAST",
"schema": "ONLINE",
}
diff --git a/sdk/python/tests/integration/feature_repos/universal/data_sources/snowflake.py b/sdk/python/tests/integration/feature_repos/universal/data_sources/snowflake.py
index ae83ea8eb0..b5fc2448d4 100644
--- a/sdk/python/tests/integration/feature_repos/universal/data_sources/snowflake.py
+++ b/sdk/python/tests/integration/feature_repos/universal/data_sources/snowflake.py
@@ -34,8 +34,10 @@ def __init__(self, project_name: str, *args, **kwargs):
warehouse=os.environ["SNOWFLAKE_CI_WAREHOUSE"],
database="FEAST",
schema="OFFLINE",
- storage_integration_name="FEAST_S3",
- blob_export_location="s3://feast-snowflake-offload/export",
+ storage_integration_name=os.getenv("BLOB_EXPORT_STORAGE_NAME", "FEAST_S3"),
+ blob_export_location=os.getenv(
+ "BLOB_EXPORT_URI", "s3://feast-snowflake-offload/export"
+ ),
)
def create_data_source(
diff --git a/sdk/python/tests/integration/registration/test_feature_store.py b/sdk/python/tests/integration/registration/test_feature_store.py
index 7b95afadba..87268d5d00 100644
--- a/sdk/python/tests/integration/registration/test_feature_store.py
+++ b/sdk/python/tests/integration/registration/test_feature_store.py
@@ -30,11 +30,7 @@
from feast.infra.online_stores.sqlite import SqliteOnlineStoreConfig
from feast.repo_config import RepoConfig
from feast.types import Array, Bytes, Float64, Int64, String
-from tests.utils.data_source_test_creator import (
- prep_file_source,
- simple_bq_source_using_query_arg,
- simple_bq_source_using_table_arg,
-)
+from tests.utils.data_source_test_creator import prep_file_source
@pytest.mark.integration
@@ -96,50 +92,19 @@ def test_feature_view_inference_success(test_feature_store, dataframe_source):
tags={},
)
- fv2 = FeatureView(
- name="fv2",
- entities=[entity],
- ttl=timedelta(minutes=5),
- online=True,
- batch_source=simple_bq_source_using_table_arg(dataframe_source, "ts_1"),
- tags={},
- )
-
- fv3 = FeatureView(
- name="fv3",
- entities=[entity],
- ttl=timedelta(minutes=5),
- online=True,
- batch_source=simple_bq_source_using_query_arg(dataframe_source, "ts_1"),
- tags={},
- )
-
- test_feature_store.apply([entity, fv1, fv2, fv3]) # Register Feature Views
+ test_feature_store.apply([entity, fv1]) # Register Feature Views
feature_view_1 = test_feature_store.list_feature_views()[0]
- feature_view_2 = test_feature_store.list_feature_views()[1]
- feature_view_3 = test_feature_store.list_feature_views()[2]
actual_file_source = {
(feature.name, feature.dtype) for feature in feature_view_1.features
}
- actual_bq_using_table_arg_source = {
- (feature.name, feature.dtype) for feature in feature_view_2.features
- }
- actual_bq_using_query_arg_source = {
- (feature.name, feature.dtype) for feature in feature_view_3.features
- }
expected = {
("float_col", Float64),
("int64_col", Int64),
("string_col", String),
}
- assert (
- expected
- == actual_file_source
- == actual_bq_using_table_arg_source
- == actual_bq_using_query_arg_source
- )
+ assert expected == actual_file_source
test_feature_store.teardown()
diff --git a/sdk/python/tests/integration/registration/test_inference.py b/sdk/python/tests/integration/registration/test_inference.py
index de02fe53fe..08ebce95e6 100644
--- a/sdk/python/tests/integration/registration/test_inference.py
+++ b/sdk/python/tests/integration/registration/test_inference.py
@@ -5,11 +5,7 @@
from feast import RepoConfig
from feast.errors import RegistryInferenceFailure
from feast.inference import update_data_sources_with_inferred_event_timestamp_col
-from tests.utils.data_source_test_creator import (
- prep_file_source,
- simple_bq_source_using_query_arg,
- simple_bq_source_using_table_arg,
-)
+from tests.utils.data_source_test_creator import prep_file_source
@pytest.mark.integration
@@ -20,8 +16,6 @@ def test_update_file_data_source_with_inferred_event_timestamp_col(simple_datase
with prep_file_source(df=simple_dataset_1) as file_source:
data_sources = [
file_source,
- simple_bq_source_using_table_arg(simple_dataset_1),
- simple_bq_source_using_query_arg(simple_dataset_1),
]
update_data_sources_with_inferred_event_timestamp_col(
data_sources, RepoConfig(provider="local", project="test")
@@ -30,7 +24,7 @@ def test_update_file_data_source_with_inferred_event_timestamp_col(simple_datase
source.timestamp_field for source in data_sources
]
- assert actual_event_timestamp_cols == ["ts_1", "ts_1", "ts_1"]
+ assert actual_event_timestamp_cols == ["ts_1"]
with prep_file_source(df=df_with_two_viable_timestamp_cols) as file_source:
with pytest.raises(RegistryInferenceFailure):
diff --git a/sdk/python/tests/integration/registration/test_sql_registry.py b/sdk/python/tests/unit/test_sql_registry.py
similarity index 93%
rename from sdk/python/tests/integration/registration/test_sql_registry.py
rename to sdk/python/tests/unit/test_sql_registry.py
index 286b1abd21..59c20ea9f6 100644
--- a/sdk/python/tests/integration/registration/test_sql_registry.py
+++ b/sdk/python/tests/unit/test_sql_registry.py
@@ -108,13 +108,27 @@ def mysql_registry():
container.stop()
+@pytest.fixture(scope="session")
+def sqlite_registry():
+ registry_config = RegistryConfig(
+ registry_type="sql",
+ path="sqlite://",
+ )
+
+ yield SqlRegistry(registry_config, None)
+
+
@pytest.mark.skipif(
sys.platform == "darwin" and "GITHUB_REF" in os.environ,
reason="does not run on mac github actions",
)
@pytest.mark.parametrize(
"sql_registry",
- [lazy_fixture("mysql_registry"), lazy_fixture("pg_registry")],
+ [
+ lazy_fixture("mysql_registry"),
+ lazy_fixture("pg_registry"),
+ lazy_fixture("sqlite_registry"),
+ ],
)
def test_apply_entity_success(sql_registry):
entity = Entity(
@@ -153,6 +167,9 @@ def test_apply_entity_success(sql_registry):
and entity.tags["team"] == "matchmaking"
)
+ # After the first apply, the created_timestamp should be the same as the last_update_timestamp.
+ assert entity.created_timestamp == entity.last_updated_timestamp
+
sql_registry.delete_entity("driver_car_id", project)
assert_project_uuid(project, project_uuid, sql_registry)
entities = sql_registry.list_entities(project)
@@ -174,7 +191,11 @@ def assert_project_uuid(project, project_uuid, sql_registry):
)
@pytest.mark.parametrize(
"sql_registry",
- [lazy_fixture("mysql_registry"), lazy_fixture("pg_registry")],
+ [
+ lazy_fixture("mysql_registry"),
+ lazy_fixture("pg_registry"),
+ lazy_fixture("sqlite_registry"),
+ ],
)
def test_apply_feature_view_success(sql_registry):
# Create Feature Views
@@ -237,6 +258,9 @@ def test_apply_feature_view_success(sql_registry):
and feature_view.entities[0] == "fs1_my_entity_1"
)
+ # After the first apply, the created_timestamp should be the same as the last_update_timestamp.
+ assert feature_view.created_timestamp == feature_view.last_updated_timestamp
+
sql_registry.delete_feature_view("my_feature_view_1", project)
feature_views = sql_registry.list_feature_views(project)
assert len(feature_views) == 0
@@ -250,7 +274,11 @@ def test_apply_feature_view_success(sql_registry):
)
@pytest.mark.parametrize(
"sql_registry",
- [lazy_fixture("mysql_registry"), lazy_fixture("pg_registry")],
+ [
+ lazy_fixture("mysql_registry"),
+ lazy_fixture("pg_registry"),
+ lazy_fixture("sqlite_registry"),
+ ],
)
def test_apply_on_demand_feature_view_success(sql_registry):
# Create Feature Views
@@ -334,7 +362,11 @@ def location_features_from_push(inputs: pd.DataFrame) -> pd.DataFrame:
)
@pytest.mark.parametrize(
"sql_registry",
- [lazy_fixture("mysql_registry"), lazy_fixture("pg_registry")],
+ [
+ lazy_fixture("mysql_registry"),
+ lazy_fixture("pg_registry"),
+ lazy_fixture("sqlite_registry"),
+ ],
)
@pytest.mark.parametrize(
"request_source_schema",
@@ -459,10 +491,13 @@ def odfv1(feature_df: pd.DataFrame) -> pd.DataFrame:
sys.platform == "darwin" and "GITHUB_REF" in os.environ,
reason="does not run on mac github actions",
)
-@pytest.mark.integration
@pytest.mark.parametrize(
"sql_registry",
- [lazy_fixture("mysql_registry"), lazy_fixture("pg_registry")],
+ [
+ lazy_fixture("mysql_registry"),
+ lazy_fixture("pg_registry"),
+ lazy_fixture("sqlite_registry"),
+ ],
)
def test_apply_data_source(sql_registry):
# Create Feature Views
diff --git a/sdk/python/tests/utils/e2e_test_validation.py b/sdk/python/tests/utils/e2e_test_validation.py
index b2eb78f3c8..f12b15d00d 100644
--- a/sdk/python/tests/utils/e2e_test_validation.py
+++ b/sdk/python/tests/utils/e2e_test_validation.py
@@ -199,7 +199,8 @@ def make_feature_store_yaml(project, test_repo_config, repo_dir_name: Path):
),
]
-if os.getenv("FEAST_IS_LOCAL_TEST", "False") == "True":
+# Only test if this is NOT a local test
+if os.getenv("FEAST_IS_LOCAL_TEST", "False") != "True":
NULLABLE_ONLINE_STORE_CONFIGS.extend(
[
IntegrationTestRepoConfig(
diff --git a/ui/.npmrc b/ui/.npmrc
new file mode 100644
index 0000000000..bd3327ab5a
--- /dev/null
+++ b/ui/.npmrc
@@ -0,0 +1 @@
+//registry.npmjs.org/:_authToken=${NPM_TOKEN}
\ No newline at end of file
diff --git a/ui/CONTRIBUTING.md b/ui/CONTRIBUTING.md
new file mode 100644
index 0000000000..970bd3676c
--- /dev/null
+++ b/ui/CONTRIBUTING.md
@@ -0,0 +1,103 @@
+Table of contents
+
+- [General contributor notes](#general-contributor-notes)
+ - [`feast ui` command](#feast-ui-command)
+ - [NPM package project structure](#npm-package-project-structure)
+ - [Tests](#tests)
+ - [Yarn commands](#yarn-commands)
+ - [`yarn install`](#yarn-install)
+ - [`yarn start`](#yarn-start)
+ - [`yarn test`](#yarn-test)
+- [Release process](#release-process)
+ - [(Advanced) Manually publishing the Feast Package to NPM](#advanced-manually-publishing-the-feast-package-to-npm)
+ - [Requirements](#requirements)
+ - [Steps for Publishing](#steps-for-publishing)
+
+# General contributor notes
+In this doc, we describe how to contribute both to the Feast Web UI NPM package as well as the embedded Feast UI in the Python SDK (i.e. what's run when you run `feast ui`)
+
+## `feast ui` command
+You can see the logic in [../sdk/python/feast/ui](../sdk/python/feast/ui/). This instance is loaded in [../sdk/python/feast/ui_server.py](../sdk/python/feast/ui_server.py).
+
+Under the hood, what happens is that the Feast SDK spins up a server which exposes an endpoint to the registry. It then mounts the UI on the server and points it to fetch data from that registry.
+
+## NPM package project structure
+The Web UI is powered by a JSON registry dump from Feast (running `feast registry-dump`). Running `yarn start` launches a UI
+powered by test data.
+- `public/` contains assets as well as demo data loaded by the Web UI.
+ - There is a `projects-list.json` which represents all Feast projects the UI shows.
+ - There is also a `registry.json` which is the registry dump for the feature repo.
+- `feature_repo/` contains a sample Feast repo which generates the `registry.json`
+- `src/` contains the Web UI source code.
+ - `src/contexts` has React context objects around project level metadata or registry path metadata to inject into pages. The contexts are static contexts provided by [FeastUISansProviders.tsx](src/FeastUISansProviders.tsx)
+ - `src/parsers` parses the `registry.json` into in memory representations of Feast objects (feature views, data sources, entities, feature services).
+ - This has ~1:1 mappings to the protobuf objects in [feast/protos/feast/core](https://github.com/feast-dev/feast/tree/master/protos/feast/core).
+ - There are also "relationships" which create an in-memory lineage graph which can be used to construct links in pages.
+ - This generates state which pages will load via React queries (to the registry path).
+ - `src/pages` has all individual web pages and their layouts. For any given Feast object (e.g. entity), there exist:
+ - an **Index page** (which is the first page you hit when you click on that object). This loads using a React query the in memory representation of all objects (parsed from `src/parsers`) and embeds:
+ - a **Listing page** (i.e. listing all the objects in the registry in a table). This creates links to the instance pages
+ - an **Instance page** (which shows details for an individual entity, feature view, etc). This embeds:
+ - a default Overview tab, which shows all the Feast metadata (e.g. for a given entity)
+ - custom tabs from `src/custom-tabs`.
+ - Other subdirectories:
+ - `src/components` has common React components that are re-used across pages
+ - `src/custom-tabs` houses custom tabs and a custom tab React context which exist on the core pages. There is a `TabsRegistryContext` which is also supplied by the [FeastUISansProviders.tsx](src/FeastUISansProviders.tsx), and if there are custom tabs, the Feast UI will embed them as a new tab in the corresponding page (e.g. feature view page).
+ - `src/graphics` houses icons that are used throughout the UI
+ - `src/hooks` has React hooks. The most complex hooks here define the bulk of the search / filter functionality.
+
+## Tests
+There are very few tests for this UI. There is a smoke test that ensures pages can load in [FeastUISansProviders.test.tsx](src/FeastUISansProviders.test.tsx)
+
+
+## Yarn commands
+
+If you would like to simply try things out and see how the UI works, you can simply run the code in this repo.
+
+> **Note**: there is an `.npmrc` which is setup for automatic releases. You'll need to comment out the line in there and continue
+
+First:
+
+### `yarn install`
+
+That will install the all the dependencies that the UI needs, as well as development dependencies. Then in the project directory, you can run:
+
+### `yarn start`
+
+Runs the app in the development mode.\
+Open [http://localhost:3000](http://localhost:3000) to view it in the browser.
+
+The page will reload if you make edits.\
+You will also see any lint errors in the console.
+
+### `yarn test`
+
+Launches the test runner in the interactive watch mode.\
+See the section about [running tests](https://facebook.github.io/create-react-app/docs/running-tests) for more information.
+
+# Release process
+There are a couple of components in Feast that are tied to the Web UI. These are all automatically handled during the release GitHub action:
+1. the npm package
+ - The release process for Feast automatically bumps the package version (see [bump_file_versions.py](../infra/scripts/release/bump_file_versions.py)) and releases the new NPM package (see [publish.yml](../.github/workflows/publish.yml) in the `publish-web-ui-npm` job)
+2. the Feast Python SDK, which bundles in a compiled version of the Feast Web UI which is run on a `feast ui` CLI command.
+ - The bundled Web UI in the Python SDK always compiles in the latest npm version
+
+## (Advanced) Manually publishing the Feast Package to NPM
+
+This generally should not be necessary, since new package versions are released with the overall Feast release workflow (see [publish.yml](../.github/workflows/publish.yml) in the `publish-web-ui-npm` job)
+
+The Feast UI is published as a module to NPM and can be found here: https://www.npmjs.com/package/@feast-dev/feast-ui
+
+### Requirements
+
+To publish a new version of the module, you will need:
+- to be part of the @feast-dev team in NPM. Ask `#feast-development` on http://slack.feast.dev to add you if necessary.
+- to [login to your NPM account on the command line](https://docs.npmjs.com/cli/v8/commands/npm-adduser).
+
+### Steps for Publishing
+
+1. Make sure tests are passing. Run tests with `yarn test` in the ui directory.
+2. Bump the version number in `package.json` as appropriate.
+3. Package the modules for distributions. Run the library build script with `yarn build:lib`. We use [Rollup](https://rollupjs.org/) for building the module, and the configs are in the `rollup.config.js` file.
+4. Publish the package to NPM. Run `npm publish`
+5. [Check NPM to see that the package was properly published](https://www.npmjs.com/package/@feast-dev/feast-ui).
\ No newline at end of file
diff --git a/ui/PUBLISHING_TO_NPM.md b/ui/PUBLISHING_TO_NPM.md
deleted file mode 100644
index 0ab1af3923..0000000000
--- a/ui/PUBLISHING_TO_NPM.md
+++ /dev/null
@@ -1,13 +0,0 @@
-# Publishing the Feast Package to NPM
-
-The Feast UI is published as a module to NPM and can be found here: https://www.npmjs.com/package/@feast-dev/feast-ui
-
-To publish a new version of the module, you will need to be part of the @feast-dev team in NPM. Ask Tony to add you if necessary. You will also need to [login to your NPM account on the command line](https://docs.npmjs.com/cli/v8/commands/npm-adduser).
-
-## Steps for Publishing
-
-1. Make sure tests are passing. Run tests with `yarn test` in the ui directory.
-2. Bump the version number in `package.json` as appropriate.
-3. Package the modules for distributions. Run the library build script with `yarn build:lib`. We use [Rollup](https://rollupjs.org/) for building the module, and the configs are in the `rollup.config.js` file.
-4. Publish the package to NPM. Run `npm publish`
-5. [Check NPM to see that the package was properly publish](https://www.npmjs.com/package/@feast-dev/feast-ui).
diff --git a/ui/README.md b/ui/README.md
index 713d1c17c5..e91a8741ec 100644
--- a/ui/README.md
+++ b/ui/README.md
@@ -4,22 +4,10 @@
This project was bootstrapped with [Create React App](https://github.com/facebook/create-react-app).
-## Project structure
-The Web UI is powered by a JSON registry dump from Feast (running `feast registry-dump`). Running `yarn start` launches a UI
-powered by test data.
-- `public/` contains assets as well as demo data loaded by the Web UI.
- - There is a `projects-list.json` which represents all Feast projects the UI shows.
- - There is also a `registry.json` which is the registry dump for the feature repo.
-- `feature_repo/` contains a sample Feast repo which generates the `registry.json`
-- `src/` contains the Web UI source code. This parses the registry json blob in `src/parsers` to make this data
-available for the rest of the UI.
-- `src/custom-tabs` includes sample custom tabs. This is a WIP plugin system where users can inject their own tabs and
-data to the UI.
-
## Usage
There are three modes of usage:
-- via the 'feast ui' CLI to view the current feature repository
+- via the `feast ui` CLI to view the current feature repository
- importing the UI as a module
- running the entire build as a React app.
@@ -124,28 +112,6 @@ const tabsRegistry = {
Examples of custom tabs can be found in the `/custom-tabs` folder.
-### Alternative: Run this Repo
-
-If you would like to simply try things out and see how the UI works, you can simply run the code in this repo. First:
-
-### `yarn install`
-
-That will install the all the dependencies that the UI needs, as well as development dependencies. Then in the project directory, you can run:
-
-### `yarn start`
-
-Runs the app in the development mode.\
-Open [http://localhost:3000](http://localhost:3000) to view it in the browser.
-
-The page will reload if you make edits.\
-You will also see any lint errors in the console.
-
-### `yarn test`
-
-Launches the test runner in the interactive watch mode.\
-See the section about [running tests](https://facebook.github.io/create-react-app/docs/running-tests) for more information.
-
-
## On React and Create React App
This project was bootstrapped with Create React App, and uses its scripts to simplify UI development. You can learn more in the [Create React App documentation](https://facebook.github.io/create-react-app/docs/getting-started).
diff --git a/ui/feature_repo/feature_store.yaml b/ui/feature_repo/feature_store.yaml
index 31b27e2385..6ecad3eb51 100644
--- a/ui/feature_repo/feature_store.yaml
+++ b/ui/feature_repo/feature_store.yaml
@@ -5,6 +5,3 @@ online_store:
type: sqlite
offline_store:
type: file
-flags:
- alpha_features: true
- on_demand_transforms: true
diff --git a/ui/package.json b/ui/package.json
index 22128cc968..21bdc65674 100644
--- a/ui/package.json
+++ b/ui/package.json
@@ -1,6 +1,6 @@
{
"name": "@feast-dev/feast-ui",
- "version": "0.20.5",
+ "version": "0.23.1",
"private": false,
"files": [
"dist"
diff --git a/ui/src/custom-tabs/data-tab/DataQuery.tsx b/ui/src/custom-tabs/data-tab/DataQuery.tsx
index f101c122e4..c79764ef99 100644
--- a/ui/src/custom-tabs/data-tab/DataQuery.tsx
+++ b/ui/src/custom-tabs/data-tab/DataQuery.tsx
@@ -1,9 +1,5 @@
import { useQuery } from "react-query";
-interface DataQueryInterface {
- featureView: string | undefined;
-}
-
const DataQuery = (featureView: string) => {
const queryKey = `data-tab-namespace:${featureView}`;
@@ -13,8 +9,7 @@ const DataQuery = (featureView: string) => {
// Customizing the URL based on your needs
const url = `/demo-custom-tabs/demo.json`;
- return fetch(url)
- .then((res) => res.json())
+ return fetch(url).then((res) => res.json());
},
{
enabled: !!featureView, // Only start the query when the variable is not undefined
diff --git a/ui/src/custom-tabs/data-tab/DataTab.tsx b/ui/src/custom-tabs/data-tab/DataTab.tsx
index 144083420a..4592d197e2 100644
--- a/ui/src/custom-tabs/data-tab/DataTab.tsx
+++ b/ui/src/custom-tabs/data-tab/DataTab.tsx
@@ -14,7 +14,6 @@ import {
EuiTableRow,
EuiTableRowCell,
} from "@elastic/eui";
-import useLoadRegularFeatureView from "../../pages/feature-views/useLoadFeatureView";
import DataQuery from "./DataQuery";
const FeatureViewDataRow = z.object({
@@ -26,29 +25,25 @@ type FeatureViewDataRowType = z.infer;
const LineHeightProp: React.CSSProperties = {
lineHeight: 1,
-}
+};
-const EuiFeatureViewDataRow = ({name, value}: FeatureViewDataRowType) => {
+const EuiFeatureViewDataRow = ({ name, value }: FeatureViewDataRowType) => {
return (
-
- {name}
-
+ {name}
-
- {value}
-
+ {value}
);
-}
+};
const FeatureViewDataTable = (data: any) => {
var items: FeatureViewDataRowType[] = [];
- for (let element in data.data){
+ for (let element in data.data) {
const row: FeatureViewDataRowType = {
name: element,
value: JSON.stringify(data.data[element], null, 2),
@@ -60,48 +55,44 @@ const FeatureViewDataTable = (data: any) => {
return (
-
- Data Item Name
-
-
- Data Item Value
-
+ Data Item Name
+ Data Item Value
{items.map((item) => {
- return
+ return ;
})}
- )
-}
+ );
+};
const DataTab = () => {
- const fName = "credit_history"
+ const fName = "credit_history";
const { isLoading, isError, isSuccess, data } = DataQuery(fName);
const isEmpty = data === undefined;
return (
- {isLoading && (
-
- Loading
-
- )}
- {isEmpty && No feature view with name: {fName}
}
- {isError && Error loading feature view: {fName}
}
- {isSuccess && data && (
-
-
-
-
-
- Properties
-
-
-
-
-
-
-
+ {isLoading && (
+
+ Loading
+
+ )}
+ {isEmpty && No feature view with name: {fName}
}
+ {isError && Error loading feature view: {fName}
}
+ {isSuccess && data && (
+
+
+
+
+
+ Properties
+
+
+
+
+
+
+
)}
);
diff --git a/ui/src/pages/feature-services/FeatureServiceListingTable.tsx b/ui/src/pages/feature-services/FeatureServiceListingTable.tsx
index c6205b020a..b865da6e23 100644
--- a/ui/src/pages/feature-services/FeatureServiceListingTable.tsx
+++ b/ui/src/pages/feature-services/FeatureServiceListingTable.tsx
@@ -53,10 +53,10 @@ const FeatureServiceListingTable = ({
},
},
{
- name: "Created at",
- field: "meta.createdTimestamp",
+ name: "Last updated",
+ field: "meta.lastUpdatedTimestamp",
render: (date: Date) => {
- return date.toLocaleDateString("en-CA");
+ return date ? date.toLocaleDateString("en-CA") : "n/a";
},
},
];
diff --git a/ui/src/pages/feature-services/FeatureServiceOverviewTab.tsx b/ui/src/pages/feature-services/FeatureServiceOverviewTab.tsx
index a3fc897325..ea62b3b3a7 100644
--- a/ui/src/pages/feature-services/FeatureServiceOverviewTab.tsx
+++ b/ui/src/pages/feature-services/FeatureServiceOverviewTab.tsx
@@ -66,14 +66,20 @@ const FeatureServiceOverviewTab = () => {
description="Feature Views"
/>
-
-
-
+ {data.meta.lastUpdatedTimestamp ? (
+
+
+
+ ) : (
+
+ No last updated timestamp specified on this feature service.
+
+ )}
diff --git a/ui/src/parsers/feastFeatureServices.ts b/ui/src/parsers/feastFeatureServices.ts
index 96c03e38ef..6812b7e02c 100644
--- a/ui/src/parsers/feastFeatureServices.ts
+++ b/ui/src/parsers/feastFeatureServices.ts
@@ -19,7 +19,8 @@ const FeastFeatureServiceSchema = z.object({
description: z.string().optional(),
}),
meta: z.object({
- createdTimestamp: z.string().transform((val) => new Date(val)),
+ createdTimestamp: z.string().transform((val) => new Date(val)).optional(),
+ lastUpdatedTimestamp: z.string().transform((val) => new Date(val)).optional(),
}),
});
diff --git a/ui/src/parsers/feastODFVS.ts b/ui/src/parsers/feastODFVS.ts
index 8341438d50..4d09cc72df 100644
--- a/ui/src/parsers/feastODFVS.ts
+++ b/ui/src/parsers/feastODFVS.ts
@@ -1,6 +1,5 @@
import { z } from "zod";
import { FeastFeatureColumnSchema } from "./feastFeatureViews";
-import { FEAST_FEATURE_VALUE_TYPES } from "./types";
const FeatureViewProjectionSchema = z.object({
featureViewProjection: z.object({