diff --git a/.github/auto_assign-issues.yml b/.github/auto_assign-issues.yml deleted file mode 100644 index cf2c7221317..00000000000 --- a/.github/auto_assign-issues.yml +++ /dev/null @@ -1,10 +0,0 @@ -# If enabled, auto-assigns users when a new issue is created -# Defaults to true, allows you to install the app globally, and disable on a per-repo basis -addAssignees: true - -# The list of users to assign to new issues. -# If empty or not provided, the repository owner is assigned -assignees: - - jklegar - - woop - - tsotnet diff --git a/.github/auto_assign.yml b/.github/auto_assign.yml new file mode 100644 index 00000000000..18151f74547 --- /dev/null +++ b/.github/auto_assign.yml @@ -0,0 +1,19 @@ +# Set to true to add reviewers to pull requests +addReviewers: false + +# Set to true to add assignees to pull requests +addAssignees: true + +# A list of assignees, overrides reviewers if set +assignees: + - woop + - tsotnet + - achals + - adchia + - felixwang9817 + +# A number of assignees to add to the pull request +# Set to 0 to add all of the assignees. +# Uses numberOfReviewers if unset. +numberOfAssignees: 1 + diff --git a/.github/workflows/assign_issues.yml b/.github/workflows/assign_issues.yml new file mode 100644 index 00000000000..f8702e2485c --- /dev/null +++ b/.github/workflows/assign_issues.yml @@ -0,0 +1,15 @@ +name: Issue assignment + +on: + issues: + types: [opened] + +jobs: + auto-assign: + runs-on: ubuntu-latest + steps: + - name: 'Auto-assign issue' + uses: pozil/auto-assign-issue@v1.1.0 + with: + assignees: achals, woop, tsotnet, felix, felixwang9817, adchia + numOfAssignee: 1 diff --git a/.github/workflows/assign_reviewers.yml b/.github/workflows/assign_reviewers.yml new file mode 100644 index 00000000000..ab776add0dc --- /dev/null +++ b/.github/workflows/assign_reviewers.yml @@ -0,0 +1,12 @@ +name: 'Assign Reviewers' +on: + pull_request_target: + types: + - opened + - ready_for_review + +jobs: + add-reviews: + runs-on: ubuntu-latest + steps: + - uses: kentaro-m/auto-assign-action@v1.2.0 diff --git a/.github/workflows/build_feature_transformation_server.yml b/.github/workflows/build_feature_transformation_server.yml new file mode 100644 index 00000000000..ff5e0aada7b --- /dev/null +++ b/.github/workflows/build_feature_transformation_server.yml @@ -0,0 +1,28 @@ +name: build_feature_transformation_server + +on: + push: + branches: + - master + +jobs: + build-docker-image: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Set up QEMU + uses: docker/setup-qemu-action@v1 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + - name: Login to DockerHub + uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + - name: Build and push + run: | + docker build \ + --file sdk/python/feast/infra/transformation_servers/Dockerfile \ + --tag feastdev/feature-transformation-server:`git rev-parse HEAD` \ + . + docker push feastdev/feature-transformation-server:`git rev-parse HEAD` diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml index 19bc08e518a..64a32bbed76 100644 --- a/.github/workflows/integration_tests.yml +++ b/.github/workflows/integration_tests.yml @@ -50,6 +50,9 @@ jobs: aws-region: us-west-2 - name: Use AWS CLI run: aws sts get-caller-identity + - name: Upgrade pip version + run: | + pip install --upgrade "pip>=21.3.1" - name: Get pip cache dir id: pip-cache run: | diff --git a/.github/workflows/linter.yml b/.github/workflows/linter.yml index aa39225be6c..e2763bf2ae1 100644 --- a/.github/workflows/linter.yml +++ b/.github/workflows/linter.yml @@ -13,6 +13,9 @@ jobs: with: python-version: "3.7" architecture: x64 + - name: Upgrade pip version + run: | + pip install --upgrade "pip>=21.3.1" - name: Get pip cache dir id: pip-cache run: | diff --git a/.github/workflows/pr_integration_tests.yml b/.github/workflows/pr_integration_tests.yml index 16e23d31bb9..816be27cbf5 100644 --- a/.github/workflows/pr_integration_tests.yml +++ b/.github/workflows/pr_integration_tests.yml @@ -61,6 +61,9 @@ jobs: aws-region: us-west-2 - name: Use AWS CLI run: aws sts get-caller-identity + - name: Upgrade pip version + run: | + pip install --upgrade "pip>=21.3.1" - name: Get pip cache dir id: pip-cache run: | diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index eea9bc23679..dd01a63a5e1 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -20,6 +20,9 @@ jobs: with: python-version: ${{ matrix.python-version }} architecture: x64 + - name: Upgrade pip version + run: | + pip install --upgrade "pip>=21.3.1" - name: Get pip cache dir id: pip-cache run: | diff --git a/.prow.yaml b/.prow.yaml index e614e4a2f96..f541f81e0e7 100644 --- a/.prow.yaml +++ b/.prow.yaml @@ -64,26 +64,6 @@ presubmits: branches: - ^v0\.(3|4)-branch$ -- name: test-usage - decorate: true - run_if_changed: "sdk/python/.*" - spec: - containers: - - image: python:3.7 - command: ["infra/scripts/test-usage.sh"] - env: - - name: GOOGLE_APPLICATION_CREDENTIALS - value: /etc/gcloud/service-account.json - volumeMounts: - - mountPath: /etc/gcloud/service-account.json - name: service-account - readOnly: true - subPath: service-account.json - volumes: - - name: service-account - secret: - secretName: feast-service-account - - name: test-golang-sdk decorate: true spec: diff --git a/CHANGELOG.md b/CHANGELOG.md index 9ebb7e1e107..6d053cc6a07 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,74 @@ # Changelog +## [v0.15.0](https://github.com/feast-dev/feast/tree/v0.15.0) (2021-11-08) + +[Full Changelog](https://github.com/feast-dev/feast/compare/v0.14.1...v0.15.0) + +**Implemented enhancements:** + +- Adding stream ingestion alpha documentation [\#2005](https://github.com/feast-dev/feast/pull/2005) ([adchia](https://github.com/adchia)) +- Feature transformation server docker image [\#1972](https://github.com/feast-dev/feast/pull/1972) ([felixwang9817](https://github.com/felixwang9817)) +- eventtime check before writing features, use pipelines, ttl [\#1961](https://github.com/feast-dev/feast/pull/1961) ([vas28r13](https://github.com/vas28r13)) +- Plugin repo universal tests [\#1946](https://github.com/feast-dev/feast/pull/1946) ([felixwang9817](https://github.com/felixwang9817)) +- direct data ingestion into Online store [\#1939](https://github.com/feast-dev/feast/pull/1939) ([vas28r13](https://github.com/vas28r13)) +- Add an interface for TransformationService and a basic implementation [\#1932](https://github.com/feast-dev/feast/pull/1932) ([achals](https://github.com/achals)) +- Allows registering of features in request data as RequestFeatureView. Refactors common logic into a BaseFeatureView class [\#1931](https://github.com/feast-dev/feast/pull/1931) ([adchia](https://github.com/adchia)) +- Add final\_output\_feature\_names in Query context to avoid SELECT \* EXCEPT [\#1911](https://github.com/feast-dev/feast/pull/1911) ([MattDelac](https://github.com/MattDelac)) +- Add Dockerfile for GCP CloudRun FeatureServer [\#1887](https://github.com/feast-dev/feast/pull/1887) ([judahrand](https://github.com/judahrand)) + +**Fixed bugs:** + +- feast=0.14.0 `query_generator()` unecessary used twice [\#1978](https://github.com/feast-dev/feast/issues/1978) +- get\_online\_features on demand transform bug fixes + local integration test mode [\#2004](https://github.com/feast-dev/feast/pull/2004) ([adchia](https://github.com/adchia)) +- Always pass full and partial feature names to ODFV [\#2003](https://github.com/feast-dev/feast/pull/2003) ([judahrand](https://github.com/judahrand)) +- Update bq\_to\_feast\_value\_type with BOOLEAN type as a legacy sql data type [\#1996](https://github.com/feast-dev/feast/pull/1996) ([mavysavydav](https://github.com/mavysavydav)) +- Fix bug where using some Pandas dtypes in the output of an ODFV fails [\#1994](https://github.com/feast-dev/feast/pull/1994) ([judahrand](https://github.com/judahrand)) +- Fix duplicate update infra [\#1990](https://github.com/feast-dev/feast/pull/1990) ([felixwang9817](https://github.com/felixwang9817)) +- Improve performance of \_convert\_arrow\_to\_proto [\#1984](https://github.com/feast-dev/feast/pull/1984) ([nossrannug](https://github.com/nossrannug)) +- Fix duplicate upload entity [\#1981](https://github.com/feast-dev/feast/pull/1981) ([achals](https://github.com/achals)) +- fix redis cluster materialization [\#1968](https://github.com/feast-dev/feast/pull/1968) ([qooba](https://github.com/qooba)) +- Allow plugin repos to actually overwrite repo configs [\#1966](https://github.com/feast-dev/feast/pull/1966) ([felixwang9817](https://github.com/felixwang9817)) +- Delete keys from Redis when tearing down online store [\#1965](https://github.com/feast-dev/feast/pull/1965) ([achals](https://github.com/achals)) +- Fix issues with lint test and upgrade pip version [\#1964](https://github.com/feast-dev/feast/pull/1964) ([felixwang9817](https://github.com/felixwang9817)) +- Move IntegrationTestRepoConfig class to another module [\#1962](https://github.com/feast-dev/feast/pull/1962) ([felixwang9817](https://github.com/felixwang9817)) +- Solve package conflict in \[gcp\] and \[ci\] [\#1955](https://github.com/feast-dev/feast/pull/1955) ([ysk24ok](https://github.com/ysk24ok)) +- Remove some paths from unit test cache [\#1944](https://github.com/feast-dev/feast/pull/1944) ([achals](https://github.com/achals)) +- Fix bug in feast alpha enable CLI command [\#1940](https://github.com/feast-dev/feast/pull/1940) ([felixwang9817](https://github.com/felixwang9817)) +- Fix conditional statements for if OnDemandFVs exist [\#1937](https://github.com/feast-dev/feast/pull/1937) ([codyjlin](https://github.com/codyjlin)) +- Fix \_\_getitem\_\_ return value for feature view and on-demand feature view [\#1936](https://github.com/feast-dev/feast/pull/1936) ([mavysavydav](https://github.com/mavysavydav)) +- Corrected setup.py BigQuery version that's needed for a contributor's merged PR 1844 [\#1934](https://github.com/feast-dev/feast/pull/1934) ([mavysavydav](https://github.com/mavysavydav)) + +**Merged pull requests:** + +- Fix protobuf version conflict in \[gcp\] and \[ci\] packages [\#1992](https://github.com/feast-dev/feast/pull/1992) ([ysk24ok](https://github.com/ysk24ok)) +- Improve aws lambda deployment \(logging, idempotency, etc\) [\#1985](https://github.com/feast-dev/feast/pull/1985) ([tsotnet](https://github.com/tsotnet)) +- Extend context for usage statistics collection & add latencies for performance analysis [\#1983](https://github.com/feast-dev/feast/pull/1983) ([pyalex](https://github.com/pyalex)) +- Update CHANGELOG for Feast v0.14.1 [\#1982](https://github.com/feast-dev/feast/pull/1982) ([felixwang9817](https://github.com/felixwang9817)) +- Document AWS Lambda permissions [\#1970](https://github.com/feast-dev/feast/pull/1970) ([tsotnet](https://github.com/tsotnet)) +- Update online store helper docstring [\#1957](https://github.com/feast-dev/feast/pull/1957) ([amommendes](https://github.com/amommendes)) +- Add public docs for entity aliasing [\#1951](https://github.com/feast-dev/feast/pull/1951) ([codyjlin](https://github.com/codyjlin)) +- Updating roadmap + hero image [\#1950](https://github.com/feast-dev/feast/pull/1950) ([adchia](https://github.com/adchia)) +- Add David and Matt as approvers as well [\#1943](https://github.com/feast-dev/feast/pull/1943) ([achals](https://github.com/achals)) +- Add David and Matt as reviewers, and add actions for issue/PR assignment [\#1942](https://github.com/feast-dev/feast/pull/1942) ([achals](https://github.com/achals)) +- Simplify BigQuery load jobs [\#1935](https://github.com/feast-dev/feast/pull/1935) ([judahrand](https://github.com/judahrand)) + +## [v0.14.1](https://github.com/feast-dev/feast/tree/v0.14.1) (2021-10-28) + +[Full Changelog](https://github.com/feast-dev/feast/compare/v0.14.0...v0.14.1) + +**Fixed bugs:** + +- Fix duplicate upload entity [\#1981](https://github.com/feast-dev/feast/pull/1981) ([achals](https://github.com/achals)) +- Fix bug in feast alpha enable CLI command [\#1940](https://github.com/feast-dev/feast/pull/1940) ([felixwang9817](https://github.com/felixwang9817)) +- Fix conditional statements for if OnDemandFVs exist [\#1937](https://github.com/feast-dev/feast/pull/1937) ([codyjlin](https://github.com/codyjlin)) +- Fix \_\_getitem\_\_ return value for feature view and on-demand feature view [\#1936](https://github.com/feast-dev/feast/pull/1936) ([mavysavydav](https://github.com/mavysavydav)) +- Corrected setup.py BigQuery version that's needed for a contributor's merged PR 1844 [\#1934](https://github.com/feast-dev/feast/pull/1934) ([mavysavydav](https://github.com/mavysavydav)) + +**Merged pull requests:** + +- Updating roadmap + hero image [\#1950](https://github.com/feast-dev/feast/pull/1950) ([adchia](https://github.com/adchia)) +- Simplify BigQuery load jobs [\#1935](https://github.com/feast-dev/feast/pull/1935) ([judahrand](https://github.com/judahrand)) + ## [v0.14.0](https://github.com/feast-dev/feast/tree/v0.14.0) (2021-10-08) [Full Changelog](https://github.com/feast-dev/feast/compare/v0.13.0...v0.14.0) @@ -322,7 +391,7 @@ **Merged pull requests:** - Use drop\_duplicates\(\) instead of groupby \(about 1.5~2x faster\) [\#1617](https://github.com/feast-dev/feast/pull/1617) ([rightx2](https://github.com/rightx2)) -- Use CONCAT\(\) instead of ROW\_NUMBER\(\) [\#1601](https://github.com/feast-dev/feast/pull/1601) ([MattDelac](https://github.com/MattDelac)) +- Use CONCAT\(\) instead of ROW\_NUMBER\(\) [\#1601](https://github.com/feast-dev/feast/pull/1601) ([MattDelac](https://github.com/MattDelac)) - Minor doc fix in the code snippet: Fix to reference the right instance for the retrieved job instance object [\#1599](https://github.com/feast-dev/feast/pull/1599) ([dmatrix](https://github.com/dmatrix)) - Append nanoseconds to dataset name in test\_historical\_retrival to prevent tests stomping over each other [\#1593](https://github.com/feast-dev/feast/pull/1593) ([achals](https://github.com/achals)) - Make start and end timestamps tz aware in the CLI [\#1590](https://github.com/feast-dev/feast/pull/1590) ([achals](https://github.com/achals)) @@ -857,7 +926,7 @@ - BQ sink produces sample of successful inserts [\#875](https://github.com/feast-dev/feast/pull/875) ([pyalex](https://github.com/pyalex)) - Add Branch and RC Awareness to Version Lint & Fix Semver Regex [\#998](https://github.com/feast-dev/feast/pull/998) ([mrzzy](https://github.com/mrzzy)) -## [v0.6.2](https://github.com/feast-dev/feast/tree/v0.6.2) (2020-08-02) +## [v0.6.2](https://github.com/feast-dev/feast/tree/v0.6.2) (2020-08-02) [Full Changelog](https://github.com/feast-dev/feast/compare/v0.6.1...v0.6.2) **Implemented enhancements:** @@ -1549,4 +1618,4 @@ See [Feast 0.5 Release Issue](https://github.com/feast-dev/feast/issues/527) for - Directory structure changes [\#7](https://github.com/feast-dev/feast/pull/7) ([zhilingc](https://github.com/zhilingc)) - Change register to apply [\#4](https://github.com/feast-dev/feast/pull/4) ([zhilingc](https://github.com/zhilingc)) - Empty response handling in serving api [\#3](https://github.com/feast-dev/feast/pull/3) ([pradithya](https://github.com/pradithya)) -- Proto file fixes [\#1](https://github.com/feast-dev/feast/pull/1) ([pradithya](https://github.com/pradithya)) +- Proto file fixes [\#1](https://github.com/feast-dev/feast/pull/1) ([pradithya](https://github.com/pradithya)) \ No newline at end of file diff --git a/Makefile b/Makefile index 1f53c390666..3ee9b1468cd 100644 --- a/Makefile +++ b/Makefile @@ -61,6 +61,12 @@ test-python: test-python-integration: FEAST_USAGE=False IS_TEST=True python -m pytest -n 8 --integration sdk/python/tests +test-python-universal-local: + FEAST_USAGE=False IS_TEST=True FEAST_IS_LOCAL_TEST=True python -m pytest -n 8 --integration --universal sdk/python/tests + +test-python-universal: + FEAST_USAGE=False IS_TEST=True python -m pytest -n 8 --integration --universal sdk/python/tests + format-python: # Sort cd ${ROOT_DIR}/sdk/python; python -m isort feast/ tests/ diff --git a/OWNERS b/OWNERS index 3ed906d2e99..199f9d19f23 100644 --- a/OWNERS +++ b/OWNERS @@ -1,15 +1,20 @@ +# This file is different from the CODEOWNERS file. +# OWNERS is used by feast-ci-bot to accept commands like `/ok-to-test` and `/lgtm` +# More info at https://www.kubernetes.dev/docs/guide/owners/ approvers: - woop - - jklegar - tsotnet - achals - adchia - felixwang9817 + - mavysavydav + - MattDelac reviewers: - woop - - jklegar - tsotnet - achals - tedhtchang - adchia - felixwang9817 + - mavysavydav + - MattDelac diff --git a/README.md b/README.md index 1d3d6831de0..91f87b52ed4 100644 --- a/README.md +++ b/README.md @@ -22,7 +22,7 @@ Feast is an open source feature store for machine learning. Feast is the fastest Please see our [documentation](https://docs.feast.dev/) for more information about the project. ## 📐 Architecture - + The above architecture is the minimal Feast deployment. Want to run the full Feast on GCP/AWS? Click [here](https://docs.feast.dev/how-to-guides/feast-gcp-aws). @@ -129,77 +129,74 @@ pprint(feature_vector) The list below contains the functionality that contributors are planning to develop for Feast -* Items below that are in development \(or planned for development\) will be indicated in parentheses. +* Items below that are in development (or planned for development) will be indicated in parentheses. * We welcome contribution to all items in the roadmap! -* Want to influence our roadmap and prioritization? Submit your feedback to [this form](https://docs.google.com/forms/d/e/1FAIpQLSfa1nRQ0sKz-JEFnMMCi4Jseag_yDssO_3nV9qMfxfrkil-wA/viewform). +* Want to influence our roadmap and prioritization? Submit your feedback to [this form](https://docs.google.com/forms/d/e/1FAIpQLSfa1nRQ0sKz-JEFnMMCi4Jseag\_yDssO\_3nV9qMfxfrkil-wA/viewform). * Want to speak to a Feast contributor? We are more than happy to jump on a call. Please schedule a time using [Calendly](https://calendly.com/d/x2ry-g5bb/meet-with-feast-team). - - - * **Data Sources** * [x] [Redshift source](https://docs.feast.dev/reference/data-sources/redshift) * [x] [BigQuery source](https://docs.feast.dev/reference/data-sources/bigquery) * [x] [Parquet file source](https://docs.feast.dev/reference/data-sources/file) - * [ ] Kafka source \(Planned for Q4 2021\) + * [ ] Kafka source (Planned for Q4 2021) + * [ ] Synapse source (Planned for Q4 2021) + * [ ] Snowflake source (Planned for Q4 2021) * [ ] HTTP source - * [ ] Snowflake source - * [ ] Synapse source * **Offline Stores** * [x] [Redshift](https://docs.feast.dev/reference/offline-stores/redshift) * [x] [BigQuery](https://docs.feast.dev/reference/offline-stores/bigquery) * [x] [In-memory / Pandas](https://docs.feast.dev/reference/offline-stores/file) * [x] [Custom offline store support](https://docs.feast.dev/how-to-guides/adding-a-new-offline-store) - * [x] [Hive \(community maintained\)](https://github.com/baineng/feast-hive) - * [ ] Snowflake - * [ ] Synapse + * [x] [Hive (community maintained)](https://github.com/baineng/feast-hive) + * [x] [Postgres (community maintained)](https://github.com/nossrannug/feast-postgres) + * [ ] Snowflake (Planned for Q4 2021) + * [ ] Synapse (Planned for Q4 2021) * **Online Stores** * [x] [DynamoDB](https://docs.feast.dev/reference/online-stores/dynamodb) * [x] [Redis](https://docs.feast.dev/reference/online-stores/redis) * [x] [Datastore](https://docs.feast.dev/reference/online-stores/datastore) * [x] [SQLite](https://docs.feast.dev/reference/online-stores/sqlite) * [x] [Custom online store support](https://docs.feast.dev/how-to-guides/adding-support-for-a-new-online-store) - * [ ] Postgres + * [x] [Postgres (community maintained)](https://github.com/nossrannug/feast-postgres) * [ ] Bigtable - * [ ] Cassandra + * [ ] Cassandra\\ * **Streaming** - * [ ] [Custom streaming ingestion job support](https://docs.feast.dev/how-to-guides/creating-a-custom-provider) - * [ ] Streaming ingestion on AWS \(Planned for Q4 2021\) - * [ ] Streaming ingestion on GCP + * [x] [Custom streaming ingestion job support](https://docs.feast.dev/how-to-guides/creating-a-custom-provider) + * [ ] Streaming ingestion on AWS (Planned for Q4 2021) + * [ ] Streaming ingestion on GCP * **Feature Engineering** - * [ ] On-demand Transformations \(Development in progress. See [RFC](https://docs.google.com/document/d/1lgfIw0Drc65LpaxbUu49RCeJgMew547meSJttnUqz7c/edit#)\) - * [ ] Batch transformation \(SQL\) - * [ ] Streaming transformation + * [x] On-demand Transformations (Alpha release. See [RFC](https://docs.google.com/document/d/1lgfIw0Drc65LpaxbUu49RCeJgMew547meSJttnUqz7c/edit#)) + * [ ] Batch transformation (SQL) + * [ ] Streaming transformation * **Deployments** - * [ ] AWS Lambda \(Development in progress. See [RFC](https://docs.google.com/document/d/1eZWKWzfBif66LDN32IajpaG-j82LSHCCOzY6R7Ax7MI/edit)\) + * [x] AWS Lambda (Alpha release. See [RFC](https://docs.google.com/document/d/1eZWKWzfBif66LDN32IajpaG-j82LSHCCOzY6R7Ax7MI/edit)) * [ ] Cloud Run * [ ] Kubernetes - * [ ] KNative + * [ ] KNative\\ * **Feature Serving** * [x] Python Client - * [ ] REST Feature Server \(Python\) \(Development in progress. See [RFC](https://docs.google.com/document/d/1iXvFhAsJ5jgAhPOpTdB3j-Wj1S9x3Ev_Wr6ZpnLzER4/edit)\) - * [ ] gRPC Feature Server \(Java\) \(See [\#1497](https://github.com/feast-dev/feast/issues/1497)\) + * [x] REST Feature Server (Python) (Alpha release. See [RFC](https://docs.google.com/document/d/1iXvFhAsJ5jgAhPOpTdB3j-Wj1S9x3Ev\_Wr6ZpnLzER4/edit)) + * [x] gRPC Feature Server (Java) (See [#1497](https://github.com/feast-dev/feast/issues/1497)) * [ ] Java Client - * [ ] Go Client + * [ ] Go Client * [ ] Push API * [ ] Delete API - * [ ] Feature Logging \(for training\) + * [ ] Feature Logging (for training) * **Data Quality Management** - * [ ] Data profiling and validation \(Great Expectations\) \(Planned for Q4 2021\) + * [ ] Data profiling and validation (Great Expectations) (Planned for Q4 2021) * [ ] Metric production * [ ] Training-serving skew detection * [ ] Drift detection - * [ ] Alerting + * [ ] Alerting * **Feature Discovery and Governance** * [x] Python SDK for browsing feature registry * [x] CLI for browsing feature registry - * [x] Model-centric feature tracking \(feature services\) + * [x] Model-centric feature tracking (feature services) * [ ] REST API for browsing feature registry - * [ ] Feast Web UI \(Planned for Q4 2021\) + * [ ] Feast Web UI (Planned for Q4 2021) * [ ] Feature versioning * [ ] Amundsen integration - ## 🎓 Important Resources Please refer to the official documentation at [Documentation](https://docs.feast.dev/) diff --git a/docs/.gitbook/assets/feast-marchitecture-211014.png b/docs/.gitbook/assets/feast-marchitecture-211014.png new file mode 100644 index 00000000000..7ae82da0211 Binary files /dev/null and b/docs/.gitbook/assets/feast-marchitecture-211014.png differ diff --git a/docs/README.md b/docs/README.md index fc7cd72ebad..1a76adbde3d 100644 --- a/docs/README.md +++ b/docs/README.md @@ -2,13 +2,13 @@ ## What is Feast? -Feast \(**Fea**ture **St**ore\) is an operational data system for managing and serving machine learning features to models in production. Feast is able to serve feature data to models from a low-latency online store \(for real-time prediction\) or from an offline store \(for scale-out batch scoring or model training\). +Feast (**Fea**ture **St**ore) is an operational data system for managing and serving machine learning features to models in production. Feast is able to serve feature data to models from a low-latency online store (for real-time prediction) or from an offline store (for scale-out batch scoring or model training). -![](.gitbook/assets/feast_hero_010.png) +![](.gitbook/assets/feast-marchitecture-211014.png) ## Problems Feast Solves -**Models need consistent access to data:** Machine Learning \(ML\) systems built on traditional data infrastructure are often coupled to databases, object stores, streams, and files. A result of this coupling, however, is that any change in data infrastructure may break dependent ML systems. Another challenge is that dual implementations of data retrieval for training and serving can lead to inconsistencies in data, which in turn can lead to training-serving skew. +**Models need consistent access to data:** Machine Learning (ML) systems built on traditional data infrastructure are often coupled to databases, object stores, streams, and files. A result of this coupling, however, is that any change in data infrastructure may break dependent ML systems. Another challenge is that dual implementations of data retrieval for training and serving can lead to inconsistencies in data, which in turn can lead to training-serving skew. Feast decouples your models from your data infrastructure by providing a single data access layer that abstracts feature storage from feature retrieval. Feast also provides a consistent means of referencing feature data for retrieval, and therefore ensures that models remain portable when moving from training to serving. @@ -34,9 +34,9 @@ Feast addresses this problem by introducing feature reuse through a centralized ## What Feast is not -[**ETL**](https://en.wikipedia.org/wiki/Extract,_transform,_load) **or** [**ELT**](https://en.wikipedia.org/wiki/Extract,_load,_transform) **system:** Feast is not \(and does not plan to become\) a general purpose data transformation or pipelining system. Feast plans to include a light-weight feature engineering toolkit, but we encourage teams to integrate Feast with upstream ETL/ELT systems that are specialized in transformation. +[**ETL**](https://en.wikipedia.org/wiki/Extract,\_transform,\_load) **or** [**ELT**](https://en.wikipedia.org/wiki/Extract,\_load,\_transform) **system:** Feast is not (and does not plan to become) a general purpose data transformation or pipelining system. Feast plans to include a light-weight feature engineering toolkit, but we encourage teams to integrate Feast with upstream ETL/ELT systems that are specialized in transformation. -**Data warehouse:** Feast is not a replacement for your data warehouse or the source of truth for all transformed data in your organization. Rather, Feast is a light-weight downstream layer that can serve data from an existing data warehouse \(or other data sources\) to models in production. +**Data warehouse:** Feast is not a replacement for your data warehouse or the source of truth for all transformed data in your organization. Rather, Feast is a light-weight downstream layer that can serve data from an existing data warehouse (or other data sources) to models in production. **Data catalog:** Feast is not a general purpose data catalog for your organization. Feast is purely focused on cataloging features for use in ML pipelines or systems, and only to the extent of facilitating the reuse of features. @@ -55,4 +55,3 @@ Explore the following resources to get started with Feast: * [Running Feast with GCP/AWS](how-to-guides/feast-gcp-aws/) provides a more in-depth guide to using Feast. * [Reference](reference/feast-cli-commands.md) contains detailed API and design documents. * [Contributing](project/contributing.md) contains resources for anyone who wants to contribute to Feast. - diff --git a/docs/SUMMARY.md b/docs/SUMMARY.md index 4aad275f3d3..987a432ac9a 100644 --- a/docs/SUMMARY.md +++ b/docs/SUMMARY.md @@ -23,6 +23,7 @@ * [Offline store](getting-started/architecture-and-components/offline-store.md) * [Online store](getting-started/architecture-and-components/online-store.md) * [Provider](getting-started/architecture-and-components/provider.md) +* [Third party integrations](getting-started/third-party-integrations.md) * [FAQ](getting-started/faq.md) ## Tutorials @@ -42,7 +43,7 @@ * [Load data into the online store](how-to-guides/feast-gcp-aws/load-data-into-the-online-store.md) * [Read features from the online store](how-to-guides/feast-gcp-aws/read-features-from-the-online-store.md) * [Running Feast in production](how-to-guides/running-feast-in-production.md) -* [Upgrading from Feast 0.9](https://docs.google.com/document/u/1/d/1AOsr_baczuARjCpmZgVd8mCqTF4AZ49OEyU4Cn-uTT0/edit) +* [Upgrading from Feast 0.9](https://docs.google.com/document/u/1/d/1AOsr\_baczuARjCpmZgVd8mCqTF4AZ49OEyU4Cn-uTT0/edit) * [Adding a custom provider](how-to-guides/creating-a-custom-provider.md) * [Adding a new online store](how-to-guides/adding-support-for-a-new-online-store.md) * [Adding a new offline store](how-to-guides/adding-a-new-offline-store.md) @@ -71,10 +72,11 @@ * [feature\_store.yaml](reference/feature-repository/feature-store-yaml.md) * [.feastignore](reference/feature-repository/feast-ignore.md) * [\[Alpha\] On demand feature view](reference/alpha-on-demand-feature-view.md) +* [\[Alpha\] Stream ingestion](reference/alpha-stream-ingestion.md) * [\[Alpha\] Local feature server](reference/feature-server.md) * [\[Alpha\] AWS Lambda feature server](reference/alpha-aws-lambda-feature-server.md) * [Feast CLI reference](reference/feast-cli-commands.md) -* [Python API reference](http://rtd.feast.dev/) +* [Python API reference](http://rtd.feast.dev) * [Usage](reference/usage.md) ## Project @@ -84,4 +86,3 @@ * [Versioning policy](project/versioning-policy.md) * [Release process](project/release-process.md) * [Feast 0.9 vs Feast 0.10+](project/feast-0.9-vs-feast-0.10+.md) - diff --git a/docs/assets/statistics-sources (1).png b/docs/assets/statistics-sources (1).png new file mode 100644 index 00000000000..02be233968d Binary files /dev/null and b/docs/assets/statistics-sources (1).png differ diff --git a/docs/assets/statistics-sources (2).png b/docs/assets/statistics-sources (2).png new file mode 100644 index 00000000000..02be233968d Binary files /dev/null and b/docs/assets/statistics-sources (2).png differ diff --git a/docs/assets/statistics-sources (3).png b/docs/assets/statistics-sources (3).png new file mode 100644 index 00000000000..02be233968d Binary files /dev/null and b/docs/assets/statistics-sources (3).png differ diff --git a/docs/assets/statistics-sources (4).png b/docs/assets/statistics-sources (4).png new file mode 100644 index 00000000000..02be233968d Binary files /dev/null and b/docs/assets/statistics-sources (4).png differ diff --git a/docs/community.md b/docs/community.md index a5ff5ba0d58..53e00eb4ebf 100644 --- a/docs/community.md +++ b/docs/community.md @@ -11,7 +11,7 @@ * Feast users should join [feast-discuss@googlegroups.com](mailto:feast-discuss@googlegroups.com) group by clicking [here](https://groups.google.com/g/feast-discuss). * Feast developers should join [feast-dev@googlegroups.com](mailto:feast-dev@googlegroups.com) group by clicking [here](https://groups.google.com/d/forum/feast-dev). * [Google Folder](https://drive.google.com/drive/u/0/folders/1jgMHOPDT2DvBlJeO9LCM79DP4lm4eOrR): This folder is used as a central repository for all Feast resources. For example: - * Design proposals in the form of Request for Comments \(RFC\). + * Design proposals in the form of Request for Comments (RFC). * User surveys and meeting minutes. * Slide decks of conferences our contributors have spoken at. * [Feast GitHub Repository](https://github.com/feast-dev/feast/): Find the complete Feast codebase on GitHub. @@ -19,25 +19,24 @@ ## How can I get help? -* **Slack:** Need to speak to a human? Come ask a question in our Slack channel \(link above\). +* **Slack:** Need to speak to a human? Come ask a question in our Slack channel (link above). * **GitHub Issues:** Found a bug or need a feature? [Create an issue on GitHub](https://github.com/feast-dev/feast/issues/new). * **StackOverflow:** Need to ask a question on how to use Feast? We also monitor and respond to [StackOverflow](https://stackoverflow.com/questions/tagged/feast). ## Community Calls -We have a user and contributor community call every two weeks \(Asia & US friendly\). +We have a user and contributor community call every two weeks (Asia & US friendly). {% hint style="info" %} Please join the above Feast user groups in order to see calendar invites to the community calls {% endhint %} -### Frequency \(alternating times every 2 weeks\) +### Frequency (alternating times every 2 weeks) -* Tuesday 18:00 pm to 18:30 pm \(US, Asia\) -* Tuesday 10:00 am to 10:30 am \(US, Europe\) +* Tuesday 18:00 pm to 18:30 pm (US, Asia) +* Tuesday 10:00 am to 10:30 am (US, Europe) ### Links * Zoom: [https://zoom.us/j/6325193230](https://zoom.us/j/6325193230) -* Meeting notes: [https://bit.ly/feast-notes](https://bit.ly/feast-notes%20) - +* Meeting notes: [https://bit.ly/feast-notes](https://bit.ly/feast-notes) diff --git a/docs/getting-started/architecture-and-components/overview.md b/docs/getting-started/architecture-and-components/overview.md index 247aa5e358b..bf5c12dcc20 100644 --- a/docs/getting-started/architecture-and-components/overview.md +++ b/docs/getting-started/architecture-and-components/overview.md @@ -1,15 +1,15 @@ # Overview -![Feast Architecture Diagram](../../.gitbook/assets/image%20%284%29.png) +![Feast Architecture Diagram](<../../.gitbook/assets/image (4).png>) ## Functionality * **Create Batch Features:** ELT/ETL systems like Spark and SQL are used to transform data in the batch store. -* **Feast Apply:** The user \(or CI\) publishes versioned controlled feature definitions using `feast apply`. This CLI command updates infrastructure and persists definitions in the object store registry. -* **Feast Materialize:** The user \(or scheduler\) executes `feast materialize` which loads features from the offline store into the online store. +* **Feast Apply:** The user (or CI) publishes versioned controlled feature definitions using `feast apply`. This CLI command updates infrastructure and persists definitions in the object store registry. +* **Feast Materialize:** The user (or scheduler) executes `feast materialize` which loads features from the offline store into the online store. * **Model Training:** A model training pipeline is launched. It uses the Feast Python SDK to retrieve a training dataset and trains a model. * **Get Historical Features:** Feast exports a point-in-time correct training dataset based on the list of features and entity dataframe provided by the model training pipeline. -* **Deploy Model:** The trained model binary \(and list of features\) are deployed into a model serving system. This step is not executed by Feast. +* **Deploy Model:** The trained model binary (and list of features) are deployed into a model serving system. This step is not executed by Feast. * **Prediction:** A backend system makes a request for a prediction from the model serving service. * **Get Online Features:** The model serving service makes a request to the Feast Online Serving service for online features using a Feast SDK. @@ -17,16 +17,15 @@ A complete Feast deployment contains the following components: -* **Feast Registry**: An object store \(GCS, S3\) based registry used to persist feature definitions that are registered with the feature store. Systems can discover feature data by interacting with the registry through the Feast SDK. +* **Feast Registry**: An object store (GCS, S3) based registry used to persist feature definitions that are registered with the feature store. Systems can discover feature data by interacting with the registry through the Feast SDK. * **Feast Python SDK/CLI:** The primary user facing SDK. Used to: * Manage version controlled feature definitions. - * Materialize \(load\) feature values into the online store. + * Materialize (load) feature values into the online store. * Build and retrieve training datasets from the offline store. * Retrieve online features. -* **Online Store:** The online store is a database that stores only the latest feature values for each entity. The online store is populated by materialization jobs. +* **Online Store:** The online store is a database that stores only the latest feature values for each entity. The online store is populated by materialization jobs and from [stream ingestion](../../reference/alpha-stream-ingestion.md). * **Offline Store:** The offline store persists batch data that has been ingested into Feast. This data is used for producing training datasets. Feast does not manage the offline store directly, but runs queries against it. {% hint style="info" %} Java and Go Clients are also available for online feature retrieval. {% endhint %} - diff --git a/docs/getting-started/concepts/feature-service.md b/docs/getting-started/concepts/feature-service.md index 7f5ca57c94c..adb4927113f 100644 --- a/docs/getting-started/concepts/feature-service.md +++ b/docs/getting-started/concepts/feature-service.md @@ -3,7 +3,7 @@ A feature service is an object that represents a logical group of features from one or more [feature views](feature-view.md#feature-view). Feature Services allows features from within a feature view to be used as needed by an ML model. Users can expect to create one feature service per model, allowing for tracking of the features used by models. {% tabs %} -{% tab title="driver\_trips\_feature\_service.py" %} +{% tab title="driver_trips_feature_service.py" %} ```python from driver_ratings_feature_view import driver_ratings_fv from driver_trips_feature_view import driver_stats_fv @@ -46,4 +46,3 @@ feature_store = FeatureStore('.') # Initialize the feature store feature_service = feature_store.get_feature_service("driver_activity") feature_store.get_historical_features(features=feature_service, entity_df=entity_df) ``` - diff --git a/docs/getting-started/concepts/feature-view.md b/docs/getting-started/concepts/feature-view.md index 7395e8957e8..535ab93b133 100644 --- a/docs/getting-started/concepts/feature-view.md +++ b/docs/getting-started/concepts/feature-view.md @@ -2,10 +2,10 @@ ## Feature views -A feature view is an object that represents a logical group of time-series feature data as it is found in a [data source](data-source.md). Feature views consist of zero or more [entities](entity.md), one or more [features](feature-view.md#feature), and a [data source](data-source.md). Feature views allow Feast to model your existing feature data in a consistent way in both an offline \(training\) and online \(serving\) environment. Feature views generally contain features that are properties of a specific object, in which case that object is defined as an entity and included in the feature view. If the features are not related to a specific object, the feature view might not have entities; see [feature views without entities](feature-view.md#feature-views-without-entities) below. +A feature view is an object that represents a logical group of time-series feature data as it is found in a [data source](data-source.md). Feature views consist of zero or more [entities](entity.md), one or more [features](feature-view.md#feature), and a [data source](data-source.md). Feature views allow Feast to model your existing feature data in a consistent way in both an offline (training) and online (serving) environment. Feature views generally contain features that are properties of a specific object, in which case that object is defined as an entity and included in the feature view. If the features are not related to a specific object, the feature view might not have entities; see [feature views without entities](feature-view.md#feature-views-without-entities) below. {% tabs %} -{% tab title="driver\_trips\_feature\_view.py" %} +{% tab title="driver_trips_feature_view.py" %} ```python driver_stats_fv = FeatureView( name="driver_activity", @@ -25,7 +25,7 @@ driver_stats_fv = FeatureView( Feature views are used during * The generation of training datasets by querying the data source of feature views in order to find historical feature values. A single training dataset may consist of features from multiple feature views. -* Loading of feature values into an online store. Feature views determine the storage schema in the online store. +* Loading of feature values into an online store. Feature views determine the storage schema in the online store. Feature values can be loaded from batch sources or from [stream sources](../../reference/alpha-stream-ingestion.md). * Retrieval of features from the online store. Feature views provide the schema definition to Feast in order to look up features from the online store. {% hint style="info" %} @@ -37,7 +37,7 @@ Feast does not generate feature values. It acts as the ingestion and serving sys If a feature view contains features that are not related to a specific entity, the feature view can be defined without entities. {% tabs %} -{% tab title="global\_stats.py" %} +{% tab title="global_stats.py" %} ```python global_stats_fv = FeatureView( name="global_stats", @@ -53,6 +53,55 @@ global_stats_fv = FeatureView( {% endtab %} {% endtabs %} +## Entity aliasing + +"Entity aliases" can be specified to join `entity_dataframe` columns that do not match the column names in the source table of a FeatureView. + +This could be used if a user has no control over these column names or if there are multiple entities are a subclass of a more general entity. For example, "spammer" and "reporter" could be aliases of a "user" entity, and "origin" and "destination" could be aliases of a "location" entity as shown below. + +It is suggested that you dynamically specify the new FeatureView name using `.with_name` and `join_key_map` override using `.with_join_key_map` instead of needing to register each new copy. + +{% tabs %} +{% tab title="location_stats_feature_view.py" %} +```python +location = Entity(name="location", join_key="location_id", value_type=ValueType.INT64) + +location_stats_fv= FeatureView( + name="location_stats", + entities=["location"], + features=[ + Feature(name="temperature", dtype=ValueType.INT32) + ], + batch_source=BigQuerySource( + table_ref="feast-oss.demo_data.location_stats" + ), +) +``` +{% endtab %} + +{% tab title="temperatures_feature_service.py" %} +```python +from location_stats_feature_view import location_stats_fv + +temperatures_fs = FeatureService( + name="temperatures", + features=[ + location_stats_feature_view + .with_name("origin_stats") + .with_join_key_map( + {"location_id": "origin_id"} + ), + location_stats_feature_view + .with_name("destination_stats") + .with_join_key_map( + {"location_id": "destination_id"} + ), + ], +) +``` +{% endtab %} +{% endtabs %} + ## Feature A feature is an individual measurable property. It is typically a property observed on a specific entity, but does not have to be associated with an entity. For example, a feature of a `customer` entity could be the number of transactions they have made on an average month, while a feature that is not observed on a specific entity could be the total number of posts made by all users in the last month. @@ -70,9 +119,9 @@ Together with [data sources](data-source.md), they indicate to Feast where to fi Feature names must be unique within a [feature view](feature-view.md#feature-view). -## \[Alpha\] On demand feature views +## \[Alpha] On demand feature views -On demand feature views allows users to use existing features and request time data \(features only available at request time\) to transform and create new features. Users define python transformation logic which is executed in both historical retrieval and online retrieval paths: +On demand feature views allows users to use existing features and request time data (features only available at request time) to transform and create new features. Users define python transformation logic which is executed in both historical retrieval and online retrieval paths: ```python # Define a request data source which encodes features / information only @@ -102,6 +151,3 @@ def transformed_conv_rate(features_df: pd.DataFrame) -> pd.DataFrame: df['conv_rate_plus_val2'] = (features_df['conv_rate'] + features_df['val_to_add_2']) return df ``` - - - diff --git a/docs/getting-started/faq.md b/docs/getting-started/faq.md index f47c3dd6d2b..e06ab7242a1 100644 --- a/docs/getting-started/faq.md +++ b/docs/getting-started/faq.md @@ -20,11 +20,11 @@ No, there are [feature views without entities](concepts/feature-view.md#feature- ### Does Feast provide security or access control? -Feast currently does not support any access control other than the access control required for the Provider's environment \(for example, GCP and AWS permissions\). +Feast currently does not support any access control other than the access control required for the Provider's environment (for example, GCP and AWS permissions). ### Does Feast support streaming sources? -Feast is actively working on this right now. Please reach out to the Feast team if you're interested in giving feedback! +Yes. In earlier versions of Feast, we used Feast Spark to manage ingestion from stream sources. In the current version of Feast, we support [push based ingestion](../reference/alpha-stream-ingestion.md). ### Does Feast support composite keys? @@ -42,12 +42,12 @@ Feast is designed to work at scale and support low latency online serving. Bench Yes. Specifically: -* Simple lists / dense embeddings: - * BigQuery supports list types natively - * Redshift does not support list types, so you'll need to serialize these features into strings \(e.g. json or protocol buffers\) - * Feast's implementation of online stores serializes features into Feast protocol buffers and supports list types \(see [reference](https://github.com/feast-dev/feast/blob/master/docs/specs/online_store_format.md#appendix-a-value-proto-format)\) -* Sparse embeddings \(e.g. one hot encodings\) - * One way to do this efficiently is to have a protobuf or string representation of [https://www.tensorflow.org/guide/sparse\_tensor](https://www.tensorflow.org/guide/sparse_tensor) +* Simple lists / dense embeddings: + * BigQuery supports list types natively + * Redshift does not support list types, so you'll need to serialize these features into strings (e.g. json or protocol buffers) + * Feast's implementation of online stores serializes features into Feast protocol buffers and supports list types (see [reference](https://github.com/feast-dev/feast/blob/master/docs/specs/online\_store\_format.md#appendix-a-value-proto-format)) +* Sparse embeddings (e.g. one hot encodings) + * One way to do this efficiently is to have a protobuf or string representation of [https://www.tensorflow.org/guide/sparse\_tensor](https://www.tensorflow.org/guide/sparse\_tensor) ### Does Feast support X storage engine? @@ -61,7 +61,7 @@ Please follow the instructions [here](../how-to-guides/adding-support-for-a-new- Yes. There are two ways to use S3 in Feast: -* Using Redshift as a data source via Spectrum \([AWS tutorial](https://docs.aws.amazon.com/redshift/latest/dg/tutorial-nested-data-create-table.html)\), and then continuing with the [Running Feast with GCP/AWS](../how-to-guides/feast-gcp-aws/) guide. See a [presentation](https://youtu.be/pMFbRJ7AnBk?t=9463) we did on this at our apply\(\) meetup. +* Using Redshift as a data source via Spectrum ([AWS tutorial](https://docs.aws.amazon.com/redshift/latest/dg/tutorial-nested-data-create-table.html)), and then continuing with the [Running Feast with GCP/AWS](../how-to-guides/feast-gcp-aws/) guide. See a [presentation](https://youtu.be/pMFbRJ7AnBk?t=9463) we did on this at our apply() meetup. * Using the `s3_endpoint_override` in a `FileSource` data source. This endpoint is more suitable for quick proof of concepts that won't necessarily scale for production use cases. ### How can I use Spark with Feast? @@ -76,11 +76,11 @@ Please see the [roadmap](../roadmap.md). ### What is the difference between Feast 0.9 and Feast 0.10+? -Feast 0.10+ is much lighter weight and more extensible than Feast 0.9. It is designed to be simple to install and use. Please see this [document](https://docs.google.com/document/d/1AOsr_baczuARjCpmZgVd8mCqTF4AZ49OEyU4Cn-uTT0) for more details. +Feast 0.10+ is much lighter weight and more extensible than Feast 0.9. It is designed to be simple to install and use. Please see this [document](https://docs.google.com/document/d/1AOsr\_baczuARjCpmZgVd8mCqTF4AZ49OEyU4Cn-uTT0) for more details. ### How do I migrate from Feast 0.9 to Feast 0.10+? -Please see this [document](https://docs.google.com/document/d/1AOsr_baczuARjCpmZgVd8mCqTF4AZ49OEyU4Cn-uTT0). If you have any questions or suggestions, feel free to leave a comment on the document! +Please see this [document](https://docs.google.com/document/d/1AOsr\_baczuARjCpmZgVd8mCqTF4AZ49OEyU4Cn-uTT0). If you have any questions or suggestions, feel free to leave a comment on the document! ### How do I contribute to Feast? @@ -93,6 +93,5 @@ Feast Core and Feast Serving were both part of Feast Java. We plan to support Fe {% hint style="info" %} **Don't see your question?** -We encourage you to ask questions on [Slack](https://slack.feast.dev/) or [Github](https://github.com/feast-dev/feast). Even better, once you get an answer, add the answer to this FAQ via a [pull request](../project/development-guide.md)! +We encourage you to ask questions on [Slack](https://slack.feast.dev) or [Github](https://github.com/feast-dev/feast). Even better, once you get an answer, add the answer to this FAQ via a [pull request](../project/development-guide.md)! {% endhint %} - diff --git a/docs/getting-started/quickstart.md b/docs/getting-started/quickstart.md index 4b04e6ca4c2..35d05990695 100644 --- a/docs/getting-started/quickstart.md +++ b/docs/getting-started/quickstart.md @@ -9,19 +9,19 @@ In this tutorial we will You can run this tutorial in Google Colab or run it on your localhost, following the guided steps below. -![](../.gitbook/assets/colab_logo_32px.png)[**Run in Google Colab**](https://colab.research.google.com/github/feast-dev/feast/blob/master/examples/quickstart/quickstart.ipynb)\*\*\*\* +![](../.gitbook/assets/colab\_logo\_32px.png)[**Run in Google Colab**](https://colab.research.google.com/github/feast-dev/feast/blob/master/examples/quickstart/quickstart.ipynb)**** ## Overview -In this tutorial, we use feature stores to generate training data and power online model inference for a ride-sharing driver satisfaction prediction model. Feast solves several common issues in this flow: +In this tutorial, we use feature stores to generate training data and power online model inference for a ride-sharing driver satisfaction prediction model. Feast solves several common issues in this flow: 1. **Training-serving skew and complex data joins:** Feature values often exist across multiple tables. Joining these datasets can be complicated, slow, and error-prone. * Feast joins these tables with battle-tested logic that ensures _point-in-time_ correctness so future feature values do not leak to models. - * _\*Upcoming_: Feast alerts users to offline / online skew with data quality monitoring. -2. **Online feature availability:** At inference time, models often need access to features that aren't readily available and need to be precomputed from other datasources. - * Feast manages deployment to a variety of online stores \(e.g. DynamoDB, Redis, Google Cloud Datastore\) and ensures necessary features are consistently _available_ and _freshly computed_ at inference time. + * _\*Upcoming_: Feast alerts users to offline / online skew with data quality monitoring. +2. **Online feature availability:** At inference time, models often need access to features that aren't readily available and need to be precomputed from other datasources. + * Feast manages deployment to a variety of online stores (e.g. DynamoDB, Redis, Google Cloud Datastore) and ensures necessary features are consistently _available_ and _freshly computed_ at inference time. 3. **Feature reusability and model versioning:** Different teams within an organization are often unable to reuse features across projects, resulting in duplicate feature creation logic. Models have data dependencies that need to be versioned, for example when running A/B tests on model versions. - * Feast enables discovery of and collaboration on previously used features and enables versioning of sets of features \(via _feature services_\). + * Feast enables discovery of and collaboration on previously used features and enables versioning of sets of features (via _feature services_). * Feast enables feature transformation so users can re-use transformation logic across online / offline usecases and across models. ## Step 1: Install Feast @@ -40,7 +40,7 @@ pip install feast ## Step 2: Create a feature repository -Bootstrap a new feature repository using `feast init` from the command line. +Bootstrap a new feature repository using `feast init` from the command line. {% tabs %} {% tab title="Bash" %} @@ -53,7 +53,7 @@ cd feature_repo {% tabs %} {% tab title="Output" %} -```text +``` Creating a new Feast repository in /home/Jovyan/feature_repo. ``` {% endtab %} @@ -66,7 +66,7 @@ Let's take a look at the resulting demo repo itself. It breaks down into * `feature_store.yaml` contains a demo setup configuring where data sources are {% tabs %} -{% tab title="feature\_store.yaml" %} +{% tab title="feature_store.yaml" %} ```yaml project: my_project registry: data/registry.db @@ -119,7 +119,7 @@ driver_hourly_stats_view = FeatureView( ![Demo parquet data: data/driver\_stats.parquet](../.gitbook/assets/screen-shot-2021-08-23-at-2.35.18-pm.png) -The key line defining the overall architecture of the feature store is the **provider**. This defines where the raw data exists \(for generating training data & feature values for serving\), and where to materialize feature values to in the online store \(for serving\). +The key line defining the overall architecture of the feature store is the **provider**. This defines where the raw data exists (for generating training data & feature values for serving), and where to materialize feature values to in the online store (for serving). Valid values for `provider` in `feature_store.yaml` are: @@ -127,11 +127,11 @@ Valid values for `provider` in `feature_store.yaml` are: * gcp: use BigQuery / Google Cloud Datastore * aws: use Redshift / DynamoDB -A custom setup \(e.g. using the built-in support for Redis\) can be made by following Creating a custom provider +To use a custom provider, see [adding a custom provider](../how-to-guides/creating-a-custom-provider.md). There are also several plugins maintained by the community: [Azure](https://github.com/Azure/feast-azure), [Postgres](https://github.com/nossrannug/feast-postgres), and [Hive](https://github.com/baineng/feast-hive). Note that the choice of provider gives sensible defaults but does not enforce those choices; for example, if you choose the AWS provider, you can use [Redis](../reference/online-stores/redis.md) as an online store alongside Redshift as an offline store. ## Step 3: Register feature definitions and deploy your feature store -The `apply` command scans python files in the current directory for feature view/entity definitions, registers the objects, and deploys infrastructure. In this example, it reads `example.py` \(shown again below for convenience\) and sets up SQLite online store tables. Note that we had specified SQLite as the default online store by using the `local` provider in `feature_store.yaml`. +The `apply` command scans python files in the current directory for feature view/entity definitions, registers the objects, and deploys infrastructure. In this example, it reads `example.py` (shown again below for convenience) and sets up SQLite online store tables. Note that we had specified SQLite as the default online store by using the `local` provider in `feature_store.yaml`. {% tabs %} {% tab title="Bash" %} @@ -183,7 +183,7 @@ driver_hourly_stats_view = FeatureView( {% tabs %} {% tab title="Output" %} -```text +``` Registered entity driver_id Registered feature view driver_hourly_stats Deploying infrastructure for driver_hourly_stats @@ -193,7 +193,7 @@ Deploying infrastructure for driver_hourly_stats ## Step 4: Generating training data -To train a model, we need features and labels. Often, this label data is stored separately \(e.g. you have one table storing user survey results and another set of tables with feature values\). +To train a model, we need features and labels. Often, this label data is stored separately (e.g. you have one table storing user survey results and another set of tables with feature values). The user can query that table of labels with timestamps and pass that into Feast as an _entity dataframe_ for training data generation. In many cases, Feast will also intelligently join relevant tables to create the relevant feature vectors. @@ -275,7 +275,7 @@ None ## Step 5: Load features into your online store -We now serialize the latest values of features since the beginning of time to prepare for serving \(note: `materialize-incremental` serializes all new features since the last `materialize` call\). +We now serialize the latest values of features since the beginning of time to prepare for serving (note: `materialize-incremental` serializes all new features since the last `materialize` call). {% tabs %} {% tab title="Bash" %} @@ -300,7 +300,7 @@ driver_hourly_stats from 2021-08-22 16:25:47+00:00 to 2021-08-23 16:25:46+00:00: ## Step 6: Fetching feature vectors for inference -At inference time, we need to quickly read the latest feature values for different drivers \(which otherwise might have existed only in batch sources\) from the online feature store using `get_online_features()`. These feature vectors can then be fed to the model. +At inference time, we need to quickly read the latest feature values for different drivers (which otherwise might have existed only in batch sources) from the online feature store using `get_online_features()`. These feature vectors can then be fed to the model. {% tabs %} {% tab title="Python" %} @@ -346,5 +346,4 @@ pprint(feature_vector) * Read the [Architecture](architecture-and-components/) page. * Check out our [Tutorials](../tutorials/tutorials-overview.md) section for more examples on how to use Feast. * Follow our [Running Feast with GCP/AWS](../how-to-guides/feast-gcp-aws/) guide for a more in-depth tutorial on using Feast. -* Join other Feast users and contributors in [Slack](https://slack.feast.dev/) and become part of the community! - +* Join other Feast users and contributors in [Slack](https://slack.feast.dev) and become part of the community! diff --git a/docs/getting-started/third-party-integrations.md b/docs/getting-started/third-party-integrations.md new file mode 100644 index 00000000000..532d86546f4 --- /dev/null +++ b/docs/getting-started/third-party-integrations.md @@ -0,0 +1,58 @@ +# Third party integrations + +We integrate with a wide set of tools and technologies so you can make Feast work in your existing stack. Many of these integrations are maintained as plugins to the main Feast repo. + +{% hint style="info" %} +Don't see your offline store or online store of choice here? Check our our guides to make a custom one! + +* [Adding a new offline store](../how-to-guides/adding-a-new-offline-store.md) +* [Adding a new online store](../how-to-guides/adding-support-for-a-new-online-store.md) +{% endhint %} + +## Integrations + +### **Data Sources** + +* [x] [Redshift source](https://docs.feast.dev/reference/data-sources/redshift) +* [x] [BigQuery source](https://docs.feast.dev/reference/data-sources/bigquery) +* [x] [Parquet file source](https://docs.feast.dev/reference/data-sources/file) +* [x] [Synapse source (community plugin)](https://github.com/Azure/feast-azure) +* [x] [Hive (community plugin)](https://github.com/baineng/feast-hive) +* [x] [Postgres (community plugin)](https://github.com/nossrannug/feast-postgres) +* [ ] Kafka source (Planned for Q4 2021) +* [ ] Snowflake source (Planned for Q4 2021) +* [ ] HTTP source + +### Offline Stores + +* [x] [Redshift](https://docs.feast.dev/reference/offline-stores/redshift) +* [x] [BigQuery](https://docs.feast.dev/reference/offline-stores/bigquery) +* [x] [Synapse (community plugin)](https://github.com/Azure/feast-azure) +* [x] [Hive (community plugin)](https://github.com/baineng/feast-hive) +* [x] [Postgres (community plugin)](https://github.com/nossrannug/feast-postgres) +* [x] [In-memory / Pandas](https://docs.feast.dev/reference/offline-stores/file) +* [x] [Custom offline store support](https://docs.feast.dev/how-to-guides/adding-a-new-offline-store) +* [ ] Snowflake (Planned for Q4 2021) +* [ ] Trino (Planned for Q4 2021) + +### Online Stores + +* [x] [DynamoDB](https://docs.feast.dev/reference/online-stores/dynamodb) +* [x] [Redis](https://docs.feast.dev/reference/online-stores/redis) +* [x] [Datastore](https://docs.feast.dev/reference/online-stores/datastore) +* [x] [SQLite](https://docs.feast.dev/reference/online-stores/sqlite) +* [x] [Azure Cache for Redis (community plugin)](https://github.com/Azure/feast-azure) +* [x] [Postgres (community plugin)](https://github.com/nossrannug/feast-postgres) +* [x] [Custom online store support](https://docs.feast.dev/how-to-guides/adding-support-for-a-new-online-store) +* [ ] Bigtable +* [ ] Cassandra + +### **Deployments** + +* [x] AWS Lambda (Alpha release. See [guide](../reference/alpha-aws-lambda-feature-server.md) and [RFC](https://docs.google.com/document/d/1eZWKWzfBif66LDN32IajpaG-j82LSHCCOzY6R7Ax7MI/edit)) +* [ ] Cloud Run +* [ ] Kubernetes +* [ ] KNative + + + diff --git a/docs/how-to-guides/adding-a-new-offline-store.md b/docs/how-to-guides/adding-a-new-offline-store.md index f996e315ae3..c77e88c7c8e 100644 --- a/docs/how-to-guides/adding-a-new-offline-store.md +++ b/docs/how-to-guides/adding-a-new-offline-store.md @@ -2,7 +2,7 @@ ## Overview -Feast makes adding support for a new offline store \(database\) easy. Developers can simply implement the [OfflineStore](https://github.com/feast-dev/feast/blob/master/sdk/python/feast/infra/offline_stores/offline_store.py#L41) interface to add support for a new store \(other than the existing stores like Parquet files, Redshift, and Bigquery\). +Feast makes adding support for a new offline store (database) easy. Developers can simply implement the [OfflineStore](../../sdk/python/feast/infra/offline\_stores/offline\_store.py#L41) interface to add support for a new store (other than the existing stores like Parquet files, Redshift, and Bigquery). In this guide, we will show you how to extend the existing File offline store and use in a feature repo. While we will be implementing a specific store, this guide should be representative for adding support for any new offline store. @@ -13,22 +13,24 @@ The process for using a custom offline store consists of 4 steps: 1. Defining an `OfflineStore` class. 2. Defining an `OfflineStoreConfig` class. 3. Defining a `RetrievalJob` class for this offline store. -4. Referencing the `OfflineStore` in a feature repo's `feature_store.yaml` file. +4. Defining a `DataSource` class for the offline store +5. Referencing the `OfflineStore` in a feature repo's `feature_store.yaml` file. +6. Testing the `OfflineStore` class. ## 1. Defining an OfflineStore class {% hint style="info" %} - OfflineStore class names must end with the OfflineStore suffix! + OfflineStore class names must end with the OfflineStore suffix! {% endhint %} -The OfflineStore class contains a couple of methods to read features from the offline store. Unlike the OnlineStore class, Feast does not manage any infrastructure for the offline store. +The OfflineStore class contains a couple of methods to read features from the offline store. Unlike the OnlineStore class, Feast does not manage any infrastructure for the offline store. There are two methods that deal with reading data from the offline stores`get_historical_features`and `pull_latest_from_table_or_query`. -* `pull_latest_from_table_or_query` is invoked when running materialization \(using the `feast materialize` or `feast materialize-incremental` commands, or the corresponding `FeatureStore.materialize()` method. This method pull data from the offline store, and the `FeatureStore` class takes care of writing this data into the online store. -* `get_historical_features` is invoked when reading values from the offline store using the `FeatureStore.get_historica_features()` method. Typically, this method is used to retrieve features when training ML models. +* `pull_latest_from_table_or_query` is invoked when running materialization (using the `feast materialize` or `feast materialize-incremental` commands, or the corresponding `FeatureStore.materialize()` method. This method pull data from the offline store, and the `FeatureStore` class takes care of writing this data into the online store. +* `get_historical_features `is invoked when reading values from the offline store using the `FeatureStore.get_historica_features()` method. Typically, this method is used to retrieve features when training ML models. -{% code title="feast\_custom\_offline\_store/file.py" %} +{% code title="feast_custom_offline_store/file.py" %} ```python def get_historical_features(self, config: RepoConfig, @@ -64,7 +66,6 @@ There are two methods that deal with reading data from the offline stores`get_hi created_timestamp_column, start_date, end_date) - ``` {% endcode %} @@ -72,39 +73,37 @@ There are two methods that deal with reading data from the offline stores`get_hi Additional configuration may be needed to allow the OfflineStore to talk to the backing store. For example, Redshift needs configuration information like the connection information for the Redshift instance, credentials for connecting to the database, etc. -To facilitate configuration, all OfflineStore implementations are **required** to also define a corresponding OfflineStoreConfig class in the same file. This OfflineStoreConfig class should inherit from the `FeastConfigBaseModel` class, which is defined [here](https://github.com/feast-dev/feast/blob/master/sdk/python/feast/repo_config.py#L44). +To facilitate configuration, all OfflineStore implementations are **required** to also define a corresponding OfflineStoreConfig class in the same file. This OfflineStoreConfig class should inherit from the `FeastConfigBaseModel` class, which is defined [here](../../sdk/python/feast/repo\_config.py#L44). -The `FeastConfigBaseModel` is a [pydantic](https://pydantic-docs.helpmanual.io/) class, which parses yaml configuration into python objects. Pydantic also allows the model classes to define validators for the config classes, to make sure that the config classes are correctly defined. +The `FeastConfigBaseModel` is a [pydantic](https://pydantic-docs.helpmanual.io) class, which parses yaml configuration into python objects. Pydantic also allows the model classes to define validators for the config classes, to make sure that the config classes are correctly defined. -This config class **must** container a `type` field, which contains the fully qualified class name of its corresponding OfflineStore class. +This config class **must** container a `type` field, which contains the fully qualified class name of its corresponding OfflineStore class. Additionally, the name of the config class must be the same as the OfflineStore class, with the `Config` suffix. An example of the config class for the custom file offline store : -{% code title="feast\_custom\_offline\_store/file.py" %} +{% code title="feast_custom_offline_store/file.py" %} ```python class CustomFileOfflineStoreConfig(FeastConfigBaseModel): """ Custom offline store config for local (file-based) store """ type: Literal["feast_custom_offline_store.file.CustomFileOfflineStore"] \ = "feast_custom_offline_store.file.CustomFileOfflineStore" - ``` {% endcode %} This configuration can be specified in the `feature_store.yaml` as follows: -{% code title="feature\_repo/feature\_store.yaml" %} +{% code title="feature_repo/feature_store.yaml" %} ```yaml - type: feast_custom_offline_store.file.CustomFileOfflineStore - +type: feast_custom_offline_store.file.CustomFileOfflineStore ``` {% endcode %} -This configuration information is available to the methods of the OfflineStore, via the`config: RepoConfig` parameter which is passed into the methods of the OfflineStore interface, specifically at the `config.offline_store` field of the `config` parameter. +This configuration information is available to the methods of the OfflineStore, via the`config: RepoConfig` parameter which is passed into the methods of the OfflineStore interface, specifically at the `config.offline_store` field of the `config` parameter. -{% code title="feast\_custom\_offline\_store/file.py" %} +{% code title="feast_custom_offline_store/file.py" %} ```python def get_historical_features(self, config: RepoConfig, @@ -117,7 +116,6 @@ This configuration information is available to the methods of the OfflineStore, offline_store_config = config.offline_store assert isinstance(offline_store_config, CustomFileOfflineStoreConfig) store_type = offline_store_config.type - ``` {% endcode %} @@ -129,7 +127,7 @@ Custom offline stores may need to implement their own instances of the `Retrieva The `RetrievalJob` interface exposes two methods - `to_df` and `to_arrow`. The expectation is for the retrieval job to be able to return the rows read from the offline store as a parquet DataFrame, or as an Arrow table respectively. -{% code title="feast\_custom\_offline\_store/file.py" %} +{% code title="feast_custom_offline_store/file.py" %} ```python class CustomFileRetrievalJob(RetrievalJob): def __init__(self, evaluation_function: Callable): @@ -149,19 +147,78 @@ class CustomFileRetrievalJob(RetrievalJob): print("Getting a pandas DataFrame from a File is easy!") df = self.evaluation_function() return pyarrow.Table.from_pandas(df) +``` +{% endcode %} + +## 4. Defining a DataSource class for the offline store +Before this offline store can be used as the batch source for a feature view in a feature repo, a subclass of the `DataSource` [base class](https://rtd.feast.dev/en/master/index.html?highlight=DataSource#feast.data\_source.DataSource) needs to be defined. This class is responsible for holding information needed by specific feature views to support reading historical values from the offline store. For example, a feature view using Redshift as the offline store may need to know which table contains historical feature values. + +The data source class should implement two methods - `from_proto`, and `to_proto`. + +For custom offline stores that are not being implemented in the main feature repo, the `custom_options` field should be used to store any configuration needed by the data source. In this case, the implementer is responsible for serializing this configuration into bytes in the `to_proto` method and reading the value back from bytes in the `from_proto` method. + +{% code title="feast_custom_offline_store/file.py" %} +```python +class CustomFileDataSource(FileSource): + """Custom data source class for local files""" + def __init__( + self, + event_timestamp_column: Optional[str] = "", + path: Optional[str] = None, + field_mapping: Optional[Dict[str, str]] = None, + created_timestamp_column: Optional[str] = "", + date_partition_column: Optional[str] = "", + ): + super(CustomFileDataSource, self).__init__( + event_timestamp_column, + created_timestamp_column, + field_mapping, + date_partition_column, + ) + self._path = path + + + @staticmethod + def from_proto(data_source: DataSourceProto): + custom_source_options = str( + data_source.custom_options.configuration, encoding="utf8" + ) + path = json.loads(custom_source_options)["path"] + return CustomFileDataSource( + field_mapping=dict(data_source.field_mapping), + path=path, + event_timestamp_column=data_source.event_timestamp_column, + created_timestamp_column=data_source.created_timestamp_column, + date_partition_column=data_source.date_partition_column, + ) + + def to_proto(self) -> DataSourceProto: + config_json = json.dumps({"path": self.path}) + data_source_proto = DataSourceProto( + type=DataSourceProto.CUSTOM_SOURCE, + custom_options=DataSourceProto.CustomSourceOptions( + configuration=bytes(config_json, encoding="utf8") + ), + ) + + data_source_proto.event_timestamp_column = self.event_timestamp_column + data_source_proto.created_timestamp_column = self.created_timestamp_column + data_source_proto.date_partition_column = self.date_partition_column + + return data_source_proto ``` {% endcode %} -## 4. Using the custom offline store +## 5. Using the custom offline store -After implementing these classes, the custom offline store can be used by referencing it in a feature repo's `feature_store.yaml` file, specifically in the `offline_store` field. The value specified should be the fully qualified class name of the OfflineStore. +After implementing these classes, the custom offline store can be used by referencing it in a feature repo's `feature_store.yaml` file, specifically in the `offline_store` field. The value specified should be the fully qualified class name of the OfflineStore. As long as your OfflineStore class is available in your Python environment, it will be imported by Feast dynamically at runtime. To use our custom file offline store, we can use the following `feature_store.yaml`: -{% code title="feature\_repo/feature\_store.yaml" %} +{% code title="feature_repo/feature_store.yaml" %} ```yaml project: test_custom registry: data/registry.db @@ -171,9 +228,9 @@ offline_store: ``` {% endcode %} -If additional configuration for the offline store is **not** required, then we can omit the other fields and only specify the `type` of the offline store class as the value for the `offline_store`. +If additional configuration for the offline store is **not **required, then we can omit the other fields and only specify the `type` of the offline store class as the value for the `offline_store`. -{% code title="feature\_repo/feature\_store.yaml" %} +{% code title="feature_repo/feature_store.yaml" %} ```yaml project: test_custom registry: data/registry.db @@ -182,3 +239,43 @@ offline_store: feast_custom_offline_store.file.CustomFileOfflineStore ``` {% endcode %} +Finally, the custom data source class can be use in the feature repo to define a data source, and refer to in a feature view definition. + +{% code title="feature_repo/repo.py" %} +```python +pdriver_hourly_stats = CustomFileDataSource( + path="feature_repo/data/driver_stats.parquet", + event_timestamp_column="event_timestamp", + created_timestamp_column="created", +) + + +driver_hourly_stats_view = FeatureView( + batch_source=driver_hourly_stats, + ... +) +``` +{% endcode %} + +## 6. Testing the OfflineStore class + +Even if you have created the `OfflineStore` class in a separate repo, you can still test your implementation against the Feast test suite, as long as you have Feast as a submodule in your repo. In the Feast submodule, we can run all the unit tests with: + +``` +make test +``` + +The universal tests, which are integration tests specifically intended to test offline and online stores, can be run with: + +``` +make test-python-universal +``` + +The unit tests should succeed, but the universal tests will likely fail. The tests are parametrized based on the `FULL_REPO_CONFIGS` variable defined in `sdk/python/tests/integration/feature_repos/repo_configuration.py`. To overwrite these configurations, you can simply create your own file that contains a `FULL_REPO_CONFIGS`, and point Feast to that file by setting the environment variable `FULL_REPO_CONFIGS_MODULE` to point to that file. The main challenge there will be to write a `DataSourceCreator` for the offline store. In this repo, the file that overwrites `FULL_REPO_CONFIGS` is `feast_custom_offline_store/feast_tests.py`, so you would run + +``` +export FULL_REPO_CONFIGS_MODULE='feast_custom_offline_store.feast_tests' +make test-python-universal +``` + +to test the offline store against the Feast universal tests. You should notice that some of the tests actually fail; this indicates that there is a mistake in the implementation of this offline store! diff --git a/docs/how-to-guides/adding-or-reusing-tests.md b/docs/how-to-guides/adding-or-reusing-tests.md index 23ca961e8c7..1730abe2096 100644 --- a/docs/how-to-guides/adding-or-reusing-tests.md +++ b/docs/how-to-guides/adding-or-reusing-tests.md @@ -10,42 +10,42 @@ This guide will go over: ## Test suite overview -Let's inspect the test setup as is: +Let's inspect the test setup in `sdk/python/tests/integration`: ```bash $ tree . ├── e2e -│   └── test_universal_e2e.py +│ └── test_universal_e2e.py ├── feature_repos -│   ├── repo_configuration.py -│   └── universal -│   ├── data_source_creator.py -│   ├── data_sources -│   │   ├── bigquery.py -│   │   ├── file.py -│   │   └── redshift.py -│   ├── entities.py -│   └── feature_views.py +│ ├── repo_configuration.py +│ └── universal +│ ├── data_source_creator.py +│ ├── data_sources +│ │ ├── bigquery.py +│ │ ├── file.py +│ │ └── redshift.py +│ ├── entities.py +│ └── feature_views.py ├── offline_store -│   ├── test_s3_custom_endpoint.py -│   └── test_universal_historical_retrieval.py +│ ├── test_s3_custom_endpoint.py +│ └── test_universal_historical_retrieval.py ├── online_store -│   ├── test_e2e_local.py -│   ├── test_feature_service_read.py -│   ├── test_online_retrieval.py -│   └── test_universal_online.py +│ ├── test_e2e_local.py +│ ├── test_feature_service_read.py +│ ├── test_online_retrieval.py +│ └── test_universal_online.py ├── registration -│   ├── test_cli.py -│   ├── test_cli_apply_duplicated_featureview_names.py -│   ├── test_cli_chdir.py -│   ├── test_feature_service_apply.py -│   ├── test_feature_store.py -│   ├── test_inference.py -│   ├── test_registry.py -│   ├── test_universal_odfv_feature_inference.py -│   └── test_universal_types.py +│ ├── test_cli.py +│ ├── test_cli_apply_duplicated_featureview_names.py +│ ├── test_cli_chdir.py +│ ├── test_feature_service_apply.py +│ ├── test_feature_store.py +│ ├── test_inference.py +│ ├── test_registry.py +│ ├── test_universal_odfv_feature_inference.py +│ └── test_universal_types.py └── scaffolding ├── test_init.py ├── test_partial_apply.py @@ -55,7 +55,7 @@ $ tree 8 directories, 27 files ``` -`feature_repos` has setup files for most tests in the test suite and sets up pytest fixtures for other tests. Crucially, this parametrizes on different offline stores, different online stores, etc and abstracts away store specific implementations so tests don't need to rewrite e.g. uploading dataframes to a specific store for setup. +`feature_repos` has setup files for most tests in the test suite and pytest fixtures for other tests. These fixtures parametrize on different offline stores, online stores, etc. and thus abstract away store specific implementations so tests don't need to rewrite e.g. uploading dataframes to a specific store for setup. ## Understanding an example test @@ -148,26 +148,34 @@ The key fixtures are the `environment` and `universal_data_sources` fixtures, wh ## Writing a new test or reusing existing tests -To: - -* Include a new offline store: - * extend `data_source_creator.py` for your offline store - * in `repo_configuration.py` add a new`IntegrationTestRepoConfig` or two \(depending on how many online stores you want to test\) - * Run the full test suite with `make test-python-integration` - * See more guidelines at [https://github.com/feast-dev/feast/issues/1892](https://github.com/feast-dev/feast/issues/1892) -* Include a new online store: - * in `repo_configuration.py` add a new config that maps to a serialized version of configuration you need in `feature_store.yaml` to setup the online store. - * in `repo_configuration.py`, add new`IntegrationTestRepoConfig` for offline stores you want to test - * Run the full test suite with `make test-python-integration` - * See more guidelines at [https://github.com/feast-dev/feast/issues/1892](https://github.com/feast-dev/feast/issues/1892) -* Add a new test to an existing test file: - * Use the same function signatures as an existing test \(e.g. have environment as an argument\) to include the relevant test fixtures. - * We prefer to expand what an individual test covers due to the cost of standing up offline / online stores -* Using custom data in a new test: - * This is used in several places such as `test_universal_types.py` +### To add a new test to an existing test file -```python +* Use the same function signatures as an existing test (e.g. use `environment` as an argument) to include the relevant test fixtures. +* If possible, expand an individual test instead of writing a new test, due to the cost of standing up offline / online stores. + +### To test a new offline / online store from a plugin repo + +* Install Feast in editable mode with `pip install -e`. +* The core tests for offline / online store behavior are parametrized by the `FULL_REPO_CONFIGS` variable defined in `feature_repos/repo_configuration.py`. To overwrite this variable without modifying the Feast repo, create your own file that contains a `FULL_REPO_CONFIGS` (which will require adding a new `IntegrationTestRepoConfig` or two) and set the environment variable `FULL_REPO_CONFIGS_MODULE` to point to that file. Then the core offline / online store tests can be run with `make test-python-universal`. +* See the [custom offline store demo](https://github.com/feast-dev/feast-custom-offline-store-demo) and the [custom online store demo](https://github.com/feast-dev/feast-custom-online-store-demo) for examples. + +### To include a new offline / online store in the main Feast repo + +* Extend `data_source_creator.py` for your offline store. +* In `repo_configuration.py` add a new`IntegrationTestRepoConfig` or two (depending on how many online stores you want to test). +* Run the full test suite with `make test-python-integration.` + +### To include a new online store +* In `repo_configuration.py` add a new config that maps to a serialized version of configuration you need in `feature_store.yaml` to setup the online store. +* In `repo_configuration.py`, add new`IntegrationTestRepoConfig` for offline stores you want to test. +* Run the full test suite with `make test-python-integration` + +### To use custom data in a new test + +* Check `test_universal_types.py` for an example of how to do this. + +```python @pytest.mark.integration def your_test(environment: Environment): df = #...# @@ -180,7 +188,4 @@ def your_test(environment: Environment): fs.apply([fv, entity]) # ... run test - - environment.data_source_creator.teardown() ``` - diff --git a/docs/how-to-guides/adding-support-for-a-new-online-store.md b/docs/how-to-guides/adding-support-for-a-new-online-store.md index f0f3192c955..fee47945bf0 100644 --- a/docs/how-to-guides/adding-support-for-a-new-online-store.md +++ b/docs/how-to-guides/adding-support-for-a-new-online-store.md @@ -2,7 +2,7 @@ ## Overview -Feast makes adding support for a new online store \(database\) easy. Developers can simply implement the [OnlineStore](https://github.com/feast-dev/feast/blob/master/sdk/python/feast/infra/online_stores/online_store.py#L26) interface to add support for a new store \(other than the existing stores like Redis, DynamoDB, SQLite, and Datastore\). +Feast makes adding support for a new online store (database) easy. Developers can simply implement the [OnlineStore](../../sdk/python/feast/infra/online\_stores/online\_store.py#L26) interface to add support for a new store (other than the existing stores like Redis, DynamoDB, SQLite, and Datastore). In this guide, we will show you how to integrate with MySQL as an online store. While we will be implementing a specific store, this guide should be representative for adding support for any new online store. @@ -13,17 +13,18 @@ The process of using a custom online store consists of 3 steps: 1. Defining the `OnlineStore` class. 2. Defining the `OnlineStoreConfig` class. 3. Referencing the `OnlineStore` in a feature repo's `feature_store.yaml` file. +4. Testing the `OnlineStore` class. ## 1. Defining an OnlineStore class {% hint style="info" %} - OnlineStore class names must end with the OnlineStore suffix! + OnlineStore class names must end with the OnlineStore suffix! {% endhint %} The OnlineStore class broadly contains two sets of methods * One set deals with managing infrastructure that the online store needed for operations -* One set deals with writing data into the store, and reading data from the store. +* One set deals with writing data into the store, and reading data from the store. ### 1.1 Infrastructure Methods @@ -31,142 +32,137 @@ There are two methods that deal with managing infrastructure for online stores, * `update` is invoked when users run `feast apply` as a CLI command, or the `FeatureStore.apply()` sdk method. -The `update` method should be used to perform any operations necessary before data can be written to or read from the store. The `update` method can be used to create MySQL tables in preparation for reads and writes to new feature views. +The `update` method should be used to perform any operations necessary before data can be written to or read from the store. The `update` method can be used to create MySQL tables in preparation for reads and writes to new feature views. * `teardown` is invoked when users run `feast teardown` or `FeatureStore.teardown()`. -The `teardown` method should be used to perform any clean-up operations. `teardown` can be used to drop MySQL indices and tables corresponding to the feature views being deleted. +The `teardown` method should be used to perform any clean-up operations. `teardown` can be used to drop MySQL indices and tables corresponding to the feature views being deleted. -{% code title="feast\_custom\_online\_store/mysql.py" %} +{% code title="feast_custom_online_store/mysql.py" %} ```python +def update( + self, + config: RepoConfig, + tables_to_delete: Sequence[Union[FeatureTable, FeatureView]], + tables_to_keep: Sequence[Union[FeatureTable, FeatureView]], + entities_to_delete: Sequence[Entity], + entities_to_keep: Sequence[Entity], + partial: bool, +): + """ + An example of creating manging the tables needed for a mysql-backed online store. + """ + conn = self._get_conn(config) + cur = conn.cursor(buffered=True) + + project = config.project + + for table in tables_to_keep: + cur.execute( + f"CREATE TABLE IF NOT EXISTS {_table_id(project, table)} (entity_key VARCHAR(512), feature_name VARCHAR(256), value BLOB, event_ts timestamp, created_ts timestamp, PRIMARY KEY(entity_key, feature_name))" + ) + cur.execute( + f"CREATE INDEX {_table_id(project, table)}_ek ON {_table_id(project, table)} (entity_key);" + ) - def update( - self, - config: RepoConfig, - tables_to_delete: Sequence[Union[FeatureTable, FeatureView]], - tables_to_keep: Sequence[Union[FeatureTable, FeatureView]], - entities_to_delete: Sequence[Entity], - entities_to_keep: Sequence[Entity], - partial: bool, - ): - """ - An example of creating manging the tables needed for a mysql-backed online store. - """ - conn = self._get_conn(config) - cur = conn.cursor(buffered=True) - - project = config.project - - for table in tables_to_keep: - cur.execute( - f"CREATE TABLE IF NOT EXISTS {_table_id(project, table)} (entity_key VARCHAR(512), feature_name VARCHAR(256), value BLOB, event_ts timestamp, created_ts timestamp, PRIMARY KEY(entity_key, feature_name))" - ) - cur.execute( - f"CREATE INDEX {_table_id(project, table)}_ek ON {_table_id(project, table)} (entity_key);" - ) - - for table in tables_to_delete: - cur.execute( - f"DROP INDEX {_table_id(project, table)}_ek ON {_table_id(project, table)};" - ) - cur.execute(f"DROP TABLE IF EXISTS {_table_id(project, table)}") - - - def teardown( - self, - config: RepoConfig, - tables: Sequence[Union[FeatureTable, FeatureView]], - entities: Sequence[Entity], - ): - """ - - """ - conn = self._get_conn(config) - cur = conn.cursor(buffered=True) - project = config.project - - for table in tables: - cur.execute( - f"DROP INDEX {_table_id(project, table)}_ek ON {_table_id(project, table)};" - ) - cur.execute(f"DROP TABLE IF EXISTS {_table_id(project, table)}") - - + for table in tables_to_delete: + cur.execute( + f"DROP INDEX {_table_id(project, table)}_ek ON {_table_id(project, table)};" + ) + cur.execute(f"DROP TABLE IF EXISTS {_table_id(project, table)}") + + +def teardown( + self, + config: RepoConfig, + tables: Sequence[Union[FeatureTable, FeatureView]], + entities: Sequence[Entity], +): + """ + + """ + conn = self._get_conn(config) + cur = conn.cursor(buffered=True) + project = config.project + + for table in tables: + cur.execute( + f"DROP INDEX {_table_id(project, table)}_ek ON {_table_id(project, table)};" + ) + cur.execute(f"DROP TABLE IF EXISTS {_table_id(project, table)}") ``` {% endcode %} ### 1.2 Read/Write Methods -There are two methods that deal with writing data to and from the online stores.`online_write_batch` and `online_read`. +There are two methods that deal with writing data to and from the online stores.`online_write_batch `and `online_read`. -* `online_write_batch` is invoked when running materialization \(using the `feast materialize` or `feast materialize-incremental` commands, or the corresponding `FeatureStore.materialize()` method. -* `online_read` is invoked when reading values from the online store using the `FeatureStore.get_online_features()` method. +* `online_write_batch `is invoked when running materialization (using the `feast materialize` or `feast materialize-incremental` commands, or the corresponding `FeatureStore.materialize()` method. +* `online_read `is invoked when reading values from the online store using the `FeatureStore.get_online_features()` method. -{% code title="feast\_custom\_online\_store/mysql.py" %} +{% code title="feast_custom_online_store/mysql.py" %} ```python - def online_write_batch( - self, - config: RepoConfig, - table: Union[FeatureTable, FeatureView], - data: List[ - Tuple[EntityKeyProto, Dict[str, ValueProto], datetime, Optional[datetime]] - ], - progress: Optional[Callable[[int], Any]], - ) -> None: - conn = self._get_conn(config) - cur = conn.cursor(buffered=True) - - project = config.project - - for entity_key, values, timestamp, created_ts in data: - entity_key_bin = serialize_entity_key(entity_key).hex() - timestamp = _to_naive_utc(timestamp) - if created_ts is not None: - created_ts = _to_naive_utc(created_ts) - - for feature_name, val in values.items(): - self.write_to_table(created_ts, cur, entity_key_bin, feature_name, project, table, timestamp, val) - self._conn.commit() - if progress: - progress(1) - - - def online_read( - self, - config: RepoConfig, - table: Union[FeatureTable, FeatureView], - entity_keys: List[EntityKeyProto], - requested_features: Optional[List[str]] = None, - ) -> List[Tuple[Optional[datetime], Optional[Dict[str, ValueProto]]]]: - conn = self._get_conn(config) - cur = conn.cursor(buffered=True) - - result: List[Tuple[Optional[datetime], Optional[Dict[str, ValueProto]]]] = [] - - project = config.project - for entity_key in entity_keys: - entity_key_bin = serialize_entity_key(entity_key).hex() - print(f"entity_key_bin: {entity_key_bin}") - - cur.execute( - f"SELECT feature_name, value, event_ts FROM {_table_id(project, table)} WHERE entity_key = %s", - (entity_key_bin,), - ) - - res = {} - res_ts = None - for feature_name, val_bin, ts in cur.fetchall(): - val = ValueProto() - val.ParseFromString(val_bin) - res[feature_name] = val - res_ts = ts - - if not res: - result.append((None, None)) - else: - result.append((res_ts, res)) - return result +def online_write_batch( + self, + config: RepoConfig, + table: Union[FeatureTable, FeatureView], + data: List[ + Tuple[EntityKeyProto, Dict[str, ValueProto], datetime, Optional[datetime]] + ], + progress: Optional[Callable[[int], Any]], +) -> None: + conn = self._get_conn(config) + cur = conn.cursor(buffered=True) + + project = config.project + + for entity_key, values, timestamp, created_ts in data: + entity_key_bin = serialize_entity_key(entity_key).hex() + timestamp = _to_naive_utc(timestamp) + if created_ts is not None: + created_ts = _to_naive_utc(created_ts) + + for feature_name, val in values.items(): + self.write_to_table(created_ts, cur, entity_key_bin, feature_name, project, table, timestamp, val) + self._conn.commit() + if progress: + progress(1) + +def online_read( + self, + config: RepoConfig, + table: Union[FeatureTable, FeatureView], + entity_keys: List[EntityKeyProto], + requested_features: Optional[List[str]] = None, +) -> List[Tuple[Optional[datetime], Optional[Dict[str, ValueProto]]]]: + conn = self._get_conn(config) + cur = conn.cursor(buffered=True) + + result: List[Tuple[Optional[datetime], Optional[Dict[str, ValueProto]]]] = [] + + project = config.project + for entity_key in entity_keys: + entity_key_bin = serialize_entity_key(entity_key).hex() + print(f"entity_key_bin: {entity_key_bin}") + + cur.execute( + f"SELECT feature_name, value, event_ts FROM {_table_id(project, table)} WHERE entity_key = %s", + (entity_key_bin,), + ) + res = {} + res_ts = None + for feature_name, val_bin, ts in cur.fetchall(): + val = ValueProto() + val.ParseFromString(val_bin) + res[feature_name] = val + res_ts = ts + + if not res: + result.append((None, None)) + else: + result.append((res_ts, res)) + return result ``` {% endcode %} @@ -174,17 +170,17 @@ There are two methods that deal with writing data to and from the online stores. Additional configuration may be needed to allow the OnlineStore to talk to the backing store. For example, MySQL may need configuration information like the host at which the MySQL instance is running, credentials for connecting to the database, etc. -To facilitate configuration, all OnlineStore implementations are **required** to also define a corresponding OnlineStoreConfig class in the same file. This OnlineStoreConfig class should inherit from the `FeastConfigBaseModel` class, which is defined [here](https://github.com/feast-dev/feast/blob/master/sdk/python/feast/repo_config.py#L44). +To facilitate configuration, all OnlineStore implementations are **required** to also define a corresponding OnlineStoreConfig class in the same file. This OnlineStoreConfig class should inherit from the `FeastConfigBaseModel` class, which is defined [here](../../sdk/python/feast/repo\_config.py#L44). -The `FeastConfigBaseModel` is a [pydantic](https://pydantic-docs.helpmanual.io/) class, which parses yaml configuration into python objects. Pydantic also allows the model classes to define validators for the config classes, to make sure that the config classes are correctly defined. +The `FeastConfigBaseModel` is a [pydantic](https://pydantic-docs.helpmanual.io) class, which parses yaml configuration into python objects. Pydantic also allows the model classes to define validators for the config classes, to make sure that the config classes are correctly defined. -This config class **must** container a `type` field, which contains the fully qualified class name of its corresponding OnlineStore class. +This config class **must** container a `type` field, which contains the fully qualified class name of its corresponding OnlineStore class. Additionally, the name of the config class must be the same as the OnlineStore class, with the `Config` suffix. An example of the config class for MySQL : -{% code title="feast\_custom\_online\_store/mysql.py" %} +{% code title="feast_custom_online_store/mysql.py" %} ```python class MySQLOnlineStoreConfig(FeastConfigBaseModel): type: Literal["feast_custom_online_store.mysql.MySQLOnlineStore"] = "feast_custom_online_store.mysql.MySQLOnlineStore" @@ -198,52 +194,51 @@ class MySQLOnlineStoreConfig(FeastConfigBaseModel): This configuration can be specified in the `feature_store.yaml` as follows: -{% code title="feature\_repo/feature\_store.yaml" %} +{% code title="feature_repo/feature_store.yaml" %} ```yaml +online_store: type: feast_custom_online_store.mysql.MySQLOnlineStore user: foo password: bar ``` {% endcode %} -This configuration information is available to the methods of the OnlineStore, via the`config: RepoConfig` parameter which is passed into all the methods of the OnlineStore interface, specifically at the `config.online_store` field of the `config` parameter. +This configuration information is available to the methods of the OnlineStore, via the`config: RepoConfig` parameter which is passed into all the methods of the OnlineStore interface, specifically at the `config.online_store` field of the `config` parameter. -{% code title="feast\_custom\_online\_store/mysql.py" %} +{% code title="feast_custom_online_store/mysql.py" %} ```python - def online_write_batch( - self, - config: RepoConfig, - table: Union[FeatureTable, FeatureView], - data: List[ - Tuple[EntityKeyProto, Dict[str, ValueProto], datetime, Optional[datetime]] - ], - progress: Optional[Callable[[int], Any]], - ) -> None: - - online_store_config = config.online_store - assert isinstance(online_store_config, MySQLOnlineStoreConfig) - - connection = mysql.connector.connect( - host=online_store_config.host or "127.0.0.1", - user=online_store_config.user or "root", - password=online_store_config.password, - database=online_store_config.database or "feast", - autocommit=True - ) - - +def online_write_batch( + self, + config: RepoConfig, + table: Union[FeatureTable, FeatureView], + data: List[ + Tuple[EntityKeyProto, Dict[str, ValueProto], datetime, Optional[datetime]] + ], + progress: Optional[Callable[[int], Any]], +) -> None: + + online_store_config = config.online_store + assert isinstance(online_store_config, MySQLOnlineStoreConfig) + + connection = mysql.connector.connect( + host=online_store_config.host or "127.0.0.1", + user=online_store_config.user or "root", + password=online_store_config.password, + database=online_store_config.database or "feast", + autocommit=True + ) ``` {% endcode %} -## 3. Using the custom online store +## 3. Using the custom online store -After implementing both these classes, the custom online store can be used by referencing it in a feature repo's `feature_store.yaml` file, specifically in the `online_store` field. The value specified should be the fully qualified class name of the OnlineStore. +After implementing both these classes, the custom online store can be used by referencing it in a feature repo's `feature_store.yaml` file, specifically in the `online_store` field. The value specified should be the fully qualified class name of the OnlineStore. As long as your OnlineStore class is available in your Python environment, it will be imported by Feast dynamically at runtime. To use our MySQL online store, we can use the following `feature_store.yaml`: -{% code title="feature\_repo/feature\_store.yaml" %} +{% code title="feature_repo/feature_store.yaml" %} ```yaml project: test_custom registry: data/registry.db @@ -255,15 +250,36 @@ online_store: ``` {% endcode %} -If additional configuration for the online store is **not** required, then we can omit the other fields and only specify the `type` of the online store class as the value for the `online_store`. +If additional configuration for the online store is **not **required, then we can omit the other fields and only specify the `type` of the online store class as the value for the `online_store`. -{% code title="feature\_repo/feature\_store.yaml" %} +{% code title="feature_repo/feature_store.yaml" %} ```yaml project: test_custom registry: data/registry.db provider: local online_store: feast_custom_online_store.mysql.MySQLOnlineStore - ``` {% endcode %} +## 4. Testing the OnlineStore class + +Even if you have created the `OnlineStore` class in a separate repo, you can still test your implementation against the Feast test suite, as long as you have Feast as a submodule in your repo. In the Feast submodule, we can run all the unit tests with: + +``` +make test +``` + +The universal tests, which are integration tests specifically intended to test offline and online stores, can be run with: + +``` +make test-python-universal +``` + +The unit tests should succeed, but the universal tests will likely fail. The tests are parametrized based on the `FULL_REPO_CONFIGS` variable defined in `sdk/python/tests/integration/feature_repos/repo_configuration.py`. To overwrite these configurations, you can simply create your own file that contains a `FULL_REPO_CONFIGS`, and point Feast to that file by setting the environment variable `FULL_REPO_CONFIGS_MODULE` to point to that file. In this repo, the file that overwrites `FULL_REPO_CONFIGS` is `feast_custom_online_store/feast_tests.py`, so you would run + +``` +export FULL_REPO_CONFIGS_MODULE='feast_custom_online_store.feast_tests' +make test-python-universal +``` + +to test the MySQL online store against the Feast universal tests. You should notice that some of the tests actually fail; this indicates that there is a mistake in the implementation of this online store! diff --git a/docs/how-to-guides/feast-gcp-aws/install-feast.md b/docs/how-to-guides/feast-gcp-aws/install-feast.md index 384da66f437..019231be095 100644 --- a/docs/how-to-guides/feast-gcp-aws/install-feast.md +++ b/docs/how-to-guides/feast-gcp-aws/install-feast.md @@ -2,19 +2,24 @@ Install Feast using [pip](https://pip.pypa.io): -```text +``` pip install feast ``` -Install Feast with GCP dependencies \(required when using BigQuery or Firestore\): +Install Feast with GCP dependencies (required when using BigQuery or Firestore): -```text +``` pip install 'feast[gcp]' ``` -Install Feast with AWS dependencies \(required when using Redshift or DynamoDB\): +Install Feast with AWS dependencies (required when using Redshift or DynamoDB): -```text +``` pip install 'feast[aws]' ``` +Install Feast with Redis dependencies (required when using Redis, either through AWS Elasticache or independently): + +``` +pip install 'feast[redis]' +``` diff --git a/docs/project/feast-0.9-vs-feast-0.10+.md b/docs/project/feast-0.9-vs-feast-0.10+.md index aac64cf6340..ed2f4ee336f 100644 --- a/docs/project/feast-0.9-vs-feast-0.10+.md +++ b/docs/project/feast-0.9-vs-feast-0.10+.md @@ -8,196 +8,58 @@ Please see [Upgrading from Feast 0.9](https://docs.google.com/document/d/1AOsr_b ### Changes introduced in Feast 0.10 -Feast contributors identified various [design challenges](https://feast.dev/blog/a-state-of-feast/) in Feast 0.9 that made deploying, operating, extending, and maintaining it challenging. These challenges applied both to users and contributors. - +Feast contributors identified various [design challenges](https://feast.dev/blog/a-state-of-feast/) in Feast 0.9 that made deploying, operating, extending, and maintaining it challenging. These challenges applied both to users and contributors. \ +\ Our goal is to make ML practitioners immediately productive in operationalizing data for machine learning. To that end, Feast 0.10+ made the following improvements on Feast 0.9: - - - - - - - - - - - - - - - - - - - - - - - - - -
Challenges in Feast 0.9 (Before) - Changed in Feast 0.10+ (After)
Hard to install because it was a heavy-weight system with many components - requiring a lot of configuration -
    -
  • Easy to install via pip install -
    • -
  • Opinionated default configurations
    • -
  • No Helm charts necessary
    • -
-
Engineering support needed to deploy/operate reliably -

-
    -
  • Feast moves from a stack of services to a CLI/SDK
    • -
  • No need for Kubernetes or Spark
    • -
  • No long running processes or orchestrators
    • -
  • Leverages globally available managed services where possible
    • -
-
Hard to develop/debug with tightly coupled components, async operations, - and hard to debug components like Spark -
    -
  • Easy to develop and debug
    • -
  • Modular components
    • -
  • Clear extension points
    • -
  • Fewer background operations
    • -
  • Faster feedback
    • -
  • Local mode
    • -
-
Inability to benefit from cloud-native technologies because of focus on - reusable technologies like Kubernetes and Spark -

<b></b>

-
    -
  • Leverages best-in-class cloud technologies so users can enjoy scalable - + powerful tech stacks without managing open source stacks themselves
    • -
-
+| Challenges in Feast 0.9** (Before)** | Changed in Feast 0.10+ (After) | +| ----------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| Hard to install because it was a heavy-weight system with many components requiring a lot of configuration | | +| Engineering support needed to deploy/operate reliably |

| +| Hard to develop/debug with tightly coupled components, async operations, and hard to debug components like Spark | | +| Inability to benefit from cloud-native technologies because of focus on reusable technologies like Kubernetes and Spark |

| ### Changes in more detail Where Feast 0.9 was a large stack of components that needed to be deployed to Kubernetes, Feast 0.10 is simply a lightweight SDK and CLI. It doesn’t need any long-running processes to operate. This SDK/CLI can deploy and configure your feature store to your infrastructure, and execute workflows like building training datasets or reading features from an online feature store. -* **Feast 0.10 introduces local mode:** Local mode allows users to try out Feast in a completely local environment \(without using any cloud technologies\). This provides users with a responsive means of trying out the software before deploying it into a production environment. -* **Feast comes with opinionated defaults:** As much as possible we are attempting to make Feast a batteries-included feature store that removes the need for users to configure infinite configuration options \(as with Feast 0.9\). Feast 0.10 comes with sane default configuration options to deploy Feast on your infrastructure. -* **Feast Core was replaced by a file-based \(S3, GCS\) registry:** Feast Core is a metadata server that maintains and exposes an API of feature definitions. With Feast 0.10, we’ve moved this entire service into a single flat file that can be stored on either the local disk or in a central object store like S3 or GCS. The benefit of this change is that users don’t need to maintain a database and a registry service, yet they can still access all the metadata they had before. +* **Feast 0.10 introduces local mode:** Local mode allows users to try out Feast in a completely local environment (without using any cloud technologies). This provides users with a responsive means of trying out the software before deploying it into a production environment. +* **Feast comes with opinionated defaults:** As much as possible we are attempting to make Feast a batteries-included feature store that removes the need for users to configure infinite configuration options (as with Feast 0.9). Feast 0.10 comes with sane default configuration options to deploy Feast on your infrastructure. +* **Feast Core was replaced by a file-based (S3, GCS) registry:** Feast Core is a metadata server that maintains and exposes an API of feature definitions. With Feast 0.10, we’ve moved this entire service into a single flat file that can be stored on either the local disk or in a central object store like S3 or GCS. The benefit of this change is that users don’t need to maintain a database and a registry service, yet they can still access all the metadata they had before. * **Materialization is a CLI operation:** Instead of having ingestion jobs be managed by a job service, users can now schedule a batch ingestion job themselves by calling “materialize”. This change was introduced because most teams already have schedulers like Airflow in their organization. By starting ingestion jobs from Airflow, teams are now able to easily track state outside of Feast and to debug failures synchronously. Similarly, streaming ingestion jobs can be launched through the “apply” command -* **Doubling down on data warehouses:** Most modern data teams are doubling down on data warehouses like BigQuery, Snowflake, and Redshift. Feast doubles down on these big data technologies as the primary interfaces through which it launches batch operations \(like training dataset generation\). This reduces the development burden on Feast contributors \(since they only need to reason about SQL\), provides users with a more responsive experience, avoids moving data from the warehouse \(to compute joins using Spark\), and provides a more serverless and scalable experience to users. +* **Doubling down on data warehouses:** Most modern data teams are doubling down on data warehouses like BigQuery, Snowflake, and Redshift. Feast doubles down on these big data technologies as the primary interfaces through which it launches batch operations (like training dataset generation). This reduces the development burden on Feast contributors (since they only need to reason about SQL), provides users with a more responsive experience, avoids moving data from the warehouse (to compute joins using Spark), and provides a more serverless and scalable experience to users. * **Temporary loss of streaming support:** Unfortunately, Feast 0.10, 0.11, and 0.12 do not support streaming feature ingestion out of the box. It is entirely possible to launch streaming ingestion jobs using these Feast versions, but it requires the use of a Feast extension point to launch these ingestion jobs. It is still a core design goal for Feast to support streaming ingestion, so this change is in the development backlog for the Feast project. -* **Addition of extension points:** Feast 0.10+ introduces various extension points. Teams can override all feature store behavior by writing \(or extending\) a provider. It is also possible for teams to add their own data storage connectors for both an offline and online store using a plugin interface that Feast provides. +* **Addition of extension points: **Feast 0.10+ introduces various extension points. Teams can override all feature store behavior by writing (or extending) a provider. It is also possible for teams to add their own data storage connectors for both an offline and online store using a plugin interface that Feast provides. ### Comparison of architectures #### Feast 0.9 -![](../.gitbook/assets/image%20%289%29.png) +![](<../.gitbook/assets/image (9).png>) #### Feast 0.10, 0.11, and 0.12 architecture -![](../.gitbook/assets/image%20%2819%29.png) +![](<../.gitbook/assets/image (19).png>) -#### Feast 1.0 architecture \(eventual goal\) +#### Feast 1.0 architecture (eventual goal) -![](../.gitbook/assets/image%20%2821%29.png) +![](<../.gitbook/assets/image (21).png>) ### Comparison of components - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
ComponentFeast 0.9Feast 0.10, 011, 0.12+
Architecture - -
    -
  • Service-oriented architecture
    • -
  • Containers and services deployed to Kubernetes
    • -
- 		-
    -
  • SDK/CLI centric software
    • -
  • Feast is able to deploy or configure infrastructure for use as a feature - store
    • -
-
Installation - Terraform and Helm -
    -
  • Pip to install SDK/CLI
    • -
  • Provider used to deploy Feast components to GCP, AWS, or other environments - during apply -
    • -
-
Required infrastructure - Kubernetes, Postgres, Spark, Docker, Object StoreNone
Batch compute - Yes (Spark based) -
    -
  • Python native (client-side) for batch data loading
    • -
  • Data warehouse for batch compute
    • -
-
Streaming support - Yes (Spark based)Planned. Streaming jobs will be launched using apply -
Offline store - None (can source data from any source Spark supports)BigQuery, Snowflake (planned), Redshift, or custom implementations
Online store - RedisDynamoDB, Firestore, Redis, and more planned.
Job Manager - YesNo
Registry - gRPC service with Postgres backendFile-based registry with accompanying SDK for exploration
Local Mode - NoYes
+| Component | Feast 0.9 | Feast 0.10, 011, 0.12+ | +| --------------------------- | ------------------------------------------------------------------------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| **Architecture** | | | +| **Installation** | Terraform and Helm | | +| **Required infrastructure** | Kubernetes, Postgres, Spark, Docker, Object Store | None | +| **Batch compute** | Yes (Spark based) | | +| **Streaming support** | Yes (Spark based) | Planned. Streaming jobs will be launched using _**apply**_ | +| **Offline store** | None (can source data from any source Spark supports) | BigQuery, Snowflake (planned), Redshift, or custom implementations | +| **Online store** | Redis | DynamoDB, Firestore, Redis, and more planned. | +| **Job Manager** | Yes | No | +| **Registry** | gRPC service with Postgres backend | File-based registry with accompanying SDK for exploration | +| **Local Mode** | No | Yes | ### Upgrading from Feast 0.9 to the latest Feast Please see the [Feast 0.9 Upgrade Guide](https://docs.google.com/document/d/1AOsr_baczuARjCpmZgVd8mCqTF4AZ49OEyU4Cn-uTT0/edit#). - diff --git a/docs/project/release-process.md b/docs/project/release-process.md index 2d3b21fc3bd..8ecd55a63f4 100644 --- a/docs/project/release-process.md +++ b/docs/project/release-process.md @@ -5,7 +5,7 @@ For Feast maintainers, these are the concrete steps for making a new release. 1. For new major or minor release, create and check out the release branch for the new stream, e.g. `v0.6-branch`. For a patch version, check out the stream's release branch. -2. Update the [CHANGELOG.md](https://github.com/feast-dev/feast/blob/master/CHANGELOG.md). See the [Creating a change log](release-process.md#creating-a-change-log) guide and commit +2. Update the [CHANGELOG.md](../../CHANGELOG.md). See the [Creating a change log](release-process.md#creating-a-change-log) guide and commit * Make to review each PR in the changelog to [flag any breaking changes and deprecation.](release-process.md#flag-breaking-changes-and-deprecations) 3. Update versions for the release/release candidate with a commit: 1. In the root `pom.xml`, remove `-SNAPSHOT` from the `` property, update versions, and commit. @@ -21,21 +21,21 @@ For Feast maintainers, these are the concrete steps for making a new release. 1. Bump to the next major/minor version and append `-SNAPSHOT` . 2. Add the change log by applying the change log commit created in step 2. 3. Check that versions are updated with `env TARGET_MERGE_BRANCH=master make lint-versions` -7. Create a [GitHub release](https://github.com/feast-dev/feast/releases) which includes a summary of im~~p~~ortant changes as well as any artifacts associated with the release. Make sure to include the same change log as added in [CHANGELOG.md](https://github.com/feast-dev/feast/blob/master/CHANGELOG.md). Use `Feast vX.Y.Z` as the title. -8. Update the[ Upgrade Guide]() to include the action required instructions for users to upgrade to this new release. Instructions should include a migration for each breaking change made to this release. +7. Create a [GitHub release](https://github.com/feast-dev/feast/releases) which includes a summary of im~~p~~ortant changes as well as any artifacts associated with the release. Make sure to include the same change log as added in [CHANGELOG.md](../../CHANGELOG.md). Use `Feast vX.Y.Z` as the title. +8. Update the[ Upgrade Guide](broken-reference) to include the action required instructions for users to upgrade to this new release. Instructions should include a migration for each breaking change made to this release. -When a tag that matches a Semantic Version string is pushed, CI will automatically build and push the relevant artifacts to their repositories or package managers \(docker images, Python wheels, etc\). JVM artifacts are promoted from Sonatype OSSRH to Maven Central, but it sometimes takes some time for them to be available. The `sdk/go/v tag` is required to version the Go SDK go module so that users can go get a specific tagged release of the Go SDK. +When a tag that matches a Semantic Version string is pushed, CI will automatically build and push the relevant artifacts to their repositories or package managers (docker images, Python wheels, etc). JVM artifacts are promoted from Sonatype OSSRH to Maven Central, but it sometimes takes some time for them to be available. The `sdk/go/v tag` is required to version the Go SDK go module so that users can go get a specific tagged release of the Go SDK. ### Creating a change log We use an [open source change log generator](https://hub.docker.com/r/ferrarimarco/github-changelog-generator/) to generate change logs. The process still requires a little bit of manual effort. -1. Create a GitHub token as [per these instructions](https://github.com/github-changelog-generator/github-changelog-generator#github-token). The token is used as an input argument \(`-t`\) to the change log generator. -2. The change log generator configuration below will look for unreleased changes on a specific branch. The branch will be `master` for a major/minor release, or a release branch \(`v0.4-branch`\) for a patch release. You will need to set the branch using the `--release-branch` argument. +1. Create a GitHub token as [per these instructions](https://github.com/github-changelog-generator/github-changelog-generator#github-token). The token is used as an input argument (`-t`) to the change log generator. +2. The change log generator configuration below will look for unreleased changes on a specific branch. The branch will be `master` for a major/minor release, or a release branch (`v0.4-branch`) for a patch release. You will need to set the branch using the `--release-branch` argument. 3. You should also set the `--future-release` argument. This is the version you are releasing. The version can still be changed at a later date. 4. Update the arguments below and run the command to generate the change log to the console. -```text +``` docker run -it --rm ferrarimarco/github-changelog-generator \ --user feast-dev \ --project feast \ @@ -52,9 +52,9 @@ docker run -it --rm ferrarimarco/github-changelog-generator \ ``` 1. Review each change log item. - * Make sure that sentences are grammatically correct and well formatted \(although we will try to enforce this at the PR review stage\). + * Make sure that sentences are grammatically correct and well formatted (although we will try to enforce this at the PR review stage). * Make sure that each item is categorised correctly. You will see the following categories: `Breaking changes`, `Implemented enhancements`, `Fixed bugs`, and `Merged pull requests`. Any unlabelled PRs will be found in `Merged pull requests`. It's important to make sure that any `breaking changes`, `enhancements`, or `bug fixes` are pulled up out of `merged pull requests` into the correct category. Housekeeping, tech debt clearing, infra changes, or refactoring do not count as `enhancements`. Only enhancements a user benefits from should be listed in that category. - * Make sure that the "Full Change log" link is actually comparing the correct tags \(normally your released version against the previously version\). + * Make sure that the "Full Change log" link is actually comparing the correct tags (normally your released version against the previously version). * Make sure that release notes and breaking changes are present. ### Flag Breaking Changes & Deprecations @@ -62,4 +62,3 @@ docker run -it --rm ferrarimarco/github-changelog-generator \ It's important to flag breaking changes and deprecation to the API for each release so that we can maintain API compatibility. Developers should have flagged PRs with breaking changes with the `compat/breaking` label. However, it's important to double check each PR's release notes and contents for changes that will break API compatibility and manually label `compat/breaking` to PRs with undeclared breaking changes. The change log will have to be regenerated if any new labels have to be added. - diff --git a/docs/reference/alpha-aws-lambda-feature-server.md b/docs/reference/alpha-aws-lambda-feature-server.md index 3b5025cc2c4..ee08931e6c2 100644 --- a/docs/reference/alpha-aws-lambda-feature-server.md +++ b/docs/reference/alpha-aws-lambda-feature-server.md @@ -37,6 +37,77 @@ feature_server: If enabled, the feature server will be deployed during `feast apply`. After it is deployed, the `feast endpoint` CLI command will indicate the server's endpoint. +## Permissions + +Feast requires the following permissions in order to deploy and teardown AWS Lambda feature server: + +| Permissions | Resources | +| ----------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------- | +|

lambda:CreateFunction

lambda:GetFunction

lambda:DeleteFunction

lambda:AddPermission

lambda:UpdateFunctionConfiguration

| arn:aws:lambda:\:\:function:feast-\* | +|

ecr:CreateRepository

ecr:DescribeRepositories

ecr:DeleteRepository

ecr:PutImage

ecr:DescribeImages

ecr:BatchDeleteImage

ecr:CompleteLayerUpload

ecr:UploadLayerPart

ecr:InitiateLayerUpload

ecr:BatchCheckLayerAvailability

ecr:GetDownloadUrlForLayer

ecr:GetRepositoryPolicy

ecr:SetRepositoryPolicy

ecr:GetAuthorizationToken

| \* | +|

iam:PassRole

| arn:aws:iam::\:role/ | +|

apigateway:*

|

arn:aws:apigateway:*::/apis/*/routes/*/routeresponses

arn:aws:apigateway:*::/apis/*/routes/*/routeresponses/*

arn:aws:apigateway:*::/apis/*/routes/*

arn:aws:apigateway:*::/apis/*/routes

arn:aws:apigateway:*::/apis/*/integrations

arn:aws:apigateway:*::/apis/*/stages/*/routesettings/*

arn:aws:apigateway:*::/apis/*

arn:aws:apigateway:*::/apis

| + +The following inline policy can be used to grant Feast the necessary permissions: + +```javascript +{ + "Statement": [ + { + Action = [ + "lambda:CreateFunction", + "lambda:GetFunction", + "lambda:DeleteFunction", + "lambda:AddPermission", + "lambda:UpdateFunctionConfiguration", + ] + Effect = "Allow" + Resource = "arn:aws:lambda:::function:feast-*" + }, + { + Action = [ + "ecr:CreateRepository", + "ecr:DescribeRepositories", + "ecr:DeleteRepository", + "ecr:PutImage", + "ecr:DescribeImages", + "ecr:BatchDeleteImage", + "ecr:CompleteLayerUpload", + "ecr:UploadLayerPart", + "ecr:InitiateLayerUpload", + "ecr:BatchCheckLayerAvailability", + "ecr:GetDownloadUrlForLayer", + "ecr:GetRepositoryPolicy", + "ecr:SetRepositoryPolicy", + "ecr:GetAuthorizationToken" + ] + Effect = "Allow" + Resource = "*" + }, + { + Action = "iam:PassRole" + Effect = "Allow" + Resource = "arn:aws:iam:::role/" + }, + { + Effect = "Allow" + Action = "apigateway:*" + Resource = [ + "arn:aws:apigateway:*::/apis/*/routes/*/routeresponses", + "arn:aws:apigateway:*::/apis/*/routes/*/routeresponses/*", + "arn:aws:apigateway:*::/apis/*/routes/*", + "arn:aws:apigateway:*::/apis/*/routes", + "arn:aws:apigateway:*::/apis/*/integrations", + "arn:aws:apigateway:*::/apis/*/stages/*/routesettings/*", + "arn:aws:apigateway:*::/apis/*", + "arn:aws:apigateway:*::/apis", + ] + }, + ], + "Version": "2012-10-17" +} +``` + ## Example After `feature_store.yaml` has been modified as described in the previous section, it can be deployed as follows: diff --git a/docs/reference/alpha-stream-ingestion.md b/docs/reference/alpha-stream-ingestion.md new file mode 100644 index 00000000000..3d2498a8c38 --- /dev/null +++ b/docs/reference/alpha-stream-ingestion.md @@ -0,0 +1,45 @@ +# \[Alpha] Stream ingestion + +**Warning**: This is an _experimental_ feature. It's intended for early testing and feedback, and could change without warnings in future releases. + +{% hint style="info" %} +To enable this feature, run **`feast alpha enable direct_ingest_to_online_store`** +{% endhint %} + +## Overview + +Streaming data sources are important sources of feature values. A typical setup with streaming data looks like: + +1. Raw events come in (stream 1) +2. Streaming transformations applied (e.g. `last_N_purchased_categories`) (stream 2) +3. Write stream 2 values to an offline store as a historical log for training +4. Write stream 2 values to an online store for low latency feature serving +5. Periodically materialize feature values from the offline store into the online store for improved correctness + +Feast now allows users to push features previously registered in a feature view to the online store. This most commonly would be done from a stream processing job (e.g. a Beam or Spark Streaming job). Future versions of Feast will allow writing features directly to the offline store as well. + +## Example + +See [https://github.com/feast-dev/feast-demo](https://github.com/feast-dev/on-demand-feature-views-demo) for an example on how to ingest stream data into Feast. + +We register a feature view as normal, and during stream processing (e.g. Kafka consumers), now we push a dataframe matching the feature view schema: + +```python +event_df = pd.DataFrame.from_dict( + { + "driver_id": [1001], + "event_timestamp": [ + datetime(2021, 5, 13, 10, 59, 42), + ], + "created": [ + datetime(2021, 5, 13, 10, 59, 42), + ], + "conv_rate": [1.0], + "acc_rate": [1.0], + "avg_daily_trips": [1000], + } +) +store.write_to_online_store("driver_hourly_stats", event_df) +``` + +Feast will coordinate between pushed stream data and regular materialization jobs to ensure only the latest feature values are written to the online store. This ensures correctness in served features for model inference. diff --git a/docs/reference/feature-repository/README.md b/docs/reference/feature-repository/README.md index fb227dd944a..90a6799c274 100644 --- a/docs/reference/feature-repository/README.md +++ b/docs/reference/feature-repository/README.md @@ -31,7 +31,7 @@ The structure of a feature repository is as follows: An example structure of a feature repository is shown below: -```text +``` $ tree -a . ├── data @@ -48,11 +48,11 @@ A couple of things to note about the feature repository: * Feast reads _all_ Python files recursively when `feast apply` is ran, including subdirectories, even if they don't contain feature definitions. * It's recommended to add `.feastignore` and add paths to all imperative scripts if you need to store them inside the feature registry. -## The feature\_store.yaml configuration file +## The feature_store.yaml configuration file The configuration for a feature store is stored in a file named `feature_store.yaml` , which must be located at the root of a feature repository. An example `feature_store.yaml` file is shown below: -{% code title="feature\_store.yaml" %} +{% code title="feature_store.yaml" %} ```yaml project: my_feature_repo_1 registry: data/metadata.db @@ -62,14 +62,14 @@ online_store: ``` {% endcode %} -The `feature_store.yaml` file configures how the feature store should run. See [feature\_store.yaml](feature-store-yaml.md) for more details. +The `feature_store.yaml` file configures how the feature store should run. See [feature_store.yaml](feature-store-yaml.md) for more details. ## The .feastignore file This file contains paths that should be ignored when running `feast apply`. An example `.feastignore` is shown below: {% code title=".feastignore" %} -```text +``` # Ignore virtual environment venv @@ -90,7 +90,7 @@ See [.feastignore](feast-ignore.md) for more details. A feature repository can also contain one or more Python files that contain feature definitions. An example feature definition file is shown below: -{% code title="driver\_features.py" %} +{% code title="driver_features.py" %} ```python from datetime import timedelta @@ -126,5 +126,4 @@ To declare new feature definitions, just add code to the feature repository, eit ### Next steps * See [Create a feature repository](../../how-to-guides/feast-gcp-aws/create-a-feature-repository.md) to get started with an example feature repository. -* See [feature\_store.yaml](feature-store-yaml.md), [.feastignore](feast-ignore.md), or [Feature Views](../../getting-started/concepts/#feature-view) for more information on the configuration files that live in a feature registry. - +* See [feature_store.yaml](feature-store-yaml.md), [.feastignore](feast-ignore.md), or [Feature Views](../../getting-started/concepts/#feature-view) for more information on the configuration files that live in a feature registry. diff --git a/docs/reference/feature-repository/feast-ignore.md b/docs/reference/feature-repository/feast-ignore.md index fb3b2be0dce..3c23ebe88bc 100644 --- a/docs/reference/feature-repository/feast-ignore.md +++ b/docs/reference/feature-repository/feast-ignore.md @@ -5,7 +5,7 @@ `.feastignore` is a file that is placed at the root of the [Feature Repository](./). This file contains paths that should be ignored when running `feast apply`. An example `.feastignore` is shown below: {% code title=".feastignore" %} -```text +``` # Ignore virtual environment venv @@ -24,10 +24,9 @@ scripts/**/foo.py ## Feast Ignore Patterns -| Pattern | Example matches | Explanation | -| :--- | :--- | :--- | -| venv | venv/foo.py venv/a/foo.py | You can specify a path to a specific directory. Everything in that directory will be ignored. | -| scripts/foo.py | scripts/foo.py | You can specify a path to a specific file. Only that file will be ignored. | -| scripts/\*.py | scripts/foo.py scripts/bar.py | You can specify an asterisk \(\*\) anywhere in the expression. An asterisk matches zero or more characters, except "/". | -| scripts/\*\*/foo.py | scripts/foo.py scripts/a/foo.py scripts/a/b/foo.py | You can specify a double asterisk \(\*\*\) anywhere in the expression. A double asterisk matches zero or more directories. | - +| Pattern | Example matches | Explanation | +| ------------------- | --------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------ | +| venv |

venv/foo.py
venv/a/foo.py

| You can specify a path to a specific directory. Everything in that directory will be ignored. | +| scripts/foo.py | scripts/foo.py | You can specify a path to a specific file. Only that file will be ignored. | +| scripts/\*.py |

scripts/foo.py
scripts/bar.py

| You can specify an asterisk (\*) anywhere in the expression. An asterisk matches zero or more characters, except "/". | +| scripts/\*\*/foo.py |

scripts/foo.py
scripts/a/foo.py
scripts/a/b/foo.py

| You can specify a double asterisk (\*\*) anywhere in the expression. A double asterisk matches zero or more directories. | diff --git a/docs/reference/feature-repository/feature-store-yaml.md b/docs/reference/feature-repository/feature-store-yaml.md index 29dcbbe2f0c..fa10149cfeb 100644 --- a/docs/reference/feature-repository/feature-store-yaml.md +++ b/docs/reference/feature-repository/feature-store-yaml.md @@ -1,10 +1,10 @@ -# feature\_store.yaml +# feature_store.yaml ## Overview `feature_store.yaml` is used to configure a feature store. The file must be located at the root of a [feature repository](./). An example `feature_store.yaml` is shown below: -{% code title="feature\_store.yaml" %} +{% code title="feature_store.yaml" %} ```yaml project: loyal_spider registry: data/registry.db @@ -21,9 +21,8 @@ The following top-level configuration options exist in the `feature_store.yaml` * **provider** — Configures the environment in which Feast will deploy and operate. * **registry** — Configures the location of the feature registry. -* **online\_store** — Configures the online store. -* **offline\_store** — Configures the offline store. +* **online_store** — Configures the online store. +* **offline_store** — Configures the offline store. * **project** — Defines a namespace for the entire feature store. Can be used to isolate multiple deployments in a single installation of Feast. Should only contain letters, numbers, and underscores. Please see the [RepoConfig](https://rtd.feast.dev/en/latest/#feast.repo_config.RepoConfig) API reference for the full list of configuration options. - diff --git a/docs/reference/offline-stores/bigquery.md b/docs/reference/offline-stores/bigquery.md index c40cdaa1296..255c587d6be 100644 --- a/docs/reference/offline-stores/bigquery.md +++ b/docs/reference/offline-stores/bigquery.md @@ -11,7 +11,7 @@ The BigQuery offline store provides support for reading [BigQuerySources](../dat ## Example -{% code title="feature\_store.yaml" %} +{% code title="feature_store.yaml" %} ```yaml project: my_feature_repo registry: gs://my-bucket/data/registry.db @@ -23,4 +23,3 @@ offline_store: {% endcode %} Configuration options are available [here](https://rtd.feast.dev/en/latest/#feast.repo_config.BigQueryOfflineStoreConfig). - diff --git a/docs/reference/offline-stores/file.md b/docs/reference/offline-stores/file.md index f986d4f9fb8..42ac821691b 100644 --- a/docs/reference/offline-stores/file.md +++ b/docs/reference/offline-stores/file.md @@ -9,7 +9,7 @@ The File offline store provides support for reading [FileSources](../data-source ## Example -{% code title="feature\_store.yaml" %} +{% code title="feature_store.yaml" %} ```yaml project: my_feature_repo registry: data/registry.db @@ -20,4 +20,3 @@ offline_store: {% endcode %} Configuration options are available [here](https://rtd.feast.dev/en/latest/#feast.repo_config.FileOfflineStoreConfig). - diff --git a/docs/reference/offline-stores/redshift.md b/docs/reference/offline-stores/redshift.md index b0928ee0f65..73148730c53 100644 --- a/docs/reference/offline-stores/redshift.md +++ b/docs/reference/offline-stores/redshift.md @@ -11,7 +11,7 @@ The Redshift offline store provides support for reading [RedshiftSources](../dat ## Example -{% code title="feature\_store.yaml" %} +{% code title="feature_store.yaml" %} ```yaml project: my_feature_repo registry: data/registry.db @@ -33,89 +33,15 @@ Configuration options are available [here](https://github.com/feast-dev/feast/bl Feast requires the following permissions in order to execute commands for Redshift offline store: - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Command - PermissionsResources
Apply - -

redshift-data:DescribeTable

-

redshift:GetClusterCredentials

- 		-

arn:aws:redshift:<region>:<account_id>:dbuser:<redshift_cluster_id>/<redshift_username>

-

arn:aws:redshift:<region>:<account_id>:dbname:<redshift_cluster_id>/<redshift_database_name>

-

arn:aws:redshift:<region>:<account_id>:cluster:<redshift_cluster_id>

-
Materialize - redshift-data:ExecuteStatementarn:aws:redshift:<region>:<account_id>:cluster:<redshift_cluster_id>
Materialize - redshift-data:DescribeStatement*
Materialize - -

s3:ListBucket

-

s3:GetObject

-

s3:DeleteObject

- 		-

arn:aws:s3:::<bucket_name>

-

arn:aws:s3:::<bucket_name>/*

-
Get Historical Features - -

redshift-data:ExecuteStatement

-

redshift:GetClusterCredentials

- 		-

arn:aws:redshift:<region>:<account_id>:dbuser:<redshift_cluster_id>/<redshift_username>

-

arn:aws:redshift:<region>:<account_id>:dbname:<redshift_cluster_id>/<redshift_database_name>

-

arn:aws:redshift:<region>:<account_id>:cluster:<redshift_cluster_id>

-
Get Historical Features - redshift-data:DescribeStatement*
Get Historical Features - -

s3:ListBucket

-

s3:GetObject

-

s3:PutObject

-

s3:DeleteObject

- 		-

arn:aws:s3:::<bucket_name>

-

arn:aws:s3:::<bucket_name>/*

-
+| **Command** | Permissions | Resources | +| --------------------------- | -------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| **Apply** |

redshift-data:DescribeTable

redshift:GetClusterCredentials

|

arn:aws:redshift:<region>:<account_id>:dbuser:<redshift_cluster_id>/<redshift_username>

arn:aws:redshift:<region>:<account_id>:dbname:<redshift_cluster_id>/<redshift_database_name>

arn:aws:redshift:<region>:<account_id>:cluster:<redshift_cluster_id>

| +| **Materialize** | redshift-data:ExecuteStatement | arn:aws:redshift:\:\:cluster:\ | +| **Materialize** | redshift-data:DescribeStatement | \* | +| **Materialize** |

s3:ListBucket

s3:GetObject

s3:DeleteObject

|

arn:aws:s3:::<bucket_name>

arn:aws:s3:::<bucket_name>/*

| +| **Get Historical Features** |

redshift-data:ExecuteStatement

redshift:GetClusterCredentials

|

arn:aws:redshift:<region>:<account_id>:dbuser:<redshift_cluster_id>/<redshift_username>

arn:aws:redshift:<region>:<account_id>:dbname:<redshift_cluster_id>/<redshift_database_name>

arn:aws:redshift:<region>:<account_id>:cluster:<redshift_cluster_id>

| +| **Get Historical Features** | redshift-data:DescribeStatement | \* | +| **Get Historical Features** |

s3:ListBucket

s3:GetObject

s3:PutObject

s3:DeleteObject

|

arn:aws:s3:::<bucket_name>

arn:aws:s3:::<bucket_name>/*

| The following inline policy can be used to grant Feast the necessary permissions: @@ -196,4 +122,3 @@ While the following trust relationship is necessary to make sure that Redshift, ] } ``` - diff --git a/docs/reference/online-stores/datastore.md b/docs/reference/online-stores/datastore.md index aab6fe0ee45..012d497f308 100644 --- a/docs/reference/online-stores/datastore.md +++ b/docs/reference/online-stores/datastore.md @@ -2,11 +2,11 @@ ## Description -The [Datastore](https://cloud.google.com/datastore) online store provides support for materializing feature values into Cloud Datastore. The data model used to store feature values in Datastore is described in more detail [here](https://github.com/feast-dev/feast/blob/master/docs/specs/online_store_format.md#google-datastore-online-store-format). +The [Datastore](https://cloud.google.com/datastore) online store provides support for materializing feature values into Cloud Datastore. The data model used to store feature values in Datastore is described in more detail [here](../../specs/online_store_format.md#google-datastore-online-store-format). ## Example -{% code title="feature\_store.yaml" %} +{% code title="feature_store.yaml" %} ```yaml project: my_feature_repo registry: data/registry.db @@ -19,4 +19,3 @@ online_store: {% endcode %} Configuration options are available [here](https://rtd.feast.dev/en/latest/#feast.repo_config.DatastoreOnlineStoreConfig). - diff --git a/docs/reference/online-stores/dynamodb.md b/docs/reference/online-stores/dynamodb.md index eb6bd84e1b9..2f260b042d9 100644 --- a/docs/reference/online-stores/dynamodb.md +++ b/docs/reference/online-stores/dynamodb.md @@ -6,7 +6,7 @@ The [DynamoDB](https://aws.amazon.com/dynamodb/) online store provides support f ## Example -{% code title="feature\_store.yaml" %} +{% code title="feature_store.yaml" %} ```yaml project: my_feature_repo registry: data/registry.db @@ -23,40 +23,11 @@ Configuration options are available [here](https://github.com/feast-dev/feast/bl Feast requires the following permissions in order to execute commands for DynamoDB online store: - - - - - - - - - - - - - - - - - - - - - - - - - -
Command - PermissionsResources
Apply - -

dynamodb:CreateTable

-

dynamodb:DescribeTable

-

dynamodb:DeleteTable

- 		arn:aws:dynamodb:<region>:<account_id>:table/*
Materialize - dynamodb.BatchWriteItemarn:aws:dynamodb:<region>:<account_id>:table/*
Get Online Features - dynamodb.GetItemarn:aws:dynamodb:<region>:<account_id>:table/*
+| **Command** | Permissions | Resources | +| ----------------------- | ----------------------------------------------------------------------------------- | ------------------------------------------------- | +| **Apply** |

dynamodb:CreateTable

dynamodb:DescribeTable

dynamodb:DeleteTable

| arn:aws:dynamodb:\:\:table/\* | +| **Materialize** | dynamodb.BatchWriteItem | arn:aws:dynamodb:\:\:table/\* | +| **Get Online Features** | dynamodb.GetItem | arn:aws:dynamodb:\:\:table/\* | The following inline policy can be used to grant Feast the necessary permissions: @@ -82,4 +53,3 @@ The following inline policy can be used to grant Feast the necessary permissions ``` Lastly, this IAM role needs to be associated with the desired Redshift cluster. Please follow the official AWS guide for the necessary steps [here](https://docs.aws.amazon.com/redshift/latest/dg/c-getting-started-using-spectrum-add-role.html). - diff --git a/docs/reference/online-stores/redis.md b/docs/reference/online-stores/redis.md index 82495c1c32c..ce1de2ad540 100644 --- a/docs/reference/online-stores/redis.md +++ b/docs/reference/online-stores/redis.md @@ -2,16 +2,16 @@ ## Description -The [Redis](https://redis.io/) online store provides support for materializing feature values into Redis. +The [Redis](https://redis.io) online store provides support for materializing feature values into Redis. * Both Redis and Redis Cluster are supported -* The data model used to store feature values in Redis is described in more detail [here](https://github.com/feast-dev/feast/blob/master/docs/specs/online_store_format.md). +* The data model used to store feature values in Redis is described in more detail [here](../../specs/online\_store\_format.md). ## Examples Connecting to a single Redis instance -{% code title="feature\_store.yaml" %} +{% code title="feature_store.yaml" %} ```yaml project: my_feature_repo registry: data/registry.db @@ -24,7 +24,7 @@ online_store: Connecting to a Redis Cluster with SSL enabled and password authentication -{% code title="feature\_store.yaml" %} +{% code title="feature_store.yaml" %} ```yaml project: my_feature_repo registry: data/registry.db @@ -36,5 +36,4 @@ online_store: ``` {% endcode %} -Configuration options are available [here](https://rtd.feast.dev/en/master/#feast.repo_config.RedisOnlineStoreConfig). - +Configuration options are available [here](https://rtd.feast.dev/en/master/#feast.infra.online\_stores.redis.RedisOnlineStoreConfig). diff --git a/docs/reference/online-stores/sqlite.md b/docs/reference/online-stores/sqlite.md index bd9fa578ff0..fd11e3439c4 100644 --- a/docs/reference/online-stores/sqlite.md +++ b/docs/reference/online-stores/sqlite.md @@ -9,7 +9,7 @@ The [SQLite](https://www.sqlite.org/index.html) online store provides support fo ## Example -{% code title="feature\_store.yaml" %} +{% code title="feature_store.yaml" %} ```yaml project: my_feature_repo registry: data/registry.db @@ -21,4 +21,3 @@ online_store: {% endcode %} Configuration options are available [here](https://rtd.feast.dev/en/latest/#feast.repo_config.SqliteOnlineStoreConfig). - diff --git a/docs/reference/providers/amazon-web-services.md b/docs/reference/providers/amazon-web-services.md index 56b72cdc16b..3135fedb740 100644 --- a/docs/reference/providers/amazon-web-services.md +++ b/docs/reference/providers/amazon-web-services.md @@ -7,7 +7,7 @@ ## Example -{% code title="feature\_store.yaml" %} +{% code title="feature_store.yaml" %} ```yaml project: my_feature_repo registry: data/registry.db @@ -25,4 +25,3 @@ offline_store: iam_role: arn:aws:iam::123456789012:role/redshift_s3_access_role ``` {% endcode %} - diff --git a/docs/reference/providers/google-cloud-platform.md b/docs/reference/providers/google-cloud-platform.md index be6a12ea307..713313d16be 100644 --- a/docs/reference/providers/google-cloud-platform.md +++ b/docs/reference/providers/google-cloud-platform.md @@ -7,7 +7,7 @@ ## Example -{% code title="feature\_store.yaml" %} +{% code title="feature_store.yaml" %} ```yaml project: my_feature_repo registry: gs://my-bucket/data/registry.db @@ -17,86 +17,11 @@ provider: gcp ## **Permissions** - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Command - ComponentPermissionsRecommended Role
Apply - BigQuery (source) -

bigquery.jobs.create

-

bigquery.readsessions.create

-

bigquery.readsessions.getData

- 		roles/bigquery.user
Apply - Datastore (destination) -

datastore.entities.allocateIds

-

datastore.entities.create

-

datastore.entities.delete

-

datastore.entities.get

-

datastore.entities.list

-

datastore.entities.update

- 		roles/datastore.owner
Materialize - BigQuery (source)bigquery.jobs.createroles/bigquery.user
Materialize - Datastore (destination) -

datastore.entities.allocateIds

-

datastore.entities.create

-

datastore.entities.delete

-

datastore.entities.get

-

datastore.entities.list

-

datastore.entities.update

-

datastore.databases.get

- 		roles/datastore.owner
Get Online Features - Datastoredatastore.entities.getroles/datastore.user
Get Historical Features - BigQuery (source) -

bigquery.datasets.get

-

bigquery.tables.get

-

bigquery.tables.create

-

bigquery.tables.updateData

-

bigquery.tables.update

-

bigquery.tables.delete

-

bigquery.tables.getData

- 		roles/bigquery.dataEditor
- +| **Command** | Component | Permissions | Recommended Role | +| --------------------------- | ----------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | ------------------------- | +| **Apply** | BigQuery (source) |

bigquery.jobs.create

bigquery.readsessions.create

bigquery.readsessions.getData

| roles/bigquery.user | +| **Apply** | Datastore (destination) |

datastore.entities.allocateIds

datastore.entities.create

datastore.entities.delete

datastore.entities.get

datastore.entities.list

datastore.entities.update

| roles/datastore.owner | +| **Materialize** | BigQuery (source) | bigquery.jobs.create | roles/bigquery.user | +| **Materialize** | Datastore (destination) |

datastore.entities.allocateIds

datastore.entities.create

datastore.entities.delete

datastore.entities.get

datastore.entities.list

datastore.entities.update

datastore.databases.get

| roles/datastore.owner | +| **Get Online Features** | Datastore | datastore.entities.get | roles/datastore.user | +| **Get Historical Features** | BigQuery (source) |

bigquery.datasets.get

bigquery.tables.get

bigquery.tables.create

bigquery.tables.updateData

bigquery.tables.update

bigquery.tables.delete

bigquery.tables.getData

| roles/bigquery.dataEditor | diff --git a/docs/reference/providers/local.md b/docs/reference/providers/local.md index 52fb18f6fa7..a93a3b8b2d1 100644 --- a/docs/reference/providers/local.md +++ b/docs/reference/providers/local.md @@ -7,11 +7,10 @@ ## Example -{% code title="feature\_store.yaml" %} +{% code title="feature_store.yaml" %} ```yaml project: my_feature_repo registry: data/registry.db provider: local ``` {% endcode %} - diff --git a/docs/roadmap.md b/docs/roadmap.md index 0e70ca416f8..c56cae98490 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -2,72 +2,74 @@ The list below contains the functionality that contributors are planning to develop for Feast -* Items below that are in development \(or planned for development\) will be indicated in parentheses. +* Items below that are in development (or planned for development) will be indicated in parentheses. * We welcome contribution to all items in the roadmap! -* Want to influence our roadmap and prioritization? Submit your feedback to [this form](https://docs.google.com/forms/d/e/1FAIpQLSfa1nRQ0sKz-JEFnMMCi4Jseag_yDssO_3nV9qMfxfrkil-wA/viewform). +* Want to influence our roadmap and prioritization? Submit your feedback to [this form](https://docs.google.com/forms/d/e/1FAIpQLSfa1nRQ0sKz-JEFnMMCi4Jseag\_yDssO\_3nV9qMfxfrkil-wA/viewform). * Want to speak to a Feast contributor? We are more than happy to jump on a call. Please schedule a time using [Calendly](https://calendly.com/d/x2ry-g5bb/meet-with-feast-team). - - - * **Data Sources** * [x] [Redshift source](https://docs.feast.dev/reference/data-sources/redshift) * [x] [BigQuery source](https://docs.feast.dev/reference/data-sources/bigquery) * [x] [Parquet file source](https://docs.feast.dev/reference/data-sources/file) - * [ ] Kafka source \(Planned for Q4 2021\) + * [x] [Synapse source (community plugin)](https://github.com/Azure/feast-azure) + * [x] [Hive (community plugin)](https://github.com/baineng/feast-hive) + * [x] [Postgres (community plugin)](https://github.com/nossrannug/feast-postgres) + * [x] Kafka source (with [push support into the online store](reference/alpha-stream-ingestion.md)) + * [ ] Snowflake source (Planned for Q4 2021) * [ ] HTTP source - * [ ] Snowflake source - * [ ] Synapse source * **Offline Stores** * [x] [Redshift](https://docs.feast.dev/reference/offline-stores/redshift) * [x] [BigQuery](https://docs.feast.dev/reference/offline-stores/bigquery) + * [x] [Synapse (community plugin)](https://github.com/Azure/feast-azure) + * [x] [Hive (community plugin)](https://github.com/baineng/feast-hive) + * [x] [Postgres (community plugin)](https://github.com/nossrannug/feast-postgres) * [x] [In-memory / Pandas](https://docs.feast.dev/reference/offline-stores/file) * [x] [Custom offline store support](https://docs.feast.dev/how-to-guides/adding-a-new-offline-store) - * [x] [Hive \(community maintained\)](https://github.com/baineng/feast-hive) - * [ ] Snowflake - * [ ] Synapse + * [ ] Snowflake (Planned for Q4 2021) + * [ ] Trino (Planned for Q4 2021) * **Online Stores** * [x] [DynamoDB](https://docs.feast.dev/reference/online-stores/dynamodb) * [x] [Redis](https://docs.feast.dev/reference/online-stores/redis) * [x] [Datastore](https://docs.feast.dev/reference/online-stores/datastore) * [x] [SQLite](https://docs.feast.dev/reference/online-stores/sqlite) + * [x] [Azure Cache for Redis (community plugin)](https://github.com/Azure/feast-azure) + * [x] [Postgres (community plugin)](https://github.com/nossrannug/feast-postgres) * [x] [Custom online store support](https://docs.feast.dev/how-to-guides/adding-support-for-a-new-online-store) - * [ ] Postgres * [ ] Bigtable - * [ ] Cassandra + * [ ] Cassandra * **Streaming** - * [ ] [Custom streaming ingestion job support](https://docs.feast.dev/how-to-guides/creating-a-custom-provider) - * [ ] Streaming ingestion on AWS \(Planned for Q4 2021\) - * [ ] Streaming ingestion on GCP + * [x] [Custom streaming ingestion job support](https://docs.feast.dev/how-to-guides/creating-a-custom-provider) + * [x] [Push based streaming data ingestion](reference/alpha-stream-ingestion.md) + * [ ] Streaming ingestion on AWS + * [ ] Streaming ingestion on GCP * **Feature Engineering** - * [ ] On-demand Transformations \(Development in progress. See [RFC](https://docs.google.com/document/d/1lgfIw0Drc65LpaxbUu49RCeJgMew547meSJttnUqz7c/edit#)\) - * [ ] Batch transformation \(SQL\) - * [ ] Streaming transformation + * [x] On-demand Transformations (Alpha release. See [RFC](https://docs.google.com/document/d/1lgfIw0Drc65LpaxbUu49RCeJgMew547meSJttnUqz7c/edit#)) + * [ ] Batch transformation (SQL) + * [ ] Streaming transformation * **Deployments** - * [ ] AWS Lambda \(Development in progress. See [RFC](https://docs.google.com/document/d/1eZWKWzfBif66LDN32IajpaG-j82LSHCCOzY6R7Ax7MI/edit)\) + * [x] AWS Lambda (Alpha release. See [RFC](https://docs.google.com/document/d/1eZWKWzfBif66LDN32IajpaG-j82LSHCCOzY6R7Ax7MI/edit)) * [ ] Cloud Run * [ ] Kubernetes - * [ ] KNative + * [ ] KNative * **Feature Serving** * [x] Python Client - * [ ] REST Feature Server \(Python\) \(Development in progress. See [RFC](https://docs.google.com/document/d/1iXvFhAsJ5jgAhPOpTdB3j-Wj1S9x3Ev_Wr6ZpnLzER4/edit)\) - * [ ] gRPC Feature Server \(Java\) \(See [\#1497](https://github.com/feast-dev/feast/issues/1497)\) + * [x] REST Feature Server (Python) (Alpha release. See [RFC](https://docs.google.com/document/d/1iXvFhAsJ5jgAhPOpTdB3j-Wj1S9x3Ev\_Wr6ZpnLzER4/edit)) + * [x] gRPC Feature Server (Java) (See [#1497](https://github.com/feast-dev/feast/issues/1497)) + * [x] Push API * [ ] Java Client - * [ ] Go Client - * [ ] Push API + * [ ] Go Client * [ ] Delete API - * [ ] Feature Logging \(for training\) + * [ ] Feature Logging (for training) * **Data Quality Management** - * [ ] Data profiling and validation \(Great Expectations\) \(Planned for Q4 2021\) + * [ ] Data profiling and validation (Great Expectations) (Planned for Q4 2021) * [ ] Metric production * [ ] Training-serving skew detection * [ ] Drift detection - * [ ] Alerting + * [ ] Alerting * **Feature Discovery and Governance** * [x] Python SDK for browsing feature registry * [x] CLI for browsing feature registry - * [x] Model-centric feature tracking \(feature services\) + * [x] Model-centric feature tracking (feature services) * [ ] REST API for browsing feature registry - * [ ] Feast Web UI \(Planned for Q4 2021\) + * [ ] Feast Web UI (Planned for Q4 2021) * [ ] Feature versioning * [ ] Amundsen integration - diff --git a/docs/specs/datastore_online_example (1).png b/docs/specs/datastore_online_example (1).png new file mode 100644 index 00000000000..9c9b644d668 Binary files /dev/null and b/docs/specs/datastore_online_example (1).png differ diff --git a/docs/specs/redis_online_example (1).png b/docs/specs/redis_online_example (1).png new file mode 100644 index 00000000000..ac426770bad Binary files /dev/null and b/docs/specs/redis_online_example (1).png differ diff --git a/go.mod b/go.mod index 6e71e5b637e..f4a14550566 100644 --- a/go.mod +++ b/go.mod @@ -9,8 +9,8 @@ require ( github.com/gogo/protobuf v1.3.1 // indirect github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e // indirect github.com/golang/mock v1.2.0 - github.com/golang/protobuf v1.4.3 - github.com/google/go-cmp v0.5.0 + github.com/golang/protobuf v1.5.2 + github.com/google/go-cmp v0.5.5 github.com/huandu/xstrings v1.2.0 // indirect github.com/lyft/protoc-gen-validate v0.1.0 // indirect github.com/mitchellh/copystructure v1.0.0 // indirect @@ -18,15 +18,18 @@ require ( github.com/mwitkow/go-proto-validators v0.2.0 // indirect github.com/pseudomuto/protoc-gen-doc v1.3.0 // indirect github.com/pseudomuto/protokit v0.2.0 // indirect + github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/spf13/cobra v0.0.4 github.com/spf13/viper v1.4.0 github.com/woop/protoc-gen-doc v1.3.0 // indirect go.opencensus.io v0.22.3 // indirect - golang.org/x/lint v0.0.0-20200302205851-738671d3881b // indirect - golang.org/x/net v0.0.0-20201021035429-f5854403a974 - golang.org/x/tools v0.0.0-20201124005743-911501bfb504 // indirect + golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 // indirect + golang.org/x/lint v0.0.0-20210508222113-6edffad5e616 // indirect + golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d + golang.org/x/tools v0.1.7 // indirect + google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013 // indirect google.golang.org/grpc v1.29.1 - google.golang.org/protobuf v1.25.0 // indirect + google.golang.org/protobuf v1.27.1 // indirect gopkg.in/russross/blackfriday.v2 v2.0.0 // indirect gopkg.in/yaml.v2 v2.2.4 istio.io/gogo-genproto v0.0.0-20191212213402-78a529a42cd8 // indirect diff --git a/go.sum b/go.sum index bd68e3f8638..5e87ccf6dbf 100644 --- a/go.sum +++ b/go.sum @@ -156,6 +156,9 @@ github.com/golang/protobuf v1.4.2 h1:+Z5KGCizgyZCbGh1KZqA0fcLLkwbsjIzS4aV2v7wJX0 github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.4.3 h1:JjCZWpVbqXDqFVmTfYWEVTMIYrL/NPdPSCHPJ0T/raM= github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= +github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/google/btree v0.0.0-20160524151835-7d79101e329e/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= @@ -168,6 +171,7 @@ github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/gofuzz v0.0.0-20161122191042-44d81051d367/go.mod h1:HP5RmnzzSNb993RKQDq4+1A4ia9nllfqcQFTQJedwGI= github.com/google/gofuzz v0.0.0-20170612174753-24818f796faf/go.mod h1:HP5RmnzzSNb993RKQDq4+1A4ia9nllfqcQFTQJedwGI= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= @@ -293,6 +297,8 @@ github.com/russross/blackfriday v1.5.2 h1:HyvC0ARfnZBqnXwABFeSZHpKvJHJJfPz81GNue github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/russross/blackfriday/v2 v2.0.1 h1:lPqVAte+HuHNfhJ/0LC98ESWRz8afy9tM/0RK8m9o+Q= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= @@ -338,6 +344,7 @@ github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1: github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.4.0/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.opencensus.io v0.21.0 h1:mU6zScU4U1YAFPHEHYk+3JC4SY7JxgkqS10ZOSyksNg= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= @@ -372,10 +379,13 @@ golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3 h1:XQyxROzUlZH+WIQwySDgnISg golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20200302205851-738671d3881b h1:Wh+f8QHJXR411sJR8/vRBTZ7YapZaRvUcLFFJhusH0k= golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20210508222113-6edffad5e616 h1:VLliZ0d+/avPrXXH+OakdXhpJuEoBZuwh1m2j7U6Iug= +golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20170114055629-f2499483f923/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d h1:g9qWBGx4puODJTMVyoPrpoxPFgVGd+z1DZwjfRu4d0I= @@ -404,6 +414,7 @@ golang.org/x/net v0.0.0-20200513185701-a91f0712d120 h1:EZ3cVSzKOlJxAd8e8YAJ7no8n golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190402181905-9f3314589c9a/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -416,6 +427,7 @@ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20170830134202-bb24a47a89ea/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -439,6 +451,10 @@ golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9 h1:YTzHMGlqJu67/uEo1lBv0n3wBXhXNeUbB1XfN2vmTm0= golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.0.0-20160726164857-2910a502d2bf/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -447,6 +463,7 @@ golang.org/x/text v0.3.1-0.20181227161524-e6919f6577db/go.mod h1:bEr9sfX3Q8Zfm5f golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/time v0.0.0-20161028155119-f51c12702a4d/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -506,6 +523,8 @@ golang.org/x/tools v0.0.0-20201017001424-6003fad69a88 h1:ZB1XYzdDo7c/O48jzjMkvIj golang.org/x/tools v0.0.0-20201017001424-6003fad69a88/go.mod h1:z6u4i615ZeAfBE4XtMziQW1fSVJXACjjbWkB/mvPzlU= golang.org/x/tools v0.0.0-20201124005743-911501bfb504 h1:jOKV2ysikH1GANB7t2LotmhyvkkPvl7HQoEXkV6slJA= golang.org/x/tools v0.0.0-20201124005743-911501bfb504/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.1.7 h1:6j8CgantCy3yc8JGBqkDLMKWqZ0RDU2g1HVgacojGWQ= +golang.org/x/tools v0.1.7/go.mod h1:LGqMHiF4EqQNHR1JncWGqT5BVaXmza+X+BDGol+dOxo= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= @@ -565,6 +584,10 @@ google.golang.org/protobuf v1.24.0 h1:UhZDfRO8JRQru4/+LlLE0BRKGF8L+PICnvYZmx/fEG google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= google.golang.org/protobuf v1.25.0 h1:Ejskq+SyPohKW+1uil0JJMtmHCgJPJ/qWTxr8qp+R4c= google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.27.1 h1:SnqbnDw1V7RiZcXPx5MEeqPv2s79L9i7BJUlG/+RurQ= +google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/infra/templates/README.md.jinja2 b/infra/templates/README.md.jinja2 index 993fd4cba44..641b1a75f59 100644 --- a/infra/templates/README.md.jinja2 +++ b/infra/templates/README.md.jinja2 @@ -20,7 +20,7 @@ Feast is an open source feature store for machine learning. Feast is the fastest Please see our [documentation](https://docs.feast.dev/) for more information about the project. ## 📐 Architecture - + The above architecture is the minimal Feast deployment. Want to run the full Feast on GCP/AWS? Click [here](https://docs.feast.dev/how-to-guides/feast-gcp-aws). diff --git a/protos/feast/core/FeatureViewProjection.proto b/protos/feast/core/FeatureViewProjection.proto index 2ede5ebc362..e81d8dad01e 100644 --- a/protos/feast/core/FeatureViewProjection.proto +++ b/protos/feast/core/FeatureViewProjection.proto @@ -19,4 +19,7 @@ message FeatureViewProjection { // The features of the feature view that are a part of the feature reference. repeated FeatureSpecV2 feature_columns = 2; + + // Map for entity join_key overrides of feature data entity join_key to entity data join_key + map join_key_map = 4; } diff --git a/protos/feast/core/Registry.proto b/protos/feast/core/Registry.proto index b8570301e96..035e87a49f8 100644 --- a/protos/feast/core/Registry.proto +++ b/protos/feast/core/Registry.proto @@ -26,6 +26,7 @@ import "feast/core/FeatureService.proto"; import "feast/core/FeatureTable.proto"; import "feast/core/FeatureView.proto"; import "feast/core/OnDemandFeatureView.proto"; +import "feast/core/RequestFeatureView.proto"; import "google/protobuf/timestamp.proto"; message Registry { @@ -33,6 +34,7 @@ message Registry { repeated FeatureTable feature_tables = 2; repeated FeatureView feature_views = 6; repeated OnDemandFeatureView on_demand_feature_views = 8; + repeated RequestFeatureView request_feature_views = 9; repeated FeatureService feature_services = 7; string registry_schema_version = 3; // to support migrations; incremented when schema is changed diff --git a/protos/feast/core/RequestFeatureView.proto b/protos/feast/core/RequestFeatureView.proto new file mode 100644 index 00000000000..c9ee540e6f9 --- /dev/null +++ b/protos/feast/core/RequestFeatureView.proto @@ -0,0 +1,43 @@ +// +// Copyright 2021 The Feast Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + + +syntax = "proto3"; +package feast.core; + +option go_package = "github.com/feast-dev/feast/sdk/go/protos/feast/core"; +option java_outer_classname = "RequestFeatureViewProto"; +option java_package = "feast.proto.core"; + +import "feast/core/FeatureView.proto"; +import "feast/core/Feature.proto"; +import "feast/core/DataSource.proto"; + +message RequestFeatureView { + // User-specified specifications of this feature view. + RequestFeatureViewSpec spec = 1; +} + +message RequestFeatureViewSpec { + // Name of the feature view. Must be unique. Not updated. + string name = 1; + + // Name of Feast project that this feature view belongs to. + string project = 2; + + // Request data which contains the underlying data schema and list of associated features + DataSource request_data_source = 3; +} diff --git a/protos/feast/serving/TransformationService.proto b/protos/feast/serving/TransformationService.proto new file mode 100644 index 00000000000..113bd120c8f --- /dev/null +++ b/protos/feast/serving/TransformationService.proto @@ -0,0 +1,67 @@ +/* + * Copyright 2021 The Feast Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +syntax = "proto3"; + +package feast.serving; + +option java_package = "feast.proto.serving"; +option java_outer_classname = "TransformationServiceAPIProto"; +option go_package = "github.com/feast-dev/feast/sdk/go/protos/feast/serving"; + +service TransformationService { + rpc GetTransformationServiceInfo (GetTransformationServiceInfoRequest) returns (GetTransformationServiceInfoResponse); + + rpc TransformFeatures (TransformFeaturesRequest) returns (TransformFeaturesResponse); +} + +message ValueType { + oneof value { + // Having a oneOf provides forward compatibility if we need to support compound types + // that are not supported by arrow natively. + bytes arrow_value = 1; + } +} + +message GetTransformationServiceInfoRequest {} + +message GetTransformationServiceInfoResponse { + // Feast version of this transformation service deployment. + string version = 1; + + // Type of transformation service deployment. This is either Python, or custom + TransformationServiceType type = 2; + + string transformation_service_type_details = 3; +} + +message TransformFeaturesRequest { + string on_demand_feature_view_name = 1; + string project = 2; + + ValueType transformation_input = 3; +} + +message TransformFeaturesResponse { + ValueType transformation_output = 3; +} + +enum TransformationServiceType { + TRANSFORMATION_SERVICE_TYPE_INVALID = 0; + TRANSFORMATION_SERVICE_TYPE_PYTHON = 1; + + TRANSFORMATION_SERVICE_TYPE_CUSTOM = 100; +} diff --git a/sdk/go/protos/feast/core/CoreService.pb.go b/sdk/go/protos/feast/core/CoreService.pb.go index 2d9b0a8e0ad..1978edce45a 100644 --- a/sdk/go/protos/feast/core/CoreService.pb.go +++ b/sdk/go/protos/feast/core/CoreService.pb.go @@ -16,8 +16,8 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.25.0 -// protoc v3.12.4 +// protoc-gen-go v1.27.1 +// protoc v3.11.2 // source: feast/core/CoreService.proto package core @@ -25,13 +25,12 @@ package core import ( context "context" _ "github.com/feast-dev/feast/sdk/go/protos/tensorflow_metadata/proto/v0" - proto "github.com/golang/protobuf/proto" - _ "github.com/golang/protobuf/ptypes/timestamp" grpc "google.golang.org/grpc" codes "google.golang.org/grpc/codes" status "google.golang.org/grpc/status" protoreflect "google.golang.org/protobuf/reflect/protoreflect" protoimpl "google.golang.org/protobuf/runtime/protoimpl" + _ "google.golang.org/protobuf/types/known/timestamppb" reflect "reflect" sync "sync" ) @@ -43,10 +42,6 @@ const ( _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) ) -// This is a compile-time assertion that a sufficiently up-to-date version -// of the legacy proto package is being used. -const _ = proto.ProtoPackageIsVersion4 - type UpdateStoreResponse_Status int32 const ( diff --git a/sdk/go/protos/feast/core/DataFormat.pb.go b/sdk/go/protos/feast/core/DataFormat.pb.go index 4a766cc8f9a..13c6cdda989 100644 --- a/sdk/go/protos/feast/core/DataFormat.pb.go +++ b/sdk/go/protos/feast/core/DataFormat.pb.go @@ -16,14 +16,13 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.25.0 -// protoc v3.12.4 +// protoc-gen-go v1.27.1 +// protoc v3.11.2 // source: feast/core/DataFormat.proto package core import ( - proto "github.com/golang/protobuf/proto" protoreflect "google.golang.org/protobuf/reflect/protoreflect" protoimpl "google.golang.org/protobuf/runtime/protoimpl" reflect "reflect" @@ -37,10 +36,6 @@ const ( _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) ) -// This is a compile-time assertion that a sufficiently up-to-date version -// of the legacy proto package is being used. -const _ = proto.ProtoPackageIsVersion4 - // Defines the file format encoding the features/entity data in files type FileFormat struct { state protoimpl.MessageState diff --git a/sdk/go/protos/feast/core/DataSource.pb.go b/sdk/go/protos/feast/core/DataSource.pb.go index 1b43972d327..8af638a834f 100644 --- a/sdk/go/protos/feast/core/DataSource.pb.go +++ b/sdk/go/protos/feast/core/DataSource.pb.go @@ -16,14 +16,14 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.25.0 -// protoc v3.12.4 +// protoc-gen-go v1.27.1 +// protoc v3.11.2 // source: feast/core/DataSource.proto package core import ( - proto "github.com/golang/protobuf/proto" + types "github.com/feast-dev/feast/sdk/go/protos/feast/types" protoreflect "google.golang.org/protobuf/reflect/protoreflect" protoimpl "google.golang.org/protobuf/runtime/protoimpl" reflect "reflect" @@ -37,10 +37,6 @@ const ( _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) ) -// This is a compile-time assertion that a sufficiently up-to-date version -// of the legacy proto package is being used. -const _ = proto.ProtoPackageIsVersion4 - // Type of Data Source. type DataSource_SourceType int32 @@ -50,6 +46,9 @@ const ( DataSource_BATCH_BIGQUERY DataSource_SourceType = 2 DataSource_STREAM_KAFKA DataSource_SourceType = 3 DataSource_STREAM_KINESIS DataSource_SourceType = 4 + DataSource_BATCH_REDSHIFT DataSource_SourceType = 5 + DataSource_CUSTOM_SOURCE DataSource_SourceType = 6 + DataSource_REQUEST_SOURCE DataSource_SourceType = 7 ) // Enum value maps for DataSource_SourceType. @@ -60,6 +59,9 @@ var ( 2: "BATCH_BIGQUERY", 3: "STREAM_KAFKA", 4: "STREAM_KINESIS", + 5: "BATCH_REDSHIFT", + 6: "CUSTOM_SOURCE", + 7: "REQUEST_SOURCE", } DataSource_SourceType_value = map[string]int32{ "INVALID": 0, @@ -67,6 +69,9 @@ var ( "BATCH_BIGQUERY": 2, "STREAM_KAFKA": 3, "STREAM_KINESIS": 4, + "BATCH_REDSHIFT": 5, + "CUSTOM_SOURCE": 6, + "REQUEST_SOURCE": 7, } ) @@ -114,6 +119,9 @@ type DataSource struct { DatePartitionColumn string `protobuf:"bytes,4,opt,name=date_partition_column,json=datePartitionColumn,proto3" json:"date_partition_column,omitempty"` // Must specify creation timestamp column name CreatedTimestampColumn string `protobuf:"bytes,5,opt,name=created_timestamp_column,json=createdTimestampColumn,proto3" json:"created_timestamp_column,omitempty"` + // This is an internal field that is represents the python class for the data source object a proto object represents. + // This should be set by feast, and not by users. + DataSourceClassType string `protobuf:"bytes,17,opt,name=data_source_class_type,json=dataSourceClassType,proto3" json:"data_source_class_type,omitempty"` // DataSource options. // // Types that are assignable to Options: @@ -121,6 +129,9 @@ type DataSource struct { // *DataSource_BigqueryOptions // *DataSource_KafkaOptions_ // *DataSource_KinesisOptions_ + // *DataSource_RedshiftOptions_ + // *DataSource_RequestDataOptions_ + // *DataSource_CustomOptions Options isDataSource_Options `protobuf_oneof:"options"` } @@ -191,6 +202,13 @@ func (x *DataSource) GetCreatedTimestampColumn() string { return "" } +func (x *DataSource) GetDataSourceClassType() string { + if x != nil { + return x.DataSourceClassType + } + return "" +} + func (m *DataSource) GetOptions() isDataSource_Options { if m != nil { return m.Options @@ -226,6 +244,27 @@ func (x *DataSource) GetKinesisOptions() *DataSource_KinesisOptions { return nil } +func (x *DataSource) GetRedshiftOptions() *DataSource_RedshiftOptions { + if x, ok := x.GetOptions().(*DataSource_RedshiftOptions_); ok { + return x.RedshiftOptions + } + return nil +} + +func (x *DataSource) GetRequestDataOptions() *DataSource_RequestDataOptions { + if x, ok := x.GetOptions().(*DataSource_RequestDataOptions_); ok { + return x.RequestDataOptions + } + return nil +} + +func (x *DataSource) GetCustomOptions() *DataSource_CustomSourceOptions { + if x, ok := x.GetOptions().(*DataSource_CustomOptions); ok { + return x.CustomOptions + } + return nil +} + type isDataSource_Options interface { isDataSource_Options() } @@ -246,6 +285,18 @@ type DataSource_KinesisOptions_ struct { KinesisOptions *DataSource_KinesisOptions `protobuf:"bytes,14,opt,name=kinesis_options,json=kinesisOptions,proto3,oneof"` } +type DataSource_RedshiftOptions_ struct { + RedshiftOptions *DataSource_RedshiftOptions `protobuf:"bytes,15,opt,name=redshift_options,json=redshiftOptions,proto3,oneof"` +} + +type DataSource_RequestDataOptions_ struct { + RequestDataOptions *DataSource_RequestDataOptions `protobuf:"bytes,18,opt,name=request_data_options,json=requestDataOptions,proto3,oneof"` +} + +type DataSource_CustomOptions struct { + CustomOptions *DataSource_CustomSourceOptions `protobuf:"bytes,16,opt,name=custom_options,json=customOptions,proto3,oneof"` +} + func (*DataSource_FileOptions_) isDataSource_Options() {} func (*DataSource_BigqueryOptions) isDataSource_Options() {} @@ -254,6 +305,12 @@ func (*DataSource_KafkaOptions_) isDataSource_Options() {} func (*DataSource_KinesisOptions_) isDataSource_Options() {} +func (*DataSource_RedshiftOptions_) isDataSource_Options() {} + +func (*DataSource_RequestDataOptions_) isDataSource_Options() {} + +func (*DataSource_CustomOptions) isDataSource_Options() {} + // Defines options for DataSource that sources features from a file type DataSource_FileOptions struct { state protoimpl.MessageState @@ -266,6 +323,8 @@ type DataSource_FileOptions struct { // gs://path/to/file for GCP GCS storage // file:///path/to/file for local storage FileUrl string `protobuf:"bytes,2,opt,name=file_url,json=fileUrl,proto3" json:"file_url,omitempty"` + // override AWS S3 storage endpoint with custom S3 endpoint + S3EndpointOverride string `protobuf:"bytes,3,opt,name=s3_endpoint_override,json=s3EndpointOverride,proto3" json:"s3_endpoint_override,omitempty"` } func (x *DataSource_FileOptions) Reset() { @@ -314,6 +373,13 @@ func (x *DataSource_FileOptions) GetFileUrl() string { return "" } +func (x *DataSource_FileOptions) GetS3EndpointOverride() string { + if x != nil { + return x.S3EndpointOverride + } + return "" +} + // Defines options for DataSource that sources features from a BigQuery Query type DataSource_BigQueryOptions struct { state protoimpl.MessageState @@ -322,6 +388,9 @@ type DataSource_BigQueryOptions struct { // Full table reference in the form of [project:dataset.table] TableRef string `protobuf:"bytes,1,opt,name=table_ref,json=tableRef,proto3" json:"table_ref,omitempty"` + // SQL query that returns a table containing feature data. Must contain an event_timestamp column, and respective + // entity columns + Query string `protobuf:"bytes,2,opt,name=query,proto3" json:"query,omitempty"` } func (x *DataSource_BigQueryOptions) Reset() { @@ -363,6 +432,13 @@ func (x *DataSource_BigQueryOptions) GetTableRef() string { return "" } +func (x *DataSource_BigQueryOptions) GetQuery() string { + if x != nil { + return x.Query + } + return "" +} + // Defines options for DataSource that sources features from Kafka messages. // Each message should be a Protobuf that can be decoded with the generated // Java Protobuf class at the given class path @@ -502,6 +578,182 @@ func (x *DataSource_KinesisOptions) GetRecordFormat() *StreamFormat { return nil } +// Defines options for DataSource that sources features from a Redshift Query +type DataSource_RedshiftOptions struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // Redshift table name + Table string `protobuf:"bytes,1,opt,name=table,proto3" json:"table,omitempty"` + // SQL query that returns a table containing feature data. Must contain an event_timestamp column, and respective + // entity columns + Query string `protobuf:"bytes,2,opt,name=query,proto3" json:"query,omitempty"` + // Redshift schema name + Schema string `protobuf:"bytes,3,opt,name=schema,proto3" json:"schema,omitempty"` +} + +func (x *DataSource_RedshiftOptions) Reset() { + *x = DataSource_RedshiftOptions{} + if protoimpl.UnsafeEnabled { + mi := &file_feast_core_DataSource_proto_msgTypes[6] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *DataSource_RedshiftOptions) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*DataSource_RedshiftOptions) ProtoMessage() {} + +func (x *DataSource_RedshiftOptions) ProtoReflect() protoreflect.Message { + mi := &file_feast_core_DataSource_proto_msgTypes[6] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use DataSource_RedshiftOptions.ProtoReflect.Descriptor instead. +func (*DataSource_RedshiftOptions) Descriptor() ([]byte, []int) { + return file_feast_core_DataSource_proto_rawDescGZIP(), []int{0, 5} +} + +func (x *DataSource_RedshiftOptions) GetTable() string { + if x != nil { + return x.Table + } + return "" +} + +func (x *DataSource_RedshiftOptions) GetQuery() string { + if x != nil { + return x.Query + } + return "" +} + +func (x *DataSource_RedshiftOptions) GetSchema() string { + if x != nil { + return x.Schema + } + return "" +} + +// Defines configuration for custom third-party data sources. +type DataSource_CustomSourceOptions struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // Serialized configuration information for the data source. The implementer of the custom data source is + // responsible for serializing and deserializing data from bytes + Configuration []byte `protobuf:"bytes,1,opt,name=configuration,proto3" json:"configuration,omitempty"` +} + +func (x *DataSource_CustomSourceOptions) Reset() { + *x = DataSource_CustomSourceOptions{} + if protoimpl.UnsafeEnabled { + mi := &file_feast_core_DataSource_proto_msgTypes[7] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *DataSource_CustomSourceOptions) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*DataSource_CustomSourceOptions) ProtoMessage() {} + +func (x *DataSource_CustomSourceOptions) ProtoReflect() protoreflect.Message { + mi := &file_feast_core_DataSource_proto_msgTypes[7] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use DataSource_CustomSourceOptions.ProtoReflect.Descriptor instead. +func (*DataSource_CustomSourceOptions) Descriptor() ([]byte, []int) { + return file_feast_core_DataSource_proto_rawDescGZIP(), []int{0, 6} +} + +func (x *DataSource_CustomSourceOptions) GetConfiguration() []byte { + if x != nil { + return x.Configuration + } + return nil +} + +// Defines options for DataSource that sources features from request data +type DataSource_RequestDataOptions struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // Name of the request data source + Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` + // Mapping of feature name to type + Schema map[string]types.ValueType_Enum `protobuf:"bytes,2,rep,name=schema,proto3" json:"schema,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"varint,2,opt,name=value,proto3,enum=feast.types.ValueType_Enum"` +} + +func (x *DataSource_RequestDataOptions) Reset() { + *x = DataSource_RequestDataOptions{} + if protoimpl.UnsafeEnabled { + mi := &file_feast_core_DataSource_proto_msgTypes[8] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *DataSource_RequestDataOptions) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*DataSource_RequestDataOptions) ProtoMessage() {} + +func (x *DataSource_RequestDataOptions) ProtoReflect() protoreflect.Message { + mi := &file_feast_core_DataSource_proto_msgTypes[8] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use DataSource_RequestDataOptions.ProtoReflect.Descriptor instead. +func (*DataSource_RequestDataOptions) Descriptor() ([]byte, []int) { + return file_feast_core_DataSource_proto_rawDescGZIP(), []int{0, 7} +} + +func (x *DataSource_RequestDataOptions) GetName() string { + if x != nil { + return x.Name + } + return "" +} + +func (x *DataSource_RequestDataOptions) GetSchema() map[string]types.ValueType_Enum { + if x != nil { + return x.Schema + } + return nil +} + var File_feast_core_DataSource_proto protoreflect.FileDescriptor var file_feast_core_DataSource_proto_rawDesc = []byte{ @@ -509,59 +761,85 @@ var file_feast_core_DataSource_proto_rawDesc = []byte{ 0x61, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x0a, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x1a, 0x1b, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2f, 0x63, 0x6f, 0x72, 0x65, 0x2f, 0x44, 0x61, 0x74, 0x61, 0x46, 0x6f, 0x72, 0x6d, 0x61, 0x74, - 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0xd6, 0x09, 0x0a, 0x0a, 0x44, 0x61, 0x74, 0x61, 0x53, - 0x6f, 0x75, 0x72, 0x63, 0x65, 0x12, 0x35, 0x0a, 0x04, 0x74, 0x79, 0x70, 0x65, 0x18, 0x01, 0x20, - 0x01, 0x28, 0x0e, 0x32, 0x21, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, - 0x2e, 0x44, 0x61, 0x74, 0x61, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x2e, 0x53, 0x6f, 0x75, 0x72, - 0x63, 0x65, 0x54, 0x79, 0x70, 0x65, 0x52, 0x04, 0x74, 0x79, 0x70, 0x65, 0x12, 0x4d, 0x0a, 0x0d, - 0x66, 0x69, 0x65, 0x6c, 0x64, 0x5f, 0x6d, 0x61, 0x70, 0x70, 0x69, 0x6e, 0x67, 0x18, 0x02, 0x20, - 0x03, 0x28, 0x0b, 0x32, 0x28, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, - 0x2e, 0x44, 0x61, 0x74, 0x61, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x2e, 0x46, 0x69, 0x65, 0x6c, - 0x64, 0x4d, 0x61, 0x70, 0x70, 0x69, 0x6e, 0x67, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x0c, 0x66, - 0x69, 0x65, 0x6c, 0x64, 0x4d, 0x61, 0x70, 0x70, 0x69, 0x6e, 0x67, 0x12, 0x34, 0x0a, 0x16, 0x65, - 0x76, 0x65, 0x6e, 0x74, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x5f, 0x63, - 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x14, 0x65, 0x76, 0x65, - 0x6e, 0x74, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x43, 0x6f, 0x6c, 0x75, 0x6d, - 0x6e, 0x12, 0x32, 0x0a, 0x15, 0x64, 0x61, 0x74, 0x65, 0x5f, 0x70, 0x61, 0x72, 0x74, 0x69, 0x74, - 0x69, 0x6f, 0x6e, 0x5f, 0x63, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x13, 0x64, 0x61, 0x74, 0x65, 0x50, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x43, - 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x12, 0x38, 0x0a, 0x18, 0x63, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, - 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x5f, 0x63, 0x6f, 0x6c, 0x75, 0x6d, - 0x6e, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x16, 0x63, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, - 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x43, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x12, - 0x47, 0x0a, 0x0c, 0x66, 0x69, 0x6c, 0x65, 0x5f, 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, - 0x0b, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x22, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, - 0x72, 0x65, 0x2e, 0x44, 0x61, 0x74, 0x61, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x2e, 0x46, 0x69, - 0x6c, 0x65, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x48, 0x00, 0x52, 0x0b, 0x66, 0x69, 0x6c, - 0x65, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x53, 0x0a, 0x10, 0x62, 0x69, 0x67, 0x71, - 0x75, 0x65, 0x72, 0x79, 0x5f, 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x0c, 0x20, 0x01, - 0x28, 0x0b, 0x32, 0x26, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, - 0x44, 0x61, 0x74, 0x61, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x2e, 0x42, 0x69, 0x67, 0x51, 0x75, - 0x65, 0x72, 0x79, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x48, 0x00, 0x52, 0x0f, 0x62, 0x69, - 0x67, 0x71, 0x75, 0x65, 0x72, 0x79, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x4a, 0x0a, - 0x0d, 0x6b, 0x61, 0x66, 0x6b, 0x61, 0x5f, 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x0d, - 0x20, 0x01, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, - 0x65, 0x2e, 0x44, 0x61, 0x74, 0x61, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x2e, 0x4b, 0x61, 0x66, - 0x6b, 0x61, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x48, 0x00, 0x52, 0x0c, 0x6b, 0x61, 0x66, - 0x6b, 0x61, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x50, 0x0a, 0x0f, 0x6b, 0x69, 0x6e, - 0x65, 0x73, 0x69, 0x73, 0x5f, 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x0e, 0x20, 0x01, - 0x28, 0x0b, 0x32, 0x25, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, - 0x44, 0x61, 0x74, 0x61, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x2e, 0x4b, 0x69, 0x6e, 0x65, 0x73, - 0x69, 0x73, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x48, 0x00, 0x52, 0x0e, 0x6b, 0x69, 0x6e, - 0x65, 0x73, 0x69, 0x73, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x1a, 0x3f, 0x0a, 0x11, 0x46, - 0x69, 0x65, 0x6c, 0x64, 0x4d, 0x61, 0x70, 0x70, 0x69, 0x6e, 0x67, 0x45, 0x6e, 0x74, 0x72, 0x79, - 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, - 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x1a, 0x61, 0x0a, 0x0b, - 0x46, 0x69, 0x6c, 0x65, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x37, 0x0a, 0x0b, 0x66, - 0x69, 0x6c, 0x65, 0x5f, 0x66, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, - 0x32, 0x16, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x46, 0x69, - 0x6c, 0x65, 0x46, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x52, 0x0a, 0x66, 0x69, 0x6c, 0x65, 0x46, 0x6f, - 0x72, 0x6d, 0x61, 0x74, 0x12, 0x19, 0x0a, 0x08, 0x66, 0x69, 0x6c, 0x65, 0x5f, 0x75, 0x72, 0x6c, - 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x66, 0x69, 0x6c, 0x65, 0x55, 0x72, 0x6c, 0x1a, - 0x2e, 0x0a, 0x0f, 0x42, 0x69, 0x67, 0x51, 0x75, 0x65, 0x72, 0x79, 0x4f, 0x70, 0x74, 0x69, 0x6f, - 0x6e, 0x73, 0x12, 0x1b, 0x0a, 0x09, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x5f, 0x72, 0x65, 0x66, 0x18, - 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x52, 0x65, 0x66, 0x1a, + 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x17, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2f, 0x74, 0x79, + 0x70, 0x65, 0x73, 0x2f, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, + 0x85, 0x10, 0x0a, 0x0a, 0x44, 0x61, 0x74, 0x61, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x12, 0x35, + 0x0a, 0x04, 0x74, 0x79, 0x70, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x21, 0x2e, 0x66, + 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x44, 0x61, 0x74, 0x61, 0x53, 0x6f, + 0x75, 0x72, 0x63, 0x65, 0x2e, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x54, 0x79, 0x70, 0x65, 0x52, + 0x04, 0x74, 0x79, 0x70, 0x65, 0x12, 0x4d, 0x0a, 0x0d, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x5f, 0x6d, + 0x61, 0x70, 0x70, 0x69, 0x6e, 0x67, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x28, 0x2e, 0x66, + 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x44, 0x61, 0x74, 0x61, 0x53, 0x6f, + 0x75, 0x72, 0x63, 0x65, 0x2e, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x4d, 0x61, 0x70, 0x70, 0x69, 0x6e, + 0x67, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x0c, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x4d, 0x61, 0x70, + 0x70, 0x69, 0x6e, 0x67, 0x12, 0x34, 0x0a, 0x16, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x5f, 0x74, 0x69, + 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x5f, 0x63, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x18, 0x03, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x14, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x54, 0x69, 0x6d, 0x65, 0x73, + 0x74, 0x61, 0x6d, 0x70, 0x43, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x12, 0x32, 0x0a, 0x15, 0x64, 0x61, + 0x74, 0x65, 0x5f, 0x70, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x63, 0x6f, 0x6c, + 0x75, 0x6d, 0x6e, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x13, 0x64, 0x61, 0x74, 0x65, 0x50, + 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x12, 0x38, + 0x0a, 0x18, 0x63, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, + 0x61, 0x6d, 0x70, 0x5f, 0x63, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x16, 0x63, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, + 0x6d, 0x70, 0x43, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x12, 0x33, 0x0a, 0x16, 0x64, 0x61, 0x74, 0x61, + 0x5f, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x63, 0x6c, 0x61, 0x73, 0x73, 0x5f, 0x74, 0x79, + 0x70, 0x65, 0x18, 0x11, 0x20, 0x01, 0x28, 0x09, 0x52, 0x13, 0x64, 0x61, 0x74, 0x61, 0x53, 0x6f, + 0x75, 0x72, 0x63, 0x65, 0x43, 0x6c, 0x61, 0x73, 0x73, 0x54, 0x79, 0x70, 0x65, 0x12, 0x47, 0x0a, + 0x0c, 0x66, 0x69, 0x6c, 0x65, 0x5f, 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x0b, 0x20, + 0x01, 0x28, 0x0b, 0x32, 0x22, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, + 0x2e, 0x44, 0x61, 0x74, 0x61, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x2e, 0x46, 0x69, 0x6c, 0x65, + 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x48, 0x00, 0x52, 0x0b, 0x66, 0x69, 0x6c, 0x65, 0x4f, + 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x53, 0x0a, 0x10, 0x62, 0x69, 0x67, 0x71, 0x75, 0x65, + 0x72, 0x79, 0x5f, 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x0c, 0x20, 0x01, 0x28, 0x0b, + 0x32, 0x26, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x44, 0x61, + 0x74, 0x61, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x2e, 0x42, 0x69, 0x67, 0x51, 0x75, 0x65, 0x72, + 0x79, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x48, 0x00, 0x52, 0x0f, 0x62, 0x69, 0x67, 0x71, + 0x75, 0x65, 0x72, 0x79, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x4a, 0x0a, 0x0d, 0x6b, + 0x61, 0x66, 0x6b, 0x61, 0x5f, 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x0d, 0x20, 0x01, + 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, + 0x44, 0x61, 0x74, 0x61, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x2e, 0x4b, 0x61, 0x66, 0x6b, 0x61, + 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x48, 0x00, 0x52, 0x0c, 0x6b, 0x61, 0x66, 0x6b, 0x61, + 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x50, 0x0a, 0x0f, 0x6b, 0x69, 0x6e, 0x65, 0x73, + 0x69, 0x73, 0x5f, 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x0e, 0x20, 0x01, 0x28, 0x0b, + 0x32, 0x25, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x44, 0x61, + 0x74, 0x61, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x2e, 0x4b, 0x69, 0x6e, 0x65, 0x73, 0x69, 0x73, + 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x48, 0x00, 0x52, 0x0e, 0x6b, 0x69, 0x6e, 0x65, 0x73, + 0x69, 0x73, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x53, 0x0a, 0x10, 0x72, 0x65, 0x64, + 0x73, 0x68, 0x69, 0x66, 0x74, 0x5f, 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x0f, 0x20, + 0x01, 0x28, 0x0b, 0x32, 0x26, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, + 0x2e, 0x44, 0x61, 0x74, 0x61, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x2e, 0x52, 0x65, 0x64, 0x73, + 0x68, 0x69, 0x66, 0x74, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x48, 0x00, 0x52, 0x0f, 0x72, + 0x65, 0x64, 0x73, 0x68, 0x69, 0x66, 0x74, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x5d, + 0x0a, 0x14, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x5f, 0x64, 0x61, 0x74, 0x61, 0x5f, 0x6f, + 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x12, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x29, 0x2e, 0x66, + 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x44, 0x61, 0x74, 0x61, 0x53, 0x6f, + 0x75, 0x72, 0x63, 0x65, 0x2e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x44, 0x61, 0x74, 0x61, + 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x48, 0x00, 0x52, 0x12, 0x72, 0x65, 0x71, 0x75, 0x65, + 0x73, 0x74, 0x44, 0x61, 0x74, 0x61, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x53, 0x0a, + 0x0e, 0x63, 0x75, 0x73, 0x74, 0x6f, 0x6d, 0x5f, 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, + 0x10, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2a, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, + 0x72, 0x65, 0x2e, 0x44, 0x61, 0x74, 0x61, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x2e, 0x43, 0x75, + 0x73, 0x74, 0x6f, 0x6d, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, + 0x73, 0x48, 0x00, 0x52, 0x0d, 0x63, 0x75, 0x73, 0x74, 0x6f, 0x6d, 0x4f, 0x70, 0x74, 0x69, 0x6f, + 0x6e, 0x73, 0x1a, 0x3f, 0x0a, 0x11, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x4d, 0x61, 0x70, 0x70, 0x69, + 0x6e, 0x67, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, + 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, + 0x02, 0x38, 0x01, 0x1a, 0x93, 0x01, 0x0a, 0x0b, 0x46, 0x69, 0x6c, 0x65, 0x4f, 0x70, 0x74, 0x69, + 0x6f, 0x6e, 0x73, 0x12, 0x37, 0x0a, 0x0b, 0x66, 0x69, 0x6c, 0x65, 0x5f, 0x66, 0x6f, 0x72, 0x6d, + 0x61, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, + 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x46, 0x69, 0x6c, 0x65, 0x46, 0x6f, 0x72, 0x6d, 0x61, 0x74, + 0x52, 0x0a, 0x66, 0x69, 0x6c, 0x65, 0x46, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x12, 0x19, 0x0a, 0x08, + 0x66, 0x69, 0x6c, 0x65, 0x5f, 0x75, 0x72, 0x6c, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, + 0x66, 0x69, 0x6c, 0x65, 0x55, 0x72, 0x6c, 0x12, 0x30, 0x0a, 0x14, 0x73, 0x33, 0x5f, 0x65, 0x6e, + 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x5f, 0x6f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x18, + 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x12, 0x73, 0x33, 0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, + 0x74, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x1a, 0x44, 0x0a, 0x0f, 0x42, 0x69, 0x67, + 0x51, 0x75, 0x65, 0x72, 0x79, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x1b, 0x0a, 0x09, + 0x74, 0x61, 0x62, 0x6c, 0x65, 0x5f, 0x72, 0x65, 0x66, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x08, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x52, 0x65, 0x66, 0x12, 0x14, 0x0a, 0x05, 0x71, 0x75, 0x65, + 0x72, 0x79, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x71, 0x75, 0x65, 0x72, 0x79, 0x1a, 0x92, 0x01, 0x0a, 0x0c, 0x4b, 0x61, 0x66, 0x6b, 0x61, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x2b, 0x0a, 0x11, 0x62, 0x6f, 0x6f, 0x74, 0x73, 0x74, 0x72, 0x61, 0x70, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x10, 0x62, 0x6f, 0x6f, @@ -579,21 +857,47 @@ var file_feast_core_DataSource_proto_rawDesc = []byte{ 0x12, 0x3d, 0x0a, 0x0d, 0x72, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x5f, 0x66, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x18, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x46, 0x6f, 0x72, 0x6d, 0x61, - 0x74, 0x52, 0x0c, 0x72, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x46, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x22, - 0x63, 0x0a, 0x0a, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x54, 0x79, 0x70, 0x65, 0x12, 0x0b, 0x0a, - 0x07, 0x49, 0x4e, 0x56, 0x41, 0x4c, 0x49, 0x44, 0x10, 0x00, 0x12, 0x0e, 0x0a, 0x0a, 0x42, 0x41, - 0x54, 0x43, 0x48, 0x5f, 0x46, 0x49, 0x4c, 0x45, 0x10, 0x01, 0x12, 0x12, 0x0a, 0x0e, 0x42, 0x41, - 0x54, 0x43, 0x48, 0x5f, 0x42, 0x49, 0x47, 0x51, 0x55, 0x45, 0x52, 0x59, 0x10, 0x02, 0x12, 0x10, - 0x0a, 0x0c, 0x53, 0x54, 0x52, 0x45, 0x41, 0x4d, 0x5f, 0x4b, 0x41, 0x46, 0x4b, 0x41, 0x10, 0x03, - 0x12, 0x12, 0x0a, 0x0e, 0x53, 0x54, 0x52, 0x45, 0x41, 0x4d, 0x5f, 0x4b, 0x49, 0x4e, 0x45, 0x53, - 0x49, 0x53, 0x10, 0x04, 0x42, 0x09, 0x0a, 0x07, 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x42, - 0x58, 0x0a, 0x10, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x63, - 0x6f, 0x72, 0x65, 0x42, 0x0f, 0x44, 0x61, 0x74, 0x61, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x50, - 0x72, 0x6f, 0x74, 0x6f, 0x5a, 0x33, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, - 0x2f, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2d, 0x64, 0x65, 0x76, 0x2f, 0x66, 0x65, 0x61, 0x73, 0x74, - 0x2f, 0x73, 0x64, 0x6b, 0x2f, 0x67, 0x6f, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x73, 0x2f, 0x66, - 0x65, 0x61, 0x73, 0x74, 0x2f, 0x63, 0x6f, 0x72, 0x65, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, - 0x33, + 0x74, 0x52, 0x0c, 0x72, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x46, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x1a, + 0x55, 0x0a, 0x0f, 0x52, 0x65, 0x64, 0x73, 0x68, 0x69, 0x66, 0x74, 0x4f, 0x70, 0x74, 0x69, 0x6f, + 0x6e, 0x73, 0x12, 0x14, 0x0a, 0x05, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x05, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x71, 0x75, 0x65, 0x72, + 0x79, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x71, 0x75, 0x65, 0x72, 0x79, 0x12, 0x16, + 0x0a, 0x06, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, + 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x1a, 0x3b, 0x0a, 0x13, 0x43, 0x75, 0x73, 0x74, 0x6f, 0x6d, + 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x24, 0x0a, + 0x0d, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0d, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x61, 0x74, + 0x69, 0x6f, 0x6e, 0x1a, 0xcf, 0x01, 0x0a, 0x12, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x44, + 0x61, 0x74, 0x61, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, + 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x4d, + 0x0a, 0x06, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x35, + 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x44, 0x61, 0x74, 0x61, + 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x2e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x44, 0x61, + 0x74, 0x61, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x2e, 0x53, 0x63, 0x68, 0x65, 0x6d, 0x61, + 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x06, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x1a, 0x56, 0x0a, + 0x0b, 0x53, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, + 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x31, + 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x1b, 0x2e, + 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x74, 0x79, 0x70, 0x65, 0x73, 0x2e, 0x56, 0x61, 0x6c, 0x75, + 0x65, 0x54, 0x79, 0x70, 0x65, 0x2e, 0x45, 0x6e, 0x75, 0x6d, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, + 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x9e, 0x01, 0x0a, 0x0a, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, + 0x54, 0x79, 0x70, 0x65, 0x12, 0x0b, 0x0a, 0x07, 0x49, 0x4e, 0x56, 0x41, 0x4c, 0x49, 0x44, 0x10, + 0x00, 0x12, 0x0e, 0x0a, 0x0a, 0x42, 0x41, 0x54, 0x43, 0x48, 0x5f, 0x46, 0x49, 0x4c, 0x45, 0x10, + 0x01, 0x12, 0x12, 0x0a, 0x0e, 0x42, 0x41, 0x54, 0x43, 0x48, 0x5f, 0x42, 0x49, 0x47, 0x51, 0x55, + 0x45, 0x52, 0x59, 0x10, 0x02, 0x12, 0x10, 0x0a, 0x0c, 0x53, 0x54, 0x52, 0x45, 0x41, 0x4d, 0x5f, + 0x4b, 0x41, 0x46, 0x4b, 0x41, 0x10, 0x03, 0x12, 0x12, 0x0a, 0x0e, 0x53, 0x54, 0x52, 0x45, 0x41, + 0x4d, 0x5f, 0x4b, 0x49, 0x4e, 0x45, 0x53, 0x49, 0x53, 0x10, 0x04, 0x12, 0x12, 0x0a, 0x0e, 0x42, + 0x41, 0x54, 0x43, 0x48, 0x5f, 0x52, 0x45, 0x44, 0x53, 0x48, 0x49, 0x46, 0x54, 0x10, 0x05, 0x12, + 0x11, 0x0a, 0x0d, 0x43, 0x55, 0x53, 0x54, 0x4f, 0x4d, 0x5f, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, + 0x10, 0x06, 0x12, 0x12, 0x0a, 0x0e, 0x52, 0x45, 0x51, 0x55, 0x45, 0x53, 0x54, 0x5f, 0x53, 0x4f, + 0x55, 0x52, 0x43, 0x45, 0x10, 0x07, 0x42, 0x09, 0x0a, 0x07, 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, + 0x73, 0x4a, 0x04, 0x08, 0x06, 0x10, 0x0b, 0x42, 0x58, 0x0a, 0x10, 0x66, 0x65, 0x61, 0x73, 0x74, + 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x42, 0x0f, 0x44, 0x61, 0x74, + 0x61, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x5a, 0x33, 0x67, 0x69, + 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2d, 0x64, + 0x65, 0x76, 0x2f, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2f, 0x73, 0x64, 0x6b, 0x2f, 0x67, 0x6f, 0x2f, + 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x73, 0x2f, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2f, 0x63, 0x6f, 0x72, + 0x65, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( @@ -609,33 +913,43 @@ func file_feast_core_DataSource_proto_rawDescGZIP() []byte { } var file_feast_core_DataSource_proto_enumTypes = make([]protoimpl.EnumInfo, 1) -var file_feast_core_DataSource_proto_msgTypes = make([]protoimpl.MessageInfo, 6) +var file_feast_core_DataSource_proto_msgTypes = make([]protoimpl.MessageInfo, 10) var file_feast_core_DataSource_proto_goTypes = []interface{}{ - (DataSource_SourceType)(0), // 0: feast.core.DataSource.SourceType - (*DataSource)(nil), // 1: feast.core.DataSource - nil, // 2: feast.core.DataSource.FieldMappingEntry - (*DataSource_FileOptions)(nil), // 3: feast.core.DataSource.FileOptions - (*DataSource_BigQueryOptions)(nil), // 4: feast.core.DataSource.BigQueryOptions - (*DataSource_KafkaOptions)(nil), // 5: feast.core.DataSource.KafkaOptions - (*DataSource_KinesisOptions)(nil), // 6: feast.core.DataSource.KinesisOptions - (*FileFormat)(nil), // 7: feast.core.FileFormat - (*StreamFormat)(nil), // 8: feast.core.StreamFormat + (DataSource_SourceType)(0), // 0: feast.core.DataSource.SourceType + (*DataSource)(nil), // 1: feast.core.DataSource + nil, // 2: feast.core.DataSource.FieldMappingEntry + (*DataSource_FileOptions)(nil), // 3: feast.core.DataSource.FileOptions + (*DataSource_BigQueryOptions)(nil), // 4: feast.core.DataSource.BigQueryOptions + (*DataSource_KafkaOptions)(nil), // 5: feast.core.DataSource.KafkaOptions + (*DataSource_KinesisOptions)(nil), // 6: feast.core.DataSource.KinesisOptions + (*DataSource_RedshiftOptions)(nil), // 7: feast.core.DataSource.RedshiftOptions + (*DataSource_CustomSourceOptions)(nil), // 8: feast.core.DataSource.CustomSourceOptions + (*DataSource_RequestDataOptions)(nil), // 9: feast.core.DataSource.RequestDataOptions + nil, // 10: feast.core.DataSource.RequestDataOptions.SchemaEntry + (*FileFormat)(nil), // 11: feast.core.FileFormat + (*StreamFormat)(nil), // 12: feast.core.StreamFormat + (types.ValueType_Enum)(0), // 13: feast.types.ValueType.Enum } var file_feast_core_DataSource_proto_depIdxs = []int32{ - 0, // 0: feast.core.DataSource.type:type_name -> feast.core.DataSource.SourceType - 2, // 1: feast.core.DataSource.field_mapping:type_name -> feast.core.DataSource.FieldMappingEntry - 3, // 2: feast.core.DataSource.file_options:type_name -> feast.core.DataSource.FileOptions - 4, // 3: feast.core.DataSource.bigquery_options:type_name -> feast.core.DataSource.BigQueryOptions - 5, // 4: feast.core.DataSource.kafka_options:type_name -> feast.core.DataSource.KafkaOptions - 6, // 5: feast.core.DataSource.kinesis_options:type_name -> feast.core.DataSource.KinesisOptions - 7, // 6: feast.core.DataSource.FileOptions.file_format:type_name -> feast.core.FileFormat - 8, // 7: feast.core.DataSource.KafkaOptions.message_format:type_name -> feast.core.StreamFormat - 8, // 8: feast.core.DataSource.KinesisOptions.record_format:type_name -> feast.core.StreamFormat - 9, // [9:9] is the sub-list for method output_type - 9, // [9:9] is the sub-list for method input_type - 9, // [9:9] is the sub-list for extension type_name - 9, // [9:9] is the sub-list for extension extendee - 0, // [0:9] is the sub-list for field type_name + 0, // 0: feast.core.DataSource.type:type_name -> feast.core.DataSource.SourceType + 2, // 1: feast.core.DataSource.field_mapping:type_name -> feast.core.DataSource.FieldMappingEntry + 3, // 2: feast.core.DataSource.file_options:type_name -> feast.core.DataSource.FileOptions + 4, // 3: feast.core.DataSource.bigquery_options:type_name -> feast.core.DataSource.BigQueryOptions + 5, // 4: feast.core.DataSource.kafka_options:type_name -> feast.core.DataSource.KafkaOptions + 6, // 5: feast.core.DataSource.kinesis_options:type_name -> feast.core.DataSource.KinesisOptions + 7, // 6: feast.core.DataSource.redshift_options:type_name -> feast.core.DataSource.RedshiftOptions + 9, // 7: feast.core.DataSource.request_data_options:type_name -> feast.core.DataSource.RequestDataOptions + 8, // 8: feast.core.DataSource.custom_options:type_name -> feast.core.DataSource.CustomSourceOptions + 11, // 9: feast.core.DataSource.FileOptions.file_format:type_name -> feast.core.FileFormat + 12, // 10: feast.core.DataSource.KafkaOptions.message_format:type_name -> feast.core.StreamFormat + 12, // 11: feast.core.DataSource.KinesisOptions.record_format:type_name -> feast.core.StreamFormat + 10, // 12: feast.core.DataSource.RequestDataOptions.schema:type_name -> feast.core.DataSource.RequestDataOptions.SchemaEntry + 13, // 13: feast.core.DataSource.RequestDataOptions.SchemaEntry.value:type_name -> feast.types.ValueType.Enum + 14, // [14:14] is the sub-list for method output_type + 14, // [14:14] is the sub-list for method input_type + 14, // [14:14] is the sub-list for extension type_name + 14, // [14:14] is the sub-list for extension extendee + 0, // [0:14] is the sub-list for field type_name } func init() { file_feast_core_DataSource_proto_init() } @@ -705,12 +1019,51 @@ func file_feast_core_DataSource_proto_init() { return nil } } + file_feast_core_DataSource_proto_msgTypes[6].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*DataSource_RedshiftOptions); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_feast_core_DataSource_proto_msgTypes[7].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*DataSource_CustomSourceOptions); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_feast_core_DataSource_proto_msgTypes[8].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*DataSource_RequestDataOptions); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } } file_feast_core_DataSource_proto_msgTypes[0].OneofWrappers = []interface{}{ (*DataSource_FileOptions_)(nil), (*DataSource_BigqueryOptions)(nil), (*DataSource_KafkaOptions_)(nil), (*DataSource_KinesisOptions_)(nil), + (*DataSource_RedshiftOptions_)(nil), + (*DataSource_RequestDataOptions_)(nil), + (*DataSource_CustomOptions)(nil), } type x struct{} out := protoimpl.TypeBuilder{ @@ -718,7 +1071,7 @@ func file_feast_core_DataSource_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: file_feast_core_DataSource_proto_rawDesc, NumEnums: 1, - NumMessages: 6, + NumMessages: 10, NumExtensions: 0, NumServices: 0, }, diff --git a/sdk/go/protos/feast/core/Entity.pb.go b/sdk/go/protos/feast/core/Entity.pb.go index 0aed9133257..c6d9014791e 100644 --- a/sdk/go/protos/feast/core/Entity.pb.go +++ b/sdk/go/protos/feast/core/Entity.pb.go @@ -16,18 +16,17 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.25.0 -// protoc v3.12.4 +// protoc-gen-go v1.27.1 +// protoc v3.11.2 // source: feast/core/Entity.proto package core import ( types "github.com/feast-dev/feast/sdk/go/protos/feast/types" - proto "github.com/golang/protobuf/proto" - timestamp "github.com/golang/protobuf/ptypes/timestamp" protoreflect "google.golang.org/protobuf/reflect/protoreflect" protoimpl "google.golang.org/protobuf/runtime/protoimpl" + timestamppb "google.golang.org/protobuf/types/known/timestamppb" reflect "reflect" sync "sync" ) @@ -39,10 +38,6 @@ const ( _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) ) -// This is a compile-time assertion that a sufficiently up-to-date version -// of the legacy proto package is being used. -const _ = proto.ProtoPackageIsVersion4 - type Entity struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -107,10 +102,14 @@ type EntitySpecV2 struct { // Name of the entity. Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` + // Name of Feast project that this feature table belongs to. + Project string `protobuf:"bytes,9,opt,name=project,proto3" json:"project,omitempty"` // Type of the entity. ValueType types.ValueType_Enum `protobuf:"varint,2,opt,name=value_type,json=valueType,proto3,enum=feast.types.ValueType_Enum" json:"value_type,omitempty"` // Description of the entity. Description string `protobuf:"bytes,3,opt,name=description,proto3" json:"description,omitempty"` + // Join key for the entity (i.e. name of the column the entity maps to). + JoinKey string `protobuf:"bytes,4,opt,name=join_key,json=joinKey,proto3" json:"join_key,omitempty"` // User defined metadata Labels map[string]string `protobuf:"bytes,8,rep,name=labels,proto3" json:"labels,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` } @@ -154,11 +153,18 @@ func (x *EntitySpecV2) GetName() string { return "" } +func (x *EntitySpecV2) GetProject() string { + if x != nil { + return x.Project + } + return "" +} + func (x *EntitySpecV2) GetValueType() types.ValueType_Enum { if x != nil { return x.ValueType } - return types.ValueType_INVALID + return types.ValueType_Enum(0) } func (x *EntitySpecV2) GetDescription() string { @@ -168,6 +174,13 @@ func (x *EntitySpecV2) GetDescription() string { return "" } +func (x *EntitySpecV2) GetJoinKey() string { + if x != nil { + return x.JoinKey + } + return "" +} + func (x *EntitySpecV2) GetLabels() map[string]string { if x != nil { return x.Labels @@ -180,8 +193,8 @@ type EntityMeta struct { sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - CreatedTimestamp *timestamp.Timestamp `protobuf:"bytes,1,opt,name=created_timestamp,json=createdTimestamp,proto3" json:"created_timestamp,omitempty"` - LastUpdatedTimestamp *timestamp.Timestamp `protobuf:"bytes,2,opt,name=last_updated_timestamp,json=lastUpdatedTimestamp,proto3" json:"last_updated_timestamp,omitempty"` + CreatedTimestamp *timestamppb.Timestamp `protobuf:"bytes,1,opt,name=created_timestamp,json=createdTimestamp,proto3" json:"created_timestamp,omitempty"` + LastUpdatedTimestamp *timestamppb.Timestamp `protobuf:"bytes,2,opt,name=last_updated_timestamp,json=lastUpdatedTimestamp,proto3" json:"last_updated_timestamp,omitempty"` } func (x *EntityMeta) Reset() { @@ -216,14 +229,14 @@ func (*EntityMeta) Descriptor() ([]byte, []int) { return file_feast_core_Entity_proto_rawDescGZIP(), []int{2} } -func (x *EntityMeta) GetCreatedTimestamp() *timestamp.Timestamp { +func (x *EntityMeta) GetCreatedTimestamp() *timestamppb.Timestamp { if x != nil { return x.CreatedTimestamp } return nil } -func (x *EntityMeta) GetLastUpdatedTimestamp() *timestamp.Timestamp { +func (x *EntityMeta) GetLastUpdatedTimestamp() *timestamppb.Timestamp { if x != nil { return x.LastUpdatedTimestamp } @@ -245,39 +258,42 @@ var file_feast_core_Entity_proto_rawDesc = []byte{ 0x32, 0x52, 0x04, 0x73, 0x70, 0x65, 0x63, 0x12, 0x2a, 0x0a, 0x04, 0x6d, 0x65, 0x74, 0x61, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x45, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x4d, 0x65, 0x74, 0x61, 0x52, 0x04, 0x6d, - 0x65, 0x74, 0x61, 0x22, 0xf9, 0x01, 0x0a, 0x0c, 0x45, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x53, 0x70, + 0x65, 0x74, 0x61, 0x22, 0xae, 0x02, 0x0a, 0x0c, 0x45, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x53, 0x70, 0x65, 0x63, 0x56, 0x32, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x3a, 0x0a, 0x0a, 0x76, 0x61, 0x6c, 0x75, - 0x65, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x1b, 0x2e, 0x66, - 0x65, 0x61, 0x73, 0x74, 0x2e, 0x74, 0x79, 0x70, 0x65, 0x73, 0x2e, 0x56, 0x61, 0x6c, 0x75, 0x65, - 0x54, 0x79, 0x70, 0x65, 0x2e, 0x45, 0x6e, 0x75, 0x6d, 0x52, 0x09, 0x76, 0x61, 0x6c, 0x75, 0x65, - 0x54, 0x79, 0x70, 0x65, 0x12, 0x20, 0x0a, 0x0b, 0x64, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, - 0x69, 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x64, 0x65, 0x73, 0x63, 0x72, - 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x3c, 0x0a, 0x06, 0x6c, 0x61, 0x62, 0x65, 0x6c, 0x73, - 0x18, 0x08, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x24, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, - 0x6f, 0x72, 0x65, 0x2e, 0x45, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x53, 0x70, 0x65, 0x63, 0x56, 0x32, - 0x2e, 0x4c, 0x61, 0x62, 0x65, 0x6c, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x06, 0x6c, 0x61, - 0x62, 0x65, 0x6c, 0x73, 0x1a, 0x39, 0x0a, 0x0b, 0x4c, 0x61, 0x62, 0x65, 0x6c, 0x73, 0x45, 0x6e, - 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, - 0xa7, 0x01, 0x0a, 0x0a, 0x45, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x4d, 0x65, 0x74, 0x61, 0x12, 0x47, - 0x0a, 0x11, 0x63, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, - 0x61, 0x6d, 0x70, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, - 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, - 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x10, 0x63, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, 0x54, 0x69, - 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x12, 0x50, 0x0a, 0x16, 0x6c, 0x61, 0x73, 0x74, 0x5f, - 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x64, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, - 0x70, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, - 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, - 0x61, 0x6d, 0x70, 0x52, 0x14, 0x6c, 0x61, 0x73, 0x74, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x64, - 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x42, 0x54, 0x0a, 0x10, 0x66, 0x65, 0x61, - 0x73, 0x74, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x42, 0x0b, 0x45, - 0x6e, 0x74, 0x69, 0x74, 0x79, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x5a, 0x33, 0x67, 0x69, 0x74, 0x68, - 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2d, 0x64, 0x65, 0x76, - 0x2f, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2f, 0x73, 0x64, 0x6b, 0x2f, 0x67, 0x6f, 0x2f, 0x70, 0x72, - 0x6f, 0x74, 0x6f, 0x73, 0x2f, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2f, 0x63, 0x6f, 0x72, 0x65, 0x62, - 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x70, 0x72, 0x6f, 0x6a, + 0x65, 0x63, 0x74, 0x18, 0x09, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x70, 0x72, 0x6f, 0x6a, 0x65, + 0x63, 0x74, 0x12, 0x3a, 0x0a, 0x0a, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x5f, 0x74, 0x79, 0x70, 0x65, + 0x18, 0x02, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x1b, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x74, + 0x79, 0x70, 0x65, 0x73, 0x2e, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x54, 0x79, 0x70, 0x65, 0x2e, 0x45, + 0x6e, 0x75, 0x6d, 0x52, 0x09, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x54, 0x79, 0x70, 0x65, 0x12, 0x20, + 0x0a, 0x0b, 0x64, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x03, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x0b, 0x64, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, + 0x12, 0x19, 0x0a, 0x08, 0x6a, 0x6f, 0x69, 0x6e, 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x04, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x07, 0x6a, 0x6f, 0x69, 0x6e, 0x4b, 0x65, 0x79, 0x12, 0x3c, 0x0a, 0x06, 0x6c, + 0x61, 0x62, 0x65, 0x6c, 0x73, 0x18, 0x08, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x24, 0x2e, 0x66, 0x65, + 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x45, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x53, + 0x70, 0x65, 0x63, 0x56, 0x32, 0x2e, 0x4c, 0x61, 0x62, 0x65, 0x6c, 0x73, 0x45, 0x6e, 0x74, 0x72, + 0x79, 0x52, 0x06, 0x6c, 0x61, 0x62, 0x65, 0x6c, 0x73, 0x1a, 0x39, 0x0a, 0x0b, 0x4c, 0x61, 0x62, + 0x65, 0x6c, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, + 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, + 0x3a, 0x02, 0x38, 0x01, 0x22, 0xa7, 0x01, 0x0a, 0x0a, 0x45, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x4d, + 0x65, 0x74, 0x61, 0x12, 0x47, 0x0a, 0x11, 0x63, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, 0x5f, 0x74, + 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, + 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, + 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x10, 0x63, 0x72, 0x65, 0x61, + 0x74, 0x65, 0x64, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x12, 0x50, 0x0a, 0x16, + 0x6c, 0x61, 0x73, 0x74, 0x5f, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x64, 0x5f, 0x74, 0x69, 0x6d, + 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, + 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, + 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x14, 0x6c, 0x61, 0x73, 0x74, 0x55, 0x70, + 0x64, 0x61, 0x74, 0x65, 0x64, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x42, 0x54, + 0x0a, 0x10, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x63, 0x6f, + 0x72, 0x65, 0x42, 0x0b, 0x45, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x5a, + 0x33, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x66, 0x65, 0x61, 0x73, + 0x74, 0x2d, 0x64, 0x65, 0x76, 0x2f, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2f, 0x73, 0x64, 0x6b, 0x2f, + 0x67, 0x6f, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x73, 0x2f, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2f, + 0x63, 0x6f, 0x72, 0x65, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( @@ -294,12 +310,12 @@ func file_feast_core_Entity_proto_rawDescGZIP() []byte { var file_feast_core_Entity_proto_msgTypes = make([]protoimpl.MessageInfo, 4) var file_feast_core_Entity_proto_goTypes = []interface{}{ - (*Entity)(nil), // 0: feast.core.Entity - (*EntitySpecV2)(nil), // 1: feast.core.EntitySpecV2 - (*EntityMeta)(nil), // 2: feast.core.EntityMeta - nil, // 3: feast.core.EntitySpecV2.LabelsEntry - (types.ValueType_Enum)(0), // 4: feast.types.ValueType.Enum - (*timestamp.Timestamp)(nil), // 5: google.protobuf.Timestamp + (*Entity)(nil), // 0: feast.core.Entity + (*EntitySpecV2)(nil), // 1: feast.core.EntitySpecV2 + (*EntityMeta)(nil), // 2: feast.core.EntityMeta + nil, // 3: feast.core.EntitySpecV2.LabelsEntry + (types.ValueType_Enum)(0), // 4: feast.types.ValueType.Enum + (*timestamppb.Timestamp)(nil), // 5: google.protobuf.Timestamp } var file_feast_core_Entity_proto_depIdxs = []int32{ 1, // 0: feast.core.Entity.spec:type_name -> feast.core.EntitySpecV2 diff --git a/sdk/go/protos/feast/core/Feature.pb.go b/sdk/go/protos/feast/core/Feature.pb.go index 1ad93ef8a1c..5d332dddff7 100644 --- a/sdk/go/protos/feast/core/Feature.pb.go +++ b/sdk/go/protos/feast/core/Feature.pb.go @@ -16,15 +16,14 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.25.0 -// protoc v3.12.4 +// protoc-gen-go v1.27.1 +// protoc v3.11.2 // source: feast/core/Feature.proto package core import ( types "github.com/feast-dev/feast/sdk/go/protos/feast/types" - proto "github.com/golang/protobuf/proto" protoreflect "google.golang.org/protobuf/reflect/protoreflect" protoimpl "google.golang.org/protobuf/runtime/protoimpl" reflect "reflect" @@ -38,10 +37,6 @@ const ( _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) ) -// This is a compile-time assertion that a sufficiently up-to-date version -// of the legacy proto package is being used. -const _ = proto.ProtoPackageIsVersion4 - type FeatureSpecV2 struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -98,7 +93,7 @@ func (x *FeatureSpecV2) GetValueType() types.ValueType_Enum { if x != nil { return x.ValueType } - return types.ValueType_INVALID + return types.ValueType_Enum(0) } func (x *FeatureSpecV2) GetLabels() map[string]string { diff --git a/sdk/go/protos/feast/core/FeatureTable.pb.go b/sdk/go/protos/feast/core/FeatureTable.pb.go index 0c4fa1b1f0c..355ef50fb86 100644 --- a/sdk/go/protos/feast/core/FeatureTable.pb.go +++ b/sdk/go/protos/feast/core/FeatureTable.pb.go @@ -16,18 +16,17 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.25.0 -// protoc v3.12.4 +// protoc-gen-go v1.27.1 +// protoc v3.11.2 // source: feast/core/FeatureTable.proto package core import ( - proto "github.com/golang/protobuf/proto" - duration "github.com/golang/protobuf/ptypes/duration" - timestamp "github.com/golang/protobuf/ptypes/timestamp" protoreflect "google.golang.org/protobuf/reflect/protoreflect" protoimpl "google.golang.org/protobuf/runtime/protoimpl" + durationpb "google.golang.org/protobuf/types/known/durationpb" + timestamppb "google.golang.org/protobuf/types/known/timestamppb" reflect "reflect" sync "sync" ) @@ -39,10 +38,6 @@ const ( _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) ) -// This is a compile-time assertion that a sufficiently up-to-date version -// of the legacy proto package is being used. -const _ = proto.ProtoPackageIsVersion4 - type FeatureTable struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -107,6 +102,8 @@ type FeatureTableSpec struct { // Name of the feature table. Must be unique. Not updated. Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` + // Name of Feast project that this feature table belongs to. + Project string `protobuf:"bytes,9,opt,name=project,proto3" json:"project,omitempty"` // List names of entities to associate with the Features defined in this // Feature Table. Not updatable. Entities []string `protobuf:"bytes,3,rep,name=entities,proto3" json:"entities,omitempty"` @@ -118,7 +115,7 @@ type FeatureTableSpec struct { // younger than max age. Age is measured as the duration of time between // the feature's event timestamp and when the feature is retrieved // Feature values outside max age will be returned as unset values and indicated to end user - MaxAge *duration.Duration `protobuf:"bytes,6,opt,name=max_age,json=maxAge,proto3" json:"max_age,omitempty"` + MaxAge *durationpb.Duration `protobuf:"bytes,6,opt,name=max_age,json=maxAge,proto3" json:"max_age,omitempty"` // Batch/Offline DataSource to source batch/offline feature data. // Only batch DataSource can be specified // (ie source type should start with 'BATCH_') @@ -168,6 +165,13 @@ func (x *FeatureTableSpec) GetName() string { return "" } +func (x *FeatureTableSpec) GetProject() string { + if x != nil { + return x.Project + } + return "" +} + func (x *FeatureTableSpec) GetEntities() []string { if x != nil { return x.Entities @@ -189,7 +193,7 @@ func (x *FeatureTableSpec) GetLabels() map[string]string { return nil } -func (x *FeatureTableSpec) GetMaxAge() *duration.Duration { +func (x *FeatureTableSpec) GetMaxAge() *durationpb.Duration { if x != nil { return x.MaxAge } @@ -216,9 +220,9 @@ type FeatureTableMeta struct { unknownFields protoimpl.UnknownFields // Time where this Feature Table is created - CreatedTimestamp *timestamp.Timestamp `protobuf:"bytes,1,opt,name=created_timestamp,json=createdTimestamp,proto3" json:"created_timestamp,omitempty"` + CreatedTimestamp *timestamppb.Timestamp `protobuf:"bytes,1,opt,name=created_timestamp,json=createdTimestamp,proto3" json:"created_timestamp,omitempty"` // Time where this Feature Table is last updated - LastUpdatedTimestamp *timestamp.Timestamp `protobuf:"bytes,2,opt,name=last_updated_timestamp,json=lastUpdatedTimestamp,proto3" json:"last_updated_timestamp,omitempty"` + LastUpdatedTimestamp *timestamppb.Timestamp `protobuf:"bytes,2,opt,name=last_updated_timestamp,json=lastUpdatedTimestamp,proto3" json:"last_updated_timestamp,omitempty"` // Auto incrementing revision no. of this Feature Table Revision int64 `protobuf:"varint,3,opt,name=revision,proto3" json:"revision,omitempty"` // Hash entities, features, batch_source and stream_source to inform JobService if @@ -258,14 +262,14 @@ func (*FeatureTableMeta) Descriptor() ([]byte, []int) { return file_feast_core_FeatureTable_proto_rawDescGZIP(), []int{2} } -func (x *FeatureTableMeta) GetCreatedTimestamp() *timestamp.Timestamp { +func (x *FeatureTableMeta) GetCreatedTimestamp() *timestamppb.Timestamp { if x != nil { return x.CreatedTimestamp } return nil } -func (x *FeatureTableMeta) GetLastUpdatedTimestamp() *timestamp.Timestamp { +func (x *FeatureTableMeta) GetLastUpdatedTimestamp() *timestamppb.Timestamp { if x != nil { return x.LastUpdatedTimestamp } @@ -306,53 +310,55 @@ var file_feast_core_FeatureTable_proto_rawDesc = []byte{ 0x04, 0x73, 0x70, 0x65, 0x63, 0x12, 0x30, 0x0a, 0x04, 0x6d, 0x65, 0x74, 0x61, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x4d, 0x65, 0x74, - 0x61, 0x52, 0x04, 0x6d, 0x65, 0x74, 0x61, 0x22, 0xa2, 0x03, 0x0a, 0x10, 0x46, 0x65, 0x61, 0x74, + 0x61, 0x52, 0x04, 0x6d, 0x65, 0x74, 0x61, 0x22, 0xbc, 0x03, 0x0a, 0x10, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x53, 0x70, 0x65, 0x63, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, - 0x12, 0x1a, 0x0a, 0x08, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x69, 0x65, 0x73, 0x18, 0x03, 0x20, 0x03, - 0x28, 0x09, 0x52, 0x08, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x69, 0x65, 0x73, 0x12, 0x35, 0x0a, 0x08, - 0x66, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x19, - 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x46, 0x65, 0x61, 0x74, - 0x75, 0x72, 0x65, 0x53, 0x70, 0x65, 0x63, 0x56, 0x32, 0x52, 0x08, 0x66, 0x65, 0x61, 0x74, 0x75, - 0x72, 0x65, 0x73, 0x12, 0x40, 0x0a, 0x06, 0x6c, 0x61, 0x62, 0x65, 0x6c, 0x73, 0x18, 0x05, 0x20, - 0x03, 0x28, 0x0b, 0x32, 0x28, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, - 0x2e, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x53, 0x70, 0x65, - 0x63, 0x2e, 0x4c, 0x61, 0x62, 0x65, 0x6c, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x06, 0x6c, - 0x61, 0x62, 0x65, 0x6c, 0x73, 0x12, 0x32, 0x0a, 0x07, 0x6d, 0x61, 0x78, 0x5f, 0x61, 0x67, 0x65, - 0x18, 0x06, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, - 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x44, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, - 0x6e, 0x52, 0x06, 0x6d, 0x61, 0x78, 0x41, 0x67, 0x65, 0x12, 0x39, 0x0a, 0x0c, 0x62, 0x61, 0x74, - 0x63, 0x68, 0x5f, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x18, 0x07, 0x20, 0x01, 0x28, 0x0b, 0x32, - 0x16, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x44, 0x61, 0x74, - 0x61, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x52, 0x0b, 0x62, 0x61, 0x74, 0x63, 0x68, 0x53, 0x6f, - 0x75, 0x72, 0x63, 0x65, 0x12, 0x3b, 0x0a, 0x0d, 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x5f, 0x73, - 0x6f, 0x75, 0x72, 0x63, 0x65, 0x18, 0x08, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x66, 0x65, - 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x44, 0x61, 0x74, 0x61, 0x53, 0x6f, 0x75, - 0x72, 0x63, 0x65, 0x52, 0x0c, 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x53, 0x6f, 0x75, 0x72, 0x63, - 0x65, 0x1a, 0x39, 0x0a, 0x0b, 0x4c, 0x61, 0x62, 0x65, 0x6c, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, - 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, - 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0xdd, 0x01, 0x0a, - 0x10, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x4d, 0x65, 0x74, - 0x61, 0x12, 0x47, 0x0a, 0x11, 0x63, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, 0x5f, 0x74, 0x69, 0x6d, - 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, - 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, - 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x10, 0x63, 0x72, 0x65, 0x61, 0x74, 0x65, - 0x64, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x12, 0x50, 0x0a, 0x16, 0x6c, 0x61, - 0x73, 0x74, 0x5f, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x64, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x73, - 0x74, 0x61, 0x6d, 0x70, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, - 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, - 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x14, 0x6c, 0x61, 0x73, 0x74, 0x55, 0x70, 0x64, 0x61, - 0x74, 0x65, 0x64, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x12, 0x1a, 0x0a, 0x08, - 0x72, 0x65, 0x76, 0x69, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x03, 0x52, 0x08, - 0x72, 0x65, 0x76, 0x69, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x12, 0x0a, 0x04, 0x68, 0x61, 0x73, 0x68, - 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x68, 0x61, 0x73, 0x68, 0x42, 0x5a, 0x0a, 0x10, - 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x63, 0x6f, 0x72, 0x65, - 0x42, 0x11, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x50, 0x72, - 0x6f, 0x74, 0x6f, 0x5a, 0x33, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, - 0x66, 0x65, 0x61, 0x73, 0x74, 0x2d, 0x64, 0x65, 0x76, 0x2f, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2f, - 0x73, 0x64, 0x6b, 0x2f, 0x67, 0x6f, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x73, 0x2f, 0x66, 0x65, - 0x61, 0x73, 0x74, 0x2f, 0x63, 0x6f, 0x72, 0x65, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x12, 0x18, 0x0a, 0x07, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x18, 0x09, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x07, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x12, 0x1a, 0x0a, 0x08, 0x65, 0x6e, + 0x74, 0x69, 0x74, 0x69, 0x65, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x09, 0x52, 0x08, 0x65, 0x6e, + 0x74, 0x69, 0x74, 0x69, 0x65, 0x73, 0x12, 0x35, 0x0a, 0x08, 0x66, 0x65, 0x61, 0x74, 0x75, 0x72, + 0x65, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, + 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x53, 0x70, 0x65, + 0x63, 0x56, 0x32, 0x52, 0x08, 0x66, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x12, 0x40, 0x0a, + 0x06, 0x6c, 0x61, 0x62, 0x65, 0x6c, 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x28, 0x2e, + 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x46, 0x65, 0x61, 0x74, 0x75, + 0x72, 0x65, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x53, 0x70, 0x65, 0x63, 0x2e, 0x4c, 0x61, 0x62, 0x65, + 0x6c, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x06, 0x6c, 0x61, 0x62, 0x65, 0x6c, 0x73, 0x12, + 0x32, 0x0a, 0x07, 0x6d, 0x61, 0x78, 0x5f, 0x61, 0x67, 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0b, + 0x32, 0x19, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, + 0x75, 0x66, 0x2e, 0x44, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x06, 0x6d, 0x61, 0x78, + 0x41, 0x67, 0x65, 0x12, 0x39, 0x0a, 0x0c, 0x62, 0x61, 0x74, 0x63, 0x68, 0x5f, 0x73, 0x6f, 0x75, + 0x72, 0x63, 0x65, 0x18, 0x07, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x66, 0x65, 0x61, 0x73, + 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x44, 0x61, 0x74, 0x61, 0x53, 0x6f, 0x75, 0x72, 0x63, + 0x65, 0x52, 0x0b, 0x62, 0x61, 0x74, 0x63, 0x68, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x12, 0x3b, + 0x0a, 0x0d, 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x5f, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x18, + 0x08, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, + 0x72, 0x65, 0x2e, 0x44, 0x61, 0x74, 0x61, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x52, 0x0c, 0x73, + 0x74, 0x72, 0x65, 0x61, 0x6d, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x1a, 0x39, 0x0a, 0x0b, 0x4c, + 0x61, 0x62, 0x65, 0x6c, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, + 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, + 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, + 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0xdd, 0x01, 0x0a, 0x10, 0x46, 0x65, 0x61, 0x74, 0x75, + 0x72, 0x65, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x4d, 0x65, 0x74, 0x61, 0x12, 0x47, 0x0a, 0x11, 0x63, + 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, + 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, + 0x6d, 0x70, 0x52, 0x10, 0x63, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, 0x54, 0x69, 0x6d, 0x65, 0x73, + 0x74, 0x61, 0x6d, 0x70, 0x12, 0x50, 0x0a, 0x16, 0x6c, 0x61, 0x73, 0x74, 0x5f, 0x75, 0x70, 0x64, + 0x61, 0x74, 0x65, 0x64, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x18, 0x02, + 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, + 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, + 0x52, 0x14, 0x6c, 0x61, 0x73, 0x74, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x64, 0x54, 0x69, 0x6d, + 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x12, 0x1a, 0x0a, 0x08, 0x72, 0x65, 0x76, 0x69, 0x73, 0x69, + 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x03, 0x52, 0x08, 0x72, 0x65, 0x76, 0x69, 0x73, 0x69, + 0x6f, 0x6e, 0x12, 0x12, 0x0a, 0x04, 0x68, 0x61, 0x73, 0x68, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x04, 0x68, 0x61, 0x73, 0x68, 0x42, 0x5a, 0x0a, 0x10, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, + 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x42, 0x11, 0x46, 0x65, 0x61, 0x74, + 0x75, 0x72, 0x65, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x5a, 0x33, 0x67, + 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2d, + 0x64, 0x65, 0x76, 0x2f, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2f, 0x73, 0x64, 0x6b, 0x2f, 0x67, 0x6f, + 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x73, 0x2f, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2f, 0x63, 0x6f, + 0x72, 0x65, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( @@ -369,14 +375,14 @@ func file_feast_core_FeatureTable_proto_rawDescGZIP() []byte { var file_feast_core_FeatureTable_proto_msgTypes = make([]protoimpl.MessageInfo, 4) var file_feast_core_FeatureTable_proto_goTypes = []interface{}{ - (*FeatureTable)(nil), // 0: feast.core.FeatureTable - (*FeatureTableSpec)(nil), // 1: feast.core.FeatureTableSpec - (*FeatureTableMeta)(nil), // 2: feast.core.FeatureTableMeta - nil, // 3: feast.core.FeatureTableSpec.LabelsEntry - (*FeatureSpecV2)(nil), // 4: feast.core.FeatureSpecV2 - (*duration.Duration)(nil), // 5: google.protobuf.Duration - (*DataSource)(nil), // 6: feast.core.DataSource - (*timestamp.Timestamp)(nil), // 7: google.protobuf.Timestamp + (*FeatureTable)(nil), // 0: feast.core.FeatureTable + (*FeatureTableSpec)(nil), // 1: feast.core.FeatureTableSpec + (*FeatureTableMeta)(nil), // 2: feast.core.FeatureTableMeta + nil, // 3: feast.core.FeatureTableSpec.LabelsEntry + (*FeatureSpecV2)(nil), // 4: feast.core.FeatureSpecV2 + (*durationpb.Duration)(nil), // 5: google.protobuf.Duration + (*DataSource)(nil), // 6: feast.core.DataSource + (*timestamppb.Timestamp)(nil), // 7: google.protobuf.Timestamp } var file_feast_core_FeatureTable_proto_depIdxs = []int32{ 1, // 0: feast.core.FeatureTable.spec:type_name -> feast.core.FeatureTableSpec diff --git a/sdk/go/protos/feast/core/Store.pb.go b/sdk/go/protos/feast/core/Store.pb.go index 9498c6a40c7..6c46f10d24a 100644 --- a/sdk/go/protos/feast/core/Store.pb.go +++ b/sdk/go/protos/feast/core/Store.pb.go @@ -16,14 +16,13 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.25.0 -// protoc v3.12.4 +// protoc-gen-go v1.27.1 +// protoc v3.11.2 // source: feast/core/Store.proto package core import ( - proto "github.com/golang/protobuf/proto" protoreflect "google.golang.org/protobuf/reflect/protoreflect" protoimpl "google.golang.org/protobuf/runtime/protoimpl" reflect "reflect" @@ -37,10 +36,6 @@ const ( _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) ) -// This is a compile-time assertion that a sufficiently up-to-date version -// of the legacy proto package is being used. -const _ = proto.ProtoPackageIsVersion4 - type Store_StoreType int32 const ( @@ -100,6 +95,59 @@ func (Store_StoreType) EnumDescriptor() ([]byte, []int) { return file_feast_core_Store_proto_rawDescGZIP(), []int{0, 0} } +// Optional. Priority of nodes when reading from cluster +type Store_RedisClusterConfig_ReadFrom int32 + +const ( + Store_RedisClusterConfig_MASTER Store_RedisClusterConfig_ReadFrom = 0 + Store_RedisClusterConfig_MASTER_PREFERRED Store_RedisClusterConfig_ReadFrom = 1 + Store_RedisClusterConfig_REPLICA Store_RedisClusterConfig_ReadFrom = 2 + Store_RedisClusterConfig_REPLICA_PREFERRED Store_RedisClusterConfig_ReadFrom = 3 +) + +// Enum value maps for Store_RedisClusterConfig_ReadFrom. +var ( + Store_RedisClusterConfig_ReadFrom_name = map[int32]string{ + 0: "MASTER", + 1: "MASTER_PREFERRED", + 2: "REPLICA", + 3: "REPLICA_PREFERRED", + } + Store_RedisClusterConfig_ReadFrom_value = map[string]int32{ + "MASTER": 0, + "MASTER_PREFERRED": 1, + "REPLICA": 2, + "REPLICA_PREFERRED": 3, + } +) + +func (x Store_RedisClusterConfig_ReadFrom) Enum() *Store_RedisClusterConfig_ReadFrom { + p := new(Store_RedisClusterConfig_ReadFrom) + *p = x + return p +} + +func (x Store_RedisClusterConfig_ReadFrom) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (Store_RedisClusterConfig_ReadFrom) Descriptor() protoreflect.EnumDescriptor { + return file_feast_core_Store_proto_enumTypes[1].Descriptor() +} + +func (Store_RedisClusterConfig_ReadFrom) Type() protoreflect.EnumType { + return &file_feast_core_Store_proto_enumTypes[1] +} + +func (x Store_RedisClusterConfig_ReadFrom) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Use Store_RedisClusterConfig_ReadFrom.Descriptor instead. +func (Store_RedisClusterConfig_ReadFrom) EnumDescriptor() ([]byte, []int) { + return file_feast_core_Store_proto_rawDescGZIP(), []int{0, 1, 0} +} + // Store provides a location where Feast reads and writes feature values. // Feature values will be written to the Store in the form of FeatureRow elements. // The way FeatureRow is encoded and decoded when it is written to and read from @@ -323,7 +371,8 @@ type Store_RedisClusterConfig struct { // Useful for migrating key prefix without re-ingestion. Disabled by default. EnableFallback bool `protobuf:"varint,6,opt,name=enable_fallback,json=enableFallback,proto3" json:"enable_fallback,omitempty"` // Optional. This would be the fallback prefix to use if enable_fallback is true. - FallbackPrefix string `protobuf:"bytes,7,opt,name=fallback_prefix,json=fallbackPrefix,proto3" json:"fallback_prefix,omitempty"` + FallbackPrefix string `protobuf:"bytes,7,opt,name=fallback_prefix,json=fallbackPrefix,proto3" json:"fallback_prefix,omitempty"` + ReadFrom Store_RedisClusterConfig_ReadFrom `protobuf:"varint,8,opt,name=read_from,json=readFrom,proto3,enum=feast.core.Store_RedisClusterConfig_ReadFrom" json:"read_from,omitempty"` } func (x *Store_RedisClusterConfig) Reset() { @@ -407,6 +456,13 @@ func (x *Store_RedisClusterConfig) GetFallbackPrefix() string { return "" } +func (x *Store_RedisClusterConfig) GetReadFrom() Store_RedisClusterConfig_ReadFrom { + if x != nil { + return x.ReadFrom + } + return Store_RedisClusterConfig_MASTER +} + type Store_Subscription struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -489,7 +545,7 @@ var File_feast_core_Store_proto protoreflect.FileDescriptor var file_feast_core_Store_proto_rawDesc = []byte{ 0x0a, 0x16, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2f, 0x63, 0x6f, 0x72, 0x65, 0x2f, 0x53, 0x74, 0x6f, 0x72, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x0a, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, - 0x63, 0x6f, 0x72, 0x65, 0x22, 0xf5, 0x07, 0x0a, 0x05, 0x53, 0x74, 0x6f, 0x72, 0x65, 0x12, 0x12, + 0x63, 0x6f, 0x72, 0x65, 0x22, 0x93, 0x09, 0x0a, 0x05, 0x53, 0x74, 0x6f, 0x72, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x2f, 0x0a, 0x04, 0x74, 0x79, 0x70, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x1b, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x53, 0x74, @@ -521,7 +577,7 @@ var file_feast_core_Store_proto_rawDesc = []byte{ 0x79, 0x5f, 0x73, 0x65, 0x63, 0x6f, 0x6e, 0x64, 0x73, 0x18, 0x05, 0x20, 0x01, 0x28, 0x05, 0x52, 0x15, 0x66, 0x6c, 0x75, 0x73, 0x68, 0x46, 0x72, 0x65, 0x71, 0x75, 0x65, 0x6e, 0x63, 0x79, 0x53, 0x65, 0x63, 0x6f, 0x6e, 0x64, 0x73, 0x12, 0x10, 0x0a, 0x03, 0x73, 0x73, 0x6c, 0x18, 0x06, 0x20, - 0x01, 0x28, 0x08, 0x52, 0x03, 0x73, 0x73, 0x6c, 0x1a, 0xb9, 0x02, 0x0a, 0x12, 0x52, 0x65, 0x64, + 0x01, 0x28, 0x08, 0x52, 0x03, 0x73, 0x73, 0x6c, 0x1a, 0xd7, 0x03, 0x0a, 0x12, 0x52, 0x65, 0x64, 0x69, 0x73, 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x2b, 0x0a, 0x11, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x73, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x10, 0x63, 0x6f, 0x6e, 0x6e, @@ -541,24 +597,34 @@ var file_feast_core_Store_proto_rawDesc = []byte{ 0x62, 0x6c, 0x65, 0x46, 0x61, 0x6c, 0x6c, 0x62, 0x61, 0x63, 0x6b, 0x12, 0x27, 0x0a, 0x0f, 0x66, 0x61, 0x6c, 0x6c, 0x62, 0x61, 0x63, 0x6b, 0x5f, 0x70, 0x72, 0x65, 0x66, 0x69, 0x78, 0x18, 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0e, 0x66, 0x61, 0x6c, 0x6c, 0x62, 0x61, 0x63, 0x6b, 0x50, 0x72, - 0x65, 0x66, 0x69, 0x78, 0x1a, 0x5c, 0x0a, 0x0c, 0x53, 0x75, 0x62, 0x73, 0x63, 0x72, 0x69, 0x70, - 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x18, 0x0a, 0x07, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x18, - 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x12, 0x12, - 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, - 0x6d, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x65, 0x78, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x18, 0x04, 0x20, - 0x01, 0x28, 0x08, 0x52, 0x07, 0x65, 0x78, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x4a, 0x04, 0x08, 0x02, - 0x10, 0x03, 0x22, 0x4e, 0x0a, 0x09, 0x53, 0x74, 0x6f, 0x72, 0x65, 0x54, 0x79, 0x70, 0x65, 0x12, - 0x0b, 0x0a, 0x07, 0x49, 0x4e, 0x56, 0x41, 0x4c, 0x49, 0x44, 0x10, 0x00, 0x12, 0x09, 0x0a, 0x05, - 0x52, 0x45, 0x44, 0x49, 0x53, 0x10, 0x01, 0x12, 0x11, 0x0a, 0x0d, 0x52, 0x45, 0x44, 0x49, 0x53, - 0x5f, 0x43, 0x4c, 0x55, 0x53, 0x54, 0x45, 0x52, 0x10, 0x04, 0x22, 0x04, 0x08, 0x02, 0x10, 0x02, - 0x22, 0x04, 0x08, 0x03, 0x10, 0x03, 0x22, 0x04, 0x08, 0x0c, 0x10, 0x0c, 0x22, 0x04, 0x08, 0x0d, - 0x10, 0x0d, 0x42, 0x08, 0x0a, 0x06, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x42, 0x53, 0x0a, 0x10, - 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x63, 0x6f, 0x72, 0x65, - 0x42, 0x0a, 0x53, 0x74, 0x6f, 0x72, 0x65, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x5a, 0x33, 0x67, 0x69, - 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2d, 0x64, - 0x65, 0x76, 0x2f, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2f, 0x73, 0x64, 0x6b, 0x2f, 0x67, 0x6f, 0x2f, - 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x73, 0x2f, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2f, 0x63, 0x6f, 0x72, - 0x65, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x65, 0x66, 0x69, 0x78, 0x12, 0x4a, 0x0a, 0x09, 0x72, 0x65, 0x61, 0x64, 0x5f, 0x66, 0x72, 0x6f, + 0x6d, 0x18, 0x08, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x2d, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, + 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x53, 0x74, 0x6f, 0x72, 0x65, 0x2e, 0x52, 0x65, 0x64, 0x69, 0x73, + 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x2e, 0x52, 0x65, + 0x61, 0x64, 0x46, 0x72, 0x6f, 0x6d, 0x52, 0x08, 0x72, 0x65, 0x61, 0x64, 0x46, 0x72, 0x6f, 0x6d, + 0x22, 0x50, 0x0a, 0x08, 0x52, 0x65, 0x61, 0x64, 0x46, 0x72, 0x6f, 0x6d, 0x12, 0x0a, 0x0a, 0x06, + 0x4d, 0x41, 0x53, 0x54, 0x45, 0x52, 0x10, 0x00, 0x12, 0x14, 0x0a, 0x10, 0x4d, 0x41, 0x53, 0x54, + 0x45, 0x52, 0x5f, 0x50, 0x52, 0x45, 0x46, 0x45, 0x52, 0x52, 0x45, 0x44, 0x10, 0x01, 0x12, 0x0b, + 0x0a, 0x07, 0x52, 0x45, 0x50, 0x4c, 0x49, 0x43, 0x41, 0x10, 0x02, 0x12, 0x15, 0x0a, 0x11, 0x52, + 0x45, 0x50, 0x4c, 0x49, 0x43, 0x41, 0x5f, 0x50, 0x52, 0x45, 0x46, 0x45, 0x52, 0x52, 0x45, 0x44, + 0x10, 0x03, 0x1a, 0x5c, 0x0a, 0x0c, 0x53, 0x75, 0x62, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, + 0x6f, 0x6e, 0x12, 0x18, 0x0a, 0x07, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x18, 0x03, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x07, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x12, 0x12, 0x0a, 0x04, + 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, + 0x12, 0x18, 0x0a, 0x07, 0x65, 0x78, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, + 0x08, 0x52, 0x07, 0x65, 0x78, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x4a, 0x04, 0x08, 0x02, 0x10, 0x03, + 0x22, 0x4e, 0x0a, 0x09, 0x53, 0x74, 0x6f, 0x72, 0x65, 0x54, 0x79, 0x70, 0x65, 0x12, 0x0b, 0x0a, + 0x07, 0x49, 0x4e, 0x56, 0x41, 0x4c, 0x49, 0x44, 0x10, 0x00, 0x12, 0x09, 0x0a, 0x05, 0x52, 0x45, + 0x44, 0x49, 0x53, 0x10, 0x01, 0x12, 0x11, 0x0a, 0x0d, 0x52, 0x45, 0x44, 0x49, 0x53, 0x5f, 0x43, + 0x4c, 0x55, 0x53, 0x54, 0x45, 0x52, 0x10, 0x04, 0x22, 0x04, 0x08, 0x02, 0x10, 0x02, 0x22, 0x04, + 0x08, 0x03, 0x10, 0x03, 0x22, 0x04, 0x08, 0x0c, 0x10, 0x0c, 0x22, 0x04, 0x08, 0x0d, 0x10, 0x0d, + 0x42, 0x08, 0x0a, 0x06, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x42, 0x53, 0x0a, 0x10, 0x66, 0x65, + 0x61, 0x73, 0x74, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x42, 0x0a, + 0x53, 0x74, 0x6f, 0x72, 0x65, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x5a, 0x33, 0x67, 0x69, 0x74, 0x68, + 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2d, 0x64, 0x65, 0x76, + 0x2f, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2f, 0x73, 0x64, 0x6b, 0x2f, 0x67, 0x6f, 0x2f, 0x70, 0x72, + 0x6f, 0x74, 0x6f, 0x73, 0x2f, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2f, 0x63, 0x6f, 0x72, 0x65, 0x62, + 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( @@ -573,25 +639,27 @@ func file_feast_core_Store_proto_rawDescGZIP() []byte { return file_feast_core_Store_proto_rawDescData } -var file_feast_core_Store_proto_enumTypes = make([]protoimpl.EnumInfo, 1) +var file_feast_core_Store_proto_enumTypes = make([]protoimpl.EnumInfo, 2) var file_feast_core_Store_proto_msgTypes = make([]protoimpl.MessageInfo, 4) var file_feast_core_Store_proto_goTypes = []interface{}{ - (Store_StoreType)(0), // 0: feast.core.Store.StoreType - (*Store)(nil), // 1: feast.core.Store - (*Store_RedisConfig)(nil), // 2: feast.core.Store.RedisConfig - (*Store_RedisClusterConfig)(nil), // 3: feast.core.Store.RedisClusterConfig - (*Store_Subscription)(nil), // 4: feast.core.Store.Subscription + (Store_StoreType)(0), // 0: feast.core.Store.StoreType + (Store_RedisClusterConfig_ReadFrom)(0), // 1: feast.core.Store.RedisClusterConfig.ReadFrom + (*Store)(nil), // 2: feast.core.Store + (*Store_RedisConfig)(nil), // 3: feast.core.Store.RedisConfig + (*Store_RedisClusterConfig)(nil), // 4: feast.core.Store.RedisClusterConfig + (*Store_Subscription)(nil), // 5: feast.core.Store.Subscription } var file_feast_core_Store_proto_depIdxs = []int32{ 0, // 0: feast.core.Store.type:type_name -> feast.core.Store.StoreType - 4, // 1: feast.core.Store.subscriptions:type_name -> feast.core.Store.Subscription - 2, // 2: feast.core.Store.redis_config:type_name -> feast.core.Store.RedisConfig - 3, // 3: feast.core.Store.redis_cluster_config:type_name -> feast.core.Store.RedisClusterConfig - 4, // [4:4] is the sub-list for method output_type - 4, // [4:4] is the sub-list for method input_type - 4, // [4:4] is the sub-list for extension type_name - 4, // [4:4] is the sub-list for extension extendee - 0, // [0:4] is the sub-list for field type_name + 5, // 1: feast.core.Store.subscriptions:type_name -> feast.core.Store.Subscription + 3, // 2: feast.core.Store.redis_config:type_name -> feast.core.Store.RedisConfig + 4, // 3: feast.core.Store.redis_cluster_config:type_name -> feast.core.Store.RedisClusterConfig + 1, // 4: feast.core.Store.RedisClusterConfig.read_from:type_name -> feast.core.Store.RedisClusterConfig.ReadFrom + 5, // [5:5] is the sub-list for method output_type + 5, // [5:5] is the sub-list for method input_type + 5, // [5:5] is the sub-list for extension type_name + 5, // [5:5] is the sub-list for extension extendee + 0, // [0:5] is the sub-list for field type_name } func init() { file_feast_core_Store_proto_init() } @@ -658,7 +726,7 @@ func file_feast_core_Store_proto_init() { File: protoimpl.DescBuilder{ GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: file_feast_core_Store_proto_rawDesc, - NumEnums: 1, + NumEnums: 2, NumMessages: 4, NumExtensions: 0, NumServices: 0, diff --git a/sdk/go/protos/feast/serving/ServingService.pb.go b/sdk/go/protos/feast/serving/ServingService.pb.go index 7d40fa45d59..32e3461dfdf 100644 --- a/sdk/go/protos/feast/serving/ServingService.pb.go +++ b/sdk/go/protos/feast/serving/ServingService.pb.go @@ -15,8 +15,8 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.25.0 -// protoc v3.12.4 +// protoc-gen-go v1.27.1 +// protoc v3.11.2 // source: feast/serving/ServingService.proto package serving @@ -25,13 +25,12 @@ import ( context "context" types "github.com/feast-dev/feast/sdk/go/protos/feast/types" _ "github.com/feast-dev/feast/sdk/go/protos/tensorflow_metadata/proto/v0" - proto "github.com/golang/protobuf/proto" - timestamp "github.com/golang/protobuf/ptypes/timestamp" grpc "google.golang.org/grpc" codes "google.golang.org/grpc/codes" status "google.golang.org/grpc/status" protoreflect "google.golang.org/protobuf/reflect/protoreflect" protoimpl "google.golang.org/protobuf/runtime/protoimpl" + timestamppb "google.golang.org/protobuf/types/known/timestamppb" reflect "reflect" sync "sync" ) @@ -43,10 +42,6 @@ const ( _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) ) -// This is a compile-time assertion that a sufficiently up-to-date version -// of the legacy proto package is being used. -const _ = proto.ProtoPackageIsVersion4 - type FeastServingType int32 const ( @@ -161,7 +156,7 @@ func (x GetOnlineFeaturesResponse_FieldStatus) Number() protoreflect.EnumNumber // Deprecated: Use GetOnlineFeaturesResponse_FieldStatus.Descriptor instead. func (GetOnlineFeaturesResponse_FieldStatus) EnumDescriptor() ([]byte, []int) { - return file_feast_serving_ServingService_proto_rawDescGZIP(), []int{4, 0} + return file_feast_serving_ServingService_proto_rawDescGZIP(), []int{6, 0} } type GetFeastServingInfoRequest struct { @@ -397,6 +392,152 @@ func (x *GetOnlineFeaturesRequestV2) GetProject() string { return "" } +// In JSON "val" field can be omitted +type FeatureList struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Val []string `protobuf:"bytes,1,rep,name=val,proto3" json:"val,omitempty"` +} + +func (x *FeatureList) Reset() { + *x = FeatureList{} + if protoimpl.UnsafeEnabled { + mi := &file_feast_serving_ServingService_proto_msgTypes[4] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *FeatureList) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*FeatureList) ProtoMessage() {} + +func (x *FeatureList) ProtoReflect() protoreflect.Message { + mi := &file_feast_serving_ServingService_proto_msgTypes[4] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use FeatureList.ProtoReflect.Descriptor instead. +func (*FeatureList) Descriptor() ([]byte, []int) { + return file_feast_serving_ServingService_proto_rawDescGZIP(), []int{4} +} + +func (x *FeatureList) GetVal() []string { + if x != nil { + return x.Val + } + return nil +} + +type GetOnlineFeaturesRequest struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // Types that are assignable to Kind: + // *GetOnlineFeaturesRequest_FeatureService + // *GetOnlineFeaturesRequest_Features + Kind isGetOnlineFeaturesRequest_Kind `protobuf_oneof:"kind"` + // The entity data is specified in a columnar format + // A map of entity name -> list of values + Entities map[string]*types.RepeatedValue `protobuf:"bytes,3,rep,name=entities,proto3" json:"entities,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` + FullFeatureNames bool `protobuf:"varint,4,opt,name=full_feature_names,json=fullFeatureNames,proto3" json:"full_feature_names,omitempty"` +} + +func (x *GetOnlineFeaturesRequest) Reset() { + *x = GetOnlineFeaturesRequest{} + if protoimpl.UnsafeEnabled { + mi := &file_feast_serving_ServingService_proto_msgTypes[5] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *GetOnlineFeaturesRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*GetOnlineFeaturesRequest) ProtoMessage() {} + +func (x *GetOnlineFeaturesRequest) ProtoReflect() protoreflect.Message { + mi := &file_feast_serving_ServingService_proto_msgTypes[5] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use GetOnlineFeaturesRequest.ProtoReflect.Descriptor instead. +func (*GetOnlineFeaturesRequest) Descriptor() ([]byte, []int) { + return file_feast_serving_ServingService_proto_rawDescGZIP(), []int{5} +} + +func (m *GetOnlineFeaturesRequest) GetKind() isGetOnlineFeaturesRequest_Kind { + if m != nil { + return m.Kind + } + return nil +} + +func (x *GetOnlineFeaturesRequest) GetFeatureService() string { + if x, ok := x.GetKind().(*GetOnlineFeaturesRequest_FeatureService); ok { + return x.FeatureService + } + return "" +} + +func (x *GetOnlineFeaturesRequest) GetFeatures() *FeatureList { + if x, ok := x.GetKind().(*GetOnlineFeaturesRequest_Features); ok { + return x.Features + } + return nil +} + +func (x *GetOnlineFeaturesRequest) GetEntities() map[string]*types.RepeatedValue { + if x != nil { + return x.Entities + } + return nil +} + +func (x *GetOnlineFeaturesRequest) GetFullFeatureNames() bool { + if x != nil { + return x.FullFeatureNames + } + return false +} + +type isGetOnlineFeaturesRequest_Kind interface { + isGetOnlineFeaturesRequest_Kind() +} + +type GetOnlineFeaturesRequest_FeatureService struct { + FeatureService string `protobuf:"bytes,1,opt,name=feature_service,json=featureService,proto3,oneof"` +} + +type GetOnlineFeaturesRequest_Features struct { + Features *FeatureList `protobuf:"bytes,2,opt,name=features,proto3,oneof"` +} + +func (*GetOnlineFeaturesRequest_FeatureService) isGetOnlineFeaturesRequest_Kind() {} + +func (*GetOnlineFeaturesRequest_Features) isGetOnlineFeaturesRequest_Kind() {} + type GetOnlineFeaturesResponse struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -409,7 +550,7 @@ type GetOnlineFeaturesResponse struct { func (x *GetOnlineFeaturesResponse) Reset() { *x = GetOnlineFeaturesResponse{} if protoimpl.UnsafeEnabled { - mi := &file_feast_serving_ServingService_proto_msgTypes[4] + mi := &file_feast_serving_ServingService_proto_msgTypes[6] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -422,7 +563,7 @@ func (x *GetOnlineFeaturesResponse) String() string { func (*GetOnlineFeaturesResponse) ProtoMessage() {} func (x *GetOnlineFeaturesResponse) ProtoReflect() protoreflect.Message { - mi := &file_feast_serving_ServingService_proto_msgTypes[4] + mi := &file_feast_serving_ServingService_proto_msgTypes[6] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -435,7 +576,7 @@ func (x *GetOnlineFeaturesResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use GetOnlineFeaturesResponse.ProtoReflect.Descriptor instead. func (*GetOnlineFeaturesResponse) Descriptor() ([]byte, []int) { - return file_feast_serving_ServingService_proto_rawDescGZIP(), []int{4} + return file_feast_serving_ServingService_proto_rawDescGZIP(), []int{6} } func (x *GetOnlineFeaturesResponse) GetFieldValues() []*GetOnlineFeaturesResponse_FieldValues { @@ -452,7 +593,7 @@ type GetOnlineFeaturesRequestV2_EntityRow struct { // Request timestamp of this row. This value will be used, // together with maxAge, to determine feature staleness. - Timestamp *timestamp.Timestamp `protobuf:"bytes,1,opt,name=timestamp,proto3" json:"timestamp,omitempty"` + Timestamp *timestamppb.Timestamp `protobuf:"bytes,1,opt,name=timestamp,proto3" json:"timestamp,omitempty"` // Map containing mapping of entity name to entity value. Fields map[string]*types.Value `protobuf:"bytes,2,rep,name=fields,proto3" json:"fields,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` } @@ -460,7 +601,7 @@ type GetOnlineFeaturesRequestV2_EntityRow struct { func (x *GetOnlineFeaturesRequestV2_EntityRow) Reset() { *x = GetOnlineFeaturesRequestV2_EntityRow{} if protoimpl.UnsafeEnabled { - mi := &file_feast_serving_ServingService_proto_msgTypes[5] + mi := &file_feast_serving_ServingService_proto_msgTypes[7] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -473,7 +614,7 @@ func (x *GetOnlineFeaturesRequestV2_EntityRow) String() string { func (*GetOnlineFeaturesRequestV2_EntityRow) ProtoMessage() {} func (x *GetOnlineFeaturesRequestV2_EntityRow) ProtoReflect() protoreflect.Message { - mi := &file_feast_serving_ServingService_proto_msgTypes[5] + mi := &file_feast_serving_ServingService_proto_msgTypes[7] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -489,7 +630,7 @@ func (*GetOnlineFeaturesRequestV2_EntityRow) Descriptor() ([]byte, []int) { return file_feast_serving_ServingService_proto_rawDescGZIP(), []int{3, 0} } -func (x *GetOnlineFeaturesRequestV2_EntityRow) GetTimestamp() *timestamp.Timestamp { +func (x *GetOnlineFeaturesRequestV2_EntityRow) GetTimestamp() *timestamppb.Timestamp { if x != nil { return x.Timestamp } @@ -518,7 +659,7 @@ type GetOnlineFeaturesResponse_FieldValues struct { func (x *GetOnlineFeaturesResponse_FieldValues) Reset() { *x = GetOnlineFeaturesResponse_FieldValues{} if protoimpl.UnsafeEnabled { - mi := &file_feast_serving_ServingService_proto_msgTypes[7] + mi := &file_feast_serving_ServingService_proto_msgTypes[10] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -531,7 +672,7 @@ func (x *GetOnlineFeaturesResponse_FieldValues) String() string { func (*GetOnlineFeaturesResponse_FieldValues) ProtoMessage() {} func (x *GetOnlineFeaturesResponse_FieldValues) ProtoReflect() protoreflect.Message { - mi := &file_feast_serving_ServingService_proto_msgTypes[7] + mi := &file_feast_serving_ServingService_proto_msgTypes[10] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -544,7 +685,7 @@ func (x *GetOnlineFeaturesResponse_FieldValues) ProtoReflect() protoreflect.Mess // Deprecated: Use GetOnlineFeaturesResponse_FieldValues.ProtoReflect.Descriptor instead. func (*GetOnlineFeaturesResponse_FieldValues) Descriptor() ([]byte, []int) { - return file_feast_serving_ServingService_proto_rawDescGZIP(), []int{4, 0} + return file_feast_serving_ServingService_proto_rawDescGZIP(), []int{6, 0} } func (x *GetOnlineFeaturesResponse_FieldValues) GetFields() map[string]*types.Value { @@ -618,73 +759,98 @@ var file_feast_serving_ServingService_proto_rawDesc = []byte{ 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x28, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x74, 0x79, 0x70, 0x65, 0x73, 0x2e, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x52, 0x05, 0x76, - 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0xdd, 0x04, 0x0a, 0x19, 0x47, 0x65, 0x74, - 0x4f, 0x6e, 0x6c, 0x69, 0x6e, 0x65, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x52, 0x65, - 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x57, 0x0a, 0x0c, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x5f, - 0x76, 0x61, 0x6c, 0x75, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x34, 0x2e, 0x66, - 0x65, 0x61, 0x73, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, 0x2e, 0x47, 0x65, 0x74, - 0x4f, 0x6e, 0x6c, 0x69, 0x6e, 0x65, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x52, 0x65, - 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x56, 0x61, 0x6c, 0x75, - 0x65, 0x73, 0x52, 0x0b, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x73, 0x1a, - 0x89, 0x03, 0x0a, 0x0b, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x73, 0x12, - 0x58, 0x0a, 0x06, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, - 0x40, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, 0x2e, + 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x1f, 0x0a, 0x0b, 0x46, 0x65, 0x61, 0x74, + 0x75, 0x72, 0x65, 0x4c, 0x69, 0x73, 0x74, 0x12, 0x10, 0x0a, 0x03, 0x76, 0x61, 0x6c, 0x18, 0x01, + 0x20, 0x03, 0x28, 0x09, 0x52, 0x03, 0x76, 0x61, 0x6c, 0x22, 0xe1, 0x02, 0x0a, 0x18, 0x47, 0x65, + 0x74, 0x4f, 0x6e, 0x6c, 0x69, 0x6e, 0x65, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x52, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x29, 0x0a, 0x0f, 0x66, 0x65, 0x61, 0x74, 0x75, 0x72, + 0x65, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x48, + 0x00, 0x52, 0x0e, 0x66, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, + 0x65, 0x12, 0x38, 0x0a, 0x08, 0x66, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x18, 0x02, 0x20, + 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, + 0x69, 0x6e, 0x67, 0x2e, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x4c, 0x69, 0x73, 0x74, 0x48, + 0x00, 0x52, 0x08, 0x66, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x12, 0x51, 0x0a, 0x08, 0x65, + 0x6e, 0x74, 0x69, 0x74, 0x69, 0x65, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x35, 0x2e, + 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, 0x2e, 0x47, 0x65, + 0x74, 0x4f, 0x6e, 0x6c, 0x69, 0x6e, 0x65, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x52, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x45, 0x6e, 0x74, 0x69, 0x74, 0x69, 0x65, 0x73, 0x45, + 0x6e, 0x74, 0x72, 0x79, 0x52, 0x08, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x69, 0x65, 0x73, 0x12, 0x2c, + 0x0a, 0x12, 0x66, 0x75, 0x6c, 0x6c, 0x5f, 0x66, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x5f, 0x6e, + 0x61, 0x6d, 0x65, 0x73, 0x18, 0x04, 0x20, 0x01, 0x28, 0x08, 0x52, 0x10, 0x66, 0x75, 0x6c, 0x6c, + 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x73, 0x1a, 0x57, 0x0a, 0x0d, + 0x45, 0x6e, 0x74, 0x69, 0x74, 0x69, 0x65, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, + 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, + 0x30, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, + 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x74, 0x79, 0x70, 0x65, 0x73, 0x2e, 0x52, 0x65, 0x70, + 0x65, 0x61, 0x74, 0x65, 0x64, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, + 0x65, 0x3a, 0x02, 0x38, 0x01, 0x42, 0x06, 0x0a, 0x04, 0x6b, 0x69, 0x6e, 0x64, 0x22, 0xdd, 0x04, + 0x0a, 0x19, 0x47, 0x65, 0x74, 0x4f, 0x6e, 0x6c, 0x69, 0x6e, 0x65, 0x46, 0x65, 0x61, 0x74, 0x75, + 0x72, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x57, 0x0a, 0x0c, 0x66, + 0x69, 0x65, 0x6c, 0x64, 0x5f, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, + 0x0b, 0x32, 0x34, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x6e, + 0x67, 0x2e, 0x47, 0x65, 0x74, 0x4f, 0x6e, 0x6c, 0x69, 0x6e, 0x65, 0x46, 0x65, 0x61, 0x74, 0x75, + 0x72, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x46, 0x69, 0x65, 0x6c, + 0x64, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x73, 0x52, 0x0b, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x56, 0x61, + 0x6c, 0x75, 0x65, 0x73, 0x1a, 0x89, 0x03, 0x0a, 0x0b, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x56, 0x61, + 0x6c, 0x75, 0x65, 0x73, 0x12, 0x58, 0x0a, 0x06, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x73, 0x18, 0x01, + 0x20, 0x03, 0x28, 0x0b, 0x32, 0x40, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x73, 0x65, 0x72, + 0x76, 0x69, 0x6e, 0x67, 0x2e, 0x47, 0x65, 0x74, 0x4f, 0x6e, 0x6c, 0x69, 0x6e, 0x65, 0x46, 0x65, + 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x46, + 0x69, 0x65, 0x6c, 0x64, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x73, 0x2e, 0x46, 0x69, 0x65, 0x6c, 0x64, + 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x06, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x73, 0x12, 0x5e, + 0x0a, 0x08, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x65, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, + 0x32, 0x42, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, + 0x2e, 0x47, 0x65, 0x74, 0x4f, 0x6e, 0x6c, 0x69, 0x6e, 0x65, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, + 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x46, 0x69, 0x65, 0x6c, 0x64, + 0x56, 0x61, 0x6c, 0x75, 0x65, 0x73, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x65, 0x73, 0x45, + 0x6e, 0x74, 0x72, 0x79, 0x52, 0x08, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x65, 0x73, 0x1a, 0x4d, + 0x0a, 0x0b, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, + 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, + 0x28, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, + 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x74, 0x79, 0x70, 0x65, 0x73, 0x2e, 0x56, 0x61, 0x6c, + 0x75, 0x65, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x1a, 0x71, 0x0a, + 0x0d, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x65, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, + 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, + 0x12, 0x4a, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0e, 0x32, + 0x34, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, 0x2e, 0x47, 0x65, 0x74, 0x4f, 0x6e, 0x6c, 0x69, 0x6e, 0x65, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, - 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x56, - 0x61, 0x6c, 0x75, 0x65, 0x73, 0x2e, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x73, 0x45, 0x6e, 0x74, 0x72, - 0x79, 0x52, 0x06, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x73, 0x12, 0x5e, 0x0a, 0x08, 0x73, 0x74, 0x61, - 0x74, 0x75, 0x73, 0x65, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x42, 0x2e, 0x66, 0x65, - 0x61, 0x73, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, 0x2e, 0x47, 0x65, 0x74, 0x4f, - 0x6e, 0x6c, 0x69, 0x6e, 0x65, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x52, 0x65, 0x73, - 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x56, 0x61, 0x6c, 0x75, 0x65, - 0x73, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x65, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, - 0x08, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x65, 0x73, 0x1a, 0x4d, 0x0a, 0x0b, 0x46, 0x69, 0x65, - 0x6c, 0x64, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, - 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x28, 0x0a, 0x05, 0x76, 0x61, - 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x66, 0x65, 0x61, 0x73, - 0x74, 0x2e, 0x74, 0x79, 0x70, 0x65, 0x73, 0x2e, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x52, 0x05, 0x76, - 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x1a, 0x71, 0x0a, 0x0d, 0x53, 0x74, 0x61, 0x74, - 0x75, 0x73, 0x65, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, - 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x4a, 0x0a, 0x05, 0x76, - 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x34, 0x2e, 0x66, 0x65, 0x61, - 0x73, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, 0x2e, 0x47, 0x65, 0x74, 0x4f, 0x6e, - 0x6c, 0x69, 0x6e, 0x65, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, - 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, - 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x5b, 0x0a, 0x0b, 0x46, - 0x69, 0x65, 0x6c, 0x64, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x0b, 0x0a, 0x07, 0x49, 0x4e, - 0x56, 0x41, 0x4c, 0x49, 0x44, 0x10, 0x00, 0x12, 0x0b, 0x0a, 0x07, 0x50, 0x52, 0x45, 0x53, 0x45, - 0x4e, 0x54, 0x10, 0x01, 0x12, 0x0e, 0x0a, 0x0a, 0x4e, 0x55, 0x4c, 0x4c, 0x5f, 0x56, 0x41, 0x4c, - 0x55, 0x45, 0x10, 0x02, 0x12, 0x0d, 0x0a, 0x09, 0x4e, 0x4f, 0x54, 0x5f, 0x46, 0x4f, 0x55, 0x4e, - 0x44, 0x10, 0x03, 0x12, 0x13, 0x0a, 0x0f, 0x4f, 0x55, 0x54, 0x53, 0x49, 0x44, 0x45, 0x5f, 0x4d, - 0x41, 0x58, 0x5f, 0x41, 0x47, 0x45, 0x10, 0x04, 0x2a, 0x6f, 0x0a, 0x10, 0x46, 0x65, 0x61, 0x73, - 0x74, 0x53, 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, 0x54, 0x79, 0x70, 0x65, 0x12, 0x1e, 0x0a, 0x1a, - 0x46, 0x45, 0x41, 0x53, 0x54, 0x5f, 0x53, 0x45, 0x52, 0x56, 0x49, 0x4e, 0x47, 0x5f, 0x54, 0x59, - 0x50, 0x45, 0x5f, 0x49, 0x4e, 0x56, 0x41, 0x4c, 0x49, 0x44, 0x10, 0x00, 0x12, 0x1d, 0x0a, 0x19, - 0x46, 0x45, 0x41, 0x53, 0x54, 0x5f, 0x53, 0x45, 0x52, 0x56, 0x49, 0x4e, 0x47, 0x5f, 0x54, 0x59, - 0x50, 0x45, 0x5f, 0x4f, 0x4e, 0x4c, 0x49, 0x4e, 0x45, 0x10, 0x01, 0x12, 0x1c, 0x0a, 0x18, 0x46, - 0x45, 0x41, 0x53, 0x54, 0x5f, 0x53, 0x45, 0x52, 0x56, 0x49, 0x4e, 0x47, 0x5f, 0x54, 0x59, 0x50, - 0x45, 0x5f, 0x42, 0x41, 0x54, 0x43, 0x48, 0x10, 0x02, 0x32, 0xea, 0x01, 0x0a, 0x0e, 0x53, 0x65, - 0x72, 0x76, 0x69, 0x6e, 0x67, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x6c, 0x0a, 0x13, - 0x47, 0x65, 0x74, 0x46, 0x65, 0x61, 0x73, 0x74, 0x53, 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, 0x49, - 0x6e, 0x66, 0x6f, 0x12, 0x29, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, + 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x53, + 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, + 0x22, 0x5b, 0x0a, 0x0b, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, + 0x0b, 0x0a, 0x07, 0x49, 0x4e, 0x56, 0x41, 0x4c, 0x49, 0x44, 0x10, 0x00, 0x12, 0x0b, 0x0a, 0x07, + 0x50, 0x52, 0x45, 0x53, 0x45, 0x4e, 0x54, 0x10, 0x01, 0x12, 0x0e, 0x0a, 0x0a, 0x4e, 0x55, 0x4c, + 0x4c, 0x5f, 0x56, 0x41, 0x4c, 0x55, 0x45, 0x10, 0x02, 0x12, 0x0d, 0x0a, 0x09, 0x4e, 0x4f, 0x54, + 0x5f, 0x46, 0x4f, 0x55, 0x4e, 0x44, 0x10, 0x03, 0x12, 0x13, 0x0a, 0x0f, 0x4f, 0x55, 0x54, 0x53, + 0x49, 0x44, 0x45, 0x5f, 0x4d, 0x41, 0x58, 0x5f, 0x41, 0x47, 0x45, 0x10, 0x04, 0x2a, 0x6f, 0x0a, + 0x10, 0x46, 0x65, 0x61, 0x73, 0x74, 0x53, 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, 0x54, 0x79, 0x70, + 0x65, 0x12, 0x1e, 0x0a, 0x1a, 0x46, 0x45, 0x41, 0x53, 0x54, 0x5f, 0x53, 0x45, 0x52, 0x56, 0x49, + 0x4e, 0x47, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x49, 0x4e, 0x56, 0x41, 0x4c, 0x49, 0x44, 0x10, + 0x00, 0x12, 0x1d, 0x0a, 0x19, 0x46, 0x45, 0x41, 0x53, 0x54, 0x5f, 0x53, 0x45, 0x52, 0x56, 0x49, + 0x4e, 0x47, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x4f, 0x4e, 0x4c, 0x49, 0x4e, 0x45, 0x10, 0x01, + 0x12, 0x1c, 0x0a, 0x18, 0x46, 0x45, 0x41, 0x53, 0x54, 0x5f, 0x53, 0x45, 0x52, 0x56, 0x49, 0x4e, + 0x47, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x42, 0x41, 0x54, 0x43, 0x48, 0x10, 0x02, 0x32, 0xea, + 0x01, 0x0a, 0x0e, 0x53, 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, + 0x65, 0x12, 0x6c, 0x0a, 0x13, 0x47, 0x65, 0x74, 0x46, 0x65, 0x61, 0x73, 0x74, 0x53, 0x65, 0x72, + 0x76, 0x69, 0x6e, 0x67, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x29, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, + 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, 0x2e, 0x47, 0x65, 0x74, 0x46, 0x65, 0x61, 0x73, + 0x74, 0x53, 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x65, 0x71, 0x75, + 0x65, 0x73, 0x74, 0x1a, 0x2a, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, 0x2e, 0x47, 0x65, 0x74, 0x46, 0x65, 0x61, 0x73, 0x74, 0x53, 0x65, 0x72, 0x76, - 0x69, 0x6e, 0x67, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x2a, - 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, 0x2e, 0x47, - 0x65, 0x74, 0x46, 0x65, 0x61, 0x73, 0x74, 0x53, 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, 0x49, 0x6e, - 0x66, 0x6f, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x6a, 0x0a, 0x13, 0x47, 0x65, - 0x74, 0x4f, 0x6e, 0x6c, 0x69, 0x6e, 0x65, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x56, - 0x32, 0x12, 0x29, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x6e, + 0x69, 0x6e, 0x67, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, + 0x6a, 0x0a, 0x13, 0x47, 0x65, 0x74, 0x4f, 0x6e, 0x6c, 0x69, 0x6e, 0x65, 0x46, 0x65, 0x61, 0x74, + 0x75, 0x72, 0x65, 0x73, 0x56, 0x32, 0x12, 0x29, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x73, + 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, 0x2e, 0x47, 0x65, 0x74, 0x4f, 0x6e, 0x6c, 0x69, 0x6e, 0x65, + 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x56, + 0x32, 0x1a, 0x28, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, 0x2e, 0x47, 0x65, 0x74, 0x4f, 0x6e, 0x6c, 0x69, 0x6e, 0x65, 0x46, 0x65, 0x61, 0x74, 0x75, - 0x72, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x56, 0x32, 0x1a, 0x28, 0x2e, 0x66, - 0x65, 0x61, 0x73, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, 0x2e, 0x47, 0x65, 0x74, - 0x4f, 0x6e, 0x6c, 0x69, 0x6e, 0x65, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x52, 0x65, - 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x42, 0x5e, 0x0a, 0x13, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, - 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, 0x42, 0x0f, 0x53, - 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, 0x41, 0x50, 0x49, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x5a, 0x36, - 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x66, 0x65, 0x61, 0x73, 0x74, - 0x2d, 0x64, 0x65, 0x76, 0x2f, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2f, 0x73, 0x64, 0x6b, 0x2f, 0x67, - 0x6f, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x73, 0x2f, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2f, 0x73, - 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x72, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x42, 0x5e, 0x0a, 0x13, 0x66, + 0x65, 0x61, 0x73, 0x74, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, + 0x6e, 0x67, 0x42, 0x0f, 0x53, 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, 0x41, 0x50, 0x49, 0x50, 0x72, + 0x6f, 0x74, 0x6f, 0x5a, 0x36, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, + 0x66, 0x65, 0x61, 0x73, 0x74, 0x2d, 0x64, 0x65, 0x76, 0x2f, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2f, + 0x73, 0x64, 0x6b, 0x2f, 0x67, 0x6f, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x73, 0x2f, 0x66, 0x65, + 0x61, 0x73, 0x74, 0x2f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, 0x62, 0x06, 0x70, 0x72, 0x6f, + 0x74, 0x6f, 0x33, } var ( @@ -700,7 +866,7 @@ func file_feast_serving_ServingService_proto_rawDescGZIP() []byte { } var file_feast_serving_ServingService_proto_enumTypes = make([]protoimpl.EnumInfo, 2) -var file_feast_serving_ServingService_proto_msgTypes = make([]protoimpl.MessageInfo, 10) +var file_feast_serving_ServingService_proto_msgTypes = make([]protoimpl.MessageInfo, 13) var file_feast_serving_ServingService_proto_goTypes = []interface{}{ (FeastServingType)(0), // 0: feast.serving.FeastServingType (GetOnlineFeaturesResponse_FieldStatus)(0), // 1: feast.serving.GetOnlineFeaturesResponse.FieldStatus @@ -708,36 +874,43 @@ var file_feast_serving_ServingService_proto_goTypes = []interface{}{ (*GetFeastServingInfoResponse)(nil), // 3: feast.serving.GetFeastServingInfoResponse (*FeatureReferenceV2)(nil), // 4: feast.serving.FeatureReferenceV2 (*GetOnlineFeaturesRequestV2)(nil), // 5: feast.serving.GetOnlineFeaturesRequestV2 - (*GetOnlineFeaturesResponse)(nil), // 6: feast.serving.GetOnlineFeaturesResponse - (*GetOnlineFeaturesRequestV2_EntityRow)(nil), // 7: feast.serving.GetOnlineFeaturesRequestV2.EntityRow - nil, // 8: feast.serving.GetOnlineFeaturesRequestV2.EntityRow.FieldsEntry - (*GetOnlineFeaturesResponse_FieldValues)(nil), // 9: feast.serving.GetOnlineFeaturesResponse.FieldValues - nil, // 10: feast.serving.GetOnlineFeaturesResponse.FieldValues.FieldsEntry - nil, // 11: feast.serving.GetOnlineFeaturesResponse.FieldValues.StatusesEntry - (*timestamp.Timestamp)(nil), // 12: google.protobuf.Timestamp - (*types.Value)(nil), // 13: feast.types.Value + (*FeatureList)(nil), // 6: feast.serving.FeatureList + (*GetOnlineFeaturesRequest)(nil), // 7: feast.serving.GetOnlineFeaturesRequest + (*GetOnlineFeaturesResponse)(nil), // 8: feast.serving.GetOnlineFeaturesResponse + (*GetOnlineFeaturesRequestV2_EntityRow)(nil), // 9: feast.serving.GetOnlineFeaturesRequestV2.EntityRow + nil, // 10: feast.serving.GetOnlineFeaturesRequestV2.EntityRow.FieldsEntry + nil, // 11: feast.serving.GetOnlineFeaturesRequest.EntitiesEntry + (*GetOnlineFeaturesResponse_FieldValues)(nil), // 12: feast.serving.GetOnlineFeaturesResponse.FieldValues + nil, // 13: feast.serving.GetOnlineFeaturesResponse.FieldValues.FieldsEntry + nil, // 14: feast.serving.GetOnlineFeaturesResponse.FieldValues.StatusesEntry + (*timestamppb.Timestamp)(nil), // 15: google.protobuf.Timestamp + (*types.Value)(nil), // 16: feast.types.Value + (*types.RepeatedValue)(nil), // 17: feast.types.RepeatedValue } var file_feast_serving_ServingService_proto_depIdxs = []int32{ 0, // 0: feast.serving.GetFeastServingInfoResponse.type:type_name -> feast.serving.FeastServingType 4, // 1: feast.serving.GetOnlineFeaturesRequestV2.features:type_name -> feast.serving.FeatureReferenceV2 - 7, // 2: feast.serving.GetOnlineFeaturesRequestV2.entity_rows:type_name -> feast.serving.GetOnlineFeaturesRequestV2.EntityRow - 9, // 3: feast.serving.GetOnlineFeaturesResponse.field_values:type_name -> feast.serving.GetOnlineFeaturesResponse.FieldValues - 12, // 4: feast.serving.GetOnlineFeaturesRequestV2.EntityRow.timestamp:type_name -> google.protobuf.Timestamp - 8, // 5: feast.serving.GetOnlineFeaturesRequestV2.EntityRow.fields:type_name -> feast.serving.GetOnlineFeaturesRequestV2.EntityRow.FieldsEntry - 13, // 6: feast.serving.GetOnlineFeaturesRequestV2.EntityRow.FieldsEntry.value:type_name -> feast.types.Value - 10, // 7: feast.serving.GetOnlineFeaturesResponse.FieldValues.fields:type_name -> feast.serving.GetOnlineFeaturesResponse.FieldValues.FieldsEntry - 11, // 8: feast.serving.GetOnlineFeaturesResponse.FieldValues.statuses:type_name -> feast.serving.GetOnlineFeaturesResponse.FieldValues.StatusesEntry - 13, // 9: feast.serving.GetOnlineFeaturesResponse.FieldValues.FieldsEntry.value:type_name -> feast.types.Value - 1, // 10: feast.serving.GetOnlineFeaturesResponse.FieldValues.StatusesEntry.value:type_name -> feast.serving.GetOnlineFeaturesResponse.FieldStatus - 2, // 11: feast.serving.ServingService.GetFeastServingInfo:input_type -> feast.serving.GetFeastServingInfoRequest - 5, // 12: feast.serving.ServingService.GetOnlineFeaturesV2:input_type -> feast.serving.GetOnlineFeaturesRequestV2 - 3, // 13: feast.serving.ServingService.GetFeastServingInfo:output_type -> feast.serving.GetFeastServingInfoResponse - 6, // 14: feast.serving.ServingService.GetOnlineFeaturesV2:output_type -> feast.serving.GetOnlineFeaturesResponse - 13, // [13:15] is the sub-list for method output_type - 11, // [11:13] is the sub-list for method input_type - 11, // [11:11] is the sub-list for extension type_name - 11, // [11:11] is the sub-list for extension extendee - 0, // [0:11] is the sub-list for field type_name + 9, // 2: feast.serving.GetOnlineFeaturesRequestV2.entity_rows:type_name -> feast.serving.GetOnlineFeaturesRequestV2.EntityRow + 6, // 3: feast.serving.GetOnlineFeaturesRequest.features:type_name -> feast.serving.FeatureList + 11, // 4: feast.serving.GetOnlineFeaturesRequest.entities:type_name -> feast.serving.GetOnlineFeaturesRequest.EntitiesEntry + 12, // 5: feast.serving.GetOnlineFeaturesResponse.field_values:type_name -> feast.serving.GetOnlineFeaturesResponse.FieldValues + 15, // 6: feast.serving.GetOnlineFeaturesRequestV2.EntityRow.timestamp:type_name -> google.protobuf.Timestamp + 10, // 7: feast.serving.GetOnlineFeaturesRequestV2.EntityRow.fields:type_name -> feast.serving.GetOnlineFeaturesRequestV2.EntityRow.FieldsEntry + 16, // 8: feast.serving.GetOnlineFeaturesRequestV2.EntityRow.FieldsEntry.value:type_name -> feast.types.Value + 17, // 9: feast.serving.GetOnlineFeaturesRequest.EntitiesEntry.value:type_name -> feast.types.RepeatedValue + 13, // 10: feast.serving.GetOnlineFeaturesResponse.FieldValues.fields:type_name -> feast.serving.GetOnlineFeaturesResponse.FieldValues.FieldsEntry + 14, // 11: feast.serving.GetOnlineFeaturesResponse.FieldValues.statuses:type_name -> feast.serving.GetOnlineFeaturesResponse.FieldValues.StatusesEntry + 16, // 12: feast.serving.GetOnlineFeaturesResponse.FieldValues.FieldsEntry.value:type_name -> feast.types.Value + 1, // 13: feast.serving.GetOnlineFeaturesResponse.FieldValues.StatusesEntry.value:type_name -> feast.serving.GetOnlineFeaturesResponse.FieldStatus + 2, // 14: feast.serving.ServingService.GetFeastServingInfo:input_type -> feast.serving.GetFeastServingInfoRequest + 5, // 15: feast.serving.ServingService.GetOnlineFeaturesV2:input_type -> feast.serving.GetOnlineFeaturesRequestV2 + 3, // 16: feast.serving.ServingService.GetFeastServingInfo:output_type -> feast.serving.GetFeastServingInfoResponse + 8, // 17: feast.serving.ServingService.GetOnlineFeaturesV2:output_type -> feast.serving.GetOnlineFeaturesResponse + 16, // [16:18] is the sub-list for method output_type + 14, // [14:16] is the sub-list for method input_type + 14, // [14:14] is the sub-list for extension type_name + 14, // [14:14] is the sub-list for extension extendee + 0, // [0:14] is the sub-list for field type_name } func init() { file_feast_serving_ServingService_proto_init() } @@ -795,7 +968,7 @@ func file_feast_serving_ServingService_proto_init() { } } file_feast_serving_ServingService_proto_msgTypes[4].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*GetOnlineFeaturesResponse); i { + switch v := v.(*FeatureList); i { case 0: return &v.state case 1: @@ -807,7 +980,19 @@ func file_feast_serving_ServingService_proto_init() { } } file_feast_serving_ServingService_proto_msgTypes[5].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*GetOnlineFeaturesRequestV2_EntityRow); i { + switch v := v.(*GetOnlineFeaturesRequest); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_feast_serving_ServingService_proto_msgTypes[6].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*GetOnlineFeaturesResponse); i { case 0: return &v.state case 1: @@ -819,6 +1004,18 @@ func file_feast_serving_ServingService_proto_init() { } } file_feast_serving_ServingService_proto_msgTypes[7].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*GetOnlineFeaturesRequestV2_EntityRow); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_feast_serving_ServingService_proto_msgTypes[10].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*GetOnlineFeaturesResponse_FieldValues); i { case 0: return &v.state @@ -831,13 +1028,17 @@ func file_feast_serving_ServingService_proto_init() { } } } + file_feast_serving_ServingService_proto_msgTypes[5].OneofWrappers = []interface{}{ + (*GetOnlineFeaturesRequest_FeatureService)(nil), + (*GetOnlineFeaturesRequest_Features)(nil), + } type x struct{} out := protoimpl.TypeBuilder{ File: protoimpl.DescBuilder{ GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: file_feast_serving_ServingService_proto_rawDesc, NumEnums: 2, - NumMessages: 10, + NumMessages: 13, NumExtensions: 0, NumServices: 1, }, diff --git a/sdk/go/protos/feast/storage/Redis.pb.go b/sdk/go/protos/feast/storage/Redis.pb.go index 761bf2bb6ea..08ee629b7c0 100644 --- a/sdk/go/protos/feast/storage/Redis.pb.go +++ b/sdk/go/protos/feast/storage/Redis.pb.go @@ -15,15 +15,14 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.25.0 -// protoc v3.12.4 +// protoc-gen-go v1.27.1 +// protoc v3.11.2 // source: feast/storage/Redis.proto package storage import ( types "github.com/feast-dev/feast/sdk/go/protos/feast/types" - proto "github.com/golang/protobuf/proto" protoreflect "google.golang.org/protobuf/reflect/protoreflect" protoimpl "google.golang.org/protobuf/runtime/protoimpl" reflect "reflect" @@ -37,10 +36,6 @@ const ( _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) ) -// This is a compile-time assertion that a sufficiently up-to-date version -// of the legacy proto package is being used. -const _ = proto.ProtoPackageIsVersion4 - type RedisKeyV2 struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache diff --git a/sdk/go/protos/feast/types/Field.pb.go b/sdk/go/protos/feast/types/Field.pb.go index 9dad77cdb91..73f46bb1ac3 100644 --- a/sdk/go/protos/feast/types/Field.pb.go +++ b/sdk/go/protos/feast/types/Field.pb.go @@ -15,14 +15,13 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.25.0 -// protoc v3.12.4 +// protoc-gen-go v1.27.1 +// protoc v3.11.2 // source: feast/types/Field.proto package types import ( - proto "github.com/golang/protobuf/proto" protoreflect "google.golang.org/protobuf/reflect/protoreflect" protoimpl "google.golang.org/protobuf/runtime/protoimpl" reflect "reflect" @@ -36,10 +35,6 @@ const ( _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) ) -// This is a compile-time assertion that a sufficiently up-to-date version -// of the legacy proto package is being used. -const _ = proto.ProtoPackageIsVersion4 - type Field struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache diff --git a/sdk/go/protos/feast/types/Value.pb.go b/sdk/go/protos/feast/types/Value.pb.go index 3b194356331..9ae2806d515 100644 --- a/sdk/go/protos/feast/types/Value.pb.go +++ b/sdk/go/protos/feast/types/Value.pb.go @@ -15,14 +15,13 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.25.0 -// protoc v3.12.4 +// protoc-gen-go v1.27.1 +// protoc v3.11.2 // source: feast/types/Value.proto package types import ( - proto "github.com/golang/protobuf/proto" protoreflect "google.golang.org/protobuf/reflect/protoreflect" protoimpl "google.golang.org/protobuf/runtime/protoimpl" reflect "reflect" @@ -36,28 +35,70 @@ const ( _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) ) -// This is a compile-time assertion that a sufficiently up-to-date version -// of the legacy proto package is being used. -const _ = proto.ProtoPackageIsVersion4 +type Null int32 + +const ( + Null_NULL Null = 0 +) + +// Enum value maps for Null. +var ( + Null_name = map[int32]string{ + 0: "NULL", + } + Null_value = map[string]int32{ + "NULL": 0, + } +) + +func (x Null) Enum() *Null { + p := new(Null) + *p = x + return p +} + +func (x Null) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (Null) Descriptor() protoreflect.EnumDescriptor { + return file_feast_types_Value_proto_enumTypes[0].Descriptor() +} + +func (Null) Type() protoreflect.EnumType { + return &file_feast_types_Value_proto_enumTypes[0] +} + +func (x Null) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Use Null.Descriptor instead. +func (Null) EnumDescriptor() ([]byte, []int) { + return file_feast_types_Value_proto_rawDescGZIP(), []int{0} +} type ValueType_Enum int32 const ( - ValueType_INVALID ValueType_Enum = 0 - ValueType_BYTES ValueType_Enum = 1 - ValueType_STRING ValueType_Enum = 2 - ValueType_INT32 ValueType_Enum = 3 - ValueType_INT64 ValueType_Enum = 4 - ValueType_DOUBLE ValueType_Enum = 5 - ValueType_FLOAT ValueType_Enum = 6 - ValueType_BOOL ValueType_Enum = 7 - ValueType_BYTES_LIST ValueType_Enum = 11 - ValueType_STRING_LIST ValueType_Enum = 12 - ValueType_INT32_LIST ValueType_Enum = 13 - ValueType_INT64_LIST ValueType_Enum = 14 - ValueType_DOUBLE_LIST ValueType_Enum = 15 - ValueType_FLOAT_LIST ValueType_Enum = 16 - ValueType_BOOL_LIST ValueType_Enum = 17 + ValueType_INVALID ValueType_Enum = 0 + ValueType_BYTES ValueType_Enum = 1 + ValueType_STRING ValueType_Enum = 2 + ValueType_INT32 ValueType_Enum = 3 + ValueType_INT64 ValueType_Enum = 4 + ValueType_DOUBLE ValueType_Enum = 5 + ValueType_FLOAT ValueType_Enum = 6 + ValueType_BOOL ValueType_Enum = 7 + ValueType_UNIX_TIMESTAMP ValueType_Enum = 8 + ValueType_BYTES_LIST ValueType_Enum = 11 + ValueType_STRING_LIST ValueType_Enum = 12 + ValueType_INT32_LIST ValueType_Enum = 13 + ValueType_INT64_LIST ValueType_Enum = 14 + ValueType_DOUBLE_LIST ValueType_Enum = 15 + ValueType_FLOAT_LIST ValueType_Enum = 16 + ValueType_BOOL_LIST ValueType_Enum = 17 + ValueType_UNIX_TIMESTAMP_LIST ValueType_Enum = 18 + ValueType_NULL ValueType_Enum = 19 ) // Enum value maps for ValueType_Enum. @@ -71,6 +112,7 @@ var ( 5: "DOUBLE", 6: "FLOAT", 7: "BOOL", + 8: "UNIX_TIMESTAMP", 11: "BYTES_LIST", 12: "STRING_LIST", 13: "INT32_LIST", @@ -78,23 +120,28 @@ var ( 15: "DOUBLE_LIST", 16: "FLOAT_LIST", 17: "BOOL_LIST", + 18: "UNIX_TIMESTAMP_LIST", + 19: "NULL", } ValueType_Enum_value = map[string]int32{ - "INVALID": 0, - "BYTES": 1, - "STRING": 2, - "INT32": 3, - "INT64": 4, - "DOUBLE": 5, - "FLOAT": 6, - "BOOL": 7, - "BYTES_LIST": 11, - "STRING_LIST": 12, - "INT32_LIST": 13, - "INT64_LIST": 14, - "DOUBLE_LIST": 15, - "FLOAT_LIST": 16, - "BOOL_LIST": 17, + "INVALID": 0, + "BYTES": 1, + "STRING": 2, + "INT32": 3, + "INT64": 4, + "DOUBLE": 5, + "FLOAT": 6, + "BOOL": 7, + "UNIX_TIMESTAMP": 8, + "BYTES_LIST": 11, + "STRING_LIST": 12, + "INT32_LIST": 13, + "INT64_LIST": 14, + "DOUBLE_LIST": 15, + "FLOAT_LIST": 16, + "BOOL_LIST": 17, + "UNIX_TIMESTAMP_LIST": 18, + "NULL": 19, } ) @@ -109,11 +156,11 @@ func (x ValueType_Enum) String() string { } func (ValueType_Enum) Descriptor() protoreflect.EnumDescriptor { - return file_feast_types_Value_proto_enumTypes[0].Descriptor() + return file_feast_types_Value_proto_enumTypes[1].Descriptor() } func (ValueType_Enum) Type() protoreflect.EnumType { - return &file_feast_types_Value_proto_enumTypes[0] + return &file_feast_types_Value_proto_enumTypes[1] } func (x ValueType_Enum) Number() protoreflect.EnumNumber { @@ -170,6 +217,7 @@ type Value struct { // ValueType is referenced by the metadata types, FeatureInfo and EntityInfo. // The enum values do not have to match the oneof val field ids, but they should. + // In JSON "*_val" field can be omitted // // Types that are assignable to Val: // *Value_BytesVal @@ -179,6 +227,7 @@ type Value struct { // *Value_DoubleVal // *Value_FloatVal // *Value_BoolVal + // *Value_UnixTimestampVal // *Value_BytesListVal // *Value_StringListVal // *Value_Int32ListVal @@ -186,6 +235,8 @@ type Value struct { // *Value_DoubleListVal // *Value_FloatListVal // *Value_BoolListVal + // *Value_UnixTimestampListVal + // *Value_NullVal Val isValue_Val `protobuf_oneof:"val"` } @@ -277,6 +328,13 @@ func (x *Value) GetBoolVal() bool { return false } +func (x *Value) GetUnixTimestampVal() int64 { + if x, ok := x.GetVal().(*Value_UnixTimestampVal); ok { + return x.UnixTimestampVal + } + return 0 +} + func (x *Value) GetBytesListVal() *BytesList { if x, ok := x.GetVal().(*Value_BytesListVal); ok { return x.BytesListVal @@ -326,6 +384,20 @@ func (x *Value) GetBoolListVal() *BoolList { return nil } +func (x *Value) GetUnixTimestampListVal() *Int64List { + if x, ok := x.GetVal().(*Value_UnixTimestampListVal); ok { + return x.UnixTimestampListVal + } + return nil +} + +func (x *Value) GetNullVal() Null { + if x, ok := x.GetVal().(*Value_NullVal); ok { + return x.NullVal + } + return Null_NULL +} + type isValue_Val interface { isValue_Val() } @@ -358,6 +430,10 @@ type Value_BoolVal struct { BoolVal bool `protobuf:"varint,7,opt,name=bool_val,json=boolVal,proto3,oneof"` } +type Value_UnixTimestampVal struct { + UnixTimestampVal int64 `protobuf:"varint,8,opt,name=unix_timestamp_val,json=unixTimestampVal,proto3,oneof"` +} + type Value_BytesListVal struct { BytesListVal *BytesList `protobuf:"bytes,11,opt,name=bytes_list_val,json=bytesListVal,proto3,oneof"` } @@ -386,6 +462,14 @@ type Value_BoolListVal struct { BoolListVal *BoolList `protobuf:"bytes,17,opt,name=bool_list_val,json=boolListVal,proto3,oneof"` } +type Value_UnixTimestampListVal struct { + UnixTimestampListVal *Int64List `protobuf:"bytes,18,opt,name=unix_timestamp_list_val,json=unixTimestampListVal,proto3,oneof"` +} + +type Value_NullVal struct { + NullVal Null `protobuf:"varint,19,opt,name=null_val,json=nullVal,proto3,enum=feast.types.Null,oneof"` +} + func (*Value_BytesVal) isValue_Val() {} func (*Value_StringVal) isValue_Val() {} @@ -400,6 +484,8 @@ func (*Value_FloatVal) isValue_Val() {} func (*Value_BoolVal) isValue_Val() {} +func (*Value_UnixTimestampVal) isValue_Val() {} + func (*Value_BytesListVal) isValue_Val() {} func (*Value_StringListVal) isValue_Val() {} @@ -414,6 +500,10 @@ func (*Value_FloatListVal) isValue_Val() {} func (*Value_BoolListVal) isValue_Val() {} +func (*Value_UnixTimestampListVal) isValue_Val() {} + +func (*Value_NullVal) isValue_Val() {} + type BytesList struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -743,89 +833,157 @@ func (x *BoolList) GetVal() []bool { return nil } +// This is to avoid an issue of being unable to specify `repeated value` in oneofs or maps +// In JSON "val" field can be omitted +type RepeatedValue struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Val []*Value `protobuf:"bytes,1,rep,name=val,proto3" json:"val,omitempty"` +} + +func (x *RepeatedValue) Reset() { + *x = RepeatedValue{} + if protoimpl.UnsafeEnabled { + mi := &file_feast_types_Value_proto_msgTypes[9] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *RepeatedValue) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*RepeatedValue) ProtoMessage() {} + +func (x *RepeatedValue) ProtoReflect() protoreflect.Message { + mi := &file_feast_types_Value_proto_msgTypes[9] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use RepeatedValue.ProtoReflect.Descriptor instead. +func (*RepeatedValue) Descriptor() ([]byte, []int) { + return file_feast_types_Value_proto_rawDescGZIP(), []int{9} +} + +func (x *RepeatedValue) GetVal() []*Value { + if x != nil { + return x.Val + } + return nil +} + var File_feast_types_Value_proto protoreflect.FileDescriptor var file_feast_types_Value_proto_rawDesc = []byte{ 0x0a, 0x17, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2f, 0x74, 0x79, 0x70, 0x65, 0x73, 0x2f, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x0b, 0x66, 0x65, 0x61, 0x73, 0x74, - 0x2e, 0x74, 0x79, 0x70, 0x65, 0x73, 0x22, 0xe0, 0x01, 0x0a, 0x09, 0x56, 0x61, 0x6c, 0x75, 0x65, - 0x54, 0x79, 0x70, 0x65, 0x22, 0xd2, 0x01, 0x0a, 0x04, 0x45, 0x6e, 0x75, 0x6d, 0x12, 0x0b, 0x0a, + 0x2e, 0x74, 0x79, 0x70, 0x65, 0x73, 0x22, 0x97, 0x02, 0x0a, 0x09, 0x56, 0x61, 0x6c, 0x75, 0x65, + 0x54, 0x79, 0x70, 0x65, 0x22, 0x89, 0x02, 0x0a, 0x04, 0x45, 0x6e, 0x75, 0x6d, 0x12, 0x0b, 0x0a, 0x07, 0x49, 0x4e, 0x56, 0x41, 0x4c, 0x49, 0x44, 0x10, 0x00, 0x12, 0x09, 0x0a, 0x05, 0x42, 0x59, 0x54, 0x45, 0x53, 0x10, 0x01, 0x12, 0x0a, 0x0a, 0x06, 0x53, 0x54, 0x52, 0x49, 0x4e, 0x47, 0x10, 0x02, 0x12, 0x09, 0x0a, 0x05, 0x49, 0x4e, 0x54, 0x33, 0x32, 0x10, 0x03, 0x12, 0x09, 0x0a, 0x05, 0x49, 0x4e, 0x54, 0x36, 0x34, 0x10, 0x04, 0x12, 0x0a, 0x0a, 0x06, 0x44, 0x4f, 0x55, 0x42, 0x4c, 0x45, 0x10, 0x05, 0x12, 0x09, 0x0a, 0x05, 0x46, 0x4c, 0x4f, 0x41, 0x54, 0x10, 0x06, 0x12, 0x08, - 0x0a, 0x04, 0x42, 0x4f, 0x4f, 0x4c, 0x10, 0x07, 0x12, 0x0e, 0x0a, 0x0a, 0x42, 0x59, 0x54, 0x45, - 0x53, 0x5f, 0x4c, 0x49, 0x53, 0x54, 0x10, 0x0b, 0x12, 0x0f, 0x0a, 0x0b, 0x53, 0x54, 0x52, 0x49, - 0x4e, 0x47, 0x5f, 0x4c, 0x49, 0x53, 0x54, 0x10, 0x0c, 0x12, 0x0e, 0x0a, 0x0a, 0x49, 0x4e, 0x54, - 0x33, 0x32, 0x5f, 0x4c, 0x49, 0x53, 0x54, 0x10, 0x0d, 0x12, 0x0e, 0x0a, 0x0a, 0x49, 0x4e, 0x54, - 0x36, 0x34, 0x5f, 0x4c, 0x49, 0x53, 0x54, 0x10, 0x0e, 0x12, 0x0f, 0x0a, 0x0b, 0x44, 0x4f, 0x55, - 0x42, 0x4c, 0x45, 0x5f, 0x4c, 0x49, 0x53, 0x54, 0x10, 0x0f, 0x12, 0x0e, 0x0a, 0x0a, 0x46, 0x4c, - 0x4f, 0x41, 0x54, 0x5f, 0x4c, 0x49, 0x53, 0x54, 0x10, 0x10, 0x12, 0x0d, 0x0a, 0x09, 0x42, 0x4f, - 0x4f, 0x4c, 0x5f, 0x4c, 0x49, 0x53, 0x54, 0x10, 0x11, 0x22, 0xac, 0x05, 0x0a, 0x05, 0x56, 0x61, - 0x6c, 0x75, 0x65, 0x12, 0x1d, 0x0a, 0x09, 0x62, 0x79, 0x74, 0x65, 0x73, 0x5f, 0x76, 0x61, 0x6c, - 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x48, 0x00, 0x52, 0x08, 0x62, 0x79, 0x74, 0x65, 0x73, 0x56, - 0x61, 0x6c, 0x12, 0x1f, 0x0a, 0x0a, 0x73, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x5f, 0x76, 0x61, 0x6c, - 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x48, 0x00, 0x52, 0x09, 0x73, 0x74, 0x72, 0x69, 0x6e, 0x67, - 0x56, 0x61, 0x6c, 0x12, 0x1d, 0x0a, 0x09, 0x69, 0x6e, 0x74, 0x33, 0x32, 0x5f, 0x76, 0x61, 0x6c, - 0x18, 0x03, 0x20, 0x01, 0x28, 0x05, 0x48, 0x00, 0x52, 0x08, 0x69, 0x6e, 0x74, 0x33, 0x32, 0x56, - 0x61, 0x6c, 0x12, 0x1d, 0x0a, 0x09, 0x69, 0x6e, 0x74, 0x36, 0x34, 0x5f, 0x76, 0x61, 0x6c, 0x18, - 0x04, 0x20, 0x01, 0x28, 0x03, 0x48, 0x00, 0x52, 0x08, 0x69, 0x6e, 0x74, 0x36, 0x34, 0x56, 0x61, - 0x6c, 0x12, 0x1f, 0x0a, 0x0a, 0x64, 0x6f, 0x75, 0x62, 0x6c, 0x65, 0x5f, 0x76, 0x61, 0x6c, 0x18, - 0x05, 0x20, 0x01, 0x28, 0x01, 0x48, 0x00, 0x52, 0x09, 0x64, 0x6f, 0x75, 0x62, 0x6c, 0x65, 0x56, - 0x61, 0x6c, 0x12, 0x1d, 0x0a, 0x09, 0x66, 0x6c, 0x6f, 0x61, 0x74, 0x5f, 0x76, 0x61, 0x6c, 0x18, - 0x06, 0x20, 0x01, 0x28, 0x02, 0x48, 0x00, 0x52, 0x08, 0x66, 0x6c, 0x6f, 0x61, 0x74, 0x56, 0x61, - 0x6c, 0x12, 0x1b, 0x0a, 0x08, 0x62, 0x6f, 0x6f, 0x6c, 0x5f, 0x76, 0x61, 0x6c, 0x18, 0x07, 0x20, - 0x01, 0x28, 0x08, 0x48, 0x00, 0x52, 0x07, 0x62, 0x6f, 0x6f, 0x6c, 0x56, 0x61, 0x6c, 0x12, 0x3e, - 0x0a, 0x0e, 0x62, 0x79, 0x74, 0x65, 0x73, 0x5f, 0x6c, 0x69, 0x73, 0x74, 0x5f, 0x76, 0x61, 0x6c, - 0x18, 0x0b, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x74, - 0x79, 0x70, 0x65, 0x73, 0x2e, 0x42, 0x79, 0x74, 0x65, 0x73, 0x4c, 0x69, 0x73, 0x74, 0x48, 0x00, - 0x52, 0x0c, 0x62, 0x79, 0x74, 0x65, 0x73, 0x4c, 0x69, 0x73, 0x74, 0x56, 0x61, 0x6c, 0x12, 0x41, - 0x0a, 0x0f, 0x73, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x5f, 0x6c, 0x69, 0x73, 0x74, 0x5f, 0x76, 0x61, - 0x6c, 0x18, 0x0c, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x17, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, - 0x74, 0x79, 0x70, 0x65, 0x73, 0x2e, 0x53, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x4c, 0x69, 0x73, 0x74, - 0x48, 0x00, 0x52, 0x0d, 0x73, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x4c, 0x69, 0x73, 0x74, 0x56, 0x61, - 0x6c, 0x12, 0x3e, 0x0a, 0x0e, 0x69, 0x6e, 0x74, 0x33, 0x32, 0x5f, 0x6c, 0x69, 0x73, 0x74, 0x5f, - 0x76, 0x61, 0x6c, 0x18, 0x0d, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x66, 0x65, 0x61, 0x73, - 0x74, 0x2e, 0x74, 0x79, 0x70, 0x65, 0x73, 0x2e, 0x49, 0x6e, 0x74, 0x33, 0x32, 0x4c, 0x69, 0x73, - 0x74, 0x48, 0x00, 0x52, 0x0c, 0x69, 0x6e, 0x74, 0x33, 0x32, 0x4c, 0x69, 0x73, 0x74, 0x56, 0x61, - 0x6c, 0x12, 0x3e, 0x0a, 0x0e, 0x69, 0x6e, 0x74, 0x36, 0x34, 0x5f, 0x6c, 0x69, 0x73, 0x74, 0x5f, - 0x76, 0x61, 0x6c, 0x18, 0x0e, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x66, 0x65, 0x61, 0x73, + 0x0a, 0x04, 0x42, 0x4f, 0x4f, 0x4c, 0x10, 0x07, 0x12, 0x12, 0x0a, 0x0e, 0x55, 0x4e, 0x49, 0x58, + 0x5f, 0x54, 0x49, 0x4d, 0x45, 0x53, 0x54, 0x41, 0x4d, 0x50, 0x10, 0x08, 0x12, 0x0e, 0x0a, 0x0a, + 0x42, 0x59, 0x54, 0x45, 0x53, 0x5f, 0x4c, 0x49, 0x53, 0x54, 0x10, 0x0b, 0x12, 0x0f, 0x0a, 0x0b, + 0x53, 0x54, 0x52, 0x49, 0x4e, 0x47, 0x5f, 0x4c, 0x49, 0x53, 0x54, 0x10, 0x0c, 0x12, 0x0e, 0x0a, + 0x0a, 0x49, 0x4e, 0x54, 0x33, 0x32, 0x5f, 0x4c, 0x49, 0x53, 0x54, 0x10, 0x0d, 0x12, 0x0e, 0x0a, + 0x0a, 0x49, 0x4e, 0x54, 0x36, 0x34, 0x5f, 0x4c, 0x49, 0x53, 0x54, 0x10, 0x0e, 0x12, 0x0f, 0x0a, + 0x0b, 0x44, 0x4f, 0x55, 0x42, 0x4c, 0x45, 0x5f, 0x4c, 0x49, 0x53, 0x54, 0x10, 0x0f, 0x12, 0x0e, + 0x0a, 0x0a, 0x46, 0x4c, 0x4f, 0x41, 0x54, 0x5f, 0x4c, 0x49, 0x53, 0x54, 0x10, 0x10, 0x12, 0x0d, + 0x0a, 0x09, 0x42, 0x4f, 0x4f, 0x4c, 0x5f, 0x4c, 0x49, 0x53, 0x54, 0x10, 0x11, 0x12, 0x17, 0x0a, + 0x13, 0x55, 0x4e, 0x49, 0x58, 0x5f, 0x54, 0x49, 0x4d, 0x45, 0x53, 0x54, 0x41, 0x4d, 0x50, 0x5f, + 0x4c, 0x49, 0x53, 0x54, 0x10, 0x12, 0x12, 0x08, 0x0a, 0x04, 0x4e, 0x55, 0x4c, 0x4c, 0x10, 0x13, + 0x22, 0xdd, 0x06, 0x0a, 0x05, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x12, 0x1d, 0x0a, 0x09, 0x62, 0x79, + 0x74, 0x65, 0x73, 0x5f, 0x76, 0x61, 0x6c, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x48, 0x00, 0x52, + 0x08, 0x62, 0x79, 0x74, 0x65, 0x73, 0x56, 0x61, 0x6c, 0x12, 0x1f, 0x0a, 0x0a, 0x73, 0x74, 0x72, + 0x69, 0x6e, 0x67, 0x5f, 0x76, 0x61, 0x6c, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x48, 0x00, 0x52, + 0x09, 0x73, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x56, 0x61, 0x6c, 0x12, 0x1d, 0x0a, 0x09, 0x69, 0x6e, + 0x74, 0x33, 0x32, 0x5f, 0x76, 0x61, 0x6c, 0x18, 0x03, 0x20, 0x01, 0x28, 0x05, 0x48, 0x00, 0x52, + 0x08, 0x69, 0x6e, 0x74, 0x33, 0x32, 0x56, 0x61, 0x6c, 0x12, 0x1d, 0x0a, 0x09, 0x69, 0x6e, 0x74, + 0x36, 0x34, 0x5f, 0x76, 0x61, 0x6c, 0x18, 0x04, 0x20, 0x01, 0x28, 0x03, 0x48, 0x00, 0x52, 0x08, + 0x69, 0x6e, 0x74, 0x36, 0x34, 0x56, 0x61, 0x6c, 0x12, 0x1f, 0x0a, 0x0a, 0x64, 0x6f, 0x75, 0x62, + 0x6c, 0x65, 0x5f, 0x76, 0x61, 0x6c, 0x18, 0x05, 0x20, 0x01, 0x28, 0x01, 0x48, 0x00, 0x52, 0x09, + 0x64, 0x6f, 0x75, 0x62, 0x6c, 0x65, 0x56, 0x61, 0x6c, 0x12, 0x1d, 0x0a, 0x09, 0x66, 0x6c, 0x6f, + 0x61, 0x74, 0x5f, 0x76, 0x61, 0x6c, 0x18, 0x06, 0x20, 0x01, 0x28, 0x02, 0x48, 0x00, 0x52, 0x08, + 0x66, 0x6c, 0x6f, 0x61, 0x74, 0x56, 0x61, 0x6c, 0x12, 0x1b, 0x0a, 0x08, 0x62, 0x6f, 0x6f, 0x6c, + 0x5f, 0x76, 0x61, 0x6c, 0x18, 0x07, 0x20, 0x01, 0x28, 0x08, 0x48, 0x00, 0x52, 0x07, 0x62, 0x6f, + 0x6f, 0x6c, 0x56, 0x61, 0x6c, 0x12, 0x2e, 0x0a, 0x12, 0x75, 0x6e, 0x69, 0x78, 0x5f, 0x74, 0x69, + 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x5f, 0x76, 0x61, 0x6c, 0x18, 0x08, 0x20, 0x01, 0x28, + 0x03, 0x48, 0x00, 0x52, 0x10, 0x75, 0x6e, 0x69, 0x78, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, + 0x6d, 0x70, 0x56, 0x61, 0x6c, 0x12, 0x3e, 0x0a, 0x0e, 0x62, 0x79, 0x74, 0x65, 0x73, 0x5f, 0x6c, + 0x69, 0x73, 0x74, 0x5f, 0x76, 0x61, 0x6c, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x16, 0x2e, + 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x74, 0x79, 0x70, 0x65, 0x73, 0x2e, 0x42, 0x79, 0x74, 0x65, + 0x73, 0x4c, 0x69, 0x73, 0x74, 0x48, 0x00, 0x52, 0x0c, 0x62, 0x79, 0x74, 0x65, 0x73, 0x4c, 0x69, + 0x73, 0x74, 0x56, 0x61, 0x6c, 0x12, 0x41, 0x0a, 0x0f, 0x73, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x5f, + 0x6c, 0x69, 0x73, 0x74, 0x5f, 0x76, 0x61, 0x6c, 0x18, 0x0c, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x17, + 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x74, 0x79, 0x70, 0x65, 0x73, 0x2e, 0x53, 0x74, 0x72, + 0x69, 0x6e, 0x67, 0x4c, 0x69, 0x73, 0x74, 0x48, 0x00, 0x52, 0x0d, 0x73, 0x74, 0x72, 0x69, 0x6e, + 0x67, 0x4c, 0x69, 0x73, 0x74, 0x56, 0x61, 0x6c, 0x12, 0x3e, 0x0a, 0x0e, 0x69, 0x6e, 0x74, 0x33, + 0x32, 0x5f, 0x6c, 0x69, 0x73, 0x74, 0x5f, 0x76, 0x61, 0x6c, 0x18, 0x0d, 0x20, 0x01, 0x28, 0x0b, + 0x32, 0x16, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x74, 0x79, 0x70, 0x65, 0x73, 0x2e, 0x49, + 0x6e, 0x74, 0x33, 0x32, 0x4c, 0x69, 0x73, 0x74, 0x48, 0x00, 0x52, 0x0c, 0x69, 0x6e, 0x74, 0x33, + 0x32, 0x4c, 0x69, 0x73, 0x74, 0x56, 0x61, 0x6c, 0x12, 0x3e, 0x0a, 0x0e, 0x69, 0x6e, 0x74, 0x36, + 0x34, 0x5f, 0x6c, 0x69, 0x73, 0x74, 0x5f, 0x76, 0x61, 0x6c, 0x18, 0x0e, 0x20, 0x01, 0x28, 0x0b, + 0x32, 0x16, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x74, 0x79, 0x70, 0x65, 0x73, 0x2e, 0x49, + 0x6e, 0x74, 0x36, 0x34, 0x4c, 0x69, 0x73, 0x74, 0x48, 0x00, 0x52, 0x0c, 0x69, 0x6e, 0x74, 0x36, + 0x34, 0x4c, 0x69, 0x73, 0x74, 0x56, 0x61, 0x6c, 0x12, 0x41, 0x0a, 0x0f, 0x64, 0x6f, 0x75, 0x62, + 0x6c, 0x65, 0x5f, 0x6c, 0x69, 0x73, 0x74, 0x5f, 0x76, 0x61, 0x6c, 0x18, 0x0f, 0x20, 0x01, 0x28, + 0x0b, 0x32, 0x17, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x74, 0x79, 0x70, 0x65, 0x73, 0x2e, + 0x44, 0x6f, 0x75, 0x62, 0x6c, 0x65, 0x4c, 0x69, 0x73, 0x74, 0x48, 0x00, 0x52, 0x0d, 0x64, 0x6f, + 0x75, 0x62, 0x6c, 0x65, 0x4c, 0x69, 0x73, 0x74, 0x56, 0x61, 0x6c, 0x12, 0x3e, 0x0a, 0x0e, 0x66, + 0x6c, 0x6f, 0x61, 0x74, 0x5f, 0x6c, 0x69, 0x73, 0x74, 0x5f, 0x76, 0x61, 0x6c, 0x18, 0x10, 0x20, + 0x01, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x74, 0x79, 0x70, 0x65, + 0x73, 0x2e, 0x46, 0x6c, 0x6f, 0x61, 0x74, 0x4c, 0x69, 0x73, 0x74, 0x48, 0x00, 0x52, 0x0c, 0x66, + 0x6c, 0x6f, 0x61, 0x74, 0x4c, 0x69, 0x73, 0x74, 0x56, 0x61, 0x6c, 0x12, 0x3b, 0x0a, 0x0d, 0x62, + 0x6f, 0x6f, 0x6c, 0x5f, 0x6c, 0x69, 0x73, 0x74, 0x5f, 0x76, 0x61, 0x6c, 0x18, 0x11, 0x20, 0x01, + 0x28, 0x0b, 0x32, 0x15, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x74, 0x79, 0x70, 0x65, 0x73, + 0x2e, 0x42, 0x6f, 0x6f, 0x6c, 0x4c, 0x69, 0x73, 0x74, 0x48, 0x00, 0x52, 0x0b, 0x62, 0x6f, 0x6f, + 0x6c, 0x4c, 0x69, 0x73, 0x74, 0x56, 0x61, 0x6c, 0x12, 0x4f, 0x0a, 0x17, 0x75, 0x6e, 0x69, 0x78, + 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x5f, 0x6c, 0x69, 0x73, 0x74, 0x5f, + 0x76, 0x61, 0x6c, 0x18, 0x12, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x74, 0x79, 0x70, 0x65, 0x73, 0x2e, 0x49, 0x6e, 0x74, 0x36, 0x34, 0x4c, 0x69, 0x73, - 0x74, 0x48, 0x00, 0x52, 0x0c, 0x69, 0x6e, 0x74, 0x36, 0x34, 0x4c, 0x69, 0x73, 0x74, 0x56, 0x61, - 0x6c, 0x12, 0x41, 0x0a, 0x0f, 0x64, 0x6f, 0x75, 0x62, 0x6c, 0x65, 0x5f, 0x6c, 0x69, 0x73, 0x74, - 0x5f, 0x76, 0x61, 0x6c, 0x18, 0x0f, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x17, 0x2e, 0x66, 0x65, 0x61, - 0x73, 0x74, 0x2e, 0x74, 0x79, 0x70, 0x65, 0x73, 0x2e, 0x44, 0x6f, 0x75, 0x62, 0x6c, 0x65, 0x4c, - 0x69, 0x73, 0x74, 0x48, 0x00, 0x52, 0x0d, 0x64, 0x6f, 0x75, 0x62, 0x6c, 0x65, 0x4c, 0x69, 0x73, - 0x74, 0x56, 0x61, 0x6c, 0x12, 0x3e, 0x0a, 0x0e, 0x66, 0x6c, 0x6f, 0x61, 0x74, 0x5f, 0x6c, 0x69, - 0x73, 0x74, 0x5f, 0x76, 0x61, 0x6c, 0x18, 0x10, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x66, - 0x65, 0x61, 0x73, 0x74, 0x2e, 0x74, 0x79, 0x70, 0x65, 0x73, 0x2e, 0x46, 0x6c, 0x6f, 0x61, 0x74, - 0x4c, 0x69, 0x73, 0x74, 0x48, 0x00, 0x52, 0x0c, 0x66, 0x6c, 0x6f, 0x61, 0x74, 0x4c, 0x69, 0x73, - 0x74, 0x56, 0x61, 0x6c, 0x12, 0x3b, 0x0a, 0x0d, 0x62, 0x6f, 0x6f, 0x6c, 0x5f, 0x6c, 0x69, 0x73, - 0x74, 0x5f, 0x76, 0x61, 0x6c, 0x18, 0x11, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x15, 0x2e, 0x66, 0x65, - 0x61, 0x73, 0x74, 0x2e, 0x74, 0x79, 0x70, 0x65, 0x73, 0x2e, 0x42, 0x6f, 0x6f, 0x6c, 0x4c, 0x69, - 0x73, 0x74, 0x48, 0x00, 0x52, 0x0b, 0x62, 0x6f, 0x6f, 0x6c, 0x4c, 0x69, 0x73, 0x74, 0x56, 0x61, - 0x6c, 0x42, 0x05, 0x0a, 0x03, 0x76, 0x61, 0x6c, 0x22, 0x1d, 0x0a, 0x09, 0x42, 0x79, 0x74, 0x65, - 0x73, 0x4c, 0x69, 0x73, 0x74, 0x12, 0x10, 0x0a, 0x03, 0x76, 0x61, 0x6c, 0x18, 0x01, 0x20, 0x03, - 0x28, 0x0c, 0x52, 0x03, 0x76, 0x61, 0x6c, 0x22, 0x1e, 0x0a, 0x0a, 0x53, 0x74, 0x72, 0x69, 0x6e, - 0x67, 0x4c, 0x69, 0x73, 0x74, 0x12, 0x10, 0x0a, 0x03, 0x76, 0x61, 0x6c, 0x18, 0x01, 0x20, 0x03, - 0x28, 0x09, 0x52, 0x03, 0x76, 0x61, 0x6c, 0x22, 0x1d, 0x0a, 0x09, 0x49, 0x6e, 0x74, 0x33, 0x32, - 0x4c, 0x69, 0x73, 0x74, 0x12, 0x10, 0x0a, 0x03, 0x76, 0x61, 0x6c, 0x18, 0x01, 0x20, 0x03, 0x28, - 0x05, 0x52, 0x03, 0x76, 0x61, 0x6c, 0x22, 0x1d, 0x0a, 0x09, 0x49, 0x6e, 0x74, 0x36, 0x34, 0x4c, - 0x69, 0x73, 0x74, 0x12, 0x10, 0x0a, 0x03, 0x76, 0x61, 0x6c, 0x18, 0x01, 0x20, 0x03, 0x28, 0x03, - 0x52, 0x03, 0x76, 0x61, 0x6c, 0x22, 0x1e, 0x0a, 0x0a, 0x44, 0x6f, 0x75, 0x62, 0x6c, 0x65, 0x4c, - 0x69, 0x73, 0x74, 0x12, 0x10, 0x0a, 0x03, 0x76, 0x61, 0x6c, 0x18, 0x01, 0x20, 0x03, 0x28, 0x01, - 0x52, 0x03, 0x76, 0x61, 0x6c, 0x22, 0x1d, 0x0a, 0x09, 0x46, 0x6c, 0x6f, 0x61, 0x74, 0x4c, 0x69, - 0x73, 0x74, 0x12, 0x10, 0x0a, 0x03, 0x76, 0x61, 0x6c, 0x18, 0x01, 0x20, 0x03, 0x28, 0x02, 0x52, - 0x03, 0x76, 0x61, 0x6c, 0x22, 0x1c, 0x0a, 0x08, 0x42, 0x6f, 0x6f, 0x6c, 0x4c, 0x69, 0x73, 0x74, - 0x12, 0x10, 0x0a, 0x03, 0x76, 0x61, 0x6c, 0x18, 0x01, 0x20, 0x03, 0x28, 0x08, 0x52, 0x03, 0x76, - 0x61, 0x6c, 0x42, 0x55, 0x0a, 0x11, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x70, 0x72, 0x6f, 0x74, - 0x6f, 0x2e, 0x74, 0x79, 0x70, 0x65, 0x73, 0x42, 0x0a, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x50, 0x72, - 0x6f, 0x74, 0x6f, 0x5a, 0x34, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, - 0x66, 0x65, 0x61, 0x73, 0x74, 0x2d, 0x64, 0x65, 0x76, 0x2f, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2f, - 0x73, 0x64, 0x6b, 0x2f, 0x67, 0x6f, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x73, 0x2f, 0x66, 0x65, - 0x61, 0x73, 0x74, 0x2f, 0x74, 0x79, 0x70, 0x65, 0x73, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, - 0x33, + 0x74, 0x48, 0x00, 0x52, 0x14, 0x75, 0x6e, 0x69, 0x78, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, + 0x6d, 0x70, 0x4c, 0x69, 0x73, 0x74, 0x56, 0x61, 0x6c, 0x12, 0x2e, 0x0a, 0x08, 0x6e, 0x75, 0x6c, + 0x6c, 0x5f, 0x76, 0x61, 0x6c, 0x18, 0x13, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x11, 0x2e, 0x66, 0x65, + 0x61, 0x73, 0x74, 0x2e, 0x74, 0x79, 0x70, 0x65, 0x73, 0x2e, 0x4e, 0x75, 0x6c, 0x6c, 0x48, 0x00, + 0x52, 0x07, 0x6e, 0x75, 0x6c, 0x6c, 0x56, 0x61, 0x6c, 0x42, 0x05, 0x0a, 0x03, 0x76, 0x61, 0x6c, + 0x22, 0x1d, 0x0a, 0x09, 0x42, 0x79, 0x74, 0x65, 0x73, 0x4c, 0x69, 0x73, 0x74, 0x12, 0x10, 0x0a, + 0x03, 0x76, 0x61, 0x6c, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0c, 0x52, 0x03, 0x76, 0x61, 0x6c, 0x22, + 0x1e, 0x0a, 0x0a, 0x53, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x4c, 0x69, 0x73, 0x74, 0x12, 0x10, 0x0a, + 0x03, 0x76, 0x61, 0x6c, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x03, 0x76, 0x61, 0x6c, 0x22, + 0x1d, 0x0a, 0x09, 0x49, 0x6e, 0x74, 0x33, 0x32, 0x4c, 0x69, 0x73, 0x74, 0x12, 0x10, 0x0a, 0x03, + 0x76, 0x61, 0x6c, 0x18, 0x01, 0x20, 0x03, 0x28, 0x05, 0x52, 0x03, 0x76, 0x61, 0x6c, 0x22, 0x1d, + 0x0a, 0x09, 0x49, 0x6e, 0x74, 0x36, 0x34, 0x4c, 0x69, 0x73, 0x74, 0x12, 0x10, 0x0a, 0x03, 0x76, + 0x61, 0x6c, 0x18, 0x01, 0x20, 0x03, 0x28, 0x03, 0x52, 0x03, 0x76, 0x61, 0x6c, 0x22, 0x1e, 0x0a, + 0x0a, 0x44, 0x6f, 0x75, 0x62, 0x6c, 0x65, 0x4c, 0x69, 0x73, 0x74, 0x12, 0x10, 0x0a, 0x03, 0x76, + 0x61, 0x6c, 0x18, 0x01, 0x20, 0x03, 0x28, 0x01, 0x52, 0x03, 0x76, 0x61, 0x6c, 0x22, 0x1d, 0x0a, + 0x09, 0x46, 0x6c, 0x6f, 0x61, 0x74, 0x4c, 0x69, 0x73, 0x74, 0x12, 0x10, 0x0a, 0x03, 0x76, 0x61, + 0x6c, 0x18, 0x01, 0x20, 0x03, 0x28, 0x02, 0x52, 0x03, 0x76, 0x61, 0x6c, 0x22, 0x1c, 0x0a, 0x08, + 0x42, 0x6f, 0x6f, 0x6c, 0x4c, 0x69, 0x73, 0x74, 0x12, 0x10, 0x0a, 0x03, 0x76, 0x61, 0x6c, 0x18, + 0x01, 0x20, 0x03, 0x28, 0x08, 0x52, 0x03, 0x76, 0x61, 0x6c, 0x22, 0x35, 0x0a, 0x0d, 0x52, 0x65, + 0x70, 0x65, 0x61, 0x74, 0x65, 0x64, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x12, 0x24, 0x0a, 0x03, 0x76, + 0x61, 0x6c, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, + 0x2e, 0x74, 0x79, 0x70, 0x65, 0x73, 0x2e, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x52, 0x03, 0x76, 0x61, + 0x6c, 0x2a, 0x10, 0x0a, 0x04, 0x4e, 0x75, 0x6c, 0x6c, 0x12, 0x08, 0x0a, 0x04, 0x4e, 0x55, 0x4c, + 0x4c, 0x10, 0x00, 0x42, 0x55, 0x0a, 0x11, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x70, 0x72, 0x6f, + 0x74, 0x6f, 0x2e, 0x74, 0x79, 0x70, 0x65, 0x73, 0x42, 0x0a, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x50, + 0x72, 0x6f, 0x74, 0x6f, 0x5a, 0x34, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, + 0x2f, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2d, 0x64, 0x65, 0x76, 0x2f, 0x66, 0x65, 0x61, 0x73, 0x74, + 0x2f, 0x73, 0x64, 0x6b, 0x2f, 0x67, 0x6f, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x73, 0x2f, 0x66, + 0x65, 0x61, 0x73, 0x74, 0x2f, 0x74, 0x79, 0x70, 0x65, 0x73, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, + 0x6f, 0x33, } var ( @@ -840,33 +998,38 @@ func file_feast_types_Value_proto_rawDescGZIP() []byte { return file_feast_types_Value_proto_rawDescData } -var file_feast_types_Value_proto_enumTypes = make([]protoimpl.EnumInfo, 1) -var file_feast_types_Value_proto_msgTypes = make([]protoimpl.MessageInfo, 9) +var file_feast_types_Value_proto_enumTypes = make([]protoimpl.EnumInfo, 2) +var file_feast_types_Value_proto_msgTypes = make([]protoimpl.MessageInfo, 10) var file_feast_types_Value_proto_goTypes = []interface{}{ - (ValueType_Enum)(0), // 0: feast.types.ValueType.Enum - (*ValueType)(nil), // 1: feast.types.ValueType - (*Value)(nil), // 2: feast.types.Value - (*BytesList)(nil), // 3: feast.types.BytesList - (*StringList)(nil), // 4: feast.types.StringList - (*Int32List)(nil), // 5: feast.types.Int32List - (*Int64List)(nil), // 6: feast.types.Int64List - (*DoubleList)(nil), // 7: feast.types.DoubleList - (*FloatList)(nil), // 8: feast.types.FloatList - (*BoolList)(nil), // 9: feast.types.BoolList + (Null)(0), // 0: feast.types.Null + (ValueType_Enum)(0), // 1: feast.types.ValueType.Enum + (*ValueType)(nil), // 2: feast.types.ValueType + (*Value)(nil), // 3: feast.types.Value + (*BytesList)(nil), // 4: feast.types.BytesList + (*StringList)(nil), // 5: feast.types.StringList + (*Int32List)(nil), // 6: feast.types.Int32List + (*Int64List)(nil), // 7: feast.types.Int64List + (*DoubleList)(nil), // 8: feast.types.DoubleList + (*FloatList)(nil), // 9: feast.types.FloatList + (*BoolList)(nil), // 10: feast.types.BoolList + (*RepeatedValue)(nil), // 11: feast.types.RepeatedValue } var file_feast_types_Value_proto_depIdxs = []int32{ - 3, // 0: feast.types.Value.bytes_list_val:type_name -> feast.types.BytesList - 4, // 1: feast.types.Value.string_list_val:type_name -> feast.types.StringList - 5, // 2: feast.types.Value.int32_list_val:type_name -> feast.types.Int32List - 6, // 3: feast.types.Value.int64_list_val:type_name -> feast.types.Int64List - 7, // 4: feast.types.Value.double_list_val:type_name -> feast.types.DoubleList - 8, // 5: feast.types.Value.float_list_val:type_name -> feast.types.FloatList - 9, // 6: feast.types.Value.bool_list_val:type_name -> feast.types.BoolList - 7, // [7:7] is the sub-list for method output_type - 7, // [7:7] is the sub-list for method input_type - 7, // [7:7] is the sub-list for extension type_name - 7, // [7:7] is the sub-list for extension extendee - 0, // [0:7] is the sub-list for field type_name + 4, // 0: feast.types.Value.bytes_list_val:type_name -> feast.types.BytesList + 5, // 1: feast.types.Value.string_list_val:type_name -> feast.types.StringList + 6, // 2: feast.types.Value.int32_list_val:type_name -> feast.types.Int32List + 7, // 3: feast.types.Value.int64_list_val:type_name -> feast.types.Int64List + 8, // 4: feast.types.Value.double_list_val:type_name -> feast.types.DoubleList + 9, // 5: feast.types.Value.float_list_val:type_name -> feast.types.FloatList + 10, // 6: feast.types.Value.bool_list_val:type_name -> feast.types.BoolList + 7, // 7: feast.types.Value.unix_timestamp_list_val:type_name -> feast.types.Int64List + 0, // 8: feast.types.Value.null_val:type_name -> feast.types.Null + 3, // 9: feast.types.RepeatedValue.val:type_name -> feast.types.Value + 10, // [10:10] is the sub-list for method output_type + 10, // [10:10] is the sub-list for method input_type + 10, // [10:10] is the sub-list for extension type_name + 10, // [10:10] is the sub-list for extension extendee + 0, // [0:10] is the sub-list for field type_name } func init() { file_feast_types_Value_proto_init() } @@ -983,6 +1146,18 @@ func file_feast_types_Value_proto_init() { return nil } } + file_feast_types_Value_proto_msgTypes[9].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*RepeatedValue); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } } file_feast_types_Value_proto_msgTypes[1].OneofWrappers = []interface{}{ (*Value_BytesVal)(nil), @@ -992,6 +1167,7 @@ func file_feast_types_Value_proto_init() { (*Value_DoubleVal)(nil), (*Value_FloatVal)(nil), (*Value_BoolVal)(nil), + (*Value_UnixTimestampVal)(nil), (*Value_BytesListVal)(nil), (*Value_StringListVal)(nil), (*Value_Int32ListVal)(nil), @@ -999,14 +1175,16 @@ func file_feast_types_Value_proto_init() { (*Value_DoubleListVal)(nil), (*Value_FloatListVal)(nil), (*Value_BoolListVal)(nil), + (*Value_UnixTimestampListVal)(nil), + (*Value_NullVal)(nil), } type x struct{} out := protoimpl.TypeBuilder{ File: protoimpl.DescBuilder{ GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: file_feast_types_Value_proto_rawDesc, - NumEnums: 1, - NumMessages: 9, + NumEnums: 2, + NumMessages: 10, NumExtensions: 0, NumServices: 0, }, diff --git a/sdk/go/protos/tensorflow_metadata/proto/v0/path.pb.go b/sdk/go/protos/tensorflow_metadata/proto/v0/path.pb.go index 1daa7687f94..5abc145a622 100644 --- a/sdk/go/protos/tensorflow_metadata/proto/v0/path.pb.go +++ b/sdk/go/protos/tensorflow_metadata/proto/v0/path.pb.go @@ -15,14 +15,13 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.25.0 -// protoc v3.12.4 +// protoc-gen-go v1.27.1 +// protoc v3.11.2 // source: tensorflow_metadata/proto/v0/path.proto package v0 import ( - proto "github.com/golang/protobuf/proto" protoreflect "google.golang.org/protobuf/reflect/protoreflect" protoimpl "google.golang.org/protobuf/runtime/protoimpl" reflect "reflect" @@ -36,10 +35,6 @@ const ( _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) ) -// This is a compile-time assertion that a sufficiently up-to-date version -// of the legacy proto package is being used. -const _ = proto.ProtoPackageIsVersion4 - // A path is a more general substitute for the name of a field or feature that // can be used for flat examples as well as structured data. For example, if // we had data in a protocol buffer: diff --git a/sdk/go/protos/tensorflow_metadata/proto/v0/schema.pb.go b/sdk/go/protos/tensorflow_metadata/proto/v0/schema.pb.go index 940779a1917..0200d3afb61 100644 --- a/sdk/go/protos/tensorflow_metadata/proto/v0/schema.pb.go +++ b/sdk/go/protos/tensorflow_metadata/proto/v0/schema.pb.go @@ -15,17 +15,16 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.25.0 -// protoc v3.12.4 +// protoc-gen-go v1.27.1 +// protoc v3.11.2 // source: tensorflow_metadata/proto/v0/schema.proto package v0 import ( - proto "github.com/golang/protobuf/proto" - any "github.com/golang/protobuf/ptypes/any" protoreflect "google.golang.org/protobuf/reflect/protoreflect" protoimpl "google.golang.org/protobuf/runtime/protoimpl" + anypb "google.golang.org/protobuf/types/known/anypb" reflect "reflect" sync "sync" ) @@ -37,10 +36,6 @@ const ( _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) ) -// This is a compile-time assertion that a sufficiently up-to-date version -// of the legacy proto package is being used. -const _ = proto.ProtoPackageIsVersion4 - // LifecycleStage. Only UNKNOWN_STAGE, BETA, and PRODUCTION features are // actually validated. // PLANNED, ALPHA, and DEBUG are treated as DEPRECATED. @@ -908,7 +903,7 @@ type Annotation struct { // developer notes etc. Comment []string `protobuf:"bytes,2,rep,name=comment" json:"comment,omitempty"` // Application-specific metadata may be attached here. - ExtraMetadata []*any.Any `protobuf:"bytes,3,rep,name=extra_metadata,json=extraMetadata" json:"extra_metadata,omitempty"` + ExtraMetadata []*anypb.Any `protobuf:"bytes,3,rep,name=extra_metadata,json=extraMetadata" json:"extra_metadata,omitempty"` } func (x *Annotation) Reset() { @@ -957,7 +952,7 @@ func (x *Annotation) GetComment() []string { return nil } -func (x *Annotation) GetExtraMetadata() []*any.Any { +func (x *Annotation) GetExtraMetadata() []*anypb.Any { if x != nil { return x.ExtraMetadata } @@ -3539,9 +3534,9 @@ var file_tensorflow_metadata_proto_v0_schema_proto_goTypes = []interface{}{ (*TensorRepresentation_DenseTensor)(nil), // 36: tensorflow.metadata.v0.TensorRepresentation.DenseTensor (*TensorRepresentation_VarLenSparseTensor)(nil), // 37: tensorflow.metadata.v0.TensorRepresentation.VarLenSparseTensor (*TensorRepresentation_SparseTensor)(nil), // 38: tensorflow.metadata.v0.TensorRepresentation.SparseTensor - nil, // 39: tensorflow.metadata.v0.TensorRepresentationGroup.TensorRepresentationEntry - (*any.Any)(nil), // 40: google.protobuf.Any - (*Path)(nil), // 41: tensorflow.metadata.v0.Path + nil, // 39: tensorflow.metadata.v0.TensorRepresentationGroup.TensorRepresentationEntry + (*anypb.Any)(nil), // 40: google.protobuf.Any + (*Path)(nil), // 41: tensorflow.metadata.v0.Path } var file_tensorflow_metadata_proto_v0_schema_proto_depIdxs = []int32{ 5, // 0: tensorflow.metadata.v0.Schema.feature:type_name -> tensorflow.metadata.v0.Feature diff --git a/sdk/go/protos/tensorflow_metadata/proto/v0/statistics.pb.go b/sdk/go/protos/tensorflow_metadata/proto/v0/statistics.pb.go index fbf6247a1d3..cd8bdf5d894 100644 --- a/sdk/go/protos/tensorflow_metadata/proto/v0/statistics.pb.go +++ b/sdk/go/protos/tensorflow_metadata/proto/v0/statistics.pb.go @@ -19,14 +19,13 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.25.0 -// protoc v3.12.4 +// protoc-gen-go v1.27.1 +// protoc v3.11.2 // source: tensorflow_metadata/proto/v0/statistics.proto package v0 import ( - proto "github.com/golang/protobuf/proto" protoreflect "google.golang.org/protobuf/reflect/protoreflect" protoimpl "google.golang.org/protobuf/runtime/protoimpl" reflect "reflect" @@ -40,10 +39,6 @@ const ( _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) ) -// This is a compile-time assertion that a sufficiently up-to-date version -// of the legacy proto package is being used. -const _ = proto.ProtoPackageIsVersion4 - // The types supported by the feature statistics. When aggregating // tf.Examples, if the bytelist contains a string, it is recommended to encode // it here as STRING instead of BYTES in order to calculate string-specific diff --git a/sdk/python/feast/base_feature_view.py b/sdk/python/feast/base_feature_view.py new file mode 100644 index 00000000000..10f949d9a1c --- /dev/null +++ b/sdk/python/feast/base_feature_view.py @@ -0,0 +1,204 @@ +# Copyright 2021 The Feast Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import warnings +from abc import ABC, abstractmethod +from typing import List, Type + +from google.protobuf.json_format import MessageToJson +from proto import Message + +from feast.feature import Feature +from feast.feature_view_projection import FeatureViewProjection + +warnings.simplefilter("once", DeprecationWarning) + + +class BaseFeatureView(ABC): + """A FeatureView defines a logical grouping of features to be served.""" + + @abstractmethod + def __init__(self, name: str, features: List[Feature]): + self._name = name + self._features = features + self._projection = FeatureViewProjection.from_definition(self) + + @property + def name(self) -> str: + return self._name + + @property + def features(self) -> List[Feature]: + return self._features + + @features.setter + def features(self, value): + self._features = value + + @property + def projection(self) -> FeatureViewProjection: + return self._projection + + @projection.setter + def projection(self, value): + self._projection = value + + @property + @abstractmethod + def proto_class(self) -> Type[Message]: + pass + + @abstractmethod + def to_proto(self) -> Message: + pass + + @classmethod + @abstractmethod + def from_proto(cls, feature_view_proto): + pass + + @abstractmethod + def __copy__(self): + """ + Generates a deep copy of this feature view + + Returns: + A copy of this FeatureView + """ + pass + + def __repr__(self): + items = (f"{k} = {v}" for k, v in self.__dict__.items()) + return f"<{self.__class__.__name__}({', '.join(items)})>" + + def __str__(self): + return str(MessageToJson(self.to_proto())) + + def __hash__(self): + return hash((id(self), self.name)) + + def __getitem__(self, item): + assert isinstance(item, list) + + referenced_features = [] + for feature in self.features: + if feature.name in item: + referenced_features.append(feature) + + cp = self.__copy__() + cp.projection.features = referenced_features + + return cp + + def __eq__(self, other): + if not isinstance(other, BaseFeatureView): + raise TypeError( + "Comparisons should only involve BaseFeatureView class objects." + ) + + if self.name != other.name: + return False + + if sorted(self.features) != sorted(other.features): + return False + + return True + + def ensure_valid(self): + """ + Validates the state of this feature view locally. + + Raises: + ValueError: The feature view is invalid. + """ + if not self.name: + raise ValueError("Feature view needs a name.") + + def with_name(self, name: str): + """ + Renames this feature view by returning a copy of this feature view with an alias + set for the feature view name. This rename operation is only used as part of query + operations and will not modify the underlying FeatureView. + + Args: + name: Name to assign to the FeatureView copy. + + Returns: + A copy of this FeatureView with the name replaced with the 'name' input. + """ + cp = self.__copy__() + cp.projection.name_alias = name + + return cp + + def set_projection(self, feature_view_projection: FeatureViewProjection) -> None: + """ + Setter for the projection object held by this FeatureView. A projection is an + object that stores the modifications to a FeatureView that is applied to the FeatureView + when the FeatureView is used such as during feature_store.get_historical_features. + This method also performs checks to ensure the projection is consistent with this + FeatureView before doing the set. + + Args: + feature_view_projection: The FeatureViewProjection object to set this FeatureView's + 'projection' field to. + """ + if feature_view_projection.name != self.name: + raise ValueError( + f"The projection for the {self.name} FeatureView cannot be applied because it differs in name. " + f"The projection is named {feature_view_projection.name} and the name indicates which " + "FeatureView the projection is for." + ) + + for feature in feature_view_projection.features: + if feature not in self.features: + raise ValueError( + f"The projection for {self.name} cannot be applied because it contains {feature.name} which the " + "FeatureView doesn't have." + ) + + self.projection = feature_view_projection + + def with_projection(self, feature_view_projection: FeatureViewProjection): + """ + Sets the feature view projection by returning a copy of this on-demand feature view + with its projection set to the given projection. A projection is an + object that stores the modifications to a feature view that is used during + query operations. + + Args: + feature_view_projection: The FeatureViewProjection object to link to this + OnDemandFeatureView. + + Returns: + A copy of this OnDemandFeatureView with its projection replaced with the + 'feature_view_projection' argument. + """ + if feature_view_projection.name != self.name: + raise ValueError( + f"The projection for the {self.name} FeatureView cannot be applied because it differs in name. " + f"The projection is named {feature_view_projection.name} and the name indicates which " + "FeatureView the projection is for." + ) + + for feature in feature_view_projection.features: + if feature not in self.features: + raise ValueError( + f"The projection for {self.name} cannot be applied because it contains {feature.name} which the " + "FeatureView doesn't have." + ) + + cp = self.__copy__() + cp.projection = feature_view_projection + + return cp diff --git a/sdk/python/feast/cli.py b/sdk/python/feast/cli.py index cf26a80d3cb..11150701a94 100644 --- a/sdk/python/feast/cli.py +++ b/sdk/python/feast/cli.py @@ -23,8 +23,11 @@ from colorama import Fore, Style from feast import flags, flags_helper, utils +from feast.constants import DEFAULT_FEATURE_TRANSFORMATION_SERVER_PORT from feast.errors import FeastObjectNotFoundException, FeastProviderLoginError from feast.feature_store import FeatureStore +from feast.feature_view import FeatureView +from feast.on_demand_feature_view import OnDemandFeatureView from feast.repo_config import load_repo_config from feast.repo_operations import ( apply_total, @@ -81,6 +84,16 @@ def cli(ctx: click.Context, chdir: Optional[str], log_level: str): datefmt="%m/%d/%Y %I:%M:%S %p", level=level, ) + # Override the logging level for already created loggers (due to loggers being created at the import time) + # Note, that format & datefmt does not need to be set, because by default child loggers don't override them + + # Also note, that mypy complains that logging.root doesn't have "manager" because of the way it's written. + # So we have to put a type ignore hint for mypy. + for logger_name in logging.root.manager.loggerDict: # type: ignore + if "feast" in logger_name: + logger = logging.getLogger(logger_name) + logger.setLevel(level) + except Exception as e: raise e pass @@ -261,12 +274,29 @@ def feature_view_list(ctx: click.Context): cli_check_repo(repo) store = FeatureStore(repo_path=str(repo)) table = [] - for feature_view in store.list_feature_views(): - table.append([feature_view.name, feature_view.entities]) + for feature_view in [ + *store.list_feature_views(), + *store.list_request_feature_views(), + *store.list_on_demand_feature_views(), + ]: + entities = set() + if isinstance(feature_view, FeatureView): + entities.update(feature_view.entities) + elif isinstance(feature_view, OnDemandFeatureView): + for backing_fv in feature_view.inputs.values(): + if isinstance(backing_fv, FeatureView): + entities.update(backing_fv.entities) + table.append( + [ + feature_view.name, + entities if len(entities) > 0 else "n/a", + type(feature_view).__name__, + ] + ) from tabulate import tabulate - print(tabulate(table, headers=["NAME", "ENTITIES"], tablefmt="plain")) + print(tabulate(table, headers=["NAME", "ENTITIES", "TYPE"], tablefmt="plain")) @cli.group(name="on-demand-feature-views") @@ -456,6 +486,24 @@ def serve_command(ctx: click.Context, port: int): store.serve(port) +@cli.command("serve_transformations") +@click.option( + "--port", + "-p", + type=click.INT, + default=DEFAULT_FEATURE_TRANSFORMATION_SERVER_PORT, + help="Specify a port for the server", +) +@click.pass_context +def serve_transformations_command(ctx: click.Context, port: int): + """[Experimental] Start a the feature consumption server locally on a given port.""" + repo = ctx.obj["CHDIR"] + cli_check_repo(repo) + store = FeatureStore(repo_path=str(repo)) + + store.serve_transformations(port) + + @cli.group(name="alpha") def alpha_cmd(): """ diff --git a/sdk/python/feast/constants.py b/sdk/python/feast/constants.py index 4cdfd46d2d8..efcea6d8f17 100644 --- a/sdk/python/feast/constants.py +++ b/sdk/python/feast/constants.py @@ -18,9 +18,22 @@ MAX_WAIT_INTERVAL: str = "60" AWS_LAMBDA_FEATURE_SERVER_IMAGE = "feastdev/feature-server:aws" +AWS_LAMBDA_FEATURE_SERVER_REPOSITORY = "feast-python-server" # feature_store.yaml environment variable name for remote feature server FEATURE_STORE_YAML_ENV_NAME: str = "FEATURE_STORE_YAML_BASE64" +# Environment variable for registry +REGISTRY_ENV_NAME: str = "REGISTRY_BASE64" + # Environment variable for toggling usage FEAST_USAGE = "FEAST_USAGE" + +# Environment variable for the path for overwriting universal test configs +FULL_REPO_CONFIGS_MODULE_ENV_NAME: str = "FULL_REPO_CONFIGS_MODULE" + +# Environment variable for overwriting FTS port +FEATURE_TRANSFORMATION_SERVER_PORT_ENV_NAME: str = "FEATURE_TRANSFORMATION_SERVER_PORT" + +# Default FTS port +DEFAULT_FEATURE_TRANSFORMATION_SERVER_PORT = 6569 diff --git a/sdk/python/feast/driver_test_data.py b/sdk/python/feast/driver_test_data.py index 6aa98b3d064..1c9a1dd20bc 100644 --- a/sdk/python/feast/driver_test_data.py +++ b/sdk/python/feast/driver_test_data.py @@ -1,4 +1,5 @@ # This module generates dummy data to be used for tests and examples. +import itertools from enum import Enum import numpy as np @@ -29,22 +30,29 @@ def _convert_event_timestamp(event_timestamp: pd.Timestamp, t: EventTimestampTyp def create_orders_df( - customers, drivers, start_date, end_date, order_count, + customers, drivers, start_date, end_date, order_count, locations=None, ) -> pd.DataFrame: """ - Example df generated by this function: + Example df generated by this function (if locations): - | order_id | driver_id | customer_id | order_is_success | event_timestamp | - +----------+-----------+-------------+------------------+---------------------+ - | 100 | 5004 | 1007 | 0 | 2021-03-10 19:31:15 | - | 101 | 5003 | 1006 | 0 | 2021-03-11 22:02:50 | - | 102 | 5010 | 1005 | 0 | 2021-03-13 00:34:24 | - | 103 | 5010 | 1001 | 1 | 2021-03-14 03:05:59 | + | order_id | driver_id | customer_id | origin_id | destination_id | order_is_success | event_timestamp | + +----------+-----------+-------------+-----------+----------------+------------------+---------------------+ + | 100 | 5004 | 1007 | 1 | 18 | 0 | 2021-03-10 19:31:15 | + | 101 | 5003 | 1006 | 24 | 42 | 0 | 2021-03-11 22:02:50 | + | 102 | 5010 | 1005 | 19 | 12 | 0 | 2021-03-13 00:34:24 | + | 103 | 5010 | 1001 | 35 | 8 | 1 | 2021-03-14 03:05:59 | """ df = pd.DataFrame() df["order_id"] = [order_id for order_id in range(100, 100 + order_count)] df["driver_id"] = np.random.choice(drivers, order_count) df["customer_id"] = np.random.choice(customers, order_count) + if locations: + location_pairs = np.array(list(itertools.permutations(locations, 2))) + locations_sample = location_pairs[ + np.random.choice(len(location_pairs), order_count) + ].T + df["origin_id"] = locations_sample[0] + df["destination_id"] = locations_sample[1] df["order_is_success"] = np.random.randint(0, 2, size=order_count).astype(np.int32) df[DEFAULT_ENTITY_DF_EVENT_TIMESTAMP_COL] = [ _convert_event_timestamp( @@ -180,6 +188,46 @@ def create_customer_daily_profile_df(customers, start_date, end_date) -> pd.Data return df_all_customers +def create_location_stats_df(locations, start_date, end_date) -> pd.DataFrame: + """ + Example df generated by this function: + + | event_timestamp | location_id | temperature | created | + +------------------+-------------+-------------+------------------+ + | 2021-03-17 19:31 | 1 | 74 | 2021-03-24 19:38 | + | 2021-03-17 20:31 | 24 | 63 | 2021-03-24 19:38 | + | 2021-03-17 21:31 | 19 | 65 | 2021-03-24 19:38 | + | 2021-03-17 22:31 | 35 | 86 | 2021-03-24 19:38 | + """ + df_hourly = pd.DataFrame( + { + "event_timestamp": [ + pd.Timestamp(dt, unit="ms", tz="UTC").round("ms") + for dt in pd.date_range( + start=start_date, end=end_date, freq="1H", closed="left" + ) + ] + } + ) + df_all_locations = pd.DataFrame() + + for location in locations: + df_hourly_copy = df_hourly.copy() + df_hourly_copy["location_id"] = location + df_all_locations = pd.concat([df_hourly_copy, df_all_locations]) + + df_all_locations.reset_index(drop=True, inplace=True) + rows = df_all_locations["event_timestamp"].count() + + df_all_locations["temperature"] = np.random.randint(50, 100, size=rows).astype( + np.int32 + ) + + # TODO: Remove created timestamp in order to test whether its really optional + df_all_locations["created"] = pd.to_datetime(pd.Timestamp.now(tz=None).round("ms")) + return df_all_locations + + def create_global_daily_stats_df(start_date, end_date) -> pd.DataFrame: """ Example df generated by this function: diff --git a/sdk/python/feast/errors.py b/sdk/python/feast/errors.py index f95e1991057..03dd2409a7f 100644 --- a/sdk/python/feast/errors.py +++ b/sdk/python/feast/errors.py @@ -53,14 +53,14 @@ def __init__(self, name, project=None): class RequestDataNotFoundInEntityDfException(FeastObjectNotFoundException): def __init__(self, feature_name, feature_view_name): super().__init__( - f"Feature {feature_name} not found in the entity dataframe, but required by on demand feature view {feature_view_name}" + f"Feature {feature_name} not found in the entity dataframe, but required by feature view {feature_view_name}" ) class RequestDataNotFoundInEntityRowsException(FeastObjectNotFoundException): def __init__(self, feature_names): super().__init__( - f"Required request data source features {feature_names} not found in the entity rows, but required by on demand feature views" + f"Required request data source features {feature_names} not found in the entity rows, but required by feature views" ) @@ -263,9 +263,10 @@ def __init__(self, entity_type: type): class ConflictingFeatureViewNames(Exception): + # TODO: print file location of conflicting feature views def __init__(self, feature_view_name: str): super().__init__( - f"The feature view name: {feature_view_name} refers to both an on-demand feature view and a feature view" + f"The feature view name: {feature_view_name} refers to feature views of different types." ) diff --git a/sdk/python/feast/feature_service.py b/sdk/python/feast/feature_service.py index 9f8e9af1fb3..46afdfff1e4 100644 --- a/sdk/python/feast/feature_service.py +++ b/sdk/python/feast/feature_service.py @@ -3,6 +3,7 @@ from google.protobuf.json_format import MessageToJson +from feast.base_feature_view import BaseFeatureView from feast.feature_table import FeatureTable from feast.feature_view import FeatureView from feast.feature_view_projection import FeatureViewProjection @@ -59,9 +60,7 @@ def __init__( self.feature_view_projections.append( FeatureViewProjection.from_definition(feature_grouping) ) - elif isinstance(feature_grouping, FeatureView) or isinstance( - feature_grouping, OnDemandFeatureView - ): + elif isinstance(feature_grouping, BaseFeatureView): self.feature_view_projections.append(feature_grouping.projection) else: raise ValueError( diff --git a/sdk/python/feast/feature_store.py b/sdk/python/feast/feature_store.py index 7a175bb5b56..15de4a5d29f 100644 --- a/sdk/python/feast/feature_store.py +++ b/sdk/python/feast/feature_store.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. import copy +import itertools import os import warnings from collections import Counter, OrderedDict, defaultdict @@ -24,7 +25,7 @@ from tqdm import tqdm from feast import feature_server, flags, flags_helper, utils -from feast.data_source import RequestDataSource +from feast.base_feature_view import BaseFeatureView from feast.entity import Entity from feast.errors import ( EntityNotFoundException, @@ -56,8 +57,9 @@ from feast.protos.feast.types.EntityKey_pb2 import EntityKey as EntityKeyProto from feast.registry import Registry from feast.repo_config import RepoConfig, load_repo_config +from feast.request_feature_view import RequestFeatureView from feast.type_map import python_value_to_proto_value -from feast.usage import UsageEvent, log_event, log_exceptions, log_exceptions_and_usage +from feast.usage import log_exceptions, log_exceptions_and_usage, set_usage_attribute from feast.value_type import ValueType from feast.version import get_version @@ -190,8 +192,25 @@ def list_feature_views(self, allow_cache: bool = False) -> List[FeatureView]: """ return self._list_feature_views(allow_cache) + @log_exceptions_and_usage + def list_request_feature_views( + self, allow_cache: bool = False + ) -> List[RequestFeatureView]: + """ + Retrieves the list of feature views from the registry. + + Args: + allow_cache: Whether to allow returning entities from a cached registry. + + Returns: + A list of feature views. + """ + return self._registry.list_request_feature_views( + self.project, allow_cache=allow_cache + ) + def _list_feature_views( - self, allow_cache: bool = False, hide_dummy_entity: bool = True + self, allow_cache: bool = False, hide_dummy_entity: bool = True, ) -> List[FeatureView]: feature_views = [] for fv in self._registry.list_feature_views( @@ -347,9 +366,31 @@ def apply( Entity, FeatureView, OnDemandFeatureView, + RequestFeatureView, FeatureService, - List[Union[FeatureView, OnDemandFeatureView, Entity, FeatureService]], + FeatureTable, + List[ + Union[ + FeatureView, + OnDemandFeatureView, + RequestFeatureView, + Entity, + FeatureService, + FeatureTable, + ] + ], ], + objects_to_delete: List[ + Union[ + FeatureView, + OnDemandFeatureView, + RequestFeatureView, + Entity, + FeatureService, + FeatureTable, + ] + ] = [], + partial: bool = True, commit: bool = True, ): """Register objects to metadata store and update related infrastructure. @@ -361,6 +402,10 @@ def apply( Args: objects: A single object, or a list of objects that should be registered with the Feature Store. + objects_to_delete: A list of objects to be deleted from the registry and removed from the + provider's infrastructure. This deletion will only be performed if partial is set to False. + partial: If True, apply will only handle the specified objects; if False, apply will also delete + all the objects in objects_to_delete, and tear down any associated cloud resources. commit: whether to commit changes to the registry Raises: @@ -393,20 +438,37 @@ def apply( assert isinstance(objects, list) + # Separate all objects into entities, feature services, and different feature view types. + entities_to_update = [ob for ob in objects if isinstance(ob, Entity)] views_to_update = [ob for ob in objects if isinstance(ob, FeatureView)] + request_views_to_update = [ + ob for ob in objects if isinstance(ob, RequestFeatureView) + ] odfvs_to_update = [ob for ob in objects if isinstance(ob, OnDemandFeatureView)] + services_to_update = [ob for ob in objects if isinstance(ob, FeatureService)] + tables_to_update = [ob for ob in objects if isinstance(ob, FeatureTable)] + + if len(entities_to_update) + len(views_to_update) + len( + request_views_to_update + ) + len(odfvs_to_update) + len(services_to_update) + len( + tables_to_update + ) != len( + objects + ): + raise ValueError("Unknown object type provided as part of apply() call") + + # Validate all types of feature views. if ( not flags_helper.enable_on_demand_feature_views(self.config) and len(odfvs_to_update) > 0 ): raise ExperimentalFeatureNotEnabled(flags.FLAG_ON_DEMAND_TRANSFORM_NAME) - if len(odfvs_to_update) > 0: - log_event(UsageEvent.APPLY_WITH_ODFV) + set_usage_attribute("odfv", bool(odfvs_to_update)) - _validate_feature_views(views_to_update) - entities_to_update = [ob for ob in objects if isinstance(ob, Entity)] - services_to_update = [ob for ob in objects if isinstance(ob, FeatureService)] + _validate_feature_views( + [*views_to_update, *odfvs_to_update, *request_views_to_update] + ) # Make inferences update_entities_with_inferred_types_from_feature_views( @@ -423,12 +485,7 @@ def apply( for odfv in odfvs_to_update: odfv.infer_features() - if len(views_to_update) + len(entities_to_update) + len( - services_to_update - ) + len(odfvs_to_update) != len(objects): - raise ValueError("Unknown object type provided as part of apply() call") - - # DUMMY_ENTITY is a placeholder entity used in entityless FeatureViews + # Handle all entityless feature views by using DUMMY_ENTITY as a placeholder entity. DUMMY_ENTITY = Entity( name=DUMMY_ENTITY_NAME, join_key=DUMMY_ENTITY_ID, @@ -436,24 +493,71 @@ def apply( ) entities_to_update.append(DUMMY_ENTITY) - for view in views_to_update: + # Add all objects to the registry and update the provider's infrastructure. + for view in itertools.chain( + views_to_update, odfvs_to_update, request_views_to_update + ): self._registry.apply_feature_view(view, project=self.project, commit=False) - for odfv in odfvs_to_update: - self._registry.apply_on_demand_feature_view( - odfv, project=self.project, commit=False - ) for ent in entities_to_update: self._registry.apply_entity(ent, project=self.project, commit=False) for feature_service in services_to_update: self._registry.apply_feature_service(feature_service, project=self.project) + for table in tables_to_update: + self._registry.apply_feature_table(table, project=self.project) + + if not partial: + # Delete all registry objects that should not exist. + entities_to_delete = [ + ob for ob in objects_to_delete if isinstance(ob, Entity) + ] + views_to_delete = [ + ob for ob in objects_to_delete if isinstance(ob, FeatureView) + ] + request_views_to_delete = [ + ob for ob in objects_to_delete if isinstance(ob, RequestFeatureView) + ] + odfvs_to_delete = [ + ob for ob in objects_to_delete if isinstance(ob, OnDemandFeatureView) + ] + services_to_delete = [ + ob for ob in objects_to_delete if isinstance(ob, FeatureService) + ] + tables_to_delete = [ + ob for ob in objects_to_delete if isinstance(ob, FeatureTable) + ] + + for entity in entities_to_delete: + self._registry.delete_entity( + entity.name, project=self.project, commit=False + ) + for view in views_to_delete: + self._registry.delete_feature_view( + view.name, project=self.project, commit=False + ) + for request_view in request_views_to_delete: + self._registry.delete_feature_view( + request_view.name, project=self.project, commit=False + ) + for odfv in odfvs_to_delete: + self._registry.delete_feature_view( + odfv.name, project=self.project, commit=False + ) + for service in services_to_delete: + self._registry.delete_feature_service( + service.name, project=self.project, commit=False + ) + for table in tables_to_delete: + self._registry.delete_feature_table( + table.name, project=self.project, commit=False + ) self._get_provider().update_infra( project=self.project, - tables_to_delete=[], - tables_to_keep=views_to_update, - entities_to_delete=[], + tables_to_delete=views_to_delete + tables_to_delete if not partial else [], + tables_to_keep=views_to_update + tables_to_update, + entities_to_delete=entities_to_delete if not partial else [], entities_to_keep=entities_to_update, - partial=True, + partial=partial, ) if commit: @@ -555,38 +659,49 @@ def get_historical_features( ) _feature_refs = self._get_features(features, feature_refs) - all_feature_views, all_on_demand_feature_views = self._get_feature_views_to_use( - features - ) + ( + all_feature_views, + all_request_feature_views, + all_on_demand_feature_views, + ) = self._get_feature_views_to_use(features) # TODO(achal): _group_feature_refs returns the on demand feature views, but it's no passed into the provider. # This is a weird interface quirk - we should revisit the `get_historical_features` to # pass in the on demand feature views as well. - fvs, odfvs = _group_feature_refs( - _feature_refs, all_feature_views, all_on_demand_feature_views + fvs, odfvs, request_fvs, request_fv_refs = _group_feature_refs( + _feature_refs, + all_feature_views, + all_request_feature_views, + all_on_demand_feature_views, ) feature_views = list(view for view, _ in fvs) on_demand_feature_views = list(view for view, _ in odfvs) - if len(on_demand_feature_views) > 0: - log_event(UsageEvent.GET_HISTORICAL_FEATURES_WITH_ODFV) + request_feature_views = list(view for view, _ in request_fvs) + + set_usage_attribute("odfv", bool(on_demand_feature_views)) + set_usage_attribute("request_fv", bool(request_feature_views)) # Check that the right request data is present in the entity_df if type(entity_df) == pd.DataFrame: entity_pd_df = cast(pd.DataFrame, entity_df) + for fv in request_feature_views: + for feature in fv.features: + if feature.name not in entity_pd_df.columns: + raise RequestDataNotFoundInEntityDfException( + feature_name=feature.name, feature_view_name=fv.name + ) for odfv in on_demand_feature_views: - odfv_inputs = odfv.inputs.values() - for odfv_input in odfv_inputs: - if type(odfv_input) == RequestDataSource: - request_data_source = cast(RequestDataSource, odfv_input) - for feature_name in request_data_source.schema.keys(): - if feature_name not in entity_pd_df.columns: - raise RequestDataNotFoundInEntityDfException( - feature_name=feature_name, - feature_view_name=odfv.name, - ) + odfv_request_data_schema = odfv.get_request_data_schema() + for feature_name in odfv_request_data_schema.keys(): + if feature_name not in entity_pd_df.columns: + raise RequestDataNotFoundInEntityDfException( + feature_name=feature_name, feature_view_name=odfv.name, + ) _validate_feature_refs(_feature_refs, full_feature_names) - + # Drop refs that refer to RequestFeatureViews since they don't need to be fetched and + # already exist in the entity_df + _feature_refs = [ref for ref in _feature_refs if ref not in request_fv_refs] provider = self._get_provider() job = provider.get_historical_features( @@ -633,7 +748,7 @@ def materialize_incremental( ... """ - feature_views_to_materialize = [] + feature_views_to_materialize: List[FeatureView] = [] if feature_views is None: feature_views_to_materialize = self._list_feature_views( hide_dummy_entity=False @@ -725,7 +840,7 @@ def materialize( f"The given start_date {start_date} is greater than the given end_date {end_date}." ) - feature_views_to_materialize = [] + feature_views_to_materialize: List[FeatureView] = [] if feature_views is None: feature_views_to_materialize = self._list_feature_views( hide_dummy_entity=False @@ -766,6 +881,26 @@ def tqdm_builder(length): feature_view, self.project, start_date, end_date ) + @log_exceptions_and_usage + def write_to_online_store( + self, feature_view_name: str, df: pd.DataFrame, + ): + """ + ingests data directly into the Online store + """ + if not flags_helper.enable_direct_ingestion_to_online_store(self.config): + raise ExperimentalFeatureNotEnabled( + flags.FLAG_DIRECT_INGEST_TO_ONLINE_STORE + ) + + # TODO: restrict this to work with online StreamFeatureViews and validate the FeatureView type + feature_view = self._registry.get_feature_view(feature_view_name, self.project) + entities = [] + for entity_name in feature_view.entities: + entities.append(self._registry.get_entity(entity_name, self.project)) + provider = self._get_provider() + provider.ingest_df(feature_view, entities, df) + @log_exceptions_and_usage def get_online_features( self, @@ -816,16 +951,28 @@ def get_online_features( >>> online_response_dict = online_response.to_dict() """ _feature_refs = self._get_features(features, feature_refs) - all_feature_views, all_on_demand_feature_views = self._get_feature_views_to_use( + ( + requested_feature_views, + requested_request_feature_views, + requested_on_demand_feature_views, + ) = self._get_feature_views_to_use( features=features, allow_cache=True, hide_dummy_entity=False ) _validate_feature_refs(_feature_refs, full_feature_names) - grouped_refs, grouped_odfv_refs = _group_feature_refs( - _feature_refs, all_feature_views, all_on_demand_feature_views + ( + grouped_refs, + grouped_odfv_refs, + grouped_request_fv_refs, + _, + ) = _group_feature_refs( + _feature_refs, + requested_feature_views, + requested_request_feature_views, + requested_on_demand_feature_views, ) - if len(grouped_odfv_refs) > 0: - log_event(UsageEvent.GET_ONLINE_FEATURES_WITH_ODFV) + set_usage_attribute("odfv", bool(grouped_odfv_refs)) + set_usage_attribute("request_fv", bool(grouped_request_fv_refs)) feature_views = list(view for view, _ in grouped_refs) entityless_case = DUMMY_ENTITY_NAME in [ @@ -836,12 +983,26 @@ def get_online_features( provider = self._get_provider() entities = self._list_entities(allow_cache=True, hide_dummy_entity=False) - entity_name_to_join_key_map = {} + entity_name_to_join_key_map: Dict[str, str] = {} for entity in entities: entity_name_to_join_key_map[entity.name] = entity.join_key + for feature_view in requested_feature_views: + for entity_name in feature_view.entities: + entity = self._registry.get_entity( + entity_name, self.project, allow_cache=True + ) + # User directly uses join_key as the entity reference in the entity_rows for the + # entity mapping case. + entity_name = feature_view.projection.join_key_map.get( + entity.join_key, entity.name + ) + join_key = feature_view.projection.join_key_map.get( + entity.join_key, entity.join_key + ) + entity_name_to_join_key_map[entity_name] = join_key - needed_request_data_features = self._get_needed_request_data_features( - grouped_odfv_refs + needed_request_data, needed_request_fv_features = self.get_needed_request_data( + grouped_odfv_refs, grouped_request_fv_refs ) join_key_rows = [] @@ -851,7 +1012,10 @@ def get_online_features( join_key_row = {} for entity_name, entity_value in row.items(): # Found request data - if entity_name in needed_request_data_features: + if ( + entity_name in needed_request_data + or entity_name in needed_request_fv_features + ): if entity_name not in request_data_features: request_data_features[entity_name] = [] request_data_features[entity_name].append(entity_value) @@ -867,10 +1031,9 @@ def get_online_features( # May be empty if this entity row was request data join_key_rows.append(join_key_row) - if len(needed_request_data_features) != len(request_data_features.keys()): - raise RequestDataNotFoundInEntityRowsException( - feature_names=needed_request_data_features - ) + self.ensure_request_data_values_exist( + needed_request_data, needed_request_fv_features, request_data_features + ) entity_row_proto_list = _infer_online_entity_rows(join_key_rows) @@ -883,20 +1046,13 @@ def get_online_features( # Also create entity values to append to the result result_rows.append(_entity_row_to_field_values(entity_row_proto)) - # Add more feature values to the existing result rows for the request data features - for feature_name, feature_values in request_data_features.items(): - for row_idx, feature_value in enumerate(feature_values): - result_row = result_rows[row_idx] - result_row.fields[feature_name].CopyFrom( - python_value_to_proto_value(feature_value) - ) - result_row.statuses[ - feature_name - ] = GetOnlineFeaturesResponse.FieldStatus.PRESENT - for table, requested_features in grouped_refs: + table_join_keys = [ + entity_name_to_join_key_map[entity_name] + for entity_name in table.entities + ] self._populate_result_rows_from_feature_view( - entity_name_to_join_key_map, + table_join_keys, full_feature_names, provider, requested_features, @@ -905,21 +1061,124 @@ def get_online_features( union_of_entity_keys, ) + requested_result_row_names = self._get_requested_result_fields( + result_rows, needed_request_fv_features + ) + self._populate_odfv_dependencies( + entity_name_to_join_key_map, + full_feature_names, + grouped_odfv_refs, + provider, + request_data_features, + result_rows, + union_of_entity_keys, + ) + initial_response = OnlineResponse( GetOnlineFeaturesResponse(field_values=result_rows) ) return self._augment_response_with_on_demand_transforms( _feature_refs, - all_on_demand_feature_views, + requested_result_row_names, + requested_on_demand_feature_views, full_feature_names, initial_response, result_rows, ) - def _populate_result_rows_from_feature_view( + def _get_requested_result_fields( + self, + result_rows: List[GetOnlineFeaturesResponse.FieldValues], + needed_request_fv_features: Set[str], + ): + # Get requested feature values so we can drop odfv dependencies that aren't requested + requested_result_row_names: Set[str] = set() + for result_row in result_rows: + for feature_name in result_row.fields.keys(): + requested_result_row_names.add(feature_name) + # Request feature view values are also request data features that should be in the + # final output + requested_result_row_names.update(needed_request_fv_features) + return requested_result_row_names + + def _populate_odfv_dependencies( self, entity_name_to_join_key_map: Dict[str, str], full_feature_names: bool, + grouped_odfv_refs: List[Tuple[OnDemandFeatureView, List[str]]], + provider: Provider, + request_data_features: Dict[str, List[Any]], + result_rows: List[GetOnlineFeaturesResponse.FieldValues], + union_of_entity_keys: List[EntityKeyProto], + ): + # Add more feature values to the existing result rows for the request data features + for feature_name, feature_values in request_data_features.items(): + for row_idx, feature_value in enumerate(feature_values): + result_row = result_rows[row_idx] + result_row.fields[feature_name].CopyFrom( + python_value_to_proto_value(feature_value) + ) + result_row.statuses[ + feature_name + ] = GetOnlineFeaturesResponse.FieldStatus.PRESENT + + # Add data if odfv requests specific feature views as dependencies + if len(grouped_odfv_refs) > 0: + for odfv, _ in grouped_odfv_refs: + for fv in odfv.input_feature_views.values(): + table_join_keys = [ + entity_name_to_join_key_map[entity_name] + for entity_name in fv.entities + ] + self._populate_result_rows_from_feature_view( + table_join_keys, + full_feature_names, + provider, + [feature.name for feature in fv.features], + result_rows, + fv, + union_of_entity_keys, + ) + + def get_needed_request_data( + self, + grouped_odfv_refs: List[Tuple[OnDemandFeatureView, List[str]]], + grouped_request_fv_refs: List[Tuple[RequestFeatureView, List[str]]], + ) -> Tuple[Set[str], Set[str]]: + needed_request_data: Set[str] = set() + needed_request_fv_features: Set[str] = set() + for odfv, _ in grouped_odfv_refs: + odfv_request_data_schema = odfv.get_request_data_schema() + needed_request_data.update(odfv_request_data_schema.keys()) + for request_fv, _ in grouped_request_fv_refs: + for feature in request_fv.features: + needed_request_fv_features.add(feature.name) + return needed_request_data, needed_request_fv_features + + def ensure_request_data_values_exist( + self, + needed_request_data: Set[str], + needed_request_fv_features: Set[str], + request_data_features: Dict[str, List[Any]], + ): + if len(needed_request_data) + len(needed_request_fv_features) != len( + request_data_features.keys() + ): + missing_features = [ + x + for x in itertools.chain( + needed_request_data, needed_request_fv_features + ) + if x not in request_data_features + ] + raise RequestDataNotFoundInEntityRowsException( + feature_names=missing_features + ) + + def _populate_result_rows_from_feature_view( + self, + table_join_keys: List[str], + full_feature_names: bool, provider: Provider, requested_features: List[str], result_rows: List[GetOnlineFeaturesResponse.FieldValues], @@ -927,7 +1186,7 @@ def _populate_result_rows_from_feature_view( union_of_entity_keys: List[EntityKeyProto], ): entity_keys = _get_table_entity_keys( - table, union_of_entity_keys, entity_name_to_join_key_map + table, union_of_entity_keys, table_join_keys ) read_rows = provider.online_read( config=self.config, @@ -965,21 +1224,10 @@ def _populate_result_rows_from_feature_view( feature_ref ] = GetOnlineFeaturesResponse.FieldStatus.PRESENT - def _get_needed_request_data_features(self, grouped_odfv_refs) -> Set[str]: - needed_request_data_features = set() - for odfv_to_feature_names in grouped_odfv_refs: - odfv, requested_feature_names = odfv_to_feature_names - odfv_inputs = odfv.inputs.values() - for odfv_input in odfv_inputs: - if type(odfv_input) == RequestDataSource: - request_data_source = cast(RequestDataSource, odfv_input) - for feature_name in request_data_source.schema.keys(): - needed_request_data_features.add(feature_name) - return needed_request_data_features - def _augment_response_with_on_demand_transforms( self, feature_refs: List[str], + requested_result_row_names: Set[str], odfvs: List[OnDemandFeatureView], full_feature_names: bool, initial_response: OnlineResponse, @@ -999,10 +1247,11 @@ def _augment_response_with_on_demand_transforms( odfv_feature_refs[view_name].append(feature_name) # Apply on demand transformations + odfv_result_names = set() for odfv_name, _feature_refs in odfv_feature_refs.items(): odfv = all_on_demand_feature_views[odfv_name] transformed_features_df = odfv.get_transformed_features_df( - full_feature_names, initial_response_df + initial_response_df ) for row_idx in range(len(result_rows)): result_row = result_rows[row_idx] @@ -1017,6 +1266,7 @@ def _augment_response_with_on_demand_transforms( if full_feature_names else transformed_feature ) + odfv_result_names.add(transformed_feature_name) proto_value = python_value_to_proto_value( transformed_features_df[transformed_feature].values[row_idx] ) @@ -1024,6 +1274,19 @@ def _augment_response_with_on_demand_transforms( result_row.statuses[ transformed_feature_name ] = GetOnlineFeaturesResponse.FieldStatus.PRESENT + + # Drop values that aren't needed + unneeded_features = [ + val + for val in result_rows[0].fields + if val not in requested_result_row_names and val not in odfv_result_names + ] + for row_idx in range(len(result_rows)): + result_row = result_rows[row_idx] + for unneeded_feature in unneeded_features: + result_row.fields.pop(unneeded_feature) + result_row.statuses.pop(unneeded_feature) + return OnlineResponse(GetOnlineFeaturesResponse(field_values=result_rows)) def _get_feature_views_to_use( @@ -1031,13 +1294,20 @@ def _get_feature_views_to_use( features: Optional[Union[List[str], FeatureService]], allow_cache=False, hide_dummy_entity: bool = True, - ) -> Tuple[List[FeatureView], List[OnDemandFeatureView]]: + ) -> Tuple[List[FeatureView], List[RequestFeatureView], List[OnDemandFeatureView]]: fvs = { fv.name: fv for fv in self._list_feature_views(allow_cache, hide_dummy_entity) } + request_fvs = { + fv.name: fv + for fv in self._registry.list_request_feature_views( + project=self.project, allow_cache=allow_cache + ) + } + od_fvs = { fv.name: fv for fv in self._registry.list_on_demand_feature_views( @@ -1045,8 +1315,8 @@ def _get_feature_views_to_use( ) } - fvs_to_use, od_fvs_to_use = [], [] if isinstance(features, FeatureService): + fvs_to_use, request_fvs_to_use, od_fvs_to_use = [], [], [] for fv_name, projection in [ (projection.name, projection) for projection in features.feature_view_projections @@ -1055,6 +1325,10 @@ def _get_feature_views_to_use( fvs_to_use.append( fvs[fv_name].with_projection(copy.copy(projection)) ) + elif fv_name in request_fvs: + request_fvs_to_use.append( + request_fvs[fv_name].with_projection(copy.copy(projection)) + ) elif fv_name in od_fvs: od_fvs_to_use.append( od_fvs[fv_name].with_projection(copy.copy(projection)) @@ -1065,9 +1339,13 @@ def _get_feature_views_to_use( f"{fv_name} which doesn't exist. Please make sure that you have created the feature view" f'{fv_name} and that you have registered it by running "apply".' ) - views_to_use = (fvs_to_use, od_fvs_to_use) + views_to_use = (fvs_to_use, request_fvs_to_use, od_fvs_to_use) else: - views_to_use = ([*fvs.values()], [*od_fvs.values()]) + views_to_use = ( + [*fvs.values()], + [*request_fvs.values()], + [*od_fvs.values()], + ) return views_to_use @@ -1084,6 +1362,18 @@ def get_feature_server_endpoint(self) -> Optional[str]: """Returns endpoint for the feature server, if it exists.""" return self._provider.get_feature_server_endpoint() + @log_exceptions_and_usage + def serve_transformations(self, port: int) -> None: + """Start the feature transformation server locally on a given port.""" + if not flags_helper.enable_python_feature_server(self.config): + raise ExperimentalFeatureNotEnabled(flags.FLAG_PYTHON_FEATURE_SERVER_NAME) + if not flags_helper.enable_on_demand_feature_views(self.config): + raise ExperimentalFeatureNotEnabled(flags.FLAG_ON_DEMAND_TRANSFORM_NAME) + + from feast import transformation_server + + transformation_server.start_server(self, port) + def _entity_row_to_key(row: GetOnlineFeaturesRequestV2.EntityRow) -> EntityKeyProto: names, values = zip(*row.fields.items()) @@ -1130,20 +1420,33 @@ def _validate_feature_refs(feature_refs: List[str], full_feature_names: bool = F def _group_feature_refs( features: List[str], all_feature_views: List[FeatureView], + all_request_feature_views: List[RequestFeatureView], all_on_demand_feature_views: List[OnDemandFeatureView], ) -> Tuple[ - List[Tuple[FeatureView, List[str]]], List[Tuple[OnDemandFeatureView, List[str]]] + List[Tuple[FeatureView, List[str]]], + List[Tuple[OnDemandFeatureView, List[str]]], + List[Tuple[RequestFeatureView, List[str]]], + Set[str], ]: """ Get list of feature views and corresponding feature names based on feature references""" # view name to view proto - view_index = {view.name: view for view in all_feature_views} + view_index = {view.projection.name_to_use(): view for view in all_feature_views} + + # request view name to proto + request_view_index = { + view.projection.name_to_use(): view for view in all_request_feature_views + } # on demand view to on demand view proto - on_demand_view_index = {view.name: view for view in all_on_demand_feature_views} + on_demand_view_index = { + view.projection.name_to_use(): view for view in all_on_demand_feature_views + } # view name to feature names views_features = defaultdict(list) + request_views_features = defaultdict(list) + request_view_refs = set() # on demand view name to feature names on_demand_view_features = defaultdict(list) @@ -1154,29 +1457,39 @@ def _group_feature_refs( views_features[view_name].append(feat_name) elif view_name in on_demand_view_index: on_demand_view_features[view_name].append(feat_name) + elif view_name in request_view_index: + request_views_features[view_name].append(feat_name) + request_view_refs.add(ref) else: raise FeatureViewNotFoundException(view_name) fvs_result: List[Tuple[FeatureView, List[str]]] = [] odfvs_result: List[Tuple[OnDemandFeatureView, List[str]]] = [] + request_fvs_result: List[Tuple[RequestFeatureView, List[str]]] = [] for view_name, feature_names in views_features.items(): fvs_result.append((view_index[view_name], feature_names)) + for view_name, feature_names in request_views_features.items(): + request_fvs_result.append((request_view_index[view_name], feature_names)) for view_name, feature_names in on_demand_view_features.items(): odfvs_result.append((on_demand_view_index[view_name], feature_names)) - return fvs_result, odfvs_result + return fvs_result, odfvs_result, request_fvs_result, request_view_refs def _get_table_entity_keys( - table: FeatureView, entity_keys: List[EntityKeyProto], join_key_map: Dict[str, str], + table: FeatureView, entity_keys: List[EntityKeyProto], table_join_keys: List[str] ) -> List[EntityKeyProto]: - table_join_keys = [join_key_map[entity_name] for entity_name in table.entities] + reverse_join_key_map = { + alias: original for original, alias in table.projection.join_key_map.items() + } required_entities = OrderedDict.fromkeys(sorted(table_join_keys)) entity_key_protos = [] for entity_key in entity_keys: required_entities_to_values = required_entities.copy() for i in range(len(entity_key.join_keys)): - entity_name = entity_key.join_keys[i] + entity_name = reverse_join_key_map.get( + entity_key.join_keys[i], entity_key.join_keys[i] + ) entity_value = entity_key.entity_values[i] if entity_name in required_entities_to_values: @@ -1221,7 +1534,7 @@ def _print_materialization_log( ) -def _validate_feature_views(feature_views: List[FeatureView]): +def _validate_feature_views(feature_views: List[BaseFeatureView]): """ Verify feature views have case-insensitively unique names""" fv_names = set() for fv in feature_views: diff --git a/sdk/python/feast/feature_table.py b/sdk/python/feast/feature_table.py index fb33223c35a..2c1022de224 100644 --- a/sdk/python/feast/feature_table.py +++ b/sdk/python/feast/feature_table.py @@ -110,7 +110,7 @@ def name(self, name: str): self._name = name @property - def entities(self): + def entities(self) -> List[str]: """ Returns the entities of this feature table """ diff --git a/sdk/python/feast/feature_view.py b/sdk/python/feast/feature_view.py index 3a5e78d83f4..a8186991cd5 100644 --- a/sdk/python/feast/feature_view.py +++ b/sdk/python/feast/feature_view.py @@ -15,12 +15,12 @@ import re import warnings from datetime import datetime, timedelta -from typing import Dict, List, Optional, Tuple, Union +from typing import Dict, List, Optional, Tuple, Type, Union from google.protobuf.duration_pb2 import Duration -from google.protobuf.json_format import MessageToJson from feast import utils +from feast.base_feature_view import BaseFeatureView from feast.data_source import DataSource from feast.errors import RegistryInferenceFailure from feast.feature import Feature @@ -47,7 +47,7 @@ DUMMY_ENTITY_VAL = "" -class FeatureView: +class FeatureView(BaseFeatureView): """ A FeatureView defines a logical grouping of serveable features. @@ -67,9 +67,7 @@ class FeatureView: FeatureViews. """ - name: str entities: List[str] - features: List[Feature] tags: Optional[Dict[str, str]] ttl: timedelta online: bool @@ -79,7 +77,6 @@ class FeatureView: created_timestamp: Optional[datetime] = None last_updated_timestamp: Optional[datetime] = None materialization_intervals: List[Tuple[datetime, datetime]] - projection: FeatureViewProjection @log_exceptions def __init__( @@ -123,9 +120,8 @@ def __init__( f"Entity or Feature name." ) - self.name = name + super().__init__(name, _features) self.entities = entities if entities else [DUMMY_ENTITY_NAME] - self.features = _features self.tags = tags if tags is not None else {} if isinstance(ttl, Duration): @@ -143,17 +139,9 @@ def __init__( self.created_timestamp: Optional[datetime] = None self.last_updated_timestamp: Optional[datetime] = None - self.projection = FeatureViewProjection.from_definition(self) - - def __repr__(self): - items = (f"{k} = {v}" for k, v in self.__dict__.items()) - return f"<{self.__class__.__name__}({', '.join(items)})>" - - def __str__(self): - return str(MessageToJson(self.to_proto())) - + # Note: Python requires redefining hash in child classes that override __eq__ def __hash__(self): - return hash((id(self), self.name)) + return super().__hash__() def __copy__(self): fv = FeatureView( @@ -170,28 +158,17 @@ def __copy__(self): fv.projection = copy.copy(self.projection) return fv - def __getitem__(self, item): - assert isinstance(item, list) - - referenced_features = [] - for feature in self.features: - if feature.name in item: - referenced_features.append(feature) - - cp = self.__copy__() - cp.projection.features = referenced_features - - return self - def __eq__(self, other): if not isinstance(other, FeatureView): raise TypeError( "Comparisons should only involve FeatureView class objects." ) + if not super().__eq__(other): + return False + if ( self.tags != other.tags - or self.name != other.name or self.ttl != other.ttl or self.online != other.online ): @@ -199,8 +176,6 @@ def __eq__(self, other): if sorted(self.entities) != sorted(other.entities): return False - if sorted(self.features) != sorted(other.features): - return False if self.batch_source != other.batch_source: return False if self.stream_source != other.stream_source: @@ -208,19 +183,22 @@ def __eq__(self, other): return True - def is_valid(self): + def ensure_valid(self): """ Validates the state of this feature view locally. Raises: ValueError: The feature view does not have a name or does not have entities. """ - if not self.name: - raise ValueError("Feature view needs a name.") + super().ensure_valid() if not self.entities: raise ValueError("Feature view has no entities.") + @property + def proto_class(self) -> Type[FeatureViewProto]: + return FeatureViewProto + def with_name(self, name: str): """ Renames this feature view by returning a copy of this feature view with an alias @@ -238,6 +216,44 @@ def with_name(self, name: str): return cp + def with_join_key_map(self, join_key_map: Dict[str, str]): + """ + Sets the join_key_map by returning a copy of this feature view with that field set. + This join_key mapping operation is only used as part of query operations and will + not modify the underlying FeatureView. + + Args: + join_key_map: A map of join keys in which the left is the join_key that + corresponds with the feature data and the right corresponds with the entity data. + + Returns: + A copy of this FeatureView with the join_key_map replaced with the 'join_key_map' input. + + Examples: + Join a location feature data table to both the origin column and destination + column of the entity data. + + temperatures_feature_service = FeatureService( + name="temperatures", + features=[ + location_stats_feature_view + .with_name("origin_stats") + .with_join_key_map( + {"location_id": "origin_id"} + ), + location_stats_feature_view + .with_name("destination_stats") + .with_join_key_map( + {"location_id": "destination_id"} + ), + ], + ) + """ + cp = self.__copy__() + cp.projection.join_key_map = join_key_map + + return cp + def with_projection(self, feature_view_projection: FeatureViewProjection): """ Sets the feature view projection by returning a copy of this feature view diff --git a/sdk/python/feast/feature_view_projection.py b/sdk/python/feast/feature_view_projection.py index e271937e0ed..97b3b0ab577 100644 --- a/sdk/python/feast/feature_view_projection.py +++ b/sdk/python/feast/feature_view_projection.py @@ -1,4 +1,4 @@ -from typing import List, Optional +from typing import Dict, List, Optional from attr import dataclass @@ -13,13 +13,16 @@ class FeatureViewProjection: name: str name_alias: Optional[str] features: List[Feature] + join_key_map: Dict[str, str] = {} def name_to_use(self): return self.name_alias or self.name def to_proto(self): feature_reference_proto = FeatureViewProjectionProto( - feature_view_name=self.name, feature_view_name_alias=self.name_alias + feature_view_name=self.name, + feature_view_name_alias=self.name_alias, + join_key_map=self.join_key_map, ) for feature in self.features: feature_reference_proto.feature_columns.append(feature.to_proto()) @@ -32,6 +35,7 @@ def from_proto(proto: FeatureViewProjectionProto): name=proto.feature_view_name, name_alias=proto.feature_view_name_alias, features=[], + join_key_map=dict(proto.join_key_map), ) for feature_column in proto.feature_columns: ref.features.append(Feature.from_proto(feature_column)) diff --git a/sdk/python/feast/flags.py b/sdk/python/feast/flags.py index c9e74168635..5c6357ec26f 100644 --- a/sdk/python/feast/flags.py +++ b/sdk/python/feast/flags.py @@ -2,6 +2,7 @@ FLAG_ON_DEMAND_TRANSFORM_NAME = "on_demand_transforms" FLAG_PYTHON_FEATURE_SERVER_NAME = "python_feature_server" FLAG_AWS_LAMBDA_FEATURE_SERVER_NAME = "aws_lambda_feature_server" +FLAG_DIRECT_INGEST_TO_ONLINE_STORE = "direct_ingest_to_online_store" ENV_FLAG_IS_TEST = "IS_TEST" FLAG_NAMES = { @@ -9,4 +10,5 @@ FLAG_ON_DEMAND_TRANSFORM_NAME, FLAG_PYTHON_FEATURE_SERVER_NAME, FLAG_AWS_LAMBDA_FEATURE_SERVER_NAME, + FLAG_DIRECT_INGEST_TO_ONLINE_STORE, } diff --git a/sdk/python/feast/flags_helper.py b/sdk/python/feast/flags_helper.py index 85770c39ec1..89784d6ecca 100644 --- a/sdk/python/feast/flags_helper.py +++ b/sdk/python/feast/flags_helper.py @@ -41,3 +41,7 @@ def enable_python_feature_server(repo_config: RepoConfig) -> bool: def enable_aws_lambda_feature_server(repo_config: RepoConfig) -> bool: return feature_flag_enabled(repo_config, flags.FLAG_AWS_LAMBDA_FEATURE_SERVER_NAME) + + +def enable_direct_ingestion_to_online_store(repo_config: RepoConfig) -> bool: + return feature_flag_enabled(repo_config, flags.FLAG_DIRECT_INGEST_TO_ONLINE_STORE) diff --git a/sdk/python/feast/infra/aws.py b/sdk/python/feast/infra/aws.py index c24c84beb6d..f0348d29014 100644 --- a/sdk/python/feast/infra/aws.py +++ b/sdk/python/feast/infra/aws.py @@ -12,6 +12,7 @@ from feast.constants import ( AWS_LAMBDA_FEATURE_SERVER_IMAGE, + AWS_LAMBDA_FEATURE_SERVER_REPOSITORY, FEAST_USAGE, FEATURE_STORE_YAML_ENV_NAME, ) @@ -29,12 +30,14 @@ from feast.feature_view import FeatureView from feast.flags import FLAG_AWS_LAMBDA_FEATURE_SERVER_NAME from feast.flags_helper import enable_aws_lambda_feature_server +from feast.infra.feature_servers.aws_lambda.config import AwsLambdaFeatureServerConfig from feast.infra.passthrough_provider import PassthroughProvider from feast.infra.utils import aws_utils from feast.protos.feast.core.Registry_pb2 import Registry as RegistryProto from feast.registry import get_registry_store_class_from_scheme from feast.registry_store import RegistryStore from feast.repo_config import RegistryConfig +from feast.usage import log_exceptions_and_usage from feast.version import get_version try: @@ -48,6 +51,7 @@ class AwsProvider(PassthroughProvider): + @log_exceptions_and_usage(provider="AwsProvider") def update_infra( self, project: str, @@ -83,90 +87,110 @@ def update_infra( registry_store_class.__name__, S3RegistryStore.__name__ ) - image_uri = self._upload_docker_image(project) - _logger.info("Deploying feature server...") + ecr_client = boto3.client("ecr") + repository_uri = self._create_or_get_repository_uri(ecr_client) + version = _get_version_for_aws() + # Only download & upload the docker image if it doesn't already exist in ECR + if not ecr_client.batch_get_image( + repositoryName=AWS_LAMBDA_FEATURE_SERVER_REPOSITORY, + imageIds=[{"imageTag": version}], + ).get("images"): + image_uri = self._upload_docker_image( + ecr_client, repository_uri, version + ) + else: + image_uri = f"{repository_uri}:{version}" - if not self.repo_config.repo_path: - raise RepoConfigPathDoesNotExist() - with open(self.repo_config.repo_path / "feature_store.yaml", "rb") as f: - config_bytes = f.read() - config_base64 = base64.b64encode(config_bytes).decode() + self._deploy_feature_server(project, image_uri) - resource_name = self._get_lambda_name(project) - lambda_client = boto3.client("lambda") - api_gateway_client = boto3.client("apigatewayv2") - function = aws_utils.get_lambda_function(lambda_client, resource_name) + def _deploy_feature_server(self, project: str, image_uri: str): + _logger.info("Deploying feature server...") + + if not self.repo_config.repo_path: + raise RepoConfigPathDoesNotExist() + with open(self.repo_config.repo_path / "feature_store.yaml", "rb") as f: + config_bytes = f.read() + config_base64 = base64.b64encode(config_bytes).decode() - if function is None: - # If the Lambda function does not exist, create it. - _logger.info(" Creating AWS Lambda...") - lambda_client.create_function( + resource_name = _get_lambda_name(project) + lambda_client = boto3.client("lambda") + api_gateway_client = boto3.client("apigatewayv2") + function = aws_utils.get_lambda_function(lambda_client, resource_name) + + if function is None: + # If the Lambda function does not exist, create it. + _logger.info(" Creating AWS Lambda...") + assert isinstance( + self.repo_config.feature_server, AwsLambdaFeatureServerConfig + ) + lambda_client.create_function( + FunctionName=resource_name, + Role=self.repo_config.feature_server.execution_role_name, + Code={"ImageUri": image_uri}, + PackageType="Image", + MemorySize=1769, + Environment={ + "Variables": { + FEATURE_STORE_YAML_ENV_NAME: config_base64, + FEAST_USAGE: "False", + } + }, + Tags={ + "feast-owned": "True", + "project": project, + "feast-sdk-version": get_version(), + }, + ) + function = aws_utils.get_lambda_function(lambda_client, resource_name) + if not function: + raise AwsLambdaDoesNotExist(resource_name) + else: + # If the feature_store.yaml has changed, need to update the environment variable. + env = function.get("Environment", {}).get("Variables", {}) + if env.get(FEATURE_STORE_YAML_ENV_NAME) != config_base64: + # Note, that this does not update Lambda gracefully (e.g. no rolling deployment). + # It's expected that feature_store.yaml is not regularly updated while the lambda + # is serving production traffic. However, the update in registry (e.g. modifying + # feature views, feature services, and other definitions does not update lambda). + _logger.info(" Updating AWS Lambda...") + + lambda_client.update_function_configuration( FunctionName=resource_name, - Role=self.repo_config.feature_server.execution_role_name, - Code={"ImageUri": image_uri}, - PackageType="Image", - MemorySize=1769, Environment={ - "Variables": { - FEATURE_STORE_YAML_ENV_NAME: config_base64, - FEAST_USAGE: "False", - } - }, - Tags={ - "feast-owned": "True", - "project": project, - "feast-sdk-version": get_version(), + "Variables": {FEATURE_STORE_YAML_ENV_NAME: config_base64} }, ) - function = aws_utils.get_lambda_function(lambda_client, resource_name) - if not function: - raise AwsLambdaDoesNotExist(resource_name) - else: - # If the feature_store.yaml has changed, need to update the environment variable. - env = function.get("Environment", {}).get("Variables", {}) - if env.get(FEATURE_STORE_YAML_ENV_NAME) != config_base64: - # Note, that this does not update Lambda gracefully (e.g. no rolling deployment). - # It's expected that feature_store.yaml is not regularly updated while the lambda - # is serving production traffic. However, the update in registry (e.g. modifying - # feature views, feature services, and other definitions does not update lambda). - _logger.info(" Updating AWS Lambda...") - - lambda_client.update_function_configuration( - FunctionName=resource_name, - Environment={ - "Variables": {FEATURE_STORE_YAML_ENV_NAME: config_base64} - }, - ) - api = aws_utils.get_first_api_gateway(api_gateway_client, resource_name) + api = aws_utils.get_first_api_gateway(api_gateway_client, resource_name) + if not api: + # If the API Gateway doesn't exist, create it + _logger.info(" Creating AWS API Gateway...") + api = api_gateway_client.create_api( + Name=resource_name, + ProtocolType="HTTP", + Target=function["FunctionArn"], + RouteKey="POST /get-online-features", + Tags={ + "feast-owned": "True", + "project": project, + "feast-sdk-version": get_version(), + }, + ) if not api: - # If the API Gateway doesn't exist, create it - _logger.info(" Creating AWS API Gateway...") - api = api_gateway_client.create_api( - Name=resource_name, - ProtocolType="HTTP", - Target=function["FunctionArn"], - RouteKey="POST /get-online-features", - Tags={ - "feast-owned": "True", - "project": project, - "feast-sdk-version": get_version(), - }, - ) - if not api: - raise AwsAPIGatewayDoesNotExist(resource_name) - # Make sure to give AWS Lambda a permission to be invoked by the newly created API Gateway - api_id = api["ApiId"] - region = lambda_client.meta.region_name - account_id = aws_utils.get_account_id() - lambda_client.add_permission( - FunctionName=function["FunctionArn"], - StatementId=str(uuid.uuid4()), - Action="lambda:InvokeFunction", - Principal="apigateway.amazonaws.com", - SourceArn=f"arn:aws:execute-api:{region}:{account_id}:{api_id}/*/*/get-online-features", - ) + raise AwsAPIGatewayDoesNotExist(resource_name) + # Make sure to give AWS Lambda a permission to be invoked by the newly created API Gateway + api_id = api["ApiId"] + region = lambda_client.meta.region_name + account_id = aws_utils.get_account_id() + lambda_client.add_permission( + FunctionName=function["FunctionArn"], + StatementId=str(uuid.uuid4()), + Action="lambda:InvokeFunction", + Principal="apigateway.amazonaws.com", + SourceArn=f"arn:aws:execute-api:{region}:{account_id}:{api_id}/*/*/get-online-features", + ) + @log_exceptions_and_usage(provider="AwsProvider") def teardown_infra( self, project: str, @@ -180,7 +204,7 @@ def teardown_infra( and self.repo_config.feature_server.enabled ): _logger.info("Tearing down feature server...") - resource_name = self._get_lambda_name(project) + resource_name = _get_lambda_name(project) lambda_client = boto3.client("lambda") api_gateway_client = boto3.client("apigatewayv2") @@ -195,9 +219,10 @@ def teardown_infra( _logger.info(" Tearing down AWS API Gateway...") aws_utils.delete_api_gateway(api_gateway_client, api["ApiId"]) + @log_exceptions_and_usage(provider="AwsProvider") def get_feature_server_endpoint(self) -> Optional[str]: project = self.repo_config.project - resource_name = self._get_lambda_name(project) + resource_name = _get_lambda_name(project) api_gateway_client = boto3.client("apigatewayv2") api = aws_utils.get_first_api_gateway(api_gateway_client, resource_name) @@ -209,25 +234,15 @@ def get_feature_server_endpoint(self) -> Optional[str]: region = lambda_client.meta.region_name return f"https://{api_id}.execute-api.{region}.amazonaws.com" - def _upload_docker_image(self, project: str) -> str: + def _upload_docker_image( + self, ecr_client, repository_uri: str, version: str + ) -> str: """ Pulls the AWS Lambda docker image from Dockerhub and uploads it to AWS ECR. - Args: - project: Feast project name - Returns: The URI of the uploaded docker image. """ - import base64 - - try: - import boto3 - except ImportError as e: - from feast.errors import FeastExtrasDependencyImportError - - raise FeastExtrasDependencyImportError("aws", str(e)) - try: import docker from docker.errors import APIError @@ -244,24 +259,12 @@ def _upload_docker_image(self, project: str) -> str: raise DockerDaemonNotRunning() _logger.info( - f"Pulling remote image {Style.BRIGHT + Fore.GREEN}{AWS_LAMBDA_FEATURE_SERVER_IMAGE}{Style.RESET_ALL}:" + f"Pulling remote image {Style.BRIGHT + Fore.GREEN}{AWS_LAMBDA_FEATURE_SERVER_IMAGE}{Style.RESET_ALL}" ) - docker_client.images.pull(AWS_LAMBDA_FEATURE_SERVER_IMAGE) - - version = self._get_version_for_aws() - repository_name = f"feast-python-server-{project}-{version}" - ecr_client = boto3.client("ecr") - try: - _logger.info( - f"Creating remote ECR repository {Style.BRIGHT + Fore.GREEN}{repository_name}{Style.RESET_ALL}:" - ) - response = ecr_client.create_repository(repositoryName=repository_name) - repository_uri = response["repository"]["repositoryUri"] - except ecr_client.exceptions.RepositoryAlreadyExistsException: - response = ecr_client.describe_repositories( - repositoryNames=[repository_name] - ) - repository_uri = response["repositories"][0]["repositoryUri"] + for line in docker_client.api.pull( + AWS_LAMBDA_FEATURE_SERVER_IMAGE, stream=True, decode=True + ): + _logger.debug(f" {line}") auth_token = ecr_client.get_authorization_token()["authorizationData"][0][ "authorizationToken" @@ -269,27 +272,59 @@ def _upload_docker_image(self, project: str) -> str: username, password = base64.b64decode(auth_token).decode("utf-8").split(":") ecr_address = repository_uri.split("/")[0] - docker_client.login(username=username, password=password, registry=ecr_address) + _logger.info( + f"Logging in Docker client to {Style.BRIGHT + Fore.GREEN}{ecr_address}{Style.RESET_ALL}" + ) + login_status = docker_client.login( + username=username, password=password, registry=ecr_address + ) + _logger.debug(f" {login_status}") image = docker_client.images.get(AWS_LAMBDA_FEATURE_SERVER_IMAGE) image_remote_name = f"{repository_uri}:{version}" _logger.info( - f"Pushing local image to remote {Style.BRIGHT + Fore.GREEN}{image_remote_name}{Style.RESET_ALL}:" + f"Pushing local image to remote {Style.BRIGHT + Fore.GREEN}{image_remote_name}{Style.RESET_ALL}" ) image.tag(image_remote_name) - docker_client.api.push(repository_uri, tag=version) + for line in docker_client.api.push( + repository_uri, tag=version, stream=True, decode=True + ): + _logger.debug(f" {line}") + return image_remote_name - def _get_lambda_name(self, project: str): - return f"feast-python-server-{project}-{self._get_version_for_aws()}" + def _create_or_get_repository_uri(self, ecr_client): + try: + return ecr_client.describe_repositories( + repositoryNames=[AWS_LAMBDA_FEATURE_SERVER_REPOSITORY] + )["repositories"][0]["repositoryUri"] + except ecr_client.exceptions.RepositoryNotFoundException: + _logger.info( + f"Creating remote ECR repository {Style.BRIGHT + Fore.GREEN}{AWS_LAMBDA_FEATURE_SERVER_REPOSITORY}{Style.RESET_ALL}" + ) + response = ecr_client.create_repository( + repositoryName=AWS_LAMBDA_FEATURE_SERVER_REPOSITORY + ) + return response["repository"]["repositoryUri"] + - @staticmethod - def _get_version_for_aws(): - """Returns Feast version with certain characters replaced. +def _get_lambda_name(project: str): + lambda_prefix = AWS_LAMBDA_FEATURE_SERVER_REPOSITORY + lambda_suffix = f"{project}-{_get_version_for_aws()}" + # AWS Lambda name can't have the length greater than 64 bytes. + # This usually occurs during integration tests or when feast is + # installed in editable mode (pip install -e), where feast version is long + if len(lambda_prefix) + len(lambda_suffix) >= 63: + lambda_suffix = base64.b64encode(lambda_suffix.encode()).decode()[:40] + return f"{lambda_prefix}-{lambda_suffix}" - This allows the version to be included in names for AWS resources. - """ - return get_version().replace(".", "_").replace("+", "_") + +def _get_version_for_aws(): + """Returns Feast version with certain characters replaced. + + This allows the version to be included in names for AWS resources. + """ + return get_version().replace(".", "_").replace("+", "_") class S3RegistryStore(RegistryStore): @@ -303,6 +338,7 @@ def __init__(self, registry_config: RegistryConfig, repo_path: Path): "s3", endpoint_url=os.environ.get("FEAST_S3_ENDPOINT_URL") ) + @log_exceptions_and_usage(registry="s3") def get_registry_proto(self): file_obj = TemporaryFile() registry_proto = RegistryProto() @@ -335,6 +371,7 @@ def get_registry_proto(self): f"Error while trying to locate Registry at path {self._uri.geturl()}" ) from e + @log_exceptions_and_usage(registry="s3") def update_registry_proto(self, registry_proto: RegistryProto): self._write_registry(registry_proto) diff --git a/sdk/python/feast/infra/feature_servers/gcp_cloudrun/Dockerfile b/sdk/python/feast/infra/feature_servers/gcp_cloudrun/Dockerfile new file mode 100644 index 00000000000..b4e1e4adb97 --- /dev/null +++ b/sdk/python/feast/infra/feature_servers/gcp_cloudrun/Dockerfile @@ -0,0 +1,28 @@ +FROM python:3.9-slim + +# Allow statements and log messages to immediately appear in the Knative logs +ENV PYTHONUNBUFFERED True + +# Copy local code to the container image. +ENV APP_HOME /app +WORKDIR $APP_HOME + +# Copy app handler code +COPY sdk/python/feast/infra/feature_servers/gcp_cloudrun/app.py ./app.py + +# Copy necessary parts of the Feast codebase +COPY sdk/python ./sdk/python +COPY protos ./protos +COPY README.md ./README.md + +# Install production dependencies. +RUN pip install --no-cache-dir \ + -e 'sdk/python[gcp,redis]' \ + -r ./sdk/python/feast/infra/feature_servers/gcp_cloudrun/requirements.txt + +# Run the web service on container startup. Here we use the gunicorn +# webserver, with one worker process and 8 threads. +# For environments with multiple CPU cores, increase the number of workers +# to be equal to the cores available. +# Timeout is set to 0 to disable the timeouts of the workers to allow Cloud Run to handle instance scaling. +CMD exec gunicorn -k uvicorn.workers.UvicornWorker --bind :$PORT --workers 1 --threads 8 --timeout 0 app:app diff --git a/sdk/python/feast/infra/feature_servers/gcp_cloudrun/app.py b/sdk/python/feast/infra/feature_servers/gcp_cloudrun/app.py new file mode 100644 index 00000000000..06749b0cd39 --- /dev/null +++ b/sdk/python/feast/infra/feature_servers/gcp_cloudrun/app.py @@ -0,0 +1,24 @@ +import base64 +import os +import tempfile +from pathlib import Path + +from feast import FeatureStore +from feast.constants import FEATURE_STORE_YAML_ENV_NAME +from feast.feature_server import get_app + +# Load RepoConfig +config_base64 = os.environ[FEATURE_STORE_YAML_ENV_NAME] +config_bytes = base64.b64decode(config_base64) + +# Create a new unique directory for writing feature_store.yaml +repo_path = Path(tempfile.mkdtemp()) + +with open(repo_path / "feature_store.yaml", "wb") as f: + f.write(config_bytes) + +# Initialize the feature store +store = FeatureStore(repo_path=str(repo_path.resolve())) + +# Create the FastAPI app +app = get_app(store) diff --git a/sdk/python/feast/infra/feature_servers/gcp_cloudrun/config.py b/sdk/python/feast/infra/feature_servers/gcp_cloudrun/config.py new file mode 100644 index 00000000000..728ac56ae29 --- /dev/null +++ b/sdk/python/feast/infra/feature_servers/gcp_cloudrun/config.py @@ -0,0 +1,20 @@ +from pydantic import StrictBool +from pydantic.typing import Literal + +from feast.repo_config import FeastConfigBaseModel + + +class GcpCloudRunFeatureServerConfig(FeastConfigBaseModel): + """Feature server config for GCP CloudRun.""" + + type: Literal["gcp_cloudrun"] = "gcp_cloudrun" + """Feature server type selector.""" + + enabled: StrictBool = False + """Whether the feature server should be launched.""" + + public: StrictBool = True + """Whether the endpoint should be publicly accessible.""" + + auth: Literal["none", "api-key"] = "none" + """Authentication method for the endpoint.""" diff --git a/sdk/python/feast/infra/feature_servers/gcp_cloudrun/requirements.txt b/sdk/python/feast/infra/feature_servers/gcp_cloudrun/requirements.txt new file mode 100644 index 00000000000..8f22dccf99a --- /dev/null +++ b/sdk/python/feast/infra/feature_servers/gcp_cloudrun/requirements.txt @@ -0,0 +1 @@ +gunicorn diff --git a/sdk/python/feast/infra/gcp.py b/sdk/python/feast/infra/gcp.py index 1dd1eefe2dd..257ae38d02c 100644 --- a/sdk/python/feast/infra/gcp.py +++ b/sdk/python/feast/infra/gcp.py @@ -8,6 +8,7 @@ from feast.protos.feast.core.Registry_pb2 import Registry as RegistryProto from feast.registry_store import RegistryStore from feast.repo_config import RegistryConfig +from feast.usage import log_exceptions_and_usage class GcpProvider(PassthroughProvider): @@ -22,7 +23,7 @@ class GCSRegistryStore(RegistryStore): def __init__(self, registry_config: RegistryConfig, repo_path: Path): uri = registry_config.path try: - from google.cloud import storage + import google.cloud.storage as storage except ImportError as e: from feast.errors import FeastExtrasDependencyImportError @@ -33,8 +34,9 @@ def __init__(self, registry_config: RegistryConfig, repo_path: Path): self._bucket = self._uri.hostname self._blob = self._uri.path.lstrip("/") + @log_exceptions_and_usage(registry="gs") def get_registry_proto(self): - from google.cloud import storage + import google.cloud.storage as storage from google.cloud.exceptions import NotFound file_obj = TemporaryFile() @@ -56,6 +58,7 @@ def get_registry_proto(self): f'Registry not found at path "{self._uri.geturl()}". Have you run "feast apply"?' ) + @log_exceptions_and_usage(registry="gs") def update_registry_proto(self, registry_proto: RegistryProto): self._write_registry(registry_proto) diff --git a/sdk/python/feast/infra/key_encoding_utils.py b/sdk/python/feast/infra/key_encoding_utils.py index eb3a5ea1cc0..8333610473b 100644 --- a/sdk/python/feast/infra/key_encoding_utils.py +++ b/sdk/python/feast/infra/key_encoding_utils.py @@ -19,6 +19,22 @@ def _serialize_val(value_type, v: ValueProto) -> Tuple[bytes, int]: raise ValueError(f"Value type not supported for Firestore: {v}") +def serialize_entity_key_prefix(entity_keys: List[str]) -> bytes: + """ + Serialize keys to a bytestring so it can be used to prefix-scan through items stored in the online store + using serialize_entity_key. + + This encoding is a partial implementation of serialize_entity_key, only operating on the keys of entities, + and not the values. + """ + sorted_keys = sorted(entity_keys) + output: List[bytes] = [] + for k in sorted_keys: + output.append(struct.pack(" bytes: """ Serialize entity key to a bytestring so it can be used as a lookup key in a hash table. diff --git a/sdk/python/feast/infra/local.py b/sdk/python/feast/infra/local.py index 6ac3ce72591..d1dbd259efd 100644 --- a/sdk/python/feast/infra/local.py +++ b/sdk/python/feast/infra/local.py @@ -11,6 +11,7 @@ from feast.protos.feast.core.Registry_pb2 import Registry as RegistryProto from feast.registry_store import RegistryStore from feast.repo_config import RegistryConfig +from feast.usage import log_exceptions_and_usage class LocalProvider(PassthroughProvider): @@ -40,6 +41,7 @@ def __init__(self, registry_config: RegistryConfig, repo_path: Path): else: self._filepath = repo_path.joinpath(registry_path) + @log_exceptions_and_usage(registry="local") def get_registry_proto(self): registry_proto = RegistryProto() if self._filepath.exists(): @@ -49,6 +51,7 @@ def get_registry_proto(self): f'Registry not found at path "{self._filepath}". Have you run "feast apply"?' ) + @log_exceptions_and_usage(registry="local") def update_registry_proto(self, registry_proto: RegistryProto): self._write_registry(registry_proto) diff --git a/sdk/python/feast/infra/offline_stores/bigquery.py b/sdk/python/feast/infra/offline_stores/bigquery.py index aa4d43047c3..15c077ecbff 100644 --- a/sdk/python/feast/infra/offline_stores/bigquery.py +++ b/sdk/python/feast/infra/offline_stores/bigquery.py @@ -26,6 +26,7 @@ from feast.registry import Registry from feast.repo_config import FeastConfigBaseModel, RepoConfig +from ...usage import log_exceptions_and_usage from .bigquery_source import BigQuerySource try: @@ -62,6 +63,7 @@ class BigQueryOfflineStoreConfig(FeastConfigBaseModel): class BigQueryOfflineStore(OfflineStore): @staticmethod + @log_exceptions_and_usage(offline_store="bigquery") def pull_latest_from_table_or_query( config: RepoConfig, data_source: DataSource, @@ -113,6 +115,7 @@ def pull_latest_from_table_or_query( ) @staticmethod + @log_exceptions_and_usage(offline_store="bigquery") def get_historical_features( config: RepoConfig, feature_views: List[FeatureView], @@ -164,6 +167,7 @@ def query_generator() -> Iterator[str]: query_context, left_table_query_string=table_reference, entity_df_event_timestamp_col=entity_df_event_timestamp_col, + entity_df_columns=entity_schema.keys(), query_template=MULTIPLE_FEATURE_VIEW_POINT_IN_TIME_JOIN, full_feature_names=full_feature_names, ) @@ -220,7 +224,7 @@ def on_demand_feature_views(self) -> Optional[List[OnDemandFeatureView]]: def _to_df_internal(self) -> pd.DataFrame: with self._query_generator() as query: - df = self.client.query(query).to_dataframe(create_bqstorage_client=True) + df = self._execute_query(query).to_dataframe(create_bqstorage_client=True) return df def to_sql(self) -> str: @@ -248,39 +252,45 @@ def to_bigquery( Returns: Returns the destination table name or returns None if job_config.dry_run is True. """ + + if not job_config: + today = date.today().strftime("%Y%m%d") + rand_id = str(uuid.uuid4())[:7] + path = f"{self.client.project}.{self.config.offline_store.dataset}.historical_{today}_{rand_id}" + job_config = bigquery.QueryJobConfig(destination=path) + + if not job_config.dry_run and self.on_demand_feature_views: + job = self.client.load_table_from_dataframe( + self.to_df(), job_config.destination + ) + job.result() + print(f"Done writing to '{job_config.destination}'.") + return str(job_config.destination) + with self._query_generator() as query: - if not job_config: - today = date.today().strftime("%Y%m%d") - rand_id = str(uuid.uuid4())[:7] - path = f"{self.client.project}.{self.config.offline_store.dataset}.historical_{today}_{rand_id}" - job_config = bigquery.QueryJobConfig(destination=path) - - if not job_config.dry_run and self.on_demand_feature_views is not None: - job = _write_pyarrow_table_to_bq( - self.client, self.to_arrow(), job_config.destination - ) - job.result() - print(f"Done writing to '{job_config.destination}'.") - return str(job_config.destination) - - bq_job = self.client.query(query, job_config=job_config) - - if job_config.dry_run: - print( - "This query will process {} bytes.".format( - bq_job.total_bytes_processed - ) - ) - return None - - block_until_done(client=self.client, bq_job=bq_job, timeout=timeout) + self._execute_query(query, job_config, timeout) print(f"Done writing to '{job_config.destination}'.") return str(job_config.destination) def _to_arrow_internal(self) -> pyarrow.Table: with self._query_generator() as query: - return self.client.query(query).to_arrow() + return self._execute_query(query).to_arrow() + + @log_exceptions_and_usage + def _execute_query( + self, query, job_config=None, timeout: int = 1800 + ) -> bigquery.job.query.QueryJob: + bq_job = self.client.query(query, job_config=job_config) + + if job_config and job_config.dry_run: + print( + "This query will process {} bytes.".format(bq_job.total_bytes_processed) + ) + return None + + block_until_done(client=self.client, bq_job=bq_job, timeout=timeout) + return bq_job def block_until_done( @@ -366,7 +376,7 @@ def _upload_entity_df_and_get_entity_schema( elif isinstance(entity_df, pd.DataFrame): # Drop the index so that we dont have unnecessary columns entity_df.reset_index(drop=True, inplace=True) - job = _write_df_to_bq(client, entity_df, table_name) + job = client.load_table_from_dataframe(entity_df, table_name) block_until_done(client, job) entity_schema = dict(zip(entity_df.columns, entity_df.dtypes)) else: @@ -400,44 +410,6 @@ def _get_bigquery_client(project: Optional[str] = None, location: Optional[str] return client -def _write_df_to_bq( - client: bigquery.Client, df: pd.DataFrame, table_name: str -) -> bigquery.LoadJob: - # It is complicated to get BQ to understand that we want an ARRAY - # https://cloud.google.com/bigquery/docs/reference/rest/v2/tables#parquetoptions - # https://github.com/googleapis/python-bigquery/issues/19 - writer = pyarrow.BufferOutputStream() - pyarrow.parquet.write_table( - pyarrow.Table.from_pandas(df), writer, use_compliant_nested_type=True - ) - return _write_pyarrow_buffer_to_bq(client, writer.getvalue(), table_name,) - - -def _write_pyarrow_table_to_bq( - client: bigquery.Client, table: pyarrow.Table, table_name: str -) -> bigquery.LoadJob: - # It is complicated to get BQ to understand that we want an ARRAY - # https://cloud.google.com/bigquery/docs/reference/rest/v2/tables#parquetoptions - # https://github.com/googleapis/python-bigquery/issues/19 - writer = pyarrow.BufferOutputStream() - pyarrow.parquet.write_table(table, writer, use_compliant_nested_type=True) - return _write_pyarrow_buffer_to_bq(client, writer.getvalue(), table_name,) - - -def _write_pyarrow_buffer_to_bq( - client: bigquery.Client, buf: pyarrow.Buffer, table_name: str -) -> bigquery.LoadJob: - reader = pyarrow.BufferReader(buf) - - parquet_options = bigquery.format_options.ParquetOptions() - parquet_options.enable_list_inference = True - job_config = bigquery.LoadJobConfig() - job_config.source_format = bigquery.SourceFormat.PARQUET - job_config.parquet_options = parquet_options - - return client.load_table_from_file(reader, table_name, job_config=job_config,) - - # TODO: Optimizations # * Use GENERATE_UUID() instead of ROW_NUMBER(), or join on entity columns directly # * Precompute ROW_NUMBER() so that it doesn't have to be recomputed for every query on entity_dataframe @@ -555,14 +527,17 @@ def _write_pyarrow_buffer_to_bq( Thus we only need to compute the latest timestamp of each feature. */ {{ featureview.name }}__latest AS ( - SELECT * EXCEPT(row_number) + SELECT + event_timestamp, + {% if featureview.created_timestamp_column %}created_timestamp,{% endif %} + {{featureview.name}}__entity_row_unique_id FROM ( SELECT *, ROW_NUMBER() OVER( PARTITION BY {{featureview.name}}__entity_row_unique_id ORDER BY event_timestamp DESC{% if featureview.created_timestamp_column %},created_timestamp DESC{% endif %} - ) AS row_number, + ) AS row_number FROM {{ featureview.name }}__base {% if featureview.created_timestamp_column %} INNER JOIN {{ featureview.name }}__dedup @@ -596,7 +571,7 @@ def _write_pyarrow_buffer_to_bq( The entity_dataframe dataset being our source of truth here. */ -SELECT * EXCEPT(entity_timestamp, {% for featureview in featureviews %} {{featureview.name}}__entity_row_unique_id{% if loop.last %}{% else %},{% endif %}{% endfor %}) +SELECT {{ final_output_feature_names | join(', ')}} FROM entity_dataframe {% for featureview in featureviews %} LEFT JOIN ( diff --git a/sdk/python/feast/infra/offline_stores/file.py b/sdk/python/feast/infra/offline_stores/file.py index 17c3ebfad67..bce32e92b31 100644 --- a/sdk/python/feast/infra/offline_stores/file.py +++ b/sdk/python/feast/infra/offline_stores/file.py @@ -20,6 +20,7 @@ ) from feast.registry import Registry from feast.repo_config import FeastConfigBaseModel, RepoConfig +from feast.usage import log_exceptions_and_usage class FileOfflineStoreConfig(FeastConfigBaseModel): @@ -51,11 +52,13 @@ def full_feature_names(self) -> bool: def on_demand_feature_views(self) -> Optional[List[OnDemandFeatureView]]: return self._on_demand_feature_views + @log_exceptions_and_usage def _to_df_internal(self) -> pd.DataFrame: # Only execute the evaluation function to build the final historical retrieval dataframe at the last moment. df = self.evaluation_function() return df + @log_exceptions_and_usage def _to_arrow_internal(self): # Only execute the evaluation function to build the final historical retrieval dataframe at the last moment. df = self.evaluation_function() @@ -64,6 +67,7 @@ def _to_arrow_internal(self): class FileOfflineStore(OfflineStore): @staticmethod + @log_exceptions_and_usage(offline_store="file") def get_historical_features( config: RepoConfig, feature_views: List[FeatureView], @@ -143,6 +147,11 @@ def evaluate_historical_retrieval(): table = _run_field_mapping( table, feature_view.batch_source.field_mapping ) + # Rename entity columns by the join_key_map dictionary if it exists + if feature_view.projection.join_key_map: + table = _run_field_mapping( + table, feature_view.projection.join_key_map + ) # Convert pyarrow table to pandas dataframe. Note, if the underlying data has missing values, # pandas will convert those values to np.nan if the dtypes are numerical (floats, ints, etc.) or boolean @@ -176,7 +185,9 @@ def evaluate_historical_retrieval(): # double underscore as separator for consistency with other databases like BigQuery, # where there are very few characters available for use as separators if full_feature_names: - formatted_feature_name = f"{feature_view.name}__{feature}" + formatted_feature_name = ( + f"{feature_view.projection.name_to_use()}__{feature}" + ) else: formatted_feature_name = feature # Add the feature name to the list of columns @@ -191,7 +202,10 @@ def evaluate_historical_retrieval(): join_keys = [] for entity_name in feature_view.entities: entity = registry.get_entity(entity_name, project) - join_keys.append(entity.join_key) + join_key = feature_view.projection.join_key_map.get( + entity.join_key, entity.join_key + ) + join_keys.append(join_key) right_entity_columns = join_keys right_entity_key_columns = [ event_timestamp_column @@ -254,6 +268,7 @@ def evaluate_historical_retrieval(): return job @staticmethod + @log_exceptions_and_usage(offline_store="file") def pull_latest_from_table_or_query( config: RepoConfig, data_source: DataSource, diff --git a/sdk/python/feast/infra/offline_stores/offline_store.py b/sdk/python/feast/infra/offline_stores/offline_store.py index 4fa108d86b0..0ab1de6a137 100644 --- a/sdk/python/feast/infra/offline_stores/offline_store.py +++ b/sdk/python/feast/infra/offline_stores/offline_store.py @@ -44,9 +44,10 @@ def to_df(self) -> pd.DataFrame: if self.on_demand_feature_views is None: return features_df + # TODO(adchia): Fix requirement to specify dependent feature views in feature_refs for odfv in self.on_demand_feature_views: features_df = features_df.join( - odfv.get_transformed_features_df(self.full_feature_names, features_df) + odfv.get_transformed_features_df(features_df) ) return features_df @@ -68,7 +69,7 @@ def to_arrow(self) -> pyarrow.Table: features_df = self._to_df_internal() for odfv in self.on_demand_feature_views: features_df = features_df.join( - odfv.get_transformed_features_df(self.full_feature_names, features_df) + odfv.get_transformed_features_df(features_df) ) return pyarrow.Table.from_pandas(features_df) diff --git a/sdk/python/feast/infra/offline_stores/offline_utils.py b/sdk/python/feast/infra/offline_stores/offline_utils.py index 05a6486293d..10c0de2f9bd 100644 --- a/sdk/python/feast/infra/offline_stores/offline_utils.py +++ b/sdk/python/feast/infra/offline_stores/offline_utils.py @@ -2,7 +2,7 @@ import uuid from dataclasses import asdict, dataclass from datetime import timedelta -from typing import Any, Dict, List, Optional, Set, Tuple +from typing import Any, Dict, KeysView, List, Optional, Set, Tuple import numpy as np import pandas as pd @@ -64,7 +64,10 @@ def get_expected_join_keys( entities = feature_view.entities for entity_name in entities: entity = registry.get_entity(entity_name, project) - join_keys.add(entity.join_key) + join_key = feature_view.projection.join_key_map.get( + entity.join_key, entity.join_key + ) + join_keys.add(join_key) return join_keys @@ -113,11 +116,11 @@ def get_feature_view_query_context( } for entity_name in feature_view.entities: entity = registry.get_entity(entity_name, project) - join_keys.append(entity.join_key) - join_key_column = reverse_field_mapping.get( + join_key = feature_view.projection.join_key_map.get( entity.join_key, entity.join_key ) - entity_selections.append(f"{join_key_column} AS {entity.join_key}") + join_keys.append(join_key) + entity_selections.append(f"{entity.join_key} AS {join_key}") if isinstance(feature_view.ttl, timedelta): ttl_seconds = int(feature_view.ttl.total_seconds()) @@ -150,12 +153,22 @@ def build_point_in_time_query( feature_view_query_contexts: List[FeatureViewQueryContext], left_table_query_string: str, entity_df_event_timestamp_col: str, + entity_df_columns: KeysView[str], query_template: str, full_feature_names: bool = False, ) -> str: """Build point-in-time query between each feature view table and the entity dataframe for Bigquery and Redshift""" template = Environment(loader=BaseLoader()).from_string(source=query_template) + final_output_feature_names = list(entity_df_columns) + final_output_feature_names.extend( + [ + (f"{fv.name}__{feature}" if full_feature_names else feature) + for fv in feature_view_query_contexts + for feature in fv.features + ] + ) + # Add additional fields to dict template_context = { "left_table_query_string": left_table_query_string, @@ -165,6 +178,7 @@ def build_point_in_time_query( ), "featureviews": [asdict(context) for context in feature_view_query_contexts], "full_feature_names": full_feature_names, + "final_output_feature_names": final_output_feature_names, } query = template.render(template_context) diff --git a/sdk/python/feast/infra/offline_stores/redshift.py b/sdk/python/feast/infra/offline_stores/redshift.py index 1b17ae15b0a..e8a7c985d47 100644 --- a/sdk/python/feast/infra/offline_stores/redshift.py +++ b/sdk/python/feast/infra/offline_stores/redshift.py @@ -18,6 +18,7 @@ from feast.infra.utils import aws_utils from feast.registry import Registry from feast.repo_config import FeastConfigBaseModel, RepoConfig +from feast.usage import log_exceptions_and_usage class RedshiftOfflineStoreConfig(FeastConfigBaseModel): @@ -47,6 +48,7 @@ class RedshiftOfflineStoreConfig(FeastConfigBaseModel): class RedshiftOfflineStore(OfflineStore): @staticmethod + @log_exceptions_and_usage(offline_store="redshift") def pull_latest_from_table_or_query( config: RepoConfig, data_source: DataSource, @@ -103,6 +105,7 @@ def pull_latest_from_table_or_query( ) @staticmethod + @log_exceptions_and_usage(offline_store="redshift") def get_historical_features( config: RepoConfig, feature_views: List[FeatureView], @@ -149,6 +152,7 @@ def query_generator() -> Iterator[str]: query_context, left_table_query_string=table_name, entity_df_event_timestamp_col=entity_df_event_timestamp_col, + entity_df_columns=entity_schema.keys(), query_template=MULTIPLE_FEATURE_VIEW_POINT_IN_TIME_JOIN, full_feature_names=full_feature_names, ) @@ -174,11 +178,6 @@ def query_generator() -> Iterator[str]: on_demand_feature_views=OnDemandFeatureView.get_requested_odfvs( feature_refs, project, registry ), - drop_columns=["entity_timestamp"] - + [ - f"{feature_view.name}__entity_row_unique_id" - for feature_view in feature_views - ], ) @@ -191,7 +190,6 @@ def __init__( config: RepoConfig, full_feature_names: bool, on_demand_feature_views: Optional[List[OnDemandFeatureView]], - drop_columns: Optional[List[str]] = None, ): """Initialize RedshiftRetrievalJob object. @@ -202,8 +200,6 @@ def __init__( config: Feast repo config full_feature_names: Whether to add the feature view prefixes to the feature names on_demand_feature_views: A list of on demand transforms to apply at retrieval time - drop_columns: Optionally a list of columns to drop before unloading to S3. - This is a convenient field, since "SELECT ... EXCEPT col" isn't supported in Redshift. """ if not isinstance(query, str): self._query_generator = query @@ -225,7 +221,6 @@ def query_generator() -> Iterator[str]: ) self._full_feature_names = full_feature_names self._on_demand_feature_views = on_demand_feature_views - self._drop_columns = drop_columns @property def full_feature_names(self) -> bool: @@ -235,6 +230,7 @@ def full_feature_names(self) -> bool: def on_demand_feature_views(self) -> Optional[List[OnDemandFeatureView]]: return self._on_demand_feature_views + @log_exceptions_and_usage def _to_df_internal(self) -> pd.DataFrame: with self._query_generator() as query: return aws_utils.unload_redshift_query_to_df( @@ -246,9 +242,9 @@ def _to_df_internal(self) -> pd.DataFrame: self._s3_path, self._config.offline_store.iam_role, query, - self._drop_columns, ) + @log_exceptions_and_usage def _to_arrow_internal(self) -> pa.Table: with self._query_generator() as query: return aws_utils.unload_redshift_query_to_pa( @@ -260,12 +256,12 @@ def _to_arrow_internal(self) -> pa.Table: self._s3_path, self._config.offline_store.iam_role, query, - self._drop_columns, ) + @log_exceptions_and_usage def to_s3(self) -> str: """ Export dataset to S3 in Parquet format and return path """ - if self.on_demand_feature_views is not None: + if self.on_demand_feature_views: transformed_df = self.to_df() aws_utils.upload_df_to_s3(self._s3_resource, self._s3_path, transformed_df) return self._s3_path @@ -279,13 +275,13 @@ def to_s3(self) -> str: self._s3_path, self._config.offline_store.iam_role, query, - self._drop_columns, ) return self._s3_path + @log_exceptions_and_usage def to_redshift(self, table_name: str) -> None: """ Save dataset as a new Redshift table """ - if self.on_demand_feature_views is not None: + if self.on_demand_feature_views: transformed_df = self.to_df() aws_utils.upload_df_to_redshift( self._redshift_client, @@ -302,9 +298,6 @@ def to_redshift(self, table_name: str) -> None: with self._query_generator() as query: query = f'CREATE TABLE "{table_name}" AS ({query});\n' - if self._drop_columns is not None: - for column in self._drop_columns: - query += f"ALTER TABLE {table_name} DROP COLUMN {column};\n" aws_utils.execute_redshift_statement( self._redshift_client, @@ -479,19 +472,23 @@ def _upload_entity_df_and_get_entity_schema( */ {{ featureview.name }}__latest AS ( SELECT - {{featureview.name}}__entity_row_unique_id, - MAX(event_timestamp) AS event_timestamp + event_timestamp, + {% if featureview.created_timestamp_column %}created_timestamp,{% endif %} + {{featureview.name}}__entity_row_unique_id + FROM + ( + SELECT *, + ROW_NUMBER() OVER( + PARTITION BY {{featureview.name}}__entity_row_unique_id + ORDER BY event_timestamp DESC{% if featureview.created_timestamp_column %},created_timestamp DESC{% endif %} + ) AS row_number + FROM {{ featureview.name }}__base {% if featureview.created_timestamp_column %} - ,MAX(created_timestamp) AS created_timestamp + INNER JOIN {{ featureview.name }}__dedup + USING ({{featureview.name}}__entity_row_unique_id, event_timestamp, created_timestamp) {% endif %} - - FROM {{ featureview.name }}__base - {% if featureview.created_timestamp_column %} - INNER JOIN {{ featureview.name }}__dedup - USING ({{featureview.name}}__entity_row_unique_id, event_timestamp, created_timestamp) - {% endif %} - - GROUP BY {{featureview.name}}__entity_row_unique_id + ) + WHERE row_number = 1 ), /* @@ -518,7 +515,7 @@ def _upload_entity_df_and_get_entity_schema( The entity_dataframe dataset being our source of truth here. */ -SELECT * +SELECT {{ final_output_feature_names | join(', ')}} FROM entity_dataframe {% for featureview in featureviews %} LEFT JOIN ( diff --git a/sdk/python/feast/infra/online_stores/datastore.py b/sdk/python/feast/infra/online_stores/datastore.py index 37011c7d510..e9e5973eddd 100644 --- a/sdk/python/feast/infra/online_stores/datastore.py +++ b/sdk/python/feast/infra/online_stores/datastore.py @@ -26,6 +26,7 @@ from feast.protos.feast.types.EntityKey_pb2 import EntityKey as EntityKeyProto from feast.protos.feast.types.Value_pb2 import Value as ValueProto from feast.repo_config import FeastConfigBaseModel, RepoConfig +from feast.usage import log_exceptions_and_usage, tracing_span try: from google.auth.exceptions import DefaultCredentialsError @@ -69,6 +70,7 @@ class DatastoreOnlineStore(OnlineStore): _client: Optional[datastore.Client] = None + @log_exceptions_and_usage(online_store="datastore") def update( self, config: RepoConfig, @@ -140,6 +142,7 @@ def _get_client(self, online_config: DatastoreOnlineStoreConfig): ) return self._client + @log_exceptions_and_usage(online_store="datastore") def online_write_batch( self, config: RepoConfig, @@ -220,6 +223,7 @@ def _write_minibatch( if progress: progress(len(entities)) + @log_exceptions_and_usage(online_store="datastore") def online_read( self, config: RepoConfig, @@ -245,7 +249,8 @@ def online_read( # NOTE: get_multi doesn't return values in the same order as the keys in the request. # Also, len(values) can be less than len(keys) in the case of missing values. - values = client.get_multi(keys) + with tracing_span(name="remote_call"): + values = client.get_multi(keys) values_dict = {v.key: v for v in values} if values is not None else {} for key in keys: if key in values_dict: diff --git a/sdk/python/feast/infra/online_stores/dynamodb.py b/sdk/python/feast/infra/online_stores/dynamodb.py index 722a081f2ee..7ff6702deae 100644 --- a/sdk/python/feast/infra/online_stores/dynamodb.py +++ b/sdk/python/feast/infra/online_stores/dynamodb.py @@ -23,6 +23,7 @@ from feast.protos.feast.types.EntityKey_pb2 import EntityKey as EntityKeyProto from feast.protos.feast.types.Value_pb2 import Value as ValueProto from feast.repo_config import FeastConfigBaseModel, RepoConfig +from feast.usage import log_exceptions_and_usage, tracing_span try: import boto3 @@ -48,6 +49,7 @@ class DynamoDBOnlineStore(OnlineStore): Online feature store for AWS DynamoDB. """ + @log_exceptions_and_usage(online_store="dynamodb") def update( self, config: RepoConfig, @@ -97,6 +99,7 @@ def teardown( self._delete_tables_idempotent(dynamodb_resource, config, tables) + @log_exceptions_and_usage(online_store="dynamodb") def online_write_batch( self, config: RepoConfig, @@ -127,6 +130,7 @@ def online_write_batch( if progress: progress(1) + @log_exceptions_and_usage(online_store="dynamodb") def online_read( self, config: RepoConfig, @@ -142,7 +146,8 @@ def online_read( for entity_key in entity_keys: table_instance = dynamodb_resource.Table(f"{config.project}.{table.name}") entity_id = compute_entity_id(entity_key) - response = table_instance.get_item(Key={"entity_id": entity_id}) + with tracing_span(name="remote_call"): + response = table_instance.get_item(Key={"entity_id": entity_id}) value = response.get("Item") if value is not None: diff --git a/sdk/python/feast/infra/online_stores/helpers.py b/sdk/python/feast/infra/online_stores/helpers.py index 391d70bfe15..5e01ddb263f 100644 --- a/sdk/python/feast/infra/online_stores/helpers.py +++ b/sdk/python/feast/infra/online_stores/helpers.py @@ -5,13 +5,16 @@ import mmh3 from feast import errors -from feast.infra.key_encoding_utils import serialize_entity_key +from feast.infra.key_encoding_utils import ( + serialize_entity_key, + serialize_entity_key_prefix, +) from feast.infra.online_stores.online_store import OnlineStore from feast.protos.feast.types.EntityKey_pb2 import EntityKey as EntityKeyProto def get_online_store_from_config(online_store_config: Any,) -> OnlineStore: - """Get the offline store from offline store config""" + """Get the online store from online store config""" module_name = online_store_config.__module__ qualified_name = type(online_store_config).__name__ @@ -41,6 +44,10 @@ def _redis_key(project: str, entity_key: EntityKeyProto) -> bytes: return b"".join(key) +def _redis_key_prefix(entity_keys: List[str]) -> bytes: + return serialize_entity_key_prefix(entity_keys) + + def _mmh3(key: str): """ Calculate murmur3_32 hash which is equal to scala version which is using little endian: diff --git a/sdk/python/feast/infra/online_stores/redis.py b/sdk/python/feast/infra/online_stores/redis.py index a226c5cd18c..c6df0c69fa7 100644 --- a/sdk/python/feast/infra/online_stores/redis.py +++ b/sdk/python/feast/infra/online_stores/redis.py @@ -12,20 +12,32 @@ # See the License for the specific language governing permissions and # limitations under the License. import json +import logging from datetime import datetime from enum import Enum -from typing import Any, Callable, Dict, List, Optional, Sequence, Tuple, Union +from typing import ( + Any, + ByteString, + Callable, + Dict, + List, + Optional, + Sequence, + Tuple, + Union, +) from google.protobuf.timestamp_pb2 import Timestamp from pydantic import StrictStr from pydantic.typing import Literal from feast import Entity, FeatureTable, FeatureView, RepoConfig, utils -from feast.infra.online_stores.helpers import _mmh3, _redis_key +from feast.infra.online_stores.helpers import _mmh3, _redis_key, _redis_key_prefix from feast.infra.online_stores.online_store import OnlineStore from feast.protos.feast.types.EntityKey_pb2 import EntityKey as EntityKeyProto from feast.protos.feast.types.Value_pb2 import Value as ValueProto from feast.repo_config import FeastConfigBaseModel +from feast.usage import log_exceptions_and_usage, tracing_span try: from redis import Redis @@ -35,7 +47,7 @@ raise FeastExtrasDependencyImportError("redis", str(e)) -EX_SECONDS = 253402300799 +logger = logging.getLogger(__name__) class RedisType(str, Enum): @@ -60,6 +72,24 @@ class RedisOnlineStoreConfig(FeastConfigBaseModel): class RedisOnlineStore(OnlineStore): _client: Optional[Union[Redis, RedisCluster]] = None + def delete_table_values( + self, config: RepoConfig, table: Union[FeatureTable, FeatureView] + ): + client = self._get_client(config.online_store) + deleted_count = 0 + pipeline = client.pipeline() + prefix = _redis_key_prefix(table.entities) + + for _k in client.scan_iter( + b"".join([prefix, b"*", config.project.encode("utf8")]) + ): + pipeline.delete(_k) + deleted_count += 1 + pipeline.execute() + + logger.debug(f"Deleted {deleted_count} keys for {table.name}") + + @log_exceptions_and_usage(online_store="redis") def update( self, config: RepoConfig, @@ -70,9 +100,10 @@ def update( partial: bool, ): """ - There's currently no setup done for Redis. + We delete the keys in redis for tables/views being removed. """ - pass + for table in tables_to_delete: + self.delete_table_values(config, table) def teardown( self, @@ -81,9 +112,10 @@ def teardown( entities: Sequence[Entity], ): """ - There's currently no teardown done for Redis. + We delete the keys in redis for tables/views being removed. """ - pass + for table in tables: + self.delete_table_values(config, table) @staticmethod def _parse_connection_string(connection_string: str): @@ -121,7 +153,7 @@ def _get_client(self, online_store_config: RedisOnlineStoreConfig): startup_nodes, kwargs = self._parse_connection_string( online_store_config.connection_string ) - if online_store_config.type == RedisType.redis_cluster: + if online_store_config.redis_type == RedisType.redis_cluster: kwargs["startup_nodes"] = startup_nodes self._client = RedisCluster(**kwargs) else: @@ -130,6 +162,7 @@ def _get_client(self, online_store_config: RedisOnlineStoreConfig): self._client = Redis(**kwargs) return self._client + @log_exceptions_and_usage(online_store="redis") def online_write_batch( self, config: RepoConfig, @@ -145,28 +178,55 @@ def online_write_batch( client = self._get_client(online_store_config) project = config.project - entity_hset = {} feature_view = table.name - - ex = Timestamp() - ex.seconds = EX_SECONDS - ex_str = ex.SerializeToString() - - for entity_key, values, timestamp, created_ts in data: - redis_key_bin = _redis_key(project, entity_key) - ts = Timestamp() - ts.seconds = int(utils.make_tzaware(timestamp).timestamp()) - entity_hset[f"_ts:{feature_view}"] = ts.SerializeToString() - entity_hset[f"_ex:{feature_view}"] = ex_str - - for feature_name, val in values.items(): - f_key = _mmh3(f"{feature_view}:{feature_name}") - entity_hset[f_key] = val.SerializeToString() - - client.hset(redis_key_bin, mapping=entity_hset) + ts_key = f"_ts:{feature_view}" + keys = [] + # redis pipelining optimization: send multiple commands to redis server without waiting for every reply + with client.pipeline() as pipe: + # check if a previous record under the key bin exists + # TODO: investigate if check and set is a better approach rather than pulling all entity ts and then setting + # it may be significantly slower but avoids potential (rare) race conditions + for entity_key, _, _, _ in data: + redis_key_bin = _redis_key(project, entity_key) + keys.append(redis_key_bin) + pipe.hmget(redis_key_bin, ts_key) + prev_event_timestamps = pipe.execute() + # flattening the list of lists. `hmget` does the lookup assuming a list of keys in the key bin + prev_event_timestamps = [i[0] for i in prev_event_timestamps] + + for redis_key_bin, prev_event_time, (_, values, timestamp, _) in zip( + keys, prev_event_timestamps, data + ): + event_time_seconds = int(utils.make_tzaware(timestamp).timestamp()) + + # ignore if event_timestamp is before the event features that are currently in the feature store + if prev_event_time: + prev_ts = Timestamp() + prev_ts.ParseFromString(prev_event_time) + if prev_ts.seconds and event_time_seconds <= prev_ts.seconds: + # TODO: somehow signal that it's not overwriting the current record? + if progress: + progress(1) + continue + + ts = Timestamp() + ts.seconds = event_time_seconds + entity_hset = dict() + entity_hset[ts_key] = ts.SerializeToString() + + for feature_name, val in values.items(): + f_key = _mmh3(f"{feature_view}:{feature_name}") + entity_hset[f_key] = val.SerializeToString() + + pipe.hset(redis_key_bin, mapping=entity_hset) + # TODO: support expiring the entity / features in Redis + # otherwise entity features remain in redis until cleaned up in separate process + # client.expire redis_key_bin based a ttl setting + results = pipe.execute() if progress: - progress(1) + progress(len(results)) + @log_exceptions_and_usage(online_store="redis") def online_read( self, config: RepoConfig, @@ -186,30 +246,50 @@ def online_read( if not requested_features: requested_features = [f.name for f in table.features] + hset_keys = [_mmh3(f"{feature_view}:{k}") for k in requested_features] + + ts_key = f"_ts:{feature_view}" + hset_keys.append(ts_key) + requested_features.append(ts_key) + + keys = [] for entity_key in entity_keys: redis_key_bin = _redis_key(project, entity_key) - hset_keys = [_mmh3(f"{feature_view}:{k}") for k in requested_features] - ts_key = f"_ts:{feature_view}" - hset_keys.append(ts_key) - values = client.hmget(redis_key_bin, hset_keys) - requested_features.append(ts_key) - res_val = dict(zip(requested_features, values)) - - res_ts = Timestamp() - ts_val = res_val.pop(ts_key) - if ts_val: - res_ts.ParseFromString(ts_val) - - res = {} - for feature_name, val_bin in res_val.items(): - val = ValueProto() - if val_bin: - val.ParseFromString(val_bin) - res[feature_name] = val - - if not res: - result.append((None, None)) - else: - timestamp = datetime.fromtimestamp(res_ts.seconds) - result.append((timestamp, res)) + keys.append(redis_key_bin) + with client.pipeline() as pipe: + for redis_key_bin in keys: + pipe.hmget(redis_key_bin, hset_keys) + with tracing_span(name="remote_call"): + redis_values = pipe.execute() + for values in redis_values: + features = self._get_features_for_entity( + values, feature_view, requested_features + ) + result.append(features) return result + + def _get_features_for_entity( + self, + values: List[ByteString], + feature_view: str, + requested_features: List[str], + ) -> Tuple[Optional[datetime], Optional[Dict[str, ValueProto]]]: + res_val = dict(zip(requested_features, values)) + + res_ts = Timestamp() + ts_val = res_val.pop(f"_ts:{feature_view}") + if ts_val: + res_ts.ParseFromString(ts_val) + + res = {} + for feature_name, val_bin in res_val.items(): + val = ValueProto() + if val_bin: + val.ParseFromString(val_bin) + res[feature_name] = val + + if not res: + return None, None + else: + timestamp = datetime.fromtimestamp(res_ts.seconds) + return timestamp, res diff --git a/sdk/python/feast/infra/online_stores/sqlite.py b/sdk/python/feast/infra/online_stores/sqlite.py index 4be90257dd4..62642153f9c 100644 --- a/sdk/python/feast/infra/online_stores/sqlite.py +++ b/sdk/python/feast/infra/online_stores/sqlite.py @@ -11,7 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - +import itertools import os import sqlite3 from datetime import datetime @@ -29,6 +29,7 @@ from feast.protos.feast.types.EntityKey_pb2 import EntityKey as EntityKeyProto from feast.protos.feast.types.Value_pb2 import Value as ValueProto from feast.repo_config import FeastConfigBaseModel, RepoConfig +from feast.usage import log_exceptions_and_usage, tracing_span class SqliteOnlineStoreConfig(FeastConfigBaseModel): @@ -73,6 +74,7 @@ def _get_conn(self, config: RepoConfig): ) return self._conn + @log_exceptions_and_usage(online_store="sqlite") def online_write_batch( self, config: RepoConfig, @@ -127,6 +129,7 @@ def online_write_batch( if progress: progress(1) + @log_exceptions_and_usage(online_store="sqlite") def online_read( self, config: RepoConfig, @@ -134,24 +137,30 @@ def online_read( entity_keys: List[EntityKeyProto], requested_features: Optional[List[str]] = None, ) -> List[Tuple[Optional[datetime], Optional[Dict[str, ValueProto]]]]: - pass conn = self._get_conn(config) cur = conn.cursor() result: List[Tuple[Optional[datetime], Optional[Dict[str, ValueProto]]]] = [] - project = config.project - for entity_key in entity_keys: - entity_key_bin = serialize_entity_key(entity_key) - + with tracing_span(name="remote_call"): + # Fetch all entities in one go cur.execute( - f"SELECT feature_name, value, event_ts FROM {_table_id(project, table)} WHERE entity_key = ?", - (entity_key_bin,), + f"SELECT entity_key, feature_name, value, event_ts " + f"FROM {_table_id(config.project, table)} " + f"WHERE entity_key IN ({','.join('?' * len(entity_keys))}) " + f"ORDER BY entity_key", + [serialize_entity_key(entity_key) for entity_key in entity_keys], ) + rows = cur.fetchall() + rows = { + k: list(group) for k, group in itertools.groupby(rows, key=lambda r: r[0]) + } + for entity_key in entity_keys: + entity_key_bin = serialize_entity_key(entity_key) res = {} res_ts = None - for feature_name, val_bin, ts in cur.fetchall(): + for _, feature_name, val_bin, ts in rows.get(entity_key_bin, []): val = ValueProto() val.ParseFromString(val_bin) res[feature_name] = val @@ -163,6 +172,7 @@ def online_read( result.append((res_ts, res)) return result + @log_exceptions_and_usage(online_store="sqlite") def update( self, config: RepoConfig, diff --git a/sdk/python/feast/infra/passthrough_provider.py b/sdk/python/feast/infra/passthrough_provider.py index 3e4c0a34858..06a9d3a8b3f 100644 --- a/sdk/python/feast/infra/passthrough_provider.py +++ b/sdk/python/feast/infra/passthrough_provider.py @@ -2,6 +2,7 @@ from typing import Any, Callable, Dict, List, Optional, Sequence, Tuple, Union import pandas +import pyarrow as pa from tqdm import tqdm from feast.entity import Entity @@ -20,6 +21,7 @@ from feast.protos.feast.types.Value_pb2 import Value as ValueProto from feast.registry import Registry from feast.repo_config import RepoConfig +from feast.usage import RatioSampler, log_exceptions_and_usage, set_usage_attribute class PassthroughProvider(Provider): @@ -43,6 +45,7 @@ def update_infra( entities_to_keep: Sequence[Entity], partial: bool, ): + set_usage_attribute("provider", self.__class__.__name__) self.online_store.update( config=self.repo_config, tables_to_delete=tables_to_delete, @@ -58,6 +61,7 @@ def teardown_infra( tables: Sequence[Union[FeatureTable, FeatureView]], entities: Sequence[Entity], ) -> None: + set_usage_attribute("provider", self.__class__.__name__) self.online_store.teardown(self.repo_config, tables, entities) def online_write_batch( @@ -69,8 +73,10 @@ def online_write_batch( ], progress: Optional[Callable[[int], Any]], ) -> None: + set_usage_attribute("provider", self.__class__.__name__) self.online_store.online_write_batch(config, table, data, progress) + @log_exceptions_and_usage(sampler=RatioSampler(ratio=0.001)) def online_read( self, config: RepoConfig, @@ -78,10 +84,27 @@ def online_read( entity_keys: List[EntityKeyProto], requested_features: List[str] = None, ) -> List[Tuple[Optional[datetime], Optional[Dict[str, ValueProto]]]]: + set_usage_attribute("provider", self.__class__.__name__) result = self.online_store.online_read(config, table, entity_keys) return result + def ingest_df( + self, feature_view: FeatureView, entities: List[Entity], df: pandas.DataFrame, + ): + set_usage_attribute("provider", self.__class__.__name__) + table = pa.Table.from_pandas(df) + + if feature_view.batch_source.field_mapping is not None: + table = _run_field_mapping(table, feature_view.batch_source.field_mapping) + + join_keys = [entity.join_key for entity in entities] + rows_to_write = _convert_arrow_to_proto(table, feature_view, join_keys) + + self.online_write_batch( + self.repo_config, feature_view, rows_to_write, progress=None + ) + def materialize_single_feature_view( self, config: RepoConfig, @@ -92,6 +115,8 @@ def materialize_single_feature_view( project: str, tqdm_builder: Callable[[int], tqdm], ) -> None: + set_usage_attribute("provider", self.__class__.__name__) + entities = [] for entity_name in feature_view.entities: entities.append(registry.get_entity(entity_name, project)) @@ -137,6 +162,8 @@ def get_historical_features( project: str, full_feature_names: bool, ) -> RetrievalJob: + set_usage_attribute("provider", self.__class__.__name__) + job = self.offline_store.get_historical_features( config=config, feature_views=feature_views, diff --git a/sdk/python/feast/infra/provider.py b/sdk/python/feast/infra/provider.py index 4debd59aa8e..5546984f71d 100644 --- a/sdk/python/feast/infra/provider.py +++ b/sdk/python/feast/infra/provider.py @@ -20,6 +20,12 @@ from feast.repo_config import RepoConfig from feast.type_map import python_value_to_proto_value +PROVIDERS_CLASS_FOR_TYPE = { + "gcp": "feast.infra.gcp.GcpProvider", + "aws": "feast.infra.aws.AwsProvider", + "local": "feast.infra.local.LocalProvider", +} + class Provider(abc.ABC): @abc.abstractmethod @@ -98,6 +104,14 @@ def online_write_batch( """ ... + def ingest_df( + self, feature_view: FeatureView, entities: List[Entity], df: pandas.DataFrame, + ): + """ + Ingests a DataFrame directly into the online store + """ + pass + @abc.abstractmethod def materialize_single_feature_view( self, @@ -150,25 +164,20 @@ def get_feature_server_endpoint(self) -> Optional[str]: def get_provider(config: RepoConfig, repo_path: Path) -> Provider: if "." not in config.provider: - if config.provider in {"gcp", "aws", "local"}: - if config.provider == "aws": - from feast.infra.aws import AwsProvider - - return AwsProvider(config) - - from feast.infra.passthrough_provider import PassthroughProvider - - return PassthroughProvider(config) - else: + if config.provider not in PROVIDERS_CLASS_FOR_TYPE: raise errors.FeastProviderNotImplementedError(config.provider) + + provider = PROVIDERS_CLASS_FOR_TYPE[config.provider] else: - # Split provider into module and class names by finding the right-most dot. - # For example, provider 'foo.bar.MyProvider' will be parsed into 'foo.bar' and 'MyProvider' - module_name, class_name = config.provider.rsplit(".", 1) + provider = config.provider - cls = importer.get_class_from_type(module_name, class_name, "Provider") + # Split provider into module and class names by finding the right-most dot. + # For example, provider 'foo.bar.MyProvider' will be parsed into 'foo.bar' and 'MyProvider' + module_name, class_name = provider.rsplit(".", 1) - return cls(config) + cls = importer.get_class_from_type(module_name, class_name, "Provider") + + return cls(config) def _get_requested_feature_views_to_features_dict( @@ -296,27 +305,28 @@ def _coerce_datetime(ts): else: return ts + column_names_idx = {k: i for i, k in enumerate(table.column_names)} for row in zip(*table.to_pydict().values()): entity_key = EntityKeyProto() for join_key in join_keys: entity_key.join_keys.append(join_key) - idx = table.column_names.index(join_key) + idx = column_names_idx[join_key] value = python_value_to_proto_value(row[idx]) entity_key.entity_values.append(value) feature_dict = {} for feature in feature_view.features: - idx = table.column_names.index(feature.name) + idx = column_names_idx[feature.name] value = python_value_to_proto_value(row[idx], feature.dtype) feature_dict[feature.name] = value - event_timestamp_idx = table.column_names.index( + event_timestamp_idx = column_names_idx[ feature_view.batch_source.event_timestamp_column - ) + ] event_timestamp = _coerce_datetime(row[event_timestamp_idx]) if feature_view.batch_source.created_timestamp_column: - created_timestamp_idx = table.column_names.index( + created_timestamp_idx = column_names_idx[ feature_view.batch_source.created_timestamp_column - ) + ] created_timestamp = _coerce_datetime(row[created_timestamp_idx]) else: created_timestamp = None diff --git a/sdk/python/feast/infra/transformation_servers/Dockerfile b/sdk/python/feast/infra/transformation_servers/Dockerfile new file mode 100644 index 00000000000..653e34cdf54 --- /dev/null +++ b/sdk/python/feast/infra/transformation_servers/Dockerfile @@ -0,0 +1,15 @@ +FROM python:3.7-slim + +# Copy app handler code +COPY sdk/python/feast/infra/transformation_servers/app.py app.py + +# Copy necessary parts of the Feast codebase +COPY sdk/python sdk/python +COPY protos protos +COPY README.md README.md + +# Install dependencies +RUN pip3 install -e 'sdk/python[ci]' + +# Start feature transformation server +CMD [ "python", "app.py" ] diff --git a/sdk/python/feast/infra/transformation_servers/app.py b/sdk/python/feast/infra/transformation_servers/app.py new file mode 100644 index 00000000000..19728429da3 --- /dev/null +++ b/sdk/python/feast/infra/transformation_servers/app.py @@ -0,0 +1,50 @@ +import base64 +import os +import tempfile +from pathlib import Path + +import yaml + +from feast import FeatureStore +from feast.constants import ( + DEFAULT_FEATURE_TRANSFORMATION_SERVER_PORT, + FEATURE_STORE_YAML_ENV_NAME, + FEATURE_TRANSFORMATION_SERVER_PORT_ENV_NAME, + REGISTRY_ENV_NAME, +) +from feast.infra.local import LocalRegistryStore +from feast.registry import get_registry_store_class_from_scheme + +# Load RepoConfig +config_base64 = os.environ[FEATURE_STORE_YAML_ENV_NAME] +config_bytes = base64.b64decode(config_base64) + +# Create a new unique directory for writing feature_store.yaml +repo_path = Path(tempfile.mkdtemp()) + +with open(repo_path / "feature_store.yaml", "wb") as f: + f.write(config_bytes) + +# Write registry contents for local registries +config_string = config_bytes.decode("utf-8") +raw_config = yaml.safe_load(config_string) +registry_path = raw_config["registry"] +registry_store_class = get_registry_store_class_from_scheme(registry_path) +if registry_store_class == LocalRegistryStore: + registry_base64 = os.environ[REGISTRY_ENV_NAME] + registry_bytes = base64.b64decode(registry_base64) + registry_dir = os.path.dirname(registry_path) + if not os.path.exists(repo_path / registry_dir): + os.makedirs(repo_path / registry_dir) + with open(repo_path / registry_path, "wb") as f: + f.write(registry_bytes) + +# Initialize the feature store +store = FeatureStore(repo_path=str(repo_path.resolve())) + +# Start the feature transformation server +port = ( + os.environ.get(FEATURE_TRANSFORMATION_SERVER_PORT_ENV_NAME) + or DEFAULT_FEATURE_TRANSFORMATION_SERVER_PORT +) +store.serve_transformations(port) diff --git a/sdk/python/feast/infra/utils/aws_utils.py b/sdk/python/feast/infra/utils/aws_utils.py index 68e5be27397..28d48a489e6 100644 --- a/sdk/python/feast/infra/utils/aws_utils.py +++ b/sdk/python/feast/infra/utils/aws_utils.py @@ -2,7 +2,7 @@ import os import tempfile import uuid -from typing import Dict, Iterator, List, Optional, Tuple +from typing import Dict, Iterator, Optional, Tuple import pandas as pd import pyarrow as pa @@ -324,7 +324,6 @@ def execute_redshift_query_and_unload_to_s3( s3_path: str, iam_role: str, query: str, - drop_columns: Optional[List[str]] = None, ) -> None: """Unload Redshift Query results to S3 @@ -337,16 +336,11 @@ def execute_redshift_query_and_unload_to_s3( iam_role: IAM Role for Redshift to assume during the UNLOAD command. The role must grant permission to write to the S3 location. query: The SQL query to execute - drop_columns: Optionally a list of columns to drop before unloading to S3. - This is a convenient field, since "SELECT ... EXCEPT col" isn't supported in Redshift. """ # Run the query, unload the results to S3 unique_table_name = "_" + str(uuid.uuid4()).replace("-", "") query = f"CREATE TEMPORARY TABLE {unique_table_name} AS ({query});\n" - if drop_columns is not None: - for column in drop_columns: - query += f"ALTER TABLE {unique_table_name} DROP COLUMN {column};\n" query += f"UNLOAD ('SELECT * FROM {unique_table_name}') TO '{s3_path}/' IAM_ROLE '{iam_role}' PARQUET" execute_redshift_statement(redshift_data_client, cluster_id, database, user, query) @@ -360,20 +354,12 @@ def unload_redshift_query_to_pa( s3_path: str, iam_role: str, query: str, - drop_columns: Optional[List[str]] = None, ) -> pa.Table: """ Unload Redshift Query results to S3 and get the results in PyArrow Table format """ bucket, key = get_bucket_and_key(s3_path) execute_redshift_query_and_unload_to_s3( - redshift_data_client, - cluster_id, - database, - user, - s3_path, - iam_role, - query, - drop_columns, + redshift_data_client, cluster_id, database, user, s3_path, iam_role, query, ) with tempfile.TemporaryDirectory() as temp_dir: @@ -391,7 +377,6 @@ def unload_redshift_query_to_df( s3_path: str, iam_role: str, query: str, - drop_columns: Optional[List[str]] = None, ) -> pd.DataFrame: """ Unload Redshift Query results to S3 and get the results in Pandas DataFrame format """ table = unload_redshift_query_to_pa( @@ -403,7 +388,6 @@ def unload_redshift_query_to_df( s3_path, iam_role, query, - drop_columns, ) return table.to_pandas() diff --git a/sdk/python/feast/on_demand_feature_view.py b/sdk/python/feast/on_demand_feature_view.py index 4ec2b138b21..c89d122dded 100644 --- a/sdk/python/feast/on_demand_feature_view.py +++ b/sdk/python/feast/on_demand_feature_view.py @@ -1,12 +1,13 @@ import copy import functools from types import MethodType -from typing import Dict, List, Union, cast +from typing import Dict, List, Type, Union import dill import pandas as pd from feast import errors +from feast.base_feature_view import BaseFeatureView from feast.data_source import RequestDataSource from feast.errors import RegistryInferenceFailure from feast.feature import Feature @@ -30,7 +31,7 @@ from feast.value_type import ValueType -class OnDemandFeatureView: +class OnDemandFeatureView(BaseFeatureView): """ [Experimental] An OnDemandFeatureView defines on demand transformations on existing feature view values and request data. @@ -42,11 +43,11 @@ class OnDemandFeatureView: udf: User defined transformation function that takes as input pandas dataframes """ - name: str - features: List[Feature] + # TODO(adchia): remove inputs from proto and declaration inputs: Dict[str, Union[FeatureView, RequestDataSource]] + input_feature_views: Dict[str, FeatureView] + input_request_data_sources: Dict[str, RequestDataSource] udf: MethodType - projection: FeatureViewProjection @log_exceptions def __init__( @@ -59,15 +60,21 @@ def __init__( """ Creates an OnDemandFeatureView object. """ - - self.name = name - self.features = features + super().__init__(name, features) self.inputs = inputs + self.input_feature_views = {} + self.input_request_data_sources = {} + for input_ref, odfv_input in inputs.items(): + if isinstance(odfv_input, RequestDataSource): + self.input_request_data_sources[input_ref] = odfv_input + else: + self.input_feature_views[input_ref] = odfv_input + self.udf = udf - self.projection = FeatureViewProjection.from_definition(self) - def __hash__(self) -> int: - return hash((id(self), self.name)) + @property + def proto_class(self) -> Type[OnDemandFeatureViewProto]: + return OnDemandFeatureViewProto def __copy__(self): fv = OnDemandFeatureView( @@ -76,70 +83,6 @@ def __copy__(self): fv.projection = copy.copy(self.projection) return fv - def __getitem__(self, item): - assert isinstance(item, list) - - referenced_features = [] - for feature in self.features: - if feature.name in item: - referenced_features.append(feature) - - cp = self.__copy__() - cp.projection.features = referenced_features - - return self - - def with_name(self, name: str): - """ - Renames this on-demand feature view by returning a copy of this feature view with an alias - set for the feature view name. This rename operation is only used as part of query - operations and will not modify the underlying OnDemandFeatureView. - - Args: - name: Name to assign to the OnDemandFeatureView copy. - - Returns: - A copy of this OnDemandFeatureView with the name replaced with the 'name' input. - """ - cp = self.__copy__() - cp.projection.name_alias = name - - return cp - - def with_projection(self, feature_view_projection: FeatureViewProjection): - """ - Sets the feature view projection by returning a copy of this on-demand feature view - with its projection set to the given projection. A projection is an - object that stores the modifications to a feature view that is used during - query operations. - - Args: - feature_view_projection: The FeatureViewProjection object to link to this - OnDemandFeatureView. - - Returns: - A copy of this OnDemandFeatureView with its projection replaced with the - 'feature_view_projection' argument. - """ - if feature_view_projection.name != self.name: - raise ValueError( - f"The projection for the {self.name} FeatureView cannot be applied because it differs in name. " - f"The projection is named {feature_view_projection.name} and the name indicates which " - "FeatureView the projection is for." - ) - - for feature in feature_view_projection.features: - if feature not in self.features: - raise ValueError( - f"The projection for {self.name} cannot be applied because it contains {feature.name} which the " - "FeatureView doesn't have." - ) - - cp = self.__copy__() - cp.projection = feature_view_projection - - return cp - def to_proto(self) -> OnDemandFeatureViewProto: """ Converts an on demand feature view object to its protobuf representation. @@ -148,15 +91,12 @@ def to_proto(self) -> OnDemandFeatureViewProto: A OnDemandFeatureViewProto protobuf. """ inputs = {} - for feature_ref, input in self.inputs.items(): - if type(input) == FeatureView: - fv = cast(FeatureView, input) - inputs[feature_ref] = OnDemandInput(feature_view=fv.to_proto()) - else: - request_data_source = cast(RequestDataSource, input) - inputs[feature_ref] = OnDemandInput( - request_data_source=request_data_source.to_proto() - ) + for input_ref, fv in self.input_feature_views.items(): + inputs[input_ref] = OnDemandInput(feature_view=fv.to_proto()) + for input_ref, request_data_source in self.input_request_data_sources.items(): + inputs[input_ref] = OnDemandInput( + request_data_source=request_data_source.to_proto() + ) spec = OnDemandFeatureViewSpec( name=self.name, @@ -217,25 +157,28 @@ def from_proto(cls, on_demand_feature_view_proto: OnDemandFeatureViewProto): return on_demand_feature_view_obj + def get_request_data_schema(self) -> Dict[str, ValueType]: + schema: Dict[str, ValueType] = {} + for request_data_source in self.input_request_data_sources.values(): + schema.update(request_data_source.schema) + return schema + def get_transformed_features_df( - self, full_feature_names: bool, df_with_features: pd.DataFrame + self, df_with_features: pd.DataFrame ) -> pd.DataFrame: # Apply on demand transformations - # TODO(adchia): Include only the feature values from the specified input FVs in the ODFV. - # Copy over un-prefixed features even if not requested since transform may need it columns_to_cleanup = [] - if full_feature_names: - for input in self.inputs.values(): - if type(input) != FeatureView: - continue - input_fv = cast(FeatureView, input) - for feature in input_fv.features: - full_feature_ref = f"{input_fv.name}__{feature.name}" - if full_feature_ref in df_with_features.keys(): - df_with_features[feature.name] = df_with_features[ - full_feature_ref - ] - columns_to_cleanup.append(feature.name) + for input_fv in self.input_feature_views.values(): + for feature in input_fv.features: + full_feature_ref = f"{input_fv.name}__{feature.name}" + if full_feature_ref in df_with_features.keys(): + # Make sure the partial feature name is always present + df_with_features[feature.name] = df_with_features[full_feature_ref] + columns_to_cleanup.append(feature.name) + elif feature.name in df_with_features.keys(): + # Make sure the full feature name is always present + df_with_features[full_feature_ref] = df_with_features[feature.name] + columns_to_cleanup.append(full_feature_ref) # Compute transformed values and apply to each result row df_with_transformed_features = self.udf.__call__(df_with_features) @@ -248,25 +191,19 @@ def infer_features(self): """ Infers the set of features associated to this feature view from the input source. - Args: - config: Configuration object used to configure the feature store. - Raises: RegistryInferenceFailure: The set of features could not be inferred. """ df = pd.DataFrame() - for input in self.inputs.values(): - if type(input) == FeatureView: - feature_view = cast(FeatureView, input) - for feature in feature_view.features: - dtype = feast_value_type_to_pandas_type(feature.dtype) - df[f"{feature_view.name}__{feature.name}"] = pd.Series(dtype=dtype) - df[f"{feature.name}"] = pd.Series(dtype=dtype) - else: - request_data = cast(RequestDataSource, input) - for feature_name, feature_type in request_data.schema.items(): - dtype = feast_value_type_to_pandas_type(feature_type) - df[f"{feature_name}"] = pd.Series(dtype=dtype) + for feature_view in self.input_feature_views.values(): + for feature in feature_view.features: + dtype = feast_value_type_to_pandas_type(feature.dtype) + df[f"{feature_view.name}__{feature.name}"] = pd.Series(dtype=dtype) + df[f"{feature.name}"] = pd.Series(dtype=dtype) + for request_data in self.input_request_data_sources.values(): + for feature_name, feature_type in request_data.schema.items(): + dtype = feast_value_type_to_pandas_type(feature_type) + df[f"{feature_name}"] = pd.Series(dtype=dtype) output_df: pd.DataFrame = self.udf.__call__(df) inferred_features = [] for f, dt in zip(output_df.columns, output_df.dtypes): diff --git a/sdk/python/feast/registry.py b/sdk/python/feast/registry.py index 0681ecb6798..72cc68df262 100644 --- a/sdk/python/feast/registry.py +++ b/sdk/python/feast/registry.py @@ -14,10 +14,14 @@ from datetime import datetime, timedelta from pathlib import Path -from typing import List, Optional, Set +from typing import Dict, List, Optional from urllib.parse import urlparse +from google.protobuf.internal.containers import RepeatedCompositeFieldContainer +from proto import Message + from feast import importer +from feast.base_feature_view import BaseFeatureView from feast.entity import Entity from feast.errors import ( ConflictingFeatureViewNames, @@ -33,6 +37,7 @@ from feast.on_demand_feature_view import OnDemandFeatureView from feast.protos.feast.core.Registry_pb2 import Registry as RegistryProto from feast.repo_config import RegistryConfig +from feast.request_feature_view import RequestFeatureView REGISTRY_SCHEMA_VERSION = "1" @@ -290,7 +295,7 @@ def apply_feature_table( self.commit() def apply_feature_view( - self, feature_view: FeatureView, project: str, commit: bool = True + self, feature_view: BaseFeatureView, project: str, commit: bool = True ): """ Registers a single feature view with Feast @@ -300,74 +305,46 @@ def apply_feature_view( project: Feast project that this feature view belongs to commit: Whether the change should be persisted immediately """ - feature_view.is_valid() + feature_view.ensure_valid() feature_view_proto = feature_view.to_proto() feature_view_proto.spec.project = project self._prepare_registry_for_changes() assert self.cached_registry_proto - if feature_view.name in self._get_existing_on_demand_feature_view_names(): - raise ConflictingFeatureViewNames(feature_view.name) + self._check_conflicting_feature_view_names(feature_view) + existing_feature_views_of_same_type: RepeatedCompositeFieldContainer + if isinstance(feature_view, FeatureView): + existing_feature_views_of_same_type = ( + self.cached_registry_proto.feature_views + ) + elif isinstance(feature_view, OnDemandFeatureView): + existing_feature_views_of_same_type = ( + self.cached_registry_proto.on_demand_feature_views + ) + elif isinstance(feature_view, RequestFeatureView): + existing_feature_views_of_same_type = ( + self.cached_registry_proto.request_feature_views + ) + else: + raise ValueError(f"Unexpected feature view type: {type(feature_view)}") for idx, existing_feature_view_proto in enumerate( - self.cached_registry_proto.feature_views + existing_feature_views_of_same_type ): if ( existing_feature_view_proto.spec.name == feature_view_proto.spec.name and existing_feature_view_proto.spec.project == project - ): - if FeatureView.from_proto(existing_feature_view_proto) == feature_view: - return - else: - del self.cached_registry_proto.feature_views[idx] - break - - self.cached_registry_proto.feature_views.append(feature_view_proto) - if commit: - self.commit() - - def apply_on_demand_feature_view( - self, - on_demand_feature_view: OnDemandFeatureView, - project: str, - commit: bool = True, - ): - """ - Registers a single on demand feature view with Feast - - Args: - on_demand_feature_view: Feature view that will be registered - project: Feast project that this feature view belongs to - commit: Whether the change should be persisted immediately - """ - on_demand_feature_view_proto = on_demand_feature_view.to_proto() - on_demand_feature_view_proto.spec.project = project - self._prepare_registry_for_changes() - assert self.cached_registry_proto - - if on_demand_feature_view.name in self._get_existing_feature_view_names(): - raise ConflictingFeatureViewNames(on_demand_feature_view.name) - - for idx, existing_feature_view_proto in enumerate( - self.cached_registry_proto.on_demand_feature_views - ): - if ( - existing_feature_view_proto.spec.name - == on_demand_feature_view_proto.spec.name - and existing_feature_view_proto.spec.project == project ): if ( - OnDemandFeatureView.from_proto(existing_feature_view_proto) - == on_demand_feature_view + feature_view.__class__.from_proto(existing_feature_view_proto) + == feature_view ): return else: - del self.cached_registry_proto.on_demand_feature_views[idx] + del self.cached_registry_proto.feature_views[idx] break - self.cached_registry_proto.on_demand_feature_views.append( - on_demand_feature_view_proto - ) + existing_feature_views_of_same_type.append(feature_view_proto) if commit: self.commit() @@ -487,18 +464,40 @@ def list_feature_views( Args: allow_cache: Allow returning feature views from the cached registry - project: Filter feature tables based on project name + project: Filter feature views based on project name Returns: List of feature views """ registry_proto = self._get_registry_proto(allow_cache=allow_cache) - feature_views = [] + feature_views: List[FeatureView] = [] for feature_view_proto in registry_proto.feature_views: if feature_view_proto.spec.project == project: feature_views.append(FeatureView.from_proto(feature_view_proto)) return feature_views + def list_request_feature_views( + self, project: str, allow_cache: bool = False + ) -> List[RequestFeatureView]: + """ + Retrieve a list of request feature views from the registry + + Args: + allow_cache: Allow returning feature views from the cached registry + project: Filter feature views based on project name + + Returns: + List of feature views + """ + registry_proto = self._get_registry_proto(allow_cache=allow_cache) + feature_views: List[RequestFeatureView] = [] + for request_feature_view_proto in registry_proto.request_feature_views: + if request_feature_view_proto.spec.project == project: + feature_views.append( + RequestFeatureView.from_proto(request_feature_view_proto) + ) + return feature_views + def get_feature_table(self, name: str, project: str) -> FeatureTable: """ Retrieves a feature table. @@ -616,8 +615,46 @@ def delete_feature_view(self, name: str, project: str, commit: bool = True): self.commit() return + for idx, existing_request_feature_view_proto in enumerate( + self.cached_registry_proto.request_feature_views + ): + if ( + existing_request_feature_view_proto.spec.name == name + and existing_request_feature_view_proto.spec.project == project + ): + del self.cached_registry_proto.request_feature_views[idx] + if commit: + self.commit() + return + raise FeatureViewNotFoundException(name, project) + def delete_entity(self, name: str, project: str, commit: bool = True): + """ + Deletes an entity or raises an exception if not found. + + Args: + name: Name of entity + project: Feast project that this entity belongs to + commit: Whether the change should be persisted immediately + """ + self._prepare_registry_for_changes() + assert self.cached_registry_proto + + for idx, existing_entity_proto in enumerate( + self.cached_registry_proto.entities + ): + if ( + existing_entity_proto.spec.name == name + and existing_entity_proto.spec.project == project + ): + del self.cached_registry_proto.entities[idx] + if commit: + self.commit() + return + + raise EntityNotFoundException(name, project) + def commit(self): """Commits the state of the registry cache to the remote registry store.""" if self.cached_registry_proto: @@ -675,15 +712,22 @@ def _get_registry_proto(self, allow_cache: bool = False) -> RegistryProto: self.cache_being_updated = False return registry_proto - def _get_existing_feature_view_names(self) -> Set[str]: - assert self.cached_registry_proto - return set([fv.spec.name for fv in self.cached_registry_proto.feature_views]) + def _check_conflicting_feature_view_names(self, feature_view: BaseFeatureView): + name_to_fv_protos = self._existing_feature_view_names_to_fvs() + if feature_view.name in name_to_fv_protos: + if not isinstance( + name_to_fv_protos.get(feature_view.name), feature_view.proto_class + ): + raise ConflictingFeatureViewNames(feature_view.name) - def _get_existing_on_demand_feature_view_names(self) -> Set[str]: + def _existing_feature_view_names_to_fvs(self) -> Dict[str, Message]: assert self.cached_registry_proto - return set( - [ - odfv.spec.name - for odfv in self.cached_registry_proto.on_demand_feature_views - ] - ) + odfvs = { + fv.spec.name: fv + for fv in self.cached_registry_proto.on_demand_feature_views + } + fvs = {fv.spec.name: fv for fv in self.cached_registry_proto.feature_views} + request_fvs = { + fv.spec.name: fv for fv in self.cached_registry_proto.request_feature_views + } + return {**odfvs, **fvs, **request_fvs} diff --git a/sdk/python/feast/repo_config.py b/sdk/python/feast/repo_config.py index 3274e3b9a63..5bbe09d8e06 100644 --- a/sdk/python/feast/repo_config.py +++ b/sdk/python/feast/repo_config.py @@ -41,6 +41,7 @@ FEATURE_SERVER_CONFIG_CLASS_FOR_TYPE = { "aws_lambda": "feast.infra.feature_servers.aws_lambda.config.AwsLambdaFeatureServerConfig", + "gcp_cloudrun": "feast.infra.feature_servers.gcp_cloudrun.config.GcpCloudRunFeatureServerConfig", } @@ -174,7 +175,6 @@ def _validate_online_store_config(cls, values): raise ValidationError( [ErrorWrapper(e, loc="online_store")], model=RepoConfig, ) - return values @root_validator(pre=True) @@ -277,14 +277,7 @@ def write_to_path(self, repo_path: Path): config_path = repo_path / "feature_store.yaml" with open(config_path, mode="w") as f: yaml.dump( - yaml.safe_load( - self.json( - exclude={"repo_path"}, - exclude_none=True, - exclude_unset=True, - exclude_defaults=True, - ) - ), + yaml.safe_load(self.json(exclude={"repo_path"}, exclude_unset=True,)), f, sort_keys=False, ) diff --git a/sdk/python/feast/repo_operations.py b/sdk/python/feast/repo_operations.py index 9378cbdf318..7fbcecc6c25 100644 --- a/sdk/python/feast/repo_operations.py +++ b/sdk/python/feast/repo_operations.py @@ -5,20 +5,21 @@ import sys from importlib.abc import Loader from pathlib import Path -from typing import List, NamedTuple, Set, Tuple, Union +from typing import List, NamedTuple, Set, Tuple, Union, cast import click from click.exceptions import BadParameter from feast import Entity, FeatureTable +from feast.base_feature_view import BaseFeatureView from feast.feature_service import FeatureService -from feast.feature_store import FeatureStore, _validate_feature_views +from feast.feature_store import FeatureStore from feast.feature_view import FeatureView -from feast.infra.provider import get_provider from feast.names import adjectives, animals from feast.on_demand_feature_view import OnDemandFeatureView from feast.registry import Registry from feast.repo_config import RepoConfig +from feast.request_feature_view import RequestFeatureView from feast.usage import log_exceptions_and_usage @@ -34,6 +35,7 @@ class ParsedRepo(NamedTuple): feature_tables: Set[FeatureTable] feature_views: Set[FeatureView] on_demand_feature_views: Set[OnDemandFeatureView] + request_feature_views: Set[RequestFeatureView] entities: Set[Entity] feature_services: Set[FeatureService] @@ -99,6 +101,7 @@ def parse_repo(repo_root: Path) -> ParsedRepo: feature_views=set(), feature_services=set(), on_demand_feature_views=set(), + request_feature_views=set(), ) for repo_file in get_repo_files(repo_root): @@ -116,6 +119,8 @@ def parse_repo(repo_root: Path) -> ParsedRepo: res.feature_services.add(obj) elif isinstance(obj, OnDemandFeatureView): res.on_demand_feature_views.add(obj) + elif isinstance(obj, RequestFeatureView): + res.request_feature_views.add(obj) return res @@ -136,7 +141,6 @@ def apply_total(repo_config: RepoConfig, repo_path: Path, skip_source_validation registry._initialize_registry() sys.dont_write_bytecode = True repo = parse_repo(repo_path) - _validate_feature_views(list(repo.feature_views)) if not skip_source_validation: data_sources = [t.batch_source for t in repo.feature_views] @@ -144,6 +148,7 @@ def apply_total(repo_config: RepoConfig, repo_path: Path, skip_source_validation for data_source in data_sources: data_source.validate(store.config) + # For each object in the registry, determine whether it should be kept or deleted. entities_to_keep, entities_to_delete = _tag_registry_entities_for_keep_delete( project, registry, repo ) @@ -157,50 +162,60 @@ def apply_total(repo_config: RepoConfig, repo_path: Path, skip_source_validation tables_to_keep, tables_to_delete = _tag_registry_tables_for_keep_delete( project, registry, repo ) - (services_to_keep, services_to_delete,) = _tag_registry_services_for_keep_delete( + services_to_keep, services_to_delete = _tag_registry_services_for_keep_delete( project, registry, repo ) sys.dont_write_bytecode = False - # Delete views that should not exist - for registry_view in views_to_delete: - registry.delete_feature_view(registry_view.name, project=project, commit=False) + # Apply all changes to the registry and infrastructure. + all_to_apply: List[ + Union[ + Entity, BaseFeatureView, FeatureService, OnDemandFeatureView, FeatureTable + ] + ] = [] + all_to_apply.extend(entities_to_keep) + all_to_apply.extend(views_to_keep) + all_to_apply.extend(services_to_keep) + all_to_apply.extend(odfvs_to_keep) + all_to_apply.extend(tables_to_keep) + all_to_delete: List[ + Union[ + Entity, BaseFeatureView, FeatureService, OnDemandFeatureView, FeatureTable + ] + ] = [] + all_to_delete.extend(entities_to_delete) + all_to_delete.extend(views_to_delete) + all_to_delete.extend(services_to_delete) + all_to_delete.extend(odfvs_to_delete) + all_to_delete.extend(tables_to_delete) + + store.apply( + all_to_apply, objects_to_delete=all_to_delete, partial=False, commit=False + ) + + for entity in entities_to_delete: click.echo( - f"Deleted feature view {Style.BRIGHT + Fore.GREEN}{registry_view.name}{Style.RESET_ALL} from registry" + f"Deleted entity {Style.BRIGHT + Fore.GREEN}{entity.name}{Style.RESET_ALL} from registry" ) - - # Delete feature services that should not exist - for feature_service_to_delete in services_to_delete: - registry.delete_feature_service( - feature_service_to_delete.name, project=project, commit=False + for view in views_to_delete: + click.echo( + f"Deleted feature view {Style.BRIGHT + Fore.GREEN}{view.name}{Style.RESET_ALL} from registry" ) + for odfv in odfvs_to_delete: click.echo( - f"Deleted feature service {Style.BRIGHT + Fore.GREEN}{feature_service_to_delete.name}{Style.RESET_ALL} " - f"from registry" + f"Deleted on demand feature view {Style.BRIGHT + Fore.GREEN}{odfv.name}{Style.RESET_ALL} from registry" ) - - # Delete tables that should not exist - for registry_table in tables_to_delete: - registry.delete_feature_table( - registry_table.name, project=project, commit=False + for table in tables_to_delete: + click.echo( + f"Deleted feature table {Style.BRIGHT + Fore.GREEN}{table.name}{Style.RESET_ALL} from registry" ) + for feature_service in services_to_delete: click.echo( - f"Deleted feature table {Style.BRIGHT + Fore.GREEN}{registry_table.name}{Style.RESET_ALL} from registry" + f"Deleted feature service {Style.BRIGHT + Fore.GREEN}{feature_service.name}{Style.RESET_ALL} " + f"from registry" ) - # TODO: delete entities from the registry too - - # Add / update views + entities + services - all_to_apply: List[ - Union[Entity, FeatureView, FeatureService, OnDemandFeatureView] - ] = [] - all_to_apply.extend(entities_to_keep) - all_to_apply.extend(views_to_keep) - all_to_apply.extend(services_to_keep) - all_to_apply.extend(odfvs_to_keep) - # TODO: delete odfvs - store.apply(all_to_apply, commit=False) for entity in entities_to_keep: click.echo( f"Registered entity {Style.BRIGHT + Fore.GREEN}{entity.name}{Style.RESET_ALL}" @@ -219,19 +234,23 @@ def apply_total(repo_config: RepoConfig, repo_path: Path, skip_source_validation ) # Create tables that should exist for table in tables_to_keep: - registry.apply_feature_table(table, project, commit=False) click.echo( f"Registered feature table {Style.BRIGHT + Fore.GREEN}{table.name}{Style.RESET_ALL}" ) - infra_provider = get_provider(repo_config, repo_path) + views_to_keep_in_infra = [ + view for view in views_to_keep if isinstance(view, FeatureView) + ] for name in [view.name for view in repo.feature_tables] + [ - table.name for table in repo.feature_views + table.name for table in views_to_keep_in_infra ]: click.echo( f"Deploying infrastructure for {Style.BRIGHT + Fore.GREEN}{name}{Style.RESET_ALL}" ) - for name in [view.name for view in views_to_delete] + [ + views_to_delete_from_infra = [ + view for view in views_to_delete if isinstance(view, FeatureView) + ] + for name in [view.name for view in views_to_delete_from_infra] + [ table.name for table in tables_to_delete ]: click.echo( @@ -239,21 +258,6 @@ def apply_total(repo_config: RepoConfig, repo_path: Path, skip_source_validation ) # TODO: consider echoing also entities being deployed/removed - all_to_delete: List[Union[FeatureTable, FeatureView]] = [] - all_to_delete.extend(tables_to_delete) - all_to_delete.extend(views_to_delete) - all_to_keep: List[Union[FeatureTable, FeatureView]] = [] - all_to_keep.extend(tables_to_keep) - all_to_keep.extend(views_to_delete) - infra_provider.update_infra( - project, - tables_to_delete=all_to_delete, - tables_to_keep=all_to_keep, - entities_to_delete=list(entities_to_delete), - entities_to_keep=list(entities_to_keep), - partial=False, - ) - # Commit the update to the registry only after successful infra update registry.commit() @@ -272,9 +276,11 @@ def _tag_registry_entities_for_keep_delete( def _tag_registry_views_for_keep_delete( project: str, registry: Registry, repo: ParsedRepo -) -> Tuple[Set[FeatureView], Set[FeatureView]]: - views_to_keep: Set[FeatureView] = repo.feature_views - views_to_delete: Set[FeatureView] = set() +) -> Tuple[Set[BaseFeatureView], Set[BaseFeatureView]]: + views_to_keep: Set[BaseFeatureView] = cast(Set[BaseFeatureView], repo.feature_views) + for request_fv in repo.request_feature_views: + views_to_keep.add(request_fv) + views_to_delete: Set[BaseFeatureView] = set() repo_feature_view_names = set(t.name for t in repo.feature_views) for registry_view in registry.list_feature_views(project=project): if registry_view.name not in repo_feature_view_names: diff --git a/sdk/python/feast/request_feature_view.py b/sdk/python/feast/request_feature_view.py new file mode 100644 index 00000000000..5d716f2f8d2 --- /dev/null +++ b/sdk/python/feast/request_feature_view.py @@ -0,0 +1,91 @@ +import copy +from typing import Type + +from feast.base_feature_view import BaseFeatureView +from feast.data_source import RequestDataSource +from feast.feature import Feature +from feast.feature_view_projection import FeatureViewProjection +from feast.protos.feast.core.RequestFeatureView_pb2 import ( + RequestFeatureView as RequestFeatureViewProto, +) +from feast.protos.feast.core.RequestFeatureView_pb2 import RequestFeatureViewSpec +from feast.usage import log_exceptions + + +class RequestFeatureView(BaseFeatureView): + """ + [Experimental] An RequestFeatureView defines a feature that is available from the inference request. + + Args: + name: Name of the group of features. + request_data_source: Request data source that specifies the schema and features + """ + + request_data_source: RequestDataSource + + @log_exceptions + def __init__( + self, name: str, request_data_source: RequestDataSource, + ): + """ + Creates an RequestFeatureView object. + """ + super().__init__( + name=name, + features=[ + Feature(name=name, dtype=dtype) + for name, dtype in request_data_source.schema.items() + ], + ) + self.request_data_source = request_data_source + + @property + def proto_class(self) -> Type[RequestFeatureViewProto]: + return RequestFeatureViewProto + + def to_proto(self) -> RequestFeatureViewProto: + """ + Converts an request feature view object to its protobuf representation. + + Returns: + A RequestFeatureViewProto protobuf. + """ + spec = RequestFeatureViewSpec( + name=self.name, request_data_source=self.request_data_source.to_proto() + ) + + return RequestFeatureViewProto(spec=spec) + + @classmethod + def from_proto(cls, request_feature_view_proto: RequestFeatureViewProto): + """ + Creates a request feature view from a protobuf representation. + + Args: + request_feature_view_proto: A protobuf representation of an request feature view. + + Returns: + A RequestFeatureView object based on the request feature view protobuf. + """ + + request_feature_view_obj = cls( + name=request_feature_view_proto.spec.name, + request_data_source=RequestDataSource.from_proto( + request_feature_view_proto.spec.request_data_source + ), + ) + + # FeatureViewProjections are not saved in the RequestFeatureView proto. + # Create the default projection. + request_feature_view_obj.projection = FeatureViewProjection.from_definition( + request_feature_view_obj + ) + + return request_feature_view_obj + + def __copy__(self): + fv = RequestFeatureView( + name=self.name, request_data_source=self.request_data_source + ) + fv.projection = copy.copy(self.projection) + return fv diff --git a/sdk/python/feast/transformation_server.py b/sdk/python/feast/transformation_server.py new file mode 100644 index 00000000000..c7e61f9d17e --- /dev/null +++ b/sdk/python/feast/transformation_server.py @@ -0,0 +1,74 @@ +import logging +import sys +from concurrent import futures + +import grpc +import pyarrow as pa +from grpc_reflection.v1alpha import reflection + +from feast.errors import OnDemandFeatureViewNotFoundException +from feast.feature_store import FeatureStore +from feast.protos.feast.serving.TransformationService_pb2 import ( + DESCRIPTOR, + TRANSFORMATION_SERVICE_TYPE_PYTHON, + GetTransformationServiceInfoResponse, + TransformFeaturesResponse, + ValueType, +) +from feast.protos.feast.serving.TransformationService_pb2_grpc import ( + TransformationServiceServicer, + add_TransformationServiceServicer_to_server, +) +from feast.version import get_version + +log = logging.getLogger(__name__) + + +class TransformationServer(TransformationServiceServicer): + def __init__(self, fs: FeatureStore) -> None: + super().__init__() + self.fs = fs + + def GetTransformationServiceInfo(self, request, context): + response = GetTransformationServiceInfoResponse( + type=TRANSFORMATION_SERVICE_TYPE_PYTHON, + transformation_service_type_details=f"Python: {sys.version}, Feast: {get_version()}", + ) + return response + + def TransformFeatures(self, request, context): + try: + odfv = self.fs.get_on_demand_feature_view( + request.on_demand_feature_view_name + ) + except OnDemandFeatureViewNotFoundException: + context.set_code(grpc.StatusCode.INVALID_ARGUMENT) + raise + + df = pa.ipc.open_file(request.transformation_input.arrow_value).read_pandas() + + result_df = odfv.get_transformed_features_df(df) + result_arrow = pa.Table.from_pandas(result_df) + sink = pa.BufferOutputStream() + writer = pa.ipc.new_file(sink, result_arrow.schema) + writer.write_table(result_arrow) + writer.close() + + buf = sink.getvalue().to_pybytes() + + return TransformFeaturesResponse( + transformation_output=ValueType(arrow_value=buf) + ) + + +def start_server(store: FeatureStore, port: int): + server = grpc.server(futures.ThreadPoolExecutor(max_workers=10)) + add_TransformationServiceServicer_to_server(TransformationServer(store), server) + service_names_available_for_reflection = ( + DESCRIPTOR.services_by_name["TransformationService"].full_name, + reflection.SERVICE_NAME, + ) + reflection.enable_server_reflection(service_names_available_for_reflection, server) + server.add_insecure_port(f"[::]:{port}") + server.start() + server.wait_for_termination() diff --git a/sdk/python/feast/type_map.py b/sdk/python/feast/type_map.py index 79fd7ed1b10..c615d2f50e9 100644 --- a/sdk/python/feast/type_map.py +++ b/sdk/python/feast/type_map.py @@ -114,7 +114,7 @@ def python_type_to_feast_value_type( Returns: Feast Value Type """ - type_name = type_name or type(value).__name__ + type_name = (type_name or type(value).__name__).lower() type_map = { "int": ValueType.INT64, @@ -131,7 +131,7 @@ def python_type_to_feast_value_type( "int8": ValueType.INT32, "bool": ValueType.BOOL, "timedelta": ValueType.UNIX_TIMESTAMP, - "Timestamp": ValueType.UNIX_TIMESTAMP, + "timestamp": ValueType.UNIX_TIMESTAMP, "datetime": ValueType.UNIX_TIMESTAMP, "datetime64[ns]": ValueType.UNIX_TIMESTAMP, "datetime64[ns, tz]": ValueType.UNIX_TIMESTAMP, @@ -402,6 +402,7 @@ def bq_to_feast_value_type(bq_type_as_str: str) -> ValueType: "FLOAT64": ValueType.DOUBLE, "BYTES": ValueType.BYTES, "BOOL": ValueType.BOOL, + "BOOLEAN": ValueType.BOOL, # legacy sql data type "ARRAY": ValueType.INT64_LIST, "ARRAY": ValueType.DOUBLE_LIST, "ARRAY": ValueType.STRING_LIST, diff --git a/sdk/python/feast/usage.py b/sdk/python/feast/usage.py index 855f622fa70..1a2bf2e2907 100644 --- a/sdk/python/feast/usage.py +++ b/sdk/python/feast/usage.py @@ -12,18 +12,20 @@ # See the License for the specific language governing permissions and # limitations under the License. import concurrent.futures +import contextlib import contextvars -import enum +import dataclasses +import hashlib import logging import os +import platform import sys +import typing import uuid -from collections import defaultdict from datetime import datetime from functools import wraps from os.path import expanduser, join from pathlib import Path -from typing import List, Optional, Tuple, Union import requests @@ -31,215 +33,346 @@ from feast.version import get_version USAGE_ENDPOINT = "https://usage.feast.dev" + _logger = logging.getLogger(__name__) +_executor = concurrent.futures.ThreadPoolExecutor(max_workers=1) -executor = concurrent.futures.ThreadPoolExecutor(max_workers=1) -call_stack: contextvars.ContextVar = contextvars.ContextVar("call_stack", default=[]) +_is_enabled = os.getenv(FEAST_USAGE, default="True") == "True" +_constant_attributes = { + "session_id": str(uuid.uuid4()), + "installation_id": None, + "version": get_version(), + "python_version": platform.python_version(), + "platform": platform.platform(), + "env_signature": hashlib.md5( + ",".join( + sorted([k for k in os.environ.keys() if not k.startswith("FEAST")]) + ).encode() + ).hexdigest(), +} -@enum.unique -class UsageEvent(enum.Enum): - """ - An event meant to be logged - """ - UNKNOWN = 0 - APPLY_WITH_ODFV = 1 - GET_HISTORICAL_FEATURES_WITH_ODFV = 2 - GET_ONLINE_FEATURES_WITH_ODFV = 3 +@dataclasses.dataclass +class FnCall: + fn_name: str + id: str - def __str__(self): - return self.name.lower() + start: datetime + end: typing.Optional[datetime] = None + parent_id: typing.Optional[str] = None -class Usage: - def __init__(self): - self._usage_enabled: bool = False - self._is_test = os.getenv("FEAST_IS_USAGE_TEST", "False") == "True" - self._usage_counter = defaultdict(lambda: 0) - self.check_env_and_configure() - - def check_env_and_configure(self): - usage_enabled = ( - os.getenv(FEAST_USAGE, default="True") == "True" - ) # written this way to turn the env var string into a boolean - - # Check if it changed - if usage_enabled != self._usage_enabled: - self._usage_enabled = usage_enabled - - if self._usage_enabled: - try: - feast_home_dir = join(expanduser("~"), ".feast") - Path(feast_home_dir).mkdir(exist_ok=True) - usage_filepath = join(feast_home_dir, "usage") - - if os.path.exists(usage_filepath): - with open(usage_filepath, "r") as f: - self._usage_id = f.read() - else: - self._usage_id = str(uuid.uuid4()) - - with open(usage_filepath, "w") as f: - f.write(self._usage_id) - print( - "Feast is an open source project that collects anonymized error reporting and usage statistics. To opt out or learn" - " more see https://docs.feast.dev/reference/usage" - ) - except Exception as e: - _logger.debug(f"Unable to configure usage {e}") + +class Sampler: + def should_record(self, event) -> bool: + raise NotImplementedError @property - def usage_id(self) -> Optional[str]: - if os.getenv("FEAST_FORCE_USAGE_UUID"): - return os.getenv("FEAST_FORCE_USAGE_UUID") - return self._usage_id - - def _send_usage_request(self, json): - try: - future = executor.submit(requests.post, USAGE_ENDPOINT, json=json) - if self._is_test: - concurrent.futures.wait([future]) - except Exception as e: - if self._is_test: - raise e - else: - pass - - def log_function(self, function_name: str): - self.check_env_and_configure() - if self._usage_enabled and self.usage_id: - if "get_online_features" in call_stack.get() and not self.should_log_for_get_online_features_event( - "get_online_features" - ): - return - json = { - "function_name": function_name, - "usage_id": self.usage_id, - "timestamp": datetime.utcnow().isoformat(), - "version": get_version(), - "os": sys.platform, - "is_test": self._is_test, - } - self._send_usage_request(json) - - def increment_event_count(self, event_name: Union[UsageEvent, str]): - self._usage_counter[event_name] += 1 - - def should_log_for_get_online_features_event(self, event_name: str): - if self._usage_counter[event_name] % 10000 != 2: - return False - self._usage_counter[event_name] = 2 # avoid overflow + def priority(self): + return 0 + + +class AlwaysSampler(Sampler): + def should_record(self, event) -> bool: return True - def log_event(self, event: UsageEvent): - self.check_env_and_configure() - if self._usage_enabled and self.usage_id: - event_name = str(event) - if ( - event == UsageEvent.GET_ONLINE_FEATURES_WITH_ODFV - and not self.should_log_for_get_online_features_event(event_name) - ): - return - json = { - "event_name": event_name, - "usage_id": self.usage_id, - "timestamp": datetime.utcnow().isoformat(), - "version": get_version(), - "os": sys.platform, - "is_test": self._is_test, - } - self._send_usage_request(json) - - def log_exception(self, error_type: str, traceback: List[Tuple[str, int, str]]): - self.check_env_and_configure() - if self._usage_enabled and self.usage_id: - json = { - "error_type": error_type, - "traceback": traceback, - "usage_id": self.usage_id, - "version": get_version(), - "os": sys.platform, - "is_test": self._is_test, - } - try: - requests.post(USAGE_ENDPOINT, json=json) - except Exception as e: - if self._is_test: - raise e - else: - pass - return - - -def log_exceptions(func): - @wraps(func) - def exception_logging_wrapper(*args, **kwargs): - try: - call_stack.set(call_stack.get() + [func.__name__]) - result = func(*args, **kwargs) - except Exception as e: - error_type = type(e).__name__ - trace_to_log = [] - tb = e.__traceback__ - while tb is not None: - trace_to_log.append( - ( - _trim_filename(tb.tb_frame.f_code.co_filename), - tb.tb_lineno, - tb.tb_frame.f_code.co_name, - ) + +class RatioSampler(Sampler): + MAX_COUNTER = (1 << 32) - 1 + + def __init__(self, ratio): + assert 0 < ratio <= 1, "Ratio must be within (0, 1]" + self.ratio = ratio + self.total_counter = 0 + self.sampled_counter = 0 + + def should_record(self, event) -> bool: + self.total_counter += 1 + if self.total_counter == self.MAX_COUNTER: + self.total_counter = 1 + self.sampled_counter = 1 + + decision = self.sampled_counter < self.ratio * self.total_counter + self.sampled_counter += int(decision) + return decision + + @property + def priority(self): + return int(1 / self.ratio) + + +class UsageContext: + attributes: typing.Dict[str, typing.Any] + + call_stack: typing.List[FnCall] + completed_calls: typing.List[FnCall] + + exception: typing.Optional[Exception] = None + traceback: typing.Optional[typing.Tuple[str, int, str]] = None + + sampler: Sampler = AlwaysSampler() + + def __init__(self): + self.attributes = {} + self.call_stack = [] + self.completed_calls = [] + + +_context = contextvars.ContextVar("usage_context", default=UsageContext()) + + +def _set_installation_id(): + if os.getenv("FEAST_FORCE_USAGE_UUID"): + _constant_attributes["installation_id"] = os.getenv("FEAST_FORCE_USAGE_UUID") + _constant_attributes["installation_ts"] = datetime.utcnow().isoformat() + return + + feast_home_dir = join(expanduser("~"), ".feast") + installation_timestamp = datetime.utcnow() + + try: + Path(feast_home_dir).mkdir(exist_ok=True) + usage_filepath = join(feast_home_dir, "usage") + + if os.path.exists(usage_filepath): + installation_timestamp = datetime.utcfromtimestamp( + os.path.getmtime(usage_filepath) + ) + with open(usage_filepath, "r") as f: + installation_id = f.read() + else: + installation_id = str(uuid.uuid4()) + + with open(usage_filepath, "w") as f: + f.write(installation_id) + print( + "Feast is an open source project that collects " + "anonymized error reporting and usage statistics. To opt out or learn" + " more see https://docs.feast.dev/reference/usage" + ) + except OSError as e: + _logger.debug(f"Unable to configure usage {e}") + installation_id = "undefined" + + _constant_attributes["installation_id"] = installation_id + _constant_attributes["installation_ts"] = installation_timestamp.isoformat() + + +_set_installation_id() + + +def _export(event: typing.Dict[str, typing.Any]): + _executor.submit(requests.post, USAGE_ENDPOINT, json=event, timeout=30) + + +def _produce_event(ctx: UsageContext): + is_test = bool({"pytest", "unittest"} & sys.modules.keys()) + event = { + "timestamp": datetime.utcnow().isoformat(), + "is_test": is_test, + "is_webserver": ( + not is_test and bool({"uwsgi", "gunicorn", "fastapi"} & sys.modules.keys()) + ), + "calls": [ + dict( + fn_name=c.fn_name, + id=c.id, + parent_id=c.parent_id, + start=c.start and c.start.isoformat(), + end=c.end and c.end.isoformat(), + ) + for c in reversed(ctx.completed_calls) + ], + "entrypoint": ctx.completed_calls[-1].fn_name, + "exception": repr(ctx.exception) if ctx.exception else None, + "traceback": ctx.traceback if ctx.exception else None, + **_constant_attributes, + } + event.update(ctx.attributes) + + if ctx.sampler and not ctx.sampler.should_record(event): + return + + _export(event) + + +@contextlib.contextmanager +def tracing_span(name): + """ + Context manager for wrapping heavy parts of code in tracing span + """ + if _is_enabled: + ctx = _context.get() + if not ctx.call_stack: + raise RuntimeError("tracing_span must be called in usage context") + + last_call = ctx.call_stack[-1] + fn_call = FnCall( + id=uuid.uuid4().hex, + parent_id=last_call.id, + fn_name=f"{last_call.fn_name}.{name}", + start=datetime.utcnow(), + ) + try: + yield + finally: + if _is_enabled: + fn_call.end = datetime.utcnow() + ctx.completed_calls.append(fn_call) + + +def log_exceptions_and_usage(*args, **attrs): + """ + This function decorator enables three components: + 1. Error tracking + 2. Usage statistic collection + 3. Time profiling + + This data is being collected, anonymized and sent to Feast Developers. + All events from nested decorated functions are being grouped into single event + to build comprehensive context useful for profiling and error tracking. + + Usage example (will result in one output event): + @log_exceptions_and_usage + def fn(...): + nested() + + @log_exceptions_and_usage(attr='value') + def nested(...): + deeply_nested() + + @log_exceptions_and_usage(attr2='value2', sample=RateSampler(rate=0.1)) + def deeply_nested(...): + ... + """ + sampler = attrs.pop("sampler", AlwaysSampler()) + + def decorator(func): + if not _is_enabled: + return func + + @wraps(func) + def wrapper(*args, **kwargs): + ctx = _context.get() + ctx.call_stack.append( + FnCall( + id=uuid.uuid4().hex, + parent_id=ctx.call_stack[-1].id if ctx.call_stack else None, + fn_name=_fn_fullname(func), + start=datetime.utcnow(), ) - tb = tb.tb_next - usage.log_exception(error_type, trace_to_log) - raise - finally: - if len(call_stack.get()) > 0: - call_stack.set(call_stack.get()[:-1]) - return result - - return exception_logging_wrapper - - -def log_exceptions_and_usage(func): - @wraps(func) - def exception_logging_wrapper(*args, **kwargs): - try: - call_stack.set(call_stack.get() + [func.__name__]) - usage.increment_event_count(func.__name__) - result = func(*args, **kwargs) - usage.log_function(func.__name__) - except Exception as e: - error_type = type(e).__name__ - trace_to_log = [] - tb = e.__traceback__ - while tb is not None: - trace_to_log.append( - ( - _trim_filename(tb.tb_frame.f_code.co_filename), - tb.tb_lineno, - tb.tb_frame.f_code.co_name, - ) + ) + ctx.attributes.update(attrs) + + try: + return func(*args, **kwargs) + except Exception: + if ctx.exception: + # exception was already recorded + raise + + _, exc, traceback = sys.exc_info() + ctx.exception = exc + ctx.traceback = _trace_to_log(traceback) + + if traceback: + raise exc.with_traceback(traceback) + + raise exc + finally: + last_call = ctx.call_stack.pop(-1) + last_call.end = datetime.utcnow() + ctx.completed_calls.append(last_call) + ctx.sampler = ( + sampler if sampler.priority > ctx.sampler.priority else ctx.sampler ) - tb = tb.tb_next - usage.log_exception(error_type, trace_to_log) - raise - finally: - if len(call_stack.get()) > 0: - call_stack.set(call_stack.get()[:-1]) - return result - return exception_logging_wrapper + if not ctx.call_stack: + # we reached the root of the stack + _context.set(UsageContext()) # reset context to default values + _produce_event(ctx) + + return wrapper + + if args: + return decorator(args[0]) + + return decorator + + +def log_exceptions(*args, **attrs): + """ + Function decorator that track errors and send them to Feast Developers + """ + + def decorator(func): + if not _is_enabled: + return func + + @wraps(func) + def wrapper(*args, **kwargs): + if _context.get().call_stack: + # we're already inside usage context + # let it handle exception + return func(*args, **kwargs) + + fn_call = FnCall( + id=uuid.uuid4().hex, fn_name=_fn_fullname(func), start=datetime.utcnow() + ) + try: + return func(*args, **kwargs) + except Exception: + _, exc, traceback = sys.exc_info() + fn_call.end = datetime.utcnow() -def log_event(event: UsageEvent): - usage.increment_event_count(event) - usage.log_event(event) + ctx = UsageContext() + ctx.exception = exc + ctx.traceback = _trace_to_log(traceback) + ctx.attributes = attrs + ctx.completed_calls.append(fn_call) + _produce_event(ctx) + + if traceback: + raise exc.with_traceback(traceback) + + raise exc + + return wrapper + + if args: + return decorator(args[0]) + + return decorator + + +def set_usage_attribute(name, value): + """ + Extend current context with custom attribute + """ + ctx = _context.get() + ctx.attributes[name] = value def _trim_filename(filename: str) -> str: return filename.split("/")[-1] -# Single global usage object -usage = Usage() +def _fn_fullname(fn: typing.Callable): + return fn.__module__ + "." + fn.__qualname__ + + +def _trace_to_log(traceback): + log = [] + while traceback is not None: + log.append( + ( + _trim_filename(traceback.tb_frame.f_code.co_filename), + traceback.tb_lineno, + traceback.tb_frame.f_code.co_name, + ) + ) + traceback = traceback.tb_next + + return log diff --git a/sdk/python/setup.py b/sdk/python/setup.py index 08a0c1db008..b5eeeecedfc 100644 --- a/sdk/python/setup.py +++ b/sdk/python/setup.py @@ -47,13 +47,14 @@ "google-api-core>=1.23.0", "googleapis-common-protos==1.52.*", "grpcio>=1.34.0", + "grpcio-reflection>=1.34.0" "Jinja2>=2.0.0", "jsonschema", "mmh3", "pandas>=1.0.0", "pandavro==1.5.*", "protobuf>=3.10", - "pyarrow>=2.0.0", + "pyarrow>=4.0.0", "pydantic>=1.0.0", "PyYAML>=5.4.*", "tabulate==0.8.*", @@ -65,10 +66,11 @@ ] GCP_REQUIRED = [ - "google-cloud-bigquery>=2.0.*", + "proto-plus<1.19.7", + "google-cloud-bigquery>=2.28.1", "google-cloud-bigquery-storage >= 2.0.0", "google-cloud-datastore>=2.1.*", - "google-cloud-storage>=1.34.*", + "google-cloud-storage>=1.34.*,<1.41", "google-cloud-core==1.4.*", ] @@ -111,10 +113,11 @@ "firebase-admin==4.5.2", "pre-commit", "assertpy==1.1", - "google-cloud-bigquery>=2.0.*", + "proto-plus<1.19.7", + "google-cloud-bigquery>=2.28.1", "google-cloud-bigquery-storage >= 2.0.0", "google-cloud-datastore>=2.1.*", - "google-cloud-storage>=1.20.*", + "google-cloud-storage>=1.20.*,<1.41", "google-cloud-core==1.4.*", "redis-py-cluster==2.1.2", "boto3==1.17.*", @@ -230,7 +233,7 @@ def run(self): ], entry_points={"console_scripts": ["feast=feast.cli:cli"]}, use_scm_version=use_scm_version, - setup_requires=["setuptools_scm", "grpcio", "grpcio-tools==1.34.0", "mypy-protobuf", "sphinx!=4.0.0"], + setup_requires=["setuptools_scm", "grpcio", "grpcio-tools==1.34.0", "mypy-protobuf==1.*", "sphinx!=4.0.0"], package_data={ "": [ "protos/feast/**/*.proto", diff --git a/sdk/python/tests/benchmarks/test_benchmark_universal_online_retrieval.py b/sdk/python/tests/benchmarks/test_benchmark_universal_online_retrieval.py index b124cd5c762..0651383880f 100644 --- a/sdk/python/tests/benchmarks/test_benchmark_universal_online_retrieval.py +++ b/sdk/python/tests/benchmarks/test_benchmark_universal_online_retrieval.py @@ -6,7 +6,11 @@ from tests.integration.feature_repos.repo_configuration import ( construct_universal_feature_views, ) -from tests.integration.feature_repos.universal.entities import customer, driver +from tests.integration.feature_repos.universal.entities import ( + customer, + driver, + location, +) @pytest.mark.benchmark @@ -24,7 +28,7 @@ def test_online_retrieval(environment, universal_data_sources, benchmark): feast_objects = [] feast_objects.extend(feature_views.values()) - feast_objects.extend([driver(), customer(), feature_service]) + feast_objects.extend([driver(), customer(), location(), feature_service]) fs.apply(feast_objects) fs.materialize(environment.start_date, environment.end_date) diff --git a/sdk/python/tests/conftest.py b/sdk/python/tests/conftest.py index df069775a1a..63bf4be6294 100644 --- a/sdk/python/tests/conftest.py +++ b/sdk/python/tests/conftest.py @@ -21,8 +21,12 @@ from _pytest.nodes import Item from tests.data.data_creator import create_dataset +from tests.integration.feature_repos.integration_test_repo_config import ( + IntegrationTestRepoConfig, +) from tests.integration.feature_repos.repo_configuration import ( FULL_REPO_CONFIGS, + REDIS_CONFIG, Environment, construct_test_environment, construct_universal_data_sources, @@ -40,6 +44,9 @@ def pytest_configure(config): "markers", "integration: mark test that has external dependencies" ) config.addinivalue_line("markers", "benchmark: mark benchmarking tests") + config.addinivalue_line( + "markers", "universal: mark tests that use the universal feature repo" + ) def pytest_addoption(parser): @@ -52,11 +59,15 @@ def pytest_addoption(parser): parser.addoption( "--benchmark", action="store_true", default=False, help="Run benchmark tests", ) + parser.addoption( + "--universal", action="store_true", default=False, help="Run universal tests", + ) def pytest_collection_modifyitems(config, items: List[Item]): should_run_integration = config.getoption("--integration") is True should_run_benchmark = config.getoption("--benchmark") is True + should_run_universal = config.getoption("--universal") is True integration_tests = [t for t in items if "integration" in t.keywords] if not should_run_integration: @@ -76,6 +87,12 @@ def pytest_collection_modifyitems(config, items: List[Item]): for t in benchmark_tests: items.append(t) + universal_tests = [t for t in items if "universal" in t.keywords] + if should_run_universal: + items.clear() + for t in universal_tests: + items.append(t) + @pytest.fixture def simple_dataset_1() -> pd.DataFrame: @@ -125,6 +142,14 @@ def environment(request): yield e +@pytest.fixture() +def local_redis_environment(): + with construct_test_environment( + IntegrationTestRepoConfig(online_store=REDIS_CONFIG) + ) as e: + yield e + + @pytest.fixture(scope="session") def universal_data_sources(environment): entities = construct_universal_entities() diff --git a/sdk/python/tests/data/data_creator.py b/sdk/python/tests/data/data_creator.py index 0b6b638711c..e5355b40bbc 100644 --- a/sdk/python/tests/data/data_creator.py +++ b/sdk/python/tests/data/data_creator.py @@ -13,7 +13,7 @@ def create_dataset( feature_is_list: bool = False, list_has_empty_list: bool = False, ) -> pd.DataFrame: - now = datetime.now().replace(microsecond=0, second=0, minute=0) + now = datetime.utcnow().replace(microsecond=0, second=0, minute=0) ts = pd.Timestamp(now).round("ms") data = { "driver_id": get_entities_for_value_type(entity_type), diff --git a/sdk/python/tests/integration/e2e/test_universal_e2e.py b/sdk/python/tests/integration/e2e/test_universal_e2e.py index e985e8ed108..9718c40d384 100644 --- a/sdk/python/tests/integration/e2e/test_universal_e2e.py +++ b/sdk/python/tests/integration/e2e/test_universal_e2e.py @@ -12,9 +12,11 @@ @pytest.mark.integration +@pytest.mark.universal @pytest.mark.parametrize("infer_features", [True, False]) def test_e2e_consistency(environment, e2e_data_sources, infer_features): fs = environment.feature_store + fs.config.project = fs.config.project + str(infer_features) df, data_source = e2e_data_sources fv = driver_feature_view(data_source=data_source, infer_features=infer_features) @@ -76,7 +78,7 @@ def check_offline_and_online_features( def run_offline_online_store_consistency_test( fs: FeatureStore, fv: FeatureView ) -> None: - now = datetime.now() + now = datetime.utcnow() full_feature_names = True check_offline_store: bool = True diff --git a/sdk/python/tests/integration/e2e/test_usage_e2e.py b/sdk/python/tests/integration/e2e/test_usage_e2e.py new file mode 100644 index 00000000000..c05ce8daf4e --- /dev/null +++ b/sdk/python/tests/integration/e2e/test_usage_e2e.py @@ -0,0 +1,135 @@ +# Copyright 2020 The Feast Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import os +import sys +import tempfile +from importlib import reload +from unittest.mock import patch + +import pytest + +from feast import Entity, RepoConfig, ValueType, usage +from feast.infra.online_stores.sqlite import SqliteOnlineStoreConfig + + +@pytest.fixture(scope="function") +def dummy_exporter(): + event_log = [] + + with patch("feast.usage._export", new=event_log.append): + yield event_log + + +@pytest.mark.integration +def test_usage_on(dummy_exporter): + usage._is_enabled = True + + _reload_feast() + from feast.feature_store import FeatureStore + + with tempfile.TemporaryDirectory() as temp_dir: + test_feature_store = FeatureStore( + config=RepoConfig( + registry=os.path.join(temp_dir, "registry.db"), + project="fake_project", + provider="local", + online_store=SqliteOnlineStoreConfig( + path=os.path.join(temp_dir, "online.db") + ), + ) + ) + entity = Entity( + name="driver_car_id", + description="Car driver id", + value_type=ValueType.STRING, + labels={"team": "matchmaking"}, + ) + + test_feature_store.apply([entity]) + + assert len(dummy_exporter) == 1 + assert { + "entrypoint": "feast.feature_store.FeatureStore.apply" + }.items() <= dummy_exporter[0].items() + + +@pytest.mark.integration +def test_usage_off(dummy_exporter): + usage._is_enabled = False + + _reload_feast() + from feast.feature_store import FeatureStore + + with tempfile.TemporaryDirectory() as temp_dir: + test_feature_store = FeatureStore( + config=RepoConfig( + registry=os.path.join(temp_dir, "registry.db"), + project="fake_project", + provider="local", + online_store=SqliteOnlineStoreConfig( + path=os.path.join(temp_dir, "online.db") + ), + ) + ) + entity = Entity( + name="driver_car_id", + description="Car driver id", + value_type=ValueType.STRING, + labels={"team": "matchmaking"}, + ) + test_feature_store.apply([entity]) + + assert not dummy_exporter + + +@pytest.mark.integration +def test_exception_usage_on(dummy_exporter): + usage._is_enabled = True + + _reload_feast() + from feast.feature_store import FeatureStore + + with pytest.raises(OSError): + FeatureStore("/tmp/non_existent_directory") + + assert len(dummy_exporter) == 1 + assert { + "entrypoint": "feast.feature_store.FeatureStore.__init__", + "exception": repr(FileNotFoundError(2, "No such file or directory")), + }.items() <= dummy_exporter[0].items() + + +@pytest.mark.integration +def test_exception_usage_off(dummy_exporter): + usage._is_enabled = False + + _reload_feast() + from feast.feature_store import FeatureStore + + with pytest.raises(OSError): + FeatureStore("/tmp/non_existent_directory") + + assert not dummy_exporter + + +def _reload_feast(): + """ After changing environment need to reload modules and rerun usage decorators """ + modules = ( + "feast.infra.local", + "feast.infra.online_stores.sqlite", + "feast.feature_store", + ) + for mod in modules: + if mod in sys.modules: + reload(sys.modules[mod]) diff --git a/sdk/python/tests/integration/feature_repos/integration_test_repo_config.py b/sdk/python/tests/integration/feature_repos/integration_test_repo_config.py new file mode 100644 index 00000000000..6e4f42cff81 --- /dev/null +++ b/sdk/python/tests/integration/feature_repos/integration_test_repo_config.py @@ -0,0 +1,35 @@ +from dataclasses import dataclass +from typing import Dict, Type, Union + +from tests.integration.feature_repos.universal.data_source_creator import ( + DataSourceCreator, +) +from tests.integration.feature_repos.universal.data_sources.file import ( + FileDataSourceCreator, +) + + +@dataclass(frozen=True) +class IntegrationTestRepoConfig: + """ + This class should hold all possible parameters that may need to be varied by individual tests. + """ + + provider: str = "local" + online_store: Union[str, Dict] = "sqlite" + + offline_store_creator: Type[DataSourceCreator] = FileDataSourceCreator + + full_feature_names: bool = True + infer_features: bool = False + + def __repr__(self) -> str: + return "-".join( + [ + f"Provider: {self.provider}", + f"{self.offline_store_creator.__name__.split('.')[-1].rstrip('DataSourceCreator')}", + self.online_store + if isinstance(self.online_store, str) + else self.online_store["type"], + ] + ) diff --git a/sdk/python/tests/integration/feature_repos/repo_configuration.py b/sdk/python/tests/integration/feature_repos/repo_configuration.py index afb66ba7f2c..67b330be491 100644 --- a/sdk/python/tests/integration/feature_repos/repo_configuration.py +++ b/sdk/python/tests/integration/feature_repos/repo_configuration.py @@ -1,24 +1,27 @@ +import importlib +import os import tempfile import uuid from contextlib import contextmanager from dataclasses import dataclass, field from datetime import datetime, timedelta from pathlib import Path -from typing import Any, Dict, List, Optional, Type, Union +from typing import Any, Dict, List, Optional import pandas as pd from feast import FeatureStore, FeatureView, RepoConfig, driver_test_data +from feast.constants import FULL_REPO_CONFIGS_MODULE_ENV_NAME from feast.data_source import DataSource +from tests.integration.feature_repos.integration_test_repo_config import ( + IntegrationTestRepoConfig, +) from tests.integration.feature_repos.universal.data_source_creator import ( DataSourceCreator, ) from tests.integration.feature_repos.universal.data_sources.bigquery import ( BigQueryDataSourceCreator, ) -from tests.integration.feature_repos.universal.data_sources.file import ( - FileDataSourceCreator, -) from tests.integration.feature_repos.universal.data_sources.redshift import ( RedshiftDataSourceCreator, ) @@ -26,71 +29,72 @@ conv_rate_plus_100_feature_view, create_conv_rate_request_data_source, create_customer_daily_profile_feature_view, + create_driver_age_request_feature_view, create_driver_hourly_stats_feature_view, create_global_stats_feature_view, + create_location_stats_feature_view, create_order_feature_view, ) - -@dataclass(frozen=True) -class IntegrationTestRepoConfig: - """ - This class should hold all possible parameters that may need to be varied by individual tests. - """ - - provider: str = "local" - online_store: Union[str, Dict] = "sqlite" - - offline_store_creator: Type[DataSourceCreator] = FileDataSourceCreator - - full_feature_names: bool = True - infer_features: bool = False - - def __repr__(self) -> str: - return "-".join( - [ - f"Provider: {self.provider}", - f"{self.offline_store_creator.__name__.split('.')[-1].rstrip('DataSourceCreator')}", - self.online_store - if isinstance(self.online_store, str) - else self.online_store["type"], - ] - ) - - DYNAMO_CONFIG = {"type": "dynamodb", "region": "us-west-2"} REDIS_CONFIG = {"type": "redis", "connection_string": "localhost:6379,db=0"} -FULL_REPO_CONFIGS: List[IntegrationTestRepoConfig] = [ + +# FULL_REPO_CONFIGS contains the repo configurations (e.g. provider, offline store, +# online store, test data, and more parameters) that most integration tests will test +# against. By default, FULL_REPO_CONFIGS uses the three providers (local, GCP, and AWS) +# with their default offline and online stores; it also tests the providers with the +# Redis online store. It can be overwritten by specifying a Python module through the +# FULL_REPO_CONFIGS_MODULE_ENV_NAME environment variable. In this case, that Python +# module will be imported and FULL_REPO_CONFIGS will be extracted from the file. +DEFAULT_FULL_REPO_CONFIGS: List[IntegrationTestRepoConfig] = [ # Local configurations IntegrationTestRepoConfig(), +] +if os.getenv("FEAST_IS_LOCAL_TEST", "False") != "True": IntegrationTestRepoConfig(online_store=REDIS_CONFIG), # GCP configurations - IntegrationTestRepoConfig( - provider="gcp", - offline_store_creator=BigQueryDataSourceCreator, - online_store="datastore", - ), - IntegrationTestRepoConfig( - provider="gcp", - offline_store_creator=BigQueryDataSourceCreator, - online_store=REDIS_CONFIG, - ), - # AWS configurations - IntegrationTestRepoConfig( - provider="aws", - offline_store_creator=RedshiftDataSourceCreator, - online_store=DYNAMO_CONFIG, - ), - IntegrationTestRepoConfig( - provider="aws", - offline_store_creator=RedshiftDataSourceCreator, - online_store=REDIS_CONFIG, - ), -] + DEFAULT_FULL_REPO_CONFIGS.extend( + [ + IntegrationTestRepoConfig( + provider="gcp", + offline_store_creator=BigQueryDataSourceCreator, + online_store="datastore", + ), + IntegrationTestRepoConfig( + provider="gcp", + offline_store_creator=BigQueryDataSourceCreator, + online_store=REDIS_CONFIG, + ), + # AWS configurations + IntegrationTestRepoConfig( + provider="aws", + offline_store_creator=RedshiftDataSourceCreator, + online_store=DYNAMO_CONFIG, + ), + IntegrationTestRepoConfig( + provider="aws", + offline_store_creator=RedshiftDataSourceCreator, + online_store=REDIS_CONFIG, + ), + ] + ) +full_repo_configs_module = os.environ.get(FULL_REPO_CONFIGS_MODULE_ENV_NAME) +if full_repo_configs_module is not None: + try: + module = importlib.import_module(full_repo_configs_module) + FULL_REPO_CONFIGS = getattr(module, "FULL_REPO_CONFIGS") + except Exception: + FULL_REPO_CONFIGS = DEFAULT_FULL_REPO_CONFIGS +else: + FULL_REPO_CONFIGS = DEFAULT_FULL_REPO_CONFIGS def construct_universal_entities() -> Dict[str, List[Any]]: - return {"customer": list(range(1001, 1020)), "driver": list(range(5001, 5020))} + return { + "customer": list(range(1001, 1020)), + "driver": list(range(5001, 5020)), + "location": list(range(1, 50)), + } def construct_universal_datasets( @@ -102,19 +106,33 @@ def construct_universal_datasets( driver_df = driver_test_data.create_driver_hourly_stats_df( entities["driver"], start_time, end_time ) + location_df = driver_test_data.create_location_stats_df( + entities["location"], start_time, end_time + ) orders_df = driver_test_data.create_orders_df( customers=entities["customer"], drivers=entities["driver"], + locations=entities["location"], start_date=start_time, end_date=end_time, order_count=20, ) global_df = driver_test_data.create_global_daily_stats_df(start_time, end_time) - entity_df = orders_df[["customer_id", "driver_id", "order_id", "event_timestamp"]] + entity_df = orders_df[ + [ + "customer_id", + "driver_id", + "order_id", + "origin_id", + "destination_id", + "event_timestamp", + ] + ] return { "customer": customer_df, "driver": driver_df, + "location": location_df, "orders": orders_df, "global": global_df, "entity": entity_df, @@ -136,6 +154,12 @@ def construct_universal_data_sources( event_timestamp_column="event_timestamp", created_timestamp_column="created", ) + location_ds = data_source_creator.create_data_source( + datasets["location"], + destination_name="location_hourly", + event_timestamp_column="event_timestamp", + created_timestamp_column="created", + ) orders_ds = data_source_creator.create_data_source( datasets["orders"], destination_name="orders", @@ -151,6 +175,7 @@ def construct_universal_data_sources( return { "customer": customer_ds, "driver": driver_ds, + "location": location_ds, "orders": orders_ds, "global": global_ds, } @@ -174,7 +199,9 @@ def construct_universal_feature_views( "input_request": create_conv_rate_request_data_source(), } ), + "driver_age_request_fv": create_driver_age_request_feature_view(), "order": create_order_feature_view(data_sources["orders"]), + "location": create_location_stats_feature_view(data_sources["location"]), } diff --git a/sdk/python/tests/integration/feature_repos/universal/data_sources/bigquery.py b/sdk/python/tests/integration/feature_repos/universal/data_sources/bigquery.py index 46e0535d736..766c31150e1 100644 --- a/sdk/python/tests/integration/feature_repos/universal/data_sources/bigquery.py +++ b/sdk/python/tests/integration/feature_repos/universal/data_sources/bigquery.py @@ -6,10 +6,7 @@ from feast import BigQuerySource from feast.data_source import DataSource -from feast.infra.offline_stores.bigquery import ( - BigQueryOfflineStoreConfig, - _write_df_to_bq, -) +from feast.infra.offline_stores.bigquery import BigQueryOfflineStoreConfig from tests.integration.feature_repos.universal.data_source_creator import ( DataSourceCreator, ) @@ -69,7 +66,7 @@ def create_data_source( f"{self.gcp_project}.{self.project_name}.{destination_name}" ) - job = _write_df_to_bq(self.client, df, destination_name) + job = self.client.load_table_from_dataframe(df, destination_name) job.result() self.tables.append(destination_name) diff --git a/sdk/python/tests/integration/feature_repos/universal/entities.py b/sdk/python/tests/integration/feature_repos/universal/entities.py index 8886e813d51..3b4ec05f5d9 100644 --- a/sdk/python/tests/integration/feature_repos/universal/entities.py +++ b/sdk/python/tests/integration/feature_repos/universal/entities.py @@ -12,3 +12,7 @@ def driver(value_type: ValueType = ValueType.INT64): def customer(): return Entity(name="customer_id", value_type=ValueType.INT64) + + +def location(): + return Entity(name="location_id", value_type=ValueType.INT64) diff --git a/sdk/python/tests/integration/feature_repos/universal/feature_views.py b/sdk/python/tests/integration/feature_repos/universal/feature_views.py index 1b701190581..23ce1b4cb19 100644 --- a/sdk/python/tests/integration/feature_repos/universal/feature_views.py +++ b/sdk/python/tests/integration/feature_repos/universal/feature_views.py @@ -5,6 +5,7 @@ from feast import Feature, FeatureView, OnDemandFeatureView, ValueType from feast.data_source import DataSource, RequestDataSource +from feast.request_feature_view import RequestFeatureView def driver_feature_view( @@ -43,6 +44,9 @@ def conv_rate_plus_100(features_df: pd.DataFrame) -> pd.DataFrame: df["conv_rate_plus_val_to_add"] = ( features_df["conv_rate"] + features_df["val_to_add"] ) + df["conv_rate_plus_100_rounded"] = ( + df["conv_rate_plus_100"].astype("float").round().astype(pd.Int32Dtype()) + ) return df @@ -54,6 +58,7 @@ def conv_rate_plus_100_feature_view( _features = features or [ Feature("conv_rate_plus_100", ValueType.DOUBLE), Feature("conv_rate_plus_val_to_add", ValueType.DOUBLE), + Feature("conv_rate_plus_100_rounded", ValueType.INT32), ] return OnDemandFeatureView( name=conv_rate_plus_100.__name__, @@ -63,6 +68,15 @@ def conv_rate_plus_100_feature_view( ) +def create_driver_age_request_feature_view(): + return RequestFeatureView( + name="driver_age", + request_data_source=RequestDataSource( + name="driver_age_source", schema={"driver_age": ValueType.INT32} + ), + ) + + def create_conv_rate_request_data_source(): return RequestDataSource( name="conv_rate_input", schema={"val_to_add": ValueType.INT32} @@ -129,3 +143,16 @@ def create_order_feature_view(source, infer_features: bool = False): batch_source=source, ttl=timedelta(days=2), ) + + +def create_location_stats_feature_view(source, infer_features: bool = False): + location_stats_feature_view = FeatureView( + name="location_stats", + entities=["location_id"], + features=None + if infer_features + else [Feature(name="temperature", dtype=ValueType.INT32)], + batch_source=source, + ttl=timedelta(days=2), + ) + return location_stats_feature_view diff --git a/sdk/python/tests/integration/offline_store/test_universal_historical_retrieval.py b/sdk/python/tests/integration/offline_store/test_universal_historical_retrieval.py index 9ad2a276041..384f3c8d4f8 100644 --- a/sdk/python/tests/integration/offline_store/test_universal_historical_retrieval.py +++ b/sdk/python/tests/integration/offline_store/test_universal_historical_retrieval.py @@ -1,4 +1,6 @@ -from datetime import datetime +import random +import time +from datetime import datetime, timedelta from typing import Any, Dict, List, Optional import numpy as np @@ -8,17 +10,27 @@ from pytz import utc from feast import utils -from feast.errors import RequestDataNotFoundInEntityDfException +from feast.entity import Entity +from feast.errors import ( + FeatureNameCollisionError, + RequestDataNotFoundInEntityDfException, +) +from feast.feature import Feature from feast.feature_service import FeatureService from feast.feature_view import FeatureView from feast.infra.offline_stores.offline_utils import ( DEFAULT_ENTITY_DF_EVENT_TIMESTAMP_COL, ) +from feast.value_type import ValueType from tests.integration.feature_repos.repo_configuration import ( construct_universal_feature_views, table_name_from_data_source, ) -from tests.integration.feature_repos.universal.entities import customer, driver +from tests.integration.feature_repos.universal.entities import ( + customer, + driver, + location, +) np.random.seed(0) @@ -66,6 +78,8 @@ def get_expected_training_df( driver_fv: FeatureView, orders_df: pd.DataFrame, order_fv: FeatureView, + location_df: pd.DataFrame, + location_fv: FeatureView, global_df: pd.DataFrame, global_fv: FeatureView, entity_df: pd.DataFrame, @@ -82,6 +96,9 @@ def get_expected_training_df( order_records = convert_timestamp_records_to_utc( orders_df.to_dict("records"), event_timestamp ) + location_records = convert_timestamp_records_to_utc( + location_df.to_dict("records"), location_fv.batch_source.event_timestamp_column + ) global_records = convert_timestamp_records_to_utc( global_df.to_dict("records"), global_fv.batch_source.event_timestamp_column ) @@ -116,6 +133,22 @@ def get_expected_training_df( filter_keys=["customer_id", "driver_id"], filter_values=[entity_row["customer_id"], entity_row["driver_id"]], ) + origin_record = find_asof_record( + location_records, + ts_key=location_fv.batch_source.event_timestamp_column, + ts_start=order_record[event_timestamp] - location_fv.ttl, + ts_end=order_record[event_timestamp], + filter_keys=["location_id"], + filter_values=[order_record["origin_id"]], + ) + destination_record = find_asof_record( + location_records, + ts_key=location_fv.batch_source.event_timestamp_column, + ts_start=order_record[event_timestamp] - location_fv.ttl, + ts_end=order_record[event_timestamp], + filter_keys=["location_id"], + filter_values=[order_record["destination_id"]], + ) global_record = find_asof_record( global_records, ts_key=global_fv.batch_source.event_timestamp_column, @@ -149,6 +182,12 @@ def get_expected_training_df( for k in ("order_is_success",) } ) + entity_row.update( + { + "origin__temperature": origin_record.get("temperature", None), + "destination__temperature": destination_record.get("temperature", None), + } + ) entity_row.update( { (f"global_stats__{k}" if full_feature_names else k): global_record.get( @@ -189,6 +228,12 @@ def get_expected_training_df( conv_feature_name = "driver_stats__conv_rate" if full_feature_names else "conv_rate" expected_df["conv_rate_plus_100"] = expected_df[conv_feature_name] + 100 + expected_df["conv_rate_plus_100_rounded"] = ( + expected_df["conv_rate_plus_100"] + .astype("float") + .round() + .astype(pd.Int32Dtype()) + ) expected_df["conv_rate_plus_val_to_add"] = ( expected_df[conv_feature_name] + expected_df["val_to_add"] ) @@ -197,6 +242,7 @@ def get_expected_training_df( @pytest.mark.integration +@pytest.mark.universal @pytest.mark.parametrize("full_feature_names", [True, False], ids=lambda v: str(v)) def test_historical_features(environment, universal_data_sources, full_feature_names): store = environment.feature_store @@ -204,9 +250,10 @@ def test_historical_features(environment, universal_data_sources, full_feature_n (entities, datasets, data_sources) = universal_data_sources feature_views = construct_universal_feature_views(data_sources) - customer_df, driver_df, orders_df, global_df, entity_df = ( + customer_df, driver_df, location_df, orders_df, global_df, entity_df = ( datasets["customer"], datasets["driver"], + datasets["location"], datasets["orders"], datasets["global"], datasets["entity"], @@ -215,18 +262,46 @@ def test_historical_features(environment, universal_data_sources, full_feature_n entity_df_with_request_data["val_to_add"] = [ i for i in range(len(entity_df_with_request_data)) ] + entity_df_with_request_data["driver_age"] = [ + i + 100 for i in range(len(entity_df_with_request_data)) + ] - customer_fv, driver_fv, driver_odfv, order_fv, global_fv = ( + ( + customer_fv, + driver_fv, + driver_odfv, + location_fv, + order_fv, + global_fv, + driver_age_request_fv, + ) = ( feature_views["customer"], feature_views["driver"], feature_views["driver_odfv"], + feature_views["location"], feature_views["order"], feature_views["global"], + feature_views["driver_age_request_fv"], ) feature_service = FeatureService( - "convrate_plus100", - features=[feature_views["driver"][["conv_rate"]], feature_views["driver_odfv"]], + name="convrate_plus100", + features=[ + feature_views["driver"][["conv_rate"]], + driver_odfv, + driver_age_request_fv, + ], + ) + feature_service_entity_mapping = FeatureService( + name="entity_mapping", + features=[ + location_fv.with_name("origin").with_join_key_map( + {"location_id": "origin_id"} + ), + location_fv.with_name("destination").with_join_key_map( + {"location_id": "destination_id"} + ), + ], ) feast_objects = [] @@ -235,11 +310,15 @@ def test_historical_features(environment, universal_data_sources, full_feature_n customer_fv, driver_fv, driver_odfv, + location_fv, order_fv, global_fv, + driver_age_request_fv, driver(), customer(), + location(), feature_service, + feature_service_entity_mapping, ] ) store.apply(feast_objects) @@ -247,20 +326,22 @@ def test_historical_features(environment, universal_data_sources, full_feature_n entity_df_query = None orders_table = table_name_from_data_source(data_sources["orders"]) if orders_table: - entity_df_query = f"SELECT customer_id, driver_id, order_id, event_timestamp FROM {orders_table}" + entity_df_query = f"SELECT customer_id, driver_id, order_id, origin_id, destination_id, event_timestamp FROM {orders_table}" event_timestamp = ( DEFAULT_ENTITY_DF_EVENT_TIMESTAMP_COL if DEFAULT_ENTITY_DF_EVENT_TIMESTAMP_COL in orders_df.columns else "e_ts" ) - expected_df = get_expected_training_df( + full_expected_df = get_expected_training_df( customer_df, customer_fv, driver_df, driver_fv, orders_df, order_fv, + location_df, + location_fv, global_df, global_fv, entity_df_with_request_data, @@ -268,6 +349,11 @@ def test_historical_features(environment, universal_data_sources, full_feature_n full_feature_names, ) + # Only need the shadow entities features in the FeatureService test + expected_df = full_expected_df.drop( + columns=["origin__temperature", "destination__temperature"], + ) + if entity_df_query: job_from_sql = store.get_historical_features( entity_df=entity_df_query, @@ -293,7 +379,13 @@ def test_historical_features(environment, universal_data_sources, full_feature_n # Not requesting the on demand transform with an entity_df query (can't add request data in them) expected_df_query = expected_df.drop( - columns=["conv_rate_plus_100", "val_to_add", "conv_rate_plus_val_to_add"] + columns=[ + "conv_rate_plus_100", + "conv_rate_plus_100_rounded", + "val_to_add", + "conv_rate_plus_val_to_add", + "driver_age", + ] ) assert sorted(expected_df_query.columns) == sorted( actual_df_from_sql_entities.columns @@ -341,10 +433,12 @@ def test_historical_features(environment, universal_data_sources, full_feature_n "customer_profile:avg_passenger_count", "customer_profile:lifetime_trip_count", "conv_rate_plus_100:conv_rate_plus_100", + "conv_rate_plus_100:conv_rate_plus_100_rounded", "conv_rate_plus_100:conv_rate_plus_val_to_add", "order:order_is_success", "global_stats:num_rides", "global_stats:avg_ride_length", + "driver_age:driver_age", ], full_feature_names=full_feature_names, ) @@ -382,6 +476,14 @@ def test_historical_features(environment, universal_data_sources, full_feature_n expected_df, event_timestamp, ) + assert_feature_service_entity_mapping_correctness( + store, + feature_service_entity_mapping, + full_feature_names, + entity_df_with_request_data, + full_expected_df, + event_timestamp, + ) table_from_df_entities: pd.DataFrame = job_from_df.to_arrow().to_pandas() @@ -412,6 +514,124 @@ def test_historical_features(environment, universal_data_sources, full_feature_n ], full_feature_names=full_feature_names, ) + # If request data is missing that's needed for a request feature view, throw an error + with pytest.raises(RequestDataNotFoundInEntityDfException): + store.get_historical_features( + entity_df=entity_df, + features=[ + "driver_stats:conv_rate", + "driver_stats:avg_daily_trips", + "customer_profile:current_balance", + "customer_profile:avg_passenger_count", + "customer_profile:lifetime_trip_count", + "driver_age:driver_age", + "global_stats:num_rides", + "global_stats:avg_ride_length", + ], + full_feature_names=full_feature_names, + ) + + +@pytest.mark.integration +@pytest.mark.universal +def test_historical_features_from_bigquery_sources_containing_backfills(environment): + store = environment.feature_store + + now = datetime.now().replace(microsecond=0, second=0, minute=0) + tomorrow = now + timedelta(days=1) + + entity_df = pd.DataFrame( + data=[ + {"driver_id": 1001, "event_timestamp": now + timedelta(days=2)}, + {"driver_id": 1002, "event_timestamp": now + timedelta(days=2)}, + ] + ) + + driver_stats_df = pd.DataFrame( + data=[ + # Duplicated rows simple case + { + "driver_id": 1001, + "avg_daily_trips": 10, + "event_timestamp": now, + "created": now, + }, + { + "driver_id": 1001, + "avg_daily_trips": 20, + "event_timestamp": now, + "created": tomorrow, + }, + # Duplicated rows after a backfill + { + "driver_id": 1002, + "avg_daily_trips": 30, + "event_timestamp": now, + "created": tomorrow, + }, + { + "driver_id": 1002, + "avg_daily_trips": 40, + "event_timestamp": tomorrow, + "created": now, + }, + ] + ) + + expected_df = pd.DataFrame( + data=[ + { + "driver_id": 1001, + "event_timestamp": now + timedelta(days=2), + "avg_daily_trips": 20, + }, + { + "driver_id": 1002, + "event_timestamp": now + timedelta(days=2), + "avg_daily_trips": 40, + }, + ] + ) + + driver_stats_data_source = environment.data_source_creator.create_data_source( + df=driver_stats_df, + destination_name=f"test_driver_stats_{int(time.time_ns())}_{random.randint(1000, 9999)}", + event_timestamp_column="event_timestamp", + created_timestamp_column="created", + ) + + driver = Entity(name="driver", join_key="driver_id", value_type=ValueType.INT64) + driver_fv = FeatureView( + name="driver_stats", + entities=["driver"], + features=[Feature(name="avg_daily_trips", dtype=ValueType.INT32)], + batch_source=driver_stats_data_source, + ttl=None, + ) + + store.apply([driver, driver_fv]) + + offline_job = store.get_historical_features( + entity_df=entity_df, + features=["driver_stats:avg_daily_trips"], + full_feature_names=False, + ) + + start_time = datetime.utcnow() + actual_df = offline_job.to_df() + + print(f"actual_df shape: {actual_df.shape}") + end_time = datetime.utcnow() + print(str(f"Time to execute job_from_df.to_df() = '{(end_time - start_time)}'\n")) + + assert sorted(expected_df.columns) == sorted(actual_df.columns) + assert_frame_equal( + expected_df.sort_values(by=["driver_id"]).reset_index(drop=True), + actual_df[expected_df.columns] + .sort_values(by=["driver_id"]) + .reset_index(drop=True), + check_dtype=False, + ) def response_feature_name(feature: str, full_feature_names: bool) -> str: @@ -451,6 +671,7 @@ def assert_feature_service_correctness( "customer_id", response_feature_name("conv_rate", full_feature_names), "conv_rate_plus_100", + "driver_age", ] ] actual_df_from_df_entities = ( @@ -463,3 +684,69 @@ def assert_feature_service_correctness( assert_frame_equal( expected_df, actual_df_from_df_entities, check_dtype=False, ) + + +def assert_feature_service_entity_mapping_correctness( + store, feature_service, full_feature_names, entity_df, expected_df, event_timestamp +): + if full_feature_names: + job_from_df = store.get_historical_features( + entity_df=entity_df, + features=feature_service, + full_feature_names=full_feature_names, + ) + actual_df_from_df_entities = job_from_df.to_df() + + expected_df: pd.DataFrame = ( + expected_df.sort_values( + by=[ + event_timestamp, + "order_id", + "driver_id", + "customer_id", + "origin_id", + "destination_id", + ] + ) + .drop_duplicates() + .reset_index(drop=True) + ) + expected_df = expected_df[ + [ + event_timestamp, + "order_id", + "driver_id", + "customer_id", + "origin_id", + "destination_id", + "origin__temperature", + "destination__temperature", + ] + ] + actual_df_from_df_entities = ( + actual_df_from_df_entities[expected_df.columns] + .sort_values( + by=[ + event_timestamp, + "order_id", + "driver_id", + "customer_id", + "origin_id", + "destination_id", + ] + ) + .drop_duplicates() + .reset_index(drop=True) + ) + + assert_frame_equal( + expected_df, actual_df_from_df_entities, check_dtype=False, + ) + else: + # using 2 of the same FeatureView without full_feature_names=True will result in collision + with pytest.raises(FeatureNameCollisionError): + job_from_df = store.get_historical_features( + entity_df=entity_df, + features=feature_service, + full_feature_names=full_feature_names, + ) diff --git a/sdk/python/tests/integration/online_store/test_universal_online.py b/sdk/python/tests/integration/online_store/test_universal_online.py index aeab91e5f88..c90021f9ce9 100644 --- a/sdk/python/tests/integration/online_store/test_universal_online.py +++ b/sdk/python/tests/integration/online_store/test_universal_online.py @@ -1,18 +1,177 @@ +import datetime +import itertools +import os import unittest from datetime import timedelta +import numpy as np import pandas as pd import pytest -from feast import FeatureService -from feast.errors import RequestDataNotFoundInEntityRowsException +from feast import Entity, Feature, FeatureService, FeatureView, ValueType +from feast.errors import ( + FeatureNameCollisionError, + RequestDataNotFoundInEntityRowsException, +) from tests.integration.feature_repos.repo_configuration import ( construct_universal_feature_views, ) -from tests.integration.feature_repos.universal.entities import customer, driver +from tests.integration.feature_repos.universal.entities import ( + customer, + driver, + location, +) +from tests.integration.feature_repos.universal.feature_views import ( + create_driver_hourly_stats_feature_view, +) +from tests.utils.data_source_utils import prep_file_source + + +# TODO: make this work with all universal (all online store types) +@pytest.mark.integration +def test_write_to_online_store_event_check(local_redis_environment): + if os.getenv("FEAST_IS_LOCAL_TEST", "False") == "True": + return + fs = local_redis_environment.feature_store + + # write same data points 3 with different timestamps + now = pd.Timestamp(datetime.datetime.utcnow()).round("ms") + hour_ago = pd.Timestamp(datetime.datetime.utcnow() - timedelta(hours=1)).round("ms") + latest = pd.Timestamp(datetime.datetime.utcnow() + timedelta(seconds=1)).round("ms") + + data = { + "id": [123, 567, 890], + "string_col": ["OLD_FEATURE", "LATEST_VALUE2", "LATEST_VALUE3"], + "ts_1": [hour_ago, now, now], + } + dataframe_source = pd.DataFrame(data) + with prep_file_source( + df=dataframe_source, event_timestamp_column="ts_1" + ) as file_source: + e = Entity(name="id", value_type=ValueType.STRING) + + # Create Feature View + fv1 = FeatureView( + name="feature_view_123", + features=[Feature(name="string_col", dtype=ValueType.STRING)], + entities=["id"], + batch_source=file_source, + ttl=timedelta(minutes=5), + ) + # Register Feature View and Entity + fs.apply([fv1, e]) + + # data to ingest into Online Store (recent) + data = { + "id": [123], + "string_col": ["hi_123"], + "ts_1": [now], + } + df_data = pd.DataFrame(data) + + # directly ingest data into the Online Store + fs.write_to_online_store("feature_view_123", df_data) + + df = fs.get_online_features( + features=["feature_view_123:string_col"], entity_rows=[{"id": 123}] + ).to_df() + assert df["string_col"].iloc[0] == "hi_123" + + # data to ingest into Online Store (1 hour delayed data) + # should now overwrite features for id=123 because it's less recent data + data = { + "id": [123, 567, 890], + "string_col": ["bye_321", "hello_123", "greetings_321"], + "ts_1": [hour_ago, hour_ago, hour_ago], + } + df_data = pd.DataFrame(data) + + # directly ingest data into the Online Store + fs.write_to_online_store("feature_view_123", df_data) + + df = fs.get_online_features( + features=["feature_view_123:string_col"], + entity_rows=[{"id": 123}, {"id": 567}, {"id": 890}], + ).to_df() + assert df["string_col"].iloc[0] == "hi_123" + assert df["string_col"].iloc[1] == "hello_123" + assert df["string_col"].iloc[2] == "greetings_321" + + # should overwrite string_col for id=123 because it's most recent based on event_timestamp + data = { + "id": [123], + "string_col": ["LATEST_VALUE"], + "ts_1": [latest], + } + df_data = pd.DataFrame(data) + + fs.write_to_online_store("feature_view_123", df_data) + + df = fs.get_online_features( + features=["feature_view_123:string_col"], + entity_rows=[{"id": 123}, {"id": 567}, {"id": 890}], + ).to_df() + assert df["string_col"].iloc[0] == "LATEST_VALUE" + assert df["string_col"].iloc[1] == "hello_123" + assert df["string_col"].iloc[2] == "greetings_321" + + # writes to online store via datasource (dataframe_source) materialization + fs.materialize( + start_date=datetime.datetime.now() - timedelta(hours=12), + end_date=datetime.datetime.utcnow(), + ) + + df = fs.get_online_features( + features=["feature_view_123:string_col"], + entity_rows=[{"id": 123}, {"id": 567}, {"id": 890}], + ).to_df() + assert df["string_col"].iloc[0] == "LATEST_VALUE" + assert df["string_col"].iloc[1] == "LATEST_VALUE2" + assert df["string_col"].iloc[2] == "LATEST_VALUE3" + + +@pytest.mark.integration +def test_write_to_online_store(environment, universal_data_sources): + fs = environment.feature_store + entities, datasets, data_sources = universal_data_sources + driver_hourly_stats = create_driver_hourly_stats_feature_view( + data_sources["driver"] + ) + driver_entity = driver() + + # Register Feature View and Entity + fs.apply([driver_hourly_stats, driver_entity]) + + # fake data to ingest into Online Store + data = { + "driver_id": [123], + "conv_rate": [0.85], + "acc_rate": [0.91], + "avg_daily_trips": [14], + "event_timestamp": [pd.Timestamp(datetime.datetime.utcnow()).round("ms")], + "created": [pd.Timestamp(datetime.datetime.utcnow()).round("ms")], + } + df_data = pd.DataFrame(data) + + # directly ingest data into the Online Store + fs.write_to_online_store("driver_stats", df_data) + + # assert the right data is in the Online Store + df = fs.get_online_features( + features=[ + "driver_stats:avg_daily_trips", + "driver_stats:acc_rate", + "driver_stats:conv_rate", + ], + entity_rows=[{"driver": 123}], + ).to_df() + assert df["avg_daily_trips"].iloc[0] == 14 + assert df["acc_rate"].iloc[0] == 0.91 + assert df["conv_rate"].iloc[0] == 0.85 @pytest.mark.integration +@pytest.mark.universal @pytest.mark.parametrize("full_feature_names", [True, False], ids=lambda v: str(v)) def test_online_retrieval(environment, universal_data_sources, full_feature_names): @@ -22,12 +181,35 @@ def test_online_retrieval(environment, universal_data_sources, full_feature_name feature_service = FeatureService( "convrate_plus100", - features=[feature_views["driver"][["conv_rate"]], feature_views["driver_odfv"]], + features=[ + feature_views["driver"][["conv_rate"]], + feature_views["driver_odfv"], + feature_views["driver_age_request_fv"], + ], + ) + feature_service_entity_mapping = FeatureService( + name="entity_mapping", + features=[ + feature_views["location"] + .with_name("origin") + .with_join_key_map({"location_id": "origin_id"}), + feature_views["location"] + .with_name("destination") + .with_join_key_map({"location_id": "destination_id"}), + ], ) feast_objects = [] feast_objects.extend(feature_views.values()) - feast_objects.extend([driver(), customer(), feature_service]) + feast_objects.extend( + [ + driver(), + customer(), + location(), + feature_service, + feature_service_entity_mapping, + ] + ) fs.apply(feast_objects) fs.materialize( environment.start_date - timedelta(days=1), @@ -54,10 +236,19 @@ def test_online_retrieval(environment, universal_data_sources, full_feature_name datasets["customer"]["customer_id"].isin(sample_customers) ] + location_pairs = np.array(list(itertools.permutations(entities["location"], 2))) + sample_location_pairs = location_pairs[np.random.choice(len(location_pairs), 10)].T + origins_df = datasets["location"][ + datasets["location"]["location_id"].isin(sample_location_pairs[0]) + ] + destinations_df = datasets["location"][ + datasets["location"]["location_id"].isin(sample_location_pairs[1]) + ] + global_df = datasets["global"] entity_rows = [ - {"driver": d, "customer_id": c, "val_to_add": 50} + {"driver": d, "customer_id": c, "val_to_add": 50, "driver_age": 25} for (d, c) in zip(sample_drivers, sample_customers) ] @@ -72,6 +263,7 @@ def test_online_retrieval(environment, universal_data_sources, full_feature_name "order:order_is_success", "global_stats:num_rides", "global_stats:avg_ride_length", + "driver_age:driver_age", ] unprefixed_feature_refs = [f.rsplit(":", 1)[-1] for f in feature_refs if ":" in f] # Remove the on demand feature view output features, since they're not present in the source dataframe @@ -85,13 +277,23 @@ def test_online_retrieval(environment, universal_data_sources, full_feature_name ) assert online_features is not None + # Test that the on demand feature views compute properly even if the dependent conv_rate + # feature isn't requested. + online_features_no_conv_rate = fs.get_online_features( + features=[ref for ref in feature_refs if ref != "driver_stats:conv_rate"], + entity_rows=entity_rows, + full_feature_names=full_feature_names, + ) + assert online_features_no_conv_rate is not None + online_features_dict = online_features.to_dict() keys = online_features_dict.keys() assert ( - len(keys) == len(feature_refs) + 3 - ) # Add three for the driver id and the customer id entity keys + val_to_add request data. + len(keys) == len(feature_refs) + 2 + ) # Add two for the driver id and the customer id entity keys for feature in feature_refs: - if full_feature_names: + # full_feature_names does not apply to request feature views + if full_feature_names and feature != "driver_age:driver_age": assert feature.replace(":", "__") in keys else: assert feature.rsplit(":", 1)[-1] in keys @@ -105,7 +307,11 @@ def test_online_retrieval(environment, universal_data_sources, full_feature_name tc = unittest.TestCase() for i, entity_row in enumerate(entity_rows): df_features = get_latest_feature_values_from_dataframes( - drivers_df, customers_df, orders_df, global_df, entity_row + driver_df=drivers_df, + customer_df=customers_df, + orders_df=orders_df, + global_df=global_df, + entity_row=entity_row, ) assert df_features["customer_id"] == online_features_dict["customer_id"][i] @@ -136,12 +342,14 @@ def test_online_retrieval(environment, universal_data_sources, full_feature_name # Check what happens for missing values missing_responses_dict = fs.get_online_features( features=feature_refs, - entity_rows=[{"driver": 0, "customer_id": 0, "val_to_add": 100}], + entity_rows=[ + {"driver": 0, "customer_id": 0, "val_to_add": 100, "driver_age": 125} + ], full_feature_names=full_feature_names, ).to_dict() assert missing_responses_dict is not None for unprefixed_feature_ref in unprefixed_feature_refs: - if unprefixed_feature_ref not in {"num_rides", "avg_ride_length"}: + if unprefixed_feature_ref not in {"num_rides", "avg_ride_length", "driver_age"}: tc.assertIsNone( missing_responses_dict[ response_feature_name(unprefixed_feature_ref, full_feature_names) @@ -156,6 +364,14 @@ def test_online_retrieval(environment, universal_data_sources, full_feature_name full_feature_names=full_feature_names, ).to_dict() + # Also with request data + with pytest.raises(RequestDataNotFoundInEntityRowsException): + fs.get_online_features( + features=feature_refs, + entity_rows=[{"driver": 0, "customer_id": 0, "val_to_add": 20}], + full_feature_names=full_feature_names, + ).to_dict() + assert_feature_service_correctness( fs, feature_service, @@ -167,6 +383,29 @@ def test_online_retrieval(environment, universal_data_sources, full_feature_name global_df, ) + entity_rows = [ + { + "driver": driver, + "customer_id": customer, + "origin_id": origin, + "destination_id": destination, + } + for (driver, customer, origin, destination) in zip( + sample_drivers, sample_customers, *sample_location_pairs + ) + ] + assert_feature_service_entity_mapping_correctness( + fs, + feature_service_entity_mapping, + entity_rows, + full_feature_names, + drivers_df, + customers_df, + orders_df, + origins_df, + destinations_df, + ) + def response_feature_name(feature: str, full_feature_names: bool) -> str: if ( @@ -193,17 +432,24 @@ def response_feature_name(feature: str, full_feature_names: bool) -> str: return feature +def get_latest_row(entity_row, df, join_key, entity_key): + rows = df[df[join_key] == entity_row[entity_key]] + return rows.loc[rows["event_timestamp"].idxmax()].to_dict() + + def get_latest_feature_values_from_dataframes( - driver_df, customer_df, orders_df, global_df, entity_row + driver_df, + customer_df, + orders_df, + entity_row, + global_df=None, + origin_df=None, + destination_df=None, ): - driver_rows = driver_df[driver_df["driver_id"] == entity_row["driver"]] - latest_driver_row: pd.DataFrame = driver_rows.loc[ - driver_rows["event_timestamp"].idxmax() - ].to_dict() - customer_rows = customer_df[customer_df["customer_id"] == entity_row["customer_id"]] - latest_customer_row = customer_rows.loc[ - customer_rows["event_timestamp"].idxmax() - ].to_dict() + latest_driver_row = get_latest_row(entity_row, driver_df, "driver_id", "driver") + latest_customer_row = get_latest_row( + entity_row, customer_df, "customer_id", "customer_id" + ) # Since the event timestamp columns may contain timestamps of different timezones, # we must first convert the timestamps to UTC before we can compare them. @@ -218,14 +464,47 @@ def get_latest_feature_values_from_dataframes( max_index = timestamps["event_timestamp"].idxmax() latest_orders_row = order_rows.loc[max_index] - latest_global_row = global_df.loc[global_df["event_timestamp"].idxmax()].to_dict() + if global_df is not None: + latest_global_row = global_df.loc[ + global_df["event_timestamp"].idxmax() + ].to_dict() + if origin_df is not None: + latest_origin_row = get_latest_row( + entity_row, origin_df, "location_id", "origin_id" + ) + latest_destination_row = get_latest_row( + entity_row, destination_df, "location_id", "destination_id" + ) + # Need full feature names for shadow entities + latest_origin_row["origin__temperature"] = latest_origin_row.pop("temperature") + latest_destination_row["destination__temperature"] = latest_destination_row.pop( + "temperature" + ) request_data_features = entity_row.copy() request_data_features.pop("driver") request_data_features.pop("customer_id") + if global_df is not None: + return { + **latest_customer_row, + **latest_driver_row, + **latest_orders_row, + **latest_global_row, + **request_data_features, + } + if origin_df is not None: + request_data_features.pop("origin_id") + request_data_features.pop("destination_id") + return { + **latest_customer_row, + **latest_driver_row, + **latest_orders_row, + **latest_origin_row, + **latest_destination_row, + **request_data_features, + } return { **latest_customer_row, **latest_driver_row, - **latest_global_row, **latest_orders_row, **request_data_features, } @@ -259,13 +538,17 @@ def assert_feature_service_correctness( for projection in feature_service.feature_view_projections ] ) - + 3 - ) # Add two for the driver id and the customer id entity keys and val_to_add request data + + 2 + ) # Add two for the driver id and the customer id entity keys tc = unittest.TestCase() for i, entity_row in enumerate(entity_rows): df_features = get_latest_feature_values_from_dataframes( - drivers_df, customers_df, orders_df, global_df, entity_row + driver_df=drivers_df, + customer_df=customers_df, + orders_df=orders_df, + global_df=global_df, + entity_row=entity_row, ) tc.assertAlmostEqual( feature_service_online_features_dict[ @@ -274,3 +557,60 @@ def assert_feature_service_correctness( df_features["conv_rate"] + 100, delta=0.0001, ) + + +def assert_feature_service_entity_mapping_correctness( + fs, + feature_service, + entity_rows, + full_feature_names, + drivers_df, + customers_df, + orders_df, + origins_df, + destinations_df, +): + if full_feature_names: + feature_service_response = fs.get_online_features( + features=feature_service, + entity_rows=entity_rows, + full_feature_names=full_feature_names, + ) + assert feature_service_response is not None + + feature_service_online_features_dict = feature_service_response.to_dict() + feature_service_keys = feature_service_online_features_dict.keys() + + assert ( + len(feature_service_keys) + == sum( + [ + len(projection.features) + for projection in feature_service.feature_view_projections + ] + ) + + 4 + ) # Add 4 for the driver_id, customer_id, origin_id, and destination_id entity keys + + for i, entity_row in enumerate(entity_rows): + df_features = get_latest_feature_values_from_dataframes( + driver_df=drivers_df, + customer_df=customers_df, + orders_df=orders_df, + origin_df=origins_df, + destination_df=destinations_df, + entity_row=entity_row, + ) + for feature_name in ["origin__temperature", "destination__temperature"]: + assert ( + feature_service_online_features_dict[feature_name][i] + == df_features[feature_name] + ) + else: + # using 2 of the same FeatureView without full_feature_names=True will result in collision + with pytest.raises(FeatureNameCollisionError): + feature_service_response = fs.get_online_features( + features=feature_service, + entity_rows=entity_rows, + full_feature_names=full_feature_names, + ) diff --git a/sdk/python/tests/integration/registration/test_registry.py b/sdk/python/tests/integration/registration/test_registry.py index b93bafb5e3d..55034c95a8c 100644 --- a/sdk/python/tests/integration/registration/test_registry.py +++ b/sdk/python/tests/integration/registration/test_registry.py @@ -101,6 +101,10 @@ def test_apply_entity_success(test_registry): and entity.labels["team"] == "matchmaking" ) + test_registry.delete_entity("driver_car_id", project) + entities = test_registry.list_entities(project) + assert len(entities) == 0 + test_registry.teardown() # Will try to reload registry, which will fail because the file has been deleted diff --git a/sdk/python/tests/integration/registration/test_universal_odfv_feature_inference.py b/sdk/python/tests/integration/registration/test_universal_odfv_feature_inference.py index f7d33a9e0a9..6d6750081bc 100644 --- a/sdk/python/tests/integration/registration/test_universal_odfv_feature_inference.py +++ b/sdk/python/tests/integration/registration/test_universal_odfv_feature_inference.py @@ -11,6 +11,7 @@ @pytest.mark.integration +@pytest.mark.universal @pytest.mark.parametrize("infer_features", [True, False], ids=lambda v: str(v)) def test_infer_odfv_features(environment, universal_data_sources, infer_features): store = environment.feature_store @@ -29,10 +30,11 @@ def test_infer_odfv_features(environment, universal_data_sources, infer_features feast_objects = [driver_hourly_stats, driver_odfv, driver(), customer()] store.apply(feast_objects) odfv = store.get_on_demand_feature_view("conv_rate_plus_100") - assert len(odfv.features) == 2 + assert len(odfv.features) == 3 @pytest.mark.integration +@pytest.mark.universal def test_infer_odfv_features_with_error(environment, universal_data_sources): store = environment.feature_store diff --git a/sdk/python/tests/integration/registration/test_universal_types.py b/sdk/python/tests/integration/registration/test_universal_types.py index fcd9c4fc520..7ef1c5c1e79 100644 --- a/sdk/python/tests/integration/registration/test_universal_types.py +++ b/sdk/python/tests/integration/registration/test_universal_types.py @@ -116,6 +116,7 @@ def get_fixtures(request): @pytest.mark.integration +@pytest.mark.universal def test_entity_inference_types_match(offline_types_test_fixtures): environment, config, data_source, fv = offline_types_test_fixtures fs = environment.feature_store @@ -139,6 +140,7 @@ def test_entity_inference_types_match(offline_types_test_fixtures): @pytest.mark.integration +@pytest.mark.universal def test_feature_get_historical_features_types_match(offline_types_test_fixtures): environment, config, data_source, fv = offline_types_test_fixtures fs = environment.feature_store @@ -185,6 +187,7 @@ def test_feature_get_historical_features_types_match(offline_types_test_fixtures @pytest.mark.integration +@pytest.mark.universal def test_feature_get_online_features_types_match(online_types_test_fixtures): environment, config, data_source, fv = online_types_test_fixtures fv = create_feature_view( diff --git a/sdk/python/tests/unit/test_usage.py b/sdk/python/tests/unit/test_usage.py new file mode 100644 index 00000000000..13988d32642 --- /dev/null +++ b/sdk/python/tests/unit/test_usage.py @@ -0,0 +1,237 @@ +import datetime +import json +import time +from unittest.mock import patch + +import pytest + +from feast.usage import ( + RatioSampler, + log_exceptions, + log_exceptions_and_usage, + set_usage_attribute, + tracing_span, +) + + +@pytest.fixture(scope="function") +def dummy_exporter(): + event_log = [] + + with patch( + "feast.usage._export", + new=lambda e: event_log.append(json.loads(json.dumps(e))), + ): + yield event_log + + +@pytest.fixture(scope="function", autouse=True) +def enabling_patch(): + with patch("feast.usage._is_enabled") as p: + p.__bool__.return_value = True + yield p + + +def test_logging_disabled(dummy_exporter, enabling_patch): + enabling_patch.__bool__.return_value = False + + @log_exceptions_and_usage(event="test-event") + def entrypoint(): + pass + + @log_exceptions(event="test-event") + def entrypoint2(): + raise ValueError(1) + + entrypoint() + with pytest.raises(ValueError): + entrypoint2() + + assert not dummy_exporter + + +def test_global_context_building(dummy_exporter): + @log_exceptions_and_usage(event="test-event") + def entrypoint(provider): + if provider == "one": + provider_one() + if provider == "two": + provider_two() + + @log_exceptions_and_usage(provider="provider-one") + def provider_one(): + dummy_layer() + + @log_exceptions_and_usage(provider="provider-two") + def provider_two(): + set_usage_attribute("new-attr", "new-val") + + @log_exceptions_and_usage + def dummy_layer(): + redis_store() + + @log_exceptions_and_usage(store="redis") + def redis_store(): + set_usage_attribute("attr", "val") + + entrypoint(provider="one") + entrypoint(provider="two") + + scope_name = "test_usage.test_global_context_building." + + assert dummy_exporter + assert { + "event": "test-event", + "provider": "provider-one", + "store": "redis", + "attr": "val", + "entrypoint": f"{scope_name}.entrypoint", + }.items() <= dummy_exporter[0].items() + assert dummy_exporter[0]["calls"][0]["fn_name"] == f"{scope_name}.entrypoint" + assert dummy_exporter[0]["calls"][1]["fn_name"] == f"{scope_name}.provider_one" + assert dummy_exporter[0]["calls"][2]["fn_name"] == f"{scope_name}.dummy_layer" + assert dummy_exporter[0]["calls"][3]["fn_name"] == f"{scope_name}.redis_store" + + assert ( + not {"store", "attr"} & dummy_exporter[1].keys() + ) # check that context was reset + assert { + "event": "test-event", + "provider": "provider-two", + "new-attr": "new-val", + }.items() <= dummy_exporter[1].items() + + +def test_exception_recording(dummy_exporter): + @log_exceptions_and_usage(event="test-event") + def entrypoint(): + provider() + + @log_exceptions_and_usage(provider="provider-one") + def provider(): + raise ValueError(1) + + with pytest.raises(ValueError): + entrypoint() + + assert dummy_exporter + assert { + "event": "test-event", + "provider": "provider-one", + "exception": repr(ValueError(1)), + "entrypoint": "test_usage.test_exception_recording..entrypoint", + }.items() <= dummy_exporter[0].items() + + +def test_only_exception_logging(dummy_exporter): + @log_exceptions(scope="exception-only") + def failing_fn(): + raise ValueError(1) + + @log_exceptions_and_usage(scope="usage-and-exception") + def entrypoint(): + failing_fn() + + with pytest.raises(ValueError): + failing_fn() + + assert { + "exception": repr(ValueError(1)), + "scope": "exception-only", + "entrypoint": "test_usage.test_only_exception_logging..failing_fn", + }.items() <= dummy_exporter[0].items() + + with pytest.raises(ValueError): + entrypoint() + + assert { + "exception": repr(ValueError(1)), + "scope": "usage-and-exception", + "entrypoint": "test_usage.test_only_exception_logging..entrypoint", + }.items() <= dummy_exporter[1].items() + + +def test_ratio_based_sampling(dummy_exporter): + @log_exceptions_and_usage() + def entrypoint(): + expensive_fn() + + @log_exceptions_and_usage(sampler=RatioSampler(ratio=0.1)) + def expensive_fn(): + pass + + for _ in range(100): + entrypoint() + + assert len(dummy_exporter) == 10 + + +def test_sampling_priority(dummy_exporter): + @log_exceptions_and_usage(sampler=RatioSampler(ratio=0.3)) + def entrypoint(): + expensive_fn() + + @log_exceptions_and_usage(sampler=RatioSampler(ratio=0.01)) + def expensive_fn(): + other_fn() + + @log_exceptions_and_usage(sampler=RatioSampler(ratio=0.1)) + def other_fn(): + pass + + for _ in range(300): + entrypoint() + + assert len(dummy_exporter) == 3 + + +def test_time_recording(dummy_exporter): + @log_exceptions_and_usage() + def entrypoint(): + time.sleep(0.1) + expensive_fn() + + @log_exceptions_and_usage() + def expensive_fn(): + time.sleep(0.5) + other_fn() + + @log_exceptions_and_usage() + def other_fn(): + time.sleep(0.2) + + entrypoint() + + assert dummy_exporter + calls = dummy_exporter[0]["calls"] + assert call_length_ms(calls[0]) >= 800 + assert call_length_ms(calls[0]) > call_length_ms(calls[1]) >= 700 + assert call_length_ms(calls[1]) > call_length_ms(calls[2]) >= 200 + + +def test_profiling_decorator(dummy_exporter): + @log_exceptions_and_usage() + def entrypoint(): + with tracing_span("custom_span"): + time.sleep(0.1) + + entrypoint() + + assert dummy_exporter + + calls = dummy_exporter[0]["calls"] + assert len(calls) + assert call_length_ms(calls[0]) >= 100 + assert call_length_ms(calls[1]) >= 100 + + assert ( + calls[1]["fn_name"] + == "test_usage.test_profiling_decorator..entrypoint.custom_span" + ) + + +def call_length_ms(call): + return ( + datetime.datetime.fromisoformat(call["end"]) + - datetime.datetime.fromisoformat(call["start"]) + ).total_seconds() * 10 ** 3 diff --git a/sdk/python/tests/utils/data_source_utils.py b/sdk/python/tests/utils/data_source_utils.py index af9663203a2..6e3d77ead0b 100644 --- a/sdk/python/tests/utils/data_source_utils.py +++ b/sdk/python/tests/utils/data_source_utils.py @@ -7,7 +7,6 @@ from feast import BigQuerySource, FileSource from feast.data_format import ParquetFormat -from feast.infra.offline_stores.bigquery import _write_df_to_bq @contextlib.contextmanager @@ -39,7 +38,7 @@ def simple_bq_source_using_table_ref_arg( client.update_dataset(dataset, ["default_table_expiration_ms"]) table_ref = f"{gcp_project}.{bigquery_dataset}.table_{random.randrange(100, 999)}" - job = _write_df_to_bq(client, df, table_ref) + job = client.load_table_from_dataframe(df, table_ref) job.result() return BigQuerySource( diff --git a/sdk/python/tests/utils/online_read_write_test.py b/sdk/python/tests/utils/online_read_write_test.py index 766f9a282cd..34ff7c7d3fa 100644 --- a/sdk/python/tests/utils/online_read_write_test.py +++ b/sdk/python/tests/utils/online_read_write_test.py @@ -67,17 +67,6 @@ def _driver_rw_test(event_ts, created_ts, write, expect_read): event_ts=time_1, created_ts=time_1, write=(1.1, "3.1"), expect_read=(1.1, "3.1") ) - # Note: This behavior has changed for performance. We should test that older - # value can't overwrite over a newer value once we add the respective flag - """ Values with an older event_ts should overwrite newer ones """ - time_2 = datetime.utcnow() - _driver_rw_test( - event_ts=time_1 - timedelta(hours=1), - created_ts=time_2, - write=(-1000, "OLD"), - expect_read=(-1000, "OLD"), - ) - """ Values with an new event_ts should overwrite older ones """ time_3 = datetime.utcnow() _driver_rw_test( @@ -86,21 +75,3 @@ def _driver_rw_test(event_ts, created_ts, write, expect_read): write=(1123, "NEWER"), expect_read=(1123, "NEWER"), ) - - # Note: This behavior has changed for performance. We should test that older - # value can't overwrite over a newer value once we add the respective flag - """ created_ts is used as a tie breaker, using older created_ts here, but we still overwrite """ - _driver_rw_test( - event_ts=time_1 + timedelta(hours=1), - created_ts=time_3 - timedelta(hours=1), - write=(54321, "I HAVE AN OLDER created_ts SO I LOSE"), - expect_read=(54321, "I HAVE AN OLDER created_ts SO I LOSE"), - ) - - """ created_ts is used as a tie breaker, using newer created_ts here so we should overwrite """ - _driver_rw_test( - event_ts=time_1 + timedelta(hours=1), - created_ts=time_3 + timedelta(hours=1), - write=(96864, "I HAVE A NEWER created_ts SO I WIN"), - expect_read=(96864, "I HAVE A NEWER created_ts SO I WIN"), - ) diff --git a/sdk/python/usage_tests/test_usage.py b/sdk/python/usage_tests/test_usage.py deleted file mode 100644 index e72e11b008e..00000000000 --- a/sdk/python/usage_tests/test_usage.py +++ /dev/null @@ -1,157 +0,0 @@ -# Copyright 2020 The Feast Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import tempfile -import uuid -from datetime import datetime - -from feast.infra.online_stores.sqlite import SqliteOnlineStoreConfig -from tenacity import retry, wait_exponential, stop_after_attempt - -from google.cloud import bigquery -import os -from time import sleep - -from feast import Entity, ValueType, FeatureStore, RepoConfig - - -USAGE_BIGQUERY_TABLE = ( - "kf-feast.feast_telemetry.cloudfunctions_googleapis_com_cloud_functions" -) - - -def test_usage_on(): - old_environ = dict(os.environ) - test_usage_id = str(uuid.uuid4()) - os.environ["FEAST_FORCE_USAGE_UUID"] = test_usage_id - os.environ["FEAST_IS_USAGE_TEST"] = "True" - os.environ["FEAST_USAGE"] = "True" - - with tempfile.TemporaryDirectory() as temp_dir: - test_feature_store = FeatureStore( - config=RepoConfig( - registry=os.path.join(temp_dir, "registry.db"), - project="fake_project", - provider="local", - online_store=SqliteOnlineStoreConfig( - path=os.path.join(temp_dir, "online.db") - ), - ) - ) - entity = Entity( - name="driver_car_id", - description="Car driver id", - value_type=ValueType.STRING, - labels={"team": "matchmaking"}, - ) - - test_feature_store.apply([entity]) - - os.environ.clear() - os.environ.update(old_environ) - ensure_bigquery_usage_id_with_retry(test_usage_id) - - -def test_usage_off(): - old_environ = dict(os.environ) - test_usage_id = str(uuid.uuid4()) - os.environ["FEAST_IS_USAGE_TEST"] = "True" - os.environ["FEAST_USAGE"] = "False" - os.environ["FEAST_FORCE_USAGE_UUID"] = test_usage_id - - with tempfile.TemporaryDirectory() as temp_dir: - test_feature_store = FeatureStore( - config=RepoConfig( - registry=os.path.join(temp_dir, "registry.db"), - project="fake_project", - provider="local", - online_store=SqliteOnlineStoreConfig( - path=os.path.join(temp_dir, "online.db") - ), - ) - ) - entity = Entity( - name="driver_car_id", - description="Car driver id", - value_type=ValueType.STRING, - labels={"team": "matchmaking"}, - ) - test_feature_store.apply([entity]) - - os.environ.clear() - os.environ.update(old_environ) - sleep(30) - rows = read_bigquery_usage_id(test_usage_id) - assert rows.total_rows == 0 - - -def test_exception_usage_on(): - old_environ = dict(os.environ) - test_usage_id = str(uuid.uuid4()) - os.environ["FEAST_FORCE_USAGE_UUID"] = test_usage_id - os.environ["FEAST_IS_USAGE_TEST"] = "True" - os.environ["FEAST_USAGE"] = "True" - - try: - test_feature_store = FeatureStore("/tmp/non_existent_directory") - except: - pass - - os.environ.clear() - os.environ.update(old_environ) - ensure_bigquery_usage_id_with_retry(test_usage_id) - - -def test_exception_usage_off(): - old_environ = dict(os.environ) - test_usage_id = str(uuid.uuid4()) - os.environ["FEAST_IS_USAGE_TEST"] = "True" - os.environ["FEAST_USAGE"] = "False" - os.environ["FEAST_FORCE_USAGE_UUID"] = test_usage_id - - try: - test_feature_store = FeatureStore("/tmp/non_existent_directory") - except: - pass - - os.environ.clear() - os.environ.update(old_environ) - sleep(30) - rows = read_bigquery_usage_id(test_usage_id) - assert rows.total_rows == 0 - - -@retry(wait=wait_exponential(multiplier=1, min=1, max=10), stop=stop_after_attempt(7)) -def ensure_bigquery_usage_id_with_retry(usage_id): - rows = read_bigquery_usage_id(usage_id) - if rows.total_rows == 0: - raise Exception(f"Could not find usage id: {usage_id}") - - -def read_bigquery_usage_id(usage_id): - bq_client = bigquery.Client() - query = f""" - SELECT - usage_id - FROM ( - SELECT - JSON_EXTRACT(textPayload, '$.usage_id') AS usage_id - FROM - `{USAGE_BIGQUERY_TABLE}` - WHERE - timestamp >= TIMESTAMP(\"{datetime.utcnow().date().isoformat()}\")) - WHERE - usage_id = '\"{usage_id}\"' - """ - query_job = bq_client.query(query) - return query_job.result()