diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml index 278be10b890..841f5da87b3 100644 --- a/.github/workflows/build_wheels.yml +++ b/.github/workflows/build_wheels.yml @@ -17,7 +17,10 @@ jobs: version_without_prefix: ${{ steps.get_release_version_without_prefix.outputs.version_without_prefix }} highest_semver_tag: ${{ steps.get_highest_semver.outputs.highest_semver_tag }} steps: - - uses: actions/checkout@v2 + - name: Checkout + uses: actions/checkout@v2 + with: + persist-credentials: false - name: Get release version id: get_release_version run: echo ::set-output name=release_version::${GITHUB_REF#refs/*/} @@ -38,6 +41,7 @@ jobs: echo ::set-output name=highest_semver_tag::$(get_tag_release -m) fi - name: Check output + id: check_output env: RELEASE_VERSION: ${{ steps.get_release_version.outputs.release_version }} VERSION_WITHOUT_PREFIX: ${{ steps.get_release_version_without_prefix.outputs.version_without_prefix }} diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 46e16657543..9587044a84d 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -49,7 +49,7 @@ jobs: needs: get-version strategy: matrix: - component: [feature-server-python, feature-server-python-aws, feature-server-java, feature-transformation-server] + component: [feature-server, feature-server-python-aws, feature-server-java, feature-transformation-server] env: MAVEN_CACHE: gs://feast-templocation-kf-feast/.m2.2020-08-19.tar REGISTRY: feastdev diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index feab7b0eef9..1fc38ff97a4 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -42,40 +42,40 @@ jobs: echo "Current version is ${CURRENT_VERSION}" echo "Next version is ${NEXT_VERSION}" - # publish-web-ui-npm: - # if: github.repository == 'feast-dev/feast' - # needs: get_dry_release_versions - # runs-on: ubuntu-latest - # env: - # # This publish is working using an NPM automation token to bypass 2FA - # NPM_TOKEN: ${{ secrets.NPM_TOKEN }} - # CURRENT_VERSION: ${{ needs.get_dry_release_versions.outputs.current_version }} - # NEXT_VERSION: ${{ needs.get_dry_release_versions.outputs.next_version }} - # steps: - # - uses: actions/checkout@v2 - # - uses: actions/setup-node@v2 - # with: - # node-version: '17.x' - # registry-url: 'https://registry.npmjs.org' - # - name: Bump file versions (temporarily for Web UI publish) - # run: python ./infra/scripts/release/bump_file_versions.py ${CURRENT_VERSION} ${NEXT_VERSION} - # - name: Install yarn dependencies - # working-directory: ./ui - # run: yarn install - # - name: Build yarn rollup - # working-directory: ./ui - # run: yarn build:lib - # - name: Publish UI package - # working-directory: ./ui - # run: npm publish - # env: - # # This publish is working using an NPM automation token to bypass 2FA - # NPM_TOKEN: ${{ secrets.NPM_TOKEN }} + publish-web-ui-npm: + if: github.repository == 'feast-dev/feast' + needs: get_dry_release_versions + runs-on: ubuntu-latest + env: + # This publish is working using an NPM automation token to bypass 2FA + NPM_TOKEN: ${{ secrets.NPM_TOKEN }} + CURRENT_VERSION: ${{ needs.get_dry_release_versions.outputs.current_version }} + NEXT_VERSION: ${{ needs.get_dry_release_versions.outputs.next_version }} + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-node@v2 + with: + node-version: '17.x' + registry-url: 'https://registry.npmjs.org' + - name: Bump file versions (temporarily for Web UI publish) + run: python ./infra/scripts/release/bump_file_versions.py ${CURRENT_VERSION} ${NEXT_VERSION} + - name: Install yarn dependencies + working-directory: ./ui + run: yarn install + - name: Build yarn rollup + working-directory: ./ui + run: yarn build:lib + - name: Publish UI package + if: github.event.inputs.dry_run == 'false' + working-directory: ./ui + run: npm publish + env: + # This publish is working using an NPM automation token to bypass 2FA + NPM_TOKEN: ${{ secrets.NPM_TOKEN }} release: name: release runs-on: ubuntu-latest - #needs: publish-web-ui-npm env: GITHUB_TOKEN: ${{ github.event.inputs.token }} GIT_AUTHOR_NAME: feast-ci-bot @@ -91,6 +91,9 @@ jobs: uses: actions/setup-node@v2 with: node-version: '16' + - name: Setup Helm-docs + run: | + brew install norwoodj/tap/helm-docs - name: Release (Dry Run) if: github.event.inputs.dry_run == 'true' run: | diff --git a/.gitignore b/.gitignore index 1edde846ff2..6a86eb2682b 100644 --- a/.gitignore +++ b/.gitignore @@ -125,8 +125,6 @@ instance/ # Sphinx documentation docs/_build/ -sdk/python/docs/source -sdk/python/docs/html # PyBuilder target/ @@ -186,6 +184,7 @@ dmypy.json *.code-workspace # Protos +sdk/python/docs/html sdk/python/feast/protos/ sdk/go/protos/ go/protos/ diff --git a/.releaserc.js b/.releaserc.js index aadc4373e91..124ad8801c2 100644 --- a/.releaserc.js +++ b/.releaserc.js @@ -40,8 +40,8 @@ module.exports = { // Validate the type of release we are doing "verifyReleaseCmd": "./infra/scripts/validate-release.sh ${nextRelease.type} " + current_branch, - // Bump all version files and build UI / update yarn.lock - "prepareCmd": "python ./infra/scripts/release/bump_file_versions.py ${lastRelease.version} ${nextRelease.version}; make build-ui" + // Bump all version files and build UI / update yarn.lock / helm charts + "prepareCmd": "python ./infra/scripts/release/bump_file_versions.py ${lastRelease.version} ${nextRelease.version}; make build-ui; make build-helm-docs" }], ["@semantic-release/release-notes-generator", { diff --git a/CHANGELOG.md b/CHANGELOG.md index 1cb54565778..2e2f6549db8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,15 @@ # Changelog +## [0.24.1](https://github.com/feast-dev/feast/compare/v0.24.0...v0.24.1) (2022-09-08) + + +### Bug Fixes + +* Fix push API to respect feature view's already inferred entity types ([#3172](https://github.com/feast-dev/feast/issues/3172)) ([5dba4a3](https://github.com/feast-dev/feast/commit/5dba4a31c18d938d9d058bf162635c3bdc5781a0)) +* Fix release workflow ([#3144](https://github.com/feast-dev/feast/issues/3144)) ([c77a655](https://github.com/feast-dev/feast/commit/c77a655c03499443e253e4cb9c7d7651685c7de3)) +* Fix Shopify timestamp bug and add warnings to help with debugging entity registration ([#3191](https://github.com/feast-dev/feast/issues/3191)) ([95066ea](https://github.com/feast-dev/feast/commit/95066ea1be9831cfac923734f3a04fac88e194eb)) +* Handle complex Spark data types in SparkSource ([#3154](https://github.com/feast-dev/feast/issues/3154)) ([0e13573](https://github.com/feast-dev/feast/commit/0e135734e313b0b579d7335d008db67dfa14b551)) + # [0.24.0](https://github.com/feast-dev/feast/compare/v0.23.0...v0.24.0) (2022-08-25) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index ae259a72fa8..2bc09150028 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,420 +1,3 @@

Development Guide: Main Feast Repository

-> Please see [Development Guide](https://docs.feast.dev/project/development-guide) for project level development instructions. - -

Maintainer's Guide

- -> Please see [Maintainer's Guide](https://docs.feast.dev/project/maintainers) for instructions for maintainers. Normal developers can also use this guide to setup their forks for localized integration tests. - -

Table of Contents

- -- [Overview](#overview) -- [Community](#community) -- [Making a pull request](#making-a-pull-request) - - [Pull request checklist](#pull-request-checklist) - - [Forking the repo](#forking-the-repo) - - [Pre-commit Hooks](#pre-commit-hooks) - - [Signing off commits](#signing-off-commits) - - [Incorporating upstream changes from master](#incorporating-upstream-changes-from-master) -- [Feast Python SDK / CLI](#feast-python-sdk--cli) - - [Environment Setup](#environment-setup) - - [Code Style & Linting](#code-style--linting) - - [Unit Tests](#unit-tests) - - [Integration Tests](#integration-tests) - - [Local integration tests](#local-integration-tests) - - [(Advanced) Full integration tests](#advanced-full-integration-tests) - - [(Advanced) Running specific provider tests or running your test against specific online or offline stores](#advanced-running-specific-provider-tests-or-running-your-test-against-specific-online-or-offline-stores) - - [(Experimental) Run full integration tests against containerized services](#experimental-run-full-integration-tests-against-containerized-services) - - [Contrib integration tests](#contrib-integration-tests) - - [(Contrib) Running tests for Spark offline store](#contrib-running-tests-for-spark-offline-store) - - [(Contrib) Running tests for Trino offline store](#contrib-running-tests-for-trino-offline-store) - - [(Contrib) Running tests for Postgres offline store](#contrib-running-tests-for-postgres-offline-store) - - [(Contrib) Running tests for Postgres online store](#contrib-running-tests-for-postgres-online-store) - - [(Contrib) Running tests for HBase online store](#contrib-running-tests-for-hbase-online-store) -- [(Experimental) Feast UI](#experimental-feast-ui) -- [Feast Java Serving](#feast-java-serving) -- [Developing the Feast Helm charts](#developing-the-feast-helm-charts) - - [Feast Java Feature Server Helm Chart](#feast-java-feature-server-helm-chart) - - [Feast Python / Go Feature Server Helm Chart](#feast-python--go-feature-server-helm-chart) -- [Feast Go Client](#feast-go-client) - - [Environment Setup](#environment-setup-1) - - [Building](#building) - - [Code Style & Linting](#code-style--linting-1) - - [Unit Tests](#unit-tests-1) - - [Testing with Github Actions workflows](#testing-with-github-actions-workflows) -- [Issues](#issues) - -## Overview -This guide is targeted at developers looking to contribute to Feast components in -the main Feast repository: -- [Feast Python SDK / CLI](#feast-python-sdk--cli) -- [Feast Java Serving](#feast-java-serving) -- [Feast Go Client](#feast-go-client) - -Please see [this page](https://docs.feast.dev/reference/codebase-structure) for more details on the structure of the entire codebase. - -## Community -See [Contribution process](https://docs.feast.dev/project/contributing) and [Community](https://docs.feast.dev/community) for details on how to get more involved in the community. - -A quick few highlights: -- [RFCs](https://drive.google.com/drive/u/0/folders/0AAe8j7ZK3sxSUk9PVA) -- [Community Slack](https://slack.feast.dev/) -- [Feast Dev Mailing List](https://groups.google.com/g/feast-dev) -- [Community Calendar](https://calendar.google.com/calendar/u/0?cid=ZTFsZHVhdGM3MDU3YTJucTBwMzNqNW5rajBAZ3JvdXAuY2FsZW5kYXIuZ29vZ2xlLmNvbQ) - - Includes biweekly community calls at 10AM PST - -## Making a pull request -We use the convention that the assignee of a PR is the person with the next action. - -This means that often, the assignee may be empty (if no reviewer has been found yet), the reviewer, or the PR writer if there are comments to be addressed. - -### Pull request checklist -A quick list of things to keep in mind as you're making changes: -- As you make changes - - Make your changes in a [forked repo](#forking-the-repo) (instead of making a branch on the main Feast repo) - - [Sign your commits](#signing-off-commits) as you go (to avoid DCO checks failing) - - [Rebase from master](#incorporating-upstream-changes-from-master) instead of using `git pull` on your PR branch - - Install [pre-commit hooks](#pre-commit-hooks) to ensure all the default linters / formatters are run when you push. -- When you make the PR - - Make a pull request from the forked repo you made - - Ensure you add a GitHub **label** (i.e. a kind tag to the PR (e.g. `kind/bug` or `kind/housekeeping`)) or else checks will fail. - - Ensure you leave a release note for any user facing changes in the PR. There is a field automatically generated in the PR request. You can write `NONE` in that field if there are no user facing changes. - - Please run tests locally before submitting a PR (e.g. for Python, the [local integration tests](#local-integration-tests)) - - Try to keep PRs smaller. This makes them easier to review. - -### Forking the repo -Fork the Feast Github repo and clone your fork locally. Then make changes to a local branch to the fork. - -See [Creating a pull request from a fork](https://docs.github.com/en/github/collaborating-with-pull-requests/proposing-changes-to-your-work-with-pull-requests/creating-a-pull-request-from-a-fork) - -### Pre-commit Hooks -Setup [`pre-commit`](https://pre-commit.com/) to automatically lint and format the codebase on commit: -1. Ensure that you have Python (3.7 and above) with `pip`, installed. -2. Install `pre-commit` with `pip` & install pre-push hooks -```sh -pip install pre-commit -pre-commit install --hook-type pre-commit --hook-type pre-push -``` -3. On push, the pre-commit hook will run. This runs `make format` and `make lint`. - -### Signing off commits -> :warning: Warning: using the default integrations with IDEs like VSCode or IntelliJ will not sign commits. -> When you submit a PR, you'll have to re-sign commits to pass the DCO check. - -Use git signoffs to sign your commits. See -https://docs.github.com/en/github/authenticating-to-github/managing-commit-signature-verification for details - -Then, you can sign off commits with the `-s` flag: -``` -git commit -s -m "My first commit" -``` - -GPG-signing commits with `-S` is optional. - -### Incorporating upstream changes from master -Our preference is the use of `git rebase [master]` instead of `git merge` : `git pull -r`. - -Note that this means if you are midway through working through a PR and rebase, you'll have to force push: -`git push --force-with-lease origin [branch name]` - -## Feast Python SDK / CLI -### Environment Setup -Setting up your development environment for Feast Python SDK / CLI: -1. Ensure that you have Docker installed in your environment. Docker is used to provision service dependencies during testing, and build images for feature servers and other components. - 1. Please note that we use [Docker with BuiltKit](https://docs.docker.com/develop/develop-images/build_enhancements/). -2. Ensure that you have `make`, Python (3.8 and above) with `pip`, installed. -3. _Recommended:_ Create a virtual environment to isolate development dependencies to be installed -```sh -# create & activate a virtual environment -python -m venv venv/ -source venv/bin/activate -``` - -3. Upgrade `pip` if outdated -```sh -pip install --upgrade pip -``` - -4. (Optional): Install Node & Yarn. Then run the following to build Feast UI artifacts for use in `feast ui` -``` -make build-ui -``` - -5. Install development dependencies for Feast Python SDK / CLI -```sh -pip install -e ".[dev]" -``` - -This will allow the installed feast version to automatically reflect changes to your local development version of Feast without needing to reinstall everytime you make code changes. - -### Code Style & Linting -Feast Python SDK / CLI codebase: -- Conforms to [Black code style](https://black.readthedocs.io/en/stable/the_black_code_style.html) -- Has type annotations as enforced by `mypy` -- Has imports sorted by `isort` -- Is lintable by `flake8` - -To ensure your Python code conforms to Feast Python code standards: -- Autoformat your code to conform to the code style: -```sh -make format-python -``` - -- Lint your Python code before submitting it for review: -```sh -make lint-python -``` - -> Setup [pre-commit hooks](#pre-commit-hooks) to automatically format and lint on commit. - -### Unit Tests -Unit tests (`pytest`) for the Feast Python SDK / CLI can run as follows: -```sh -make test-python -``` - -> :warning: Local configuration can interfere with Unit tests and cause them to fail: -> - Ensure [no AWS configuration is present](https://boto3.amazonaws.com/v1/documentation/api/latest/guide/configuration.html) -> and [no AWS credentials can be accessed](https://boto3.amazonaws.com/v1/documentation/api/latest/guide/credentials.html#configuring-credentials) by `boto3` -> - Ensure Feast Python SDK / CLI is not configured with configuration overrides (ie `~/.feast/config` should be empty). - -### Integration Tests -There are two sets of tests you can run: -1. Local integration tests (for faster development, tests file offline store & key online stores) -2. Full integration tests (requires cloud environment setups) - -#### Local integration tests -For this approach of running tests, you'll need to have docker set up locally: [Get Docker](https://docs.docker.com/get-docker/) - -It leverages a file based offline store to test against emulated versions of Datastore, DynamoDB, and Redis, using ephemeral containers. - -These tests create new temporary tables / datasets locally only, and they are cleaned up. when the containers are torn down. - -```sh -make test-python-integration-local -``` - -#### (Advanced) Full integration tests -To test across clouds, on top of setting up Redis, you also need GCP / AWS / Snowflake setup. - -> Note: you can manually control what tests are run today by inspecting -> [RepoConfiguration](https://github.com/feast-dev/feast/blob/master/sdk/python/tests/integration/feature_repos/repo_configuration.py) -> and commenting out tests that are added to `DEFAULT_FULL_REPO_CONFIGS` - -**GCP** -1. You can get free credits [here](https://cloud.google.com/free/docs/free-cloud-features#free-trial). -2. You will need to setup a service account, enable the BigQuery API, and create a staging location for a bucket. - * Setup your service account and project using steps 1-5 [here](https://codelabs.developers.google.com/codelabs/cloud-bigquery-python#0). - * Remember to save your `PROJECT_ID` and your `key.json`. These will be your secrets that you will need to configure in Github actions. Namely, `secrets.GCP_PROJECT_ID` and `secrets.GCP_SA_KEY`. The `GCP_SA_KEY` value is the contents of your `key.json` file. - * Follow these [instructions](https://cloud.google.com/storage/docs/creating-buckets) in your project to create a bucket for running GCP tests and remember to save the bucket name. - * Make sure to add the service account email that you created in the previous step to the users that can access your bucket. Then, make sure to give the account the correct access roles, namely `objectCreator`, `objectViewer`, `objectAdmin`, and `admin`, so that your tests can use the bucket. -3. Install the [Cloud SDK](https://cloud.google.com/sdk/docs/install). -4. Login to gcloud if you haven't already: - ``` - gcloud auth login - gcloud auth application-default login - ``` - - When you run `gcloud auth application-default login`, you should see some output of the form: - ``` - Credentials saved to file: [$HOME/.config/gcloud/application_default_credentials.json] - ``` - - You should run `export GOOGLE_APPLICATION_CREDENTIALS="$HOME/.config/gcloud/application_default_credentials.json”` to add the application credentials to your .zshrc or .bashrc. -5. Run `export GCLOUD_PROJECT=[your project id from step 2]` to your .zshrc or .bashrc. -6. Running `gcloud config list` should give you something like this: - ```sh - $ gcloud config list - [core] - account = [your email] - disable_usage_reporting = True - project = [your project id] - - Your active configuration is: [default] - ``` -7. Export GCP specific environment variables in your workflow. Namely, - ```sh - export GCS_REGION='[your gcs region e.g US]' - export GCS_STAGING_LOCATION='[your gcs staging location]' - ``` - **NOTE**: Your `GCS_STAGING_LOCATION` should be in the form `gs://` where the bucket name is from step 2. - -8. Once authenticated, you should be able to run the integration tests for BigQuery without any failures. - -**AWS** -1. Setup AWS by creating an account, database, and cluster. You will need to enable Redshift and Dynamo. - * You can get free credits [here](https://aws.amazon.com/free/?all-free-tier.sort-by=item.additionalFields.SortRank&al[…]f.Free%20Tier%20Types=*all&awsf.Free%20Tier%20Categories=*all). -2. To run the AWS Redshift and Dynamo integration tests you will have to export your own AWS credentials. Namely, - -```sh -export AWS_REGION='[your aws region]' -export AWS_CLUSTER_ID='[your aws cluster id]' -export AWS_USER='[your aws user]' -export AWS_DB='[your aws database]' -export AWS_STAGING_LOCATION='[your s3 staging location uri]' -export AWS_IAM_ROLE='[redshift and s3 access role]' -export AWS_LAMBDA_ROLE='[your aws lambda execution role]' -export AWS_REGISTRY_PATH='[your aws registry path]' -``` - -**Snowflake** -1. See https://signup.snowflake.com/ to setup a trial. -2. Setup your account and if you are not an `ACCOUNTADMIN` (if you created your own account, you should be), give yourself the `SYSADMIN` role. - ```sql - grant role accountadmin, sysadmin to user user2; - ``` - * Also remember to save your [account name](https://docs.snowflake.com/en/user-guide/admin-account-identifier.html#:~:text=organization_name%20is%20the%20name%20of,your%20account%20within%20your%20organization), username, and role. - * Your account name can be found under -3. Create Dashboard and add a Tile. -4. Create a warehouse and database named `FEAST` with the schemas `OFFLINE` and `ONLINE`. - ```sql - create or replace warehouse feast_tests_wh with - warehouse_size='MEDIUM' --set your warehouse size to whatever your budget allows-- - auto_suspend = 180 - auto_resume = true - initially_suspended=true; - - create or replace database FEAST; - use database FEAST; - create schema OFFLINE; - create schema ONLINE; - ``` -5. You will need to create a data unloading location(either on S3, GCP, or Azure). Detailed instructions [here](https://docs.snowflake.com/en/user-guide/data-unload-overview.html). You will need to save the storage export location and the storage export name. You will need to create a [storage integration ](https://docs.snowflake.com/en/sql-reference/sql/create-storage-integration.html) in your warehouse to make this work. Name this storage integration `FEAST_S3`. -6. Then to run successfully, you'll need some environment variables setup: - ```sh - export SNOWFLAKE_CI_DEPLOYMENT='[your snowflake account name]' - export SNOWFLAKE_CI_USER='[your snowflake username]' - export SNOWFLAKE_CI_PASSWORD='[your snowflake pw]' - export SNOWFLAKE_CI_ROLE='[your CI role e.g. SYSADMIN]' - export SNOWFLAKE_CI_WAREHOUSE='[your warehouse]' - export BLOB_EXPORT_STORAGE_NAME='[your data unloading storage name]' - export BLOB_EXPORT_URI='[your data unloading blob uri]` - ``` -7. Once everything is setup, running snowflake integration tests should pass without failures. - -Note that for Snowflake / GCP / AWS, running `make test-python-integration` will create new temporary tables / datasets in your cloud storage tables. - -#### (Advanced) Running specific provider tests or running your test against specific online or offline stores - -1. If you don't need to have your test run against all of the providers(`gcp`, `aws`, and `snowflake`) or don't need to run against all of the online stores, you can tag your test with specific providers or stores that you need(`@pytest.mark.universal_online_stores` or `@pytest.mark.universal_online_stores` with the `only` parameter). The `only` parameter selects specific offline providers and online stores that your test will test against. Example: - -```python -# Only parametrizes this test with the sqlite online store -@pytest.mark.universal_online_stores(only=["sqlite"]) -def test_feature_get_online_features_types_match(): -``` - -2. You can also filter tests to run by using pytest's cli filtering. Instead of using the make commands to test Feast, you can filter tests by name with the `-k` parameter. The parametrized integration tests are all uniquely identified by their provider and online store so the `-k` option can select only the tests that you need to run. For example, to run only Redshift related tests, you can use the following command: - -```sh -python -m pytest -n 8 --integration -k Redshift sdk/python/tests -``` - -#### (Experimental) Run full integration tests against containerized services -Test across clouds requires existing accounts on GCP / AWS / Snowflake, and may incur costs when using these services. - -For this approach of running tests, you'll need to have docker set up locally: [Get Docker](https://docs.docker.com/get-docker/) - -It's possible to run some integration tests against emulated local versions of these services, using ephemeral containers. -These tests create new temporary tables / datasets locally only, and they are cleaned up. when the containers are torn down. - -The services with containerized replacements currently implemented are: -- Datastore -- DynamoDB -- Redis -- Trino -- HBase -- Postgres -- Cassandra - -You can run `make test-python-integration-container` to run tests against the containerized versions of dependencies. - -### Contrib integration tests -#### (Contrib) Running tests for Spark offline store -You can run `make test-python-universal-spark` to run all tests against the Spark offline store. (Note: you'll have to run `pip install -e ".[dev]"` first). - -Not all tests are passing yet - -#### (Contrib) Running tests for Trino offline store -You can run `make test-python-universal-trino` to run all tests against the Trino offline store. (Note: you'll have to run `pip install -e ".[dev]"` first) - -#### (Contrib) Running tests for Postgres offline store -You can run `test-python-universal-postgres-offline` to run all tests against the Postgres offline store. (Note: you'll have to run `pip install -e ".[dev]"` first) - -#### (Contrib) Running tests for Postgres online store -You can run `test-python-universal-postgres-online` to run all tests against the Postgres offline store. (Note: you'll have to run `pip install -e ".[dev]"` first) - -#### (Contrib) Running tests for HBase online store -TODO - -## (Experimental) Feast UI -See [Feast contributing guide](ui/CONTRIBUTING.md) - -## Feast Java Serving -See [Java contributing guide](java/CONTRIBUTING.md) - -See also development instructions related to the helm chart below at [Developing the Feast Helm charts](#developing-the-feast-helm-charts) - -## Developing the Feast Helm charts -There are 3 helm charts: -- Feast Java feature server -- Feast Python / Go feature server -- (deprecated) Feast Python feature server - -Generally, you can override the images in the helm charts with locally built Docker images, and install the local helm -chart. - -All README's for helm charts are generated using [helm-docs](https://github.com/norwoodj/helm-docs). You can install it -(e.g. with `brew install norwoodj/tap/helm-docs`) and then run `make build-helm-docs`. - -### Feast Java Feature Server Helm Chart -See the Java demo example (it has development instructions too using minikube) [here](examples/java-demo/README.md) - -It will: -- run `make build-java-docker-dev` to build local Java feature server binaries -- configure the included `application-override.yaml` to override the image tag to use the locally built dev images. -- install the local chart with `helm install feast-release ../../../infra/charts/feast --values application-override.yaml` - -### Feast Python / Go Feature Server Helm Chart -See the Python demo example (it has development instructions too using minikube) [here](examples/python-helm-demo/README.md) - -It will: -- run `make build-feature-server-dev` to build a local python feature server binary -- install the local chart with `helm install feast-release ../../../infra/charts/feast-feature-server --set image.tag=dev --set feature_store_yaml_base64=$(base64 feature_store.yaml)` - -## Feast Go Client -### Environment Setup -Setting up your development environment for Feast Go SDK: - -- Install Golang, [`protoc` with the Golang & grpc plugins](https://developers.google.com/protocol-buffers/docs/gotutorial#compiling-your-protocol-buffers) - -### Building -Build the Feast Go Client with the `go` toolchain: -```sh -make compile-go-lib -``` - -### Code Style & Linting -Feast Go Client codebase: -- Conforms to the code style enforced by `go fmt`. -- Is lintable by `go vet`. - -Autoformat your Go code to satisfy the Code Style standard: -```sh -go fmt -``` - -Lint your Go code: -```sh -go vet -``` - -> Setup [pre-commit hooks](#pre-commit-hooks) to automatically format and lint on commit. - -### Unit Tests -Unit tests for the Feast Go Client can be run as follows: -```sh -make test-go -``` - -### Testing with Github Actions workflows - -Please refer to the maintainers [doc](./docs/project/maintainers.md) if you would like to locally test out the github actions workflow changes. This document will help you setup your fork to test the ci integration tests and other workflows without needing to make a pull request against feast-dev master. +> Please see [Development Guide](https://docs.feast.dev/project/development-guide) for project level development instructions, including instructions for Maintainers. diff --git a/Makefile b/Makefile index 8e03ed5349f..cb20484a7d9 100644 --- a/Makefile +++ b/Makefile @@ -353,19 +353,15 @@ lint-go: compile-protos-go compile-go-lib # Docker -build-docker: build-ci-docker build-feature-server-python-docker build-feature-server-python-aws-docker build-feature-transformation-server-docker build-feature-server-java-docker +build-docker: build-feature-server-python-docker build-feature-server-python-aws-docker build-feature-transformation-server-docker build-feature-server-java-docker push-ci-docker: docker push $(REGISTRY)/feast-ci:$(VERSION) -# TODO(adchia): consider removing. This doesn't run successfully right now -build-ci-docker: - docker buildx build -t $(REGISTRY)/feast-ci:$(VERSION) -f infra/docker/ci/Dockerfile --load . - -push-feature-server-python-docker: +push-feature-server-docker: docker push $(REGISTRY)/feature-server:$$VERSION -build-feature-server-python-docker: +build-feature-server-docker: docker buildx build --build-arg VERSION=$$VERSION \ -t $(REGISTRY)/feature-server:$$VERSION \ -f sdk/python/feast/infra/feature_servers/multicloud/Dockerfile --load . diff --git a/README.md b/README.md index b663533710b..c8adfa5f22c 100644 --- a/README.md +++ b/README.md @@ -185,8 +185,8 @@ The list below contains the functionality that contributors are planning to deve * [ ] Batch transformation (In progress. See [RFC](https://docs.google.com/document/d/1964OkzuBljifDvkV-0fakp2uaijnVzdwWNGdz7Vz50A/edit)) * **Streaming** * [x] [Custom streaming ingestion job support](https://docs.feast.dev/how-to-guides/creating-a-custom-provider) - * [x] [Push based streaming data ingestion to online store (Alpha)](https://docs.feast.dev/reference/data-sources/push) - * [x] [Push based streaming data ingestion to offline store (Alpha)](https://docs.feast.dev/reference/data-sources/push) + * [x] [Push based streaming data ingestion to online store](https://docs.feast.dev/reference/data-sources/push) + * [x] [Push based streaming data ingestion to offline store](https://docs.feast.dev/reference/data-sources/push) * **Deployments** * [x] AWS Lambda (Alpha release. See [RFC](https://docs.google.com/document/d/1eZWKWzfBif66LDN32IajpaG-j82LSHCCOzY6R7Ax7MI/edit)) * [x] Kubernetes (See [guide](https://docs.feast.dev/how-to-guides/running-feast-in-production#4.3.-java-based-feature-server-deployed-on-kubernetes)) @@ -202,7 +202,7 @@ The list below contains the functionality that contributors are planning to deve * [x] Model-centric feature tracking (feature services) * [x] Amundsen integration (see [Feast extractor](https://github.com/amundsen-io/amundsen/blob/main/databuilder/databuilder/extractor/feast_extractor.py)) * [x] DataHub integration (see [DataHub Feast docs](https://datahubproject.io/docs/generated/ingestion/sources/feast/)) - * [x] Feast Web UI (Alpha release. See [docs](https://docs.feast.dev/reference/alpha-web-ui)) + * [x] Feast Web UI (Beta release. See [docs](https://docs.feast.dev/reference/alpha-web-ui)) ## 🎓 Important Resources diff --git a/docs/README.md b/docs/README.md index b838e5fe5b1..f387406c3fd 100644 --- a/docs/README.md +++ b/docs/README.md @@ -57,7 +57,7 @@ Many companies have used Feast to power real-world ML use cases such as: ## How can I get started? {% hint style="info" %} -The best way to learn Feast is to use it. Head over to our [Quickstart](getting-started/quickstart.md) and try it out! +The best way to learn Feast is to use it. Join our [Slack channel](http://slack.feast.dev) and head over to our [Quickstart](getting-started/quickstart.md) and try it out! {% endhint %} Explore the following resources to get started with Feast: diff --git a/docs/SUMMARY.md b/docs/SUMMARY.md index 8ee48677302..430f9a745d7 100644 --- a/docs/SUMMARY.md +++ b/docs/SUMMARY.md @@ -1,7 +1,7 @@ # Table of contents * [Introduction](README.md) -* [Community](community.md) +* [Community & getting help](community.md) * [Roadmap](roadmap.md) * [Changelog](https://github.com/feast-dev/feast/blob/master/CHANGELOG.md) @@ -24,7 +24,6 @@ * [Online store](getting-started/architecture-and-components/online-store.md) * [Batch Materialization Engine](getting-started/architecture-and-components/batch-materialization-engine.md) * [Provider](getting-started/architecture-and-components/provider.md) -* [Learning by example](getting-started/feast-workshop.md) * [Third party integrations](getting-started/third-party-integrations.md) * [FAQ](getting-started/faq.md) @@ -50,7 +49,7 @@ * [Read features from the online store](how-to-guides/feast-snowflake-gcp-aws/read-features-from-the-online-store.md) * [Scaling Feast](how-to-guides/scaling-feast.md) * [Structuring Feature Repos](how-to-guides/structuring-repos.md) -* [Running Feast in production](how-to-guides/running-feast-in-production.md) +* [Running Feast in production (e.g. on Kubernetes)](how-to-guides/running-feast-in-production.md) * [Upgrading for Feast 0.20+](how-to-guides/automated-feast-upgrade.md) * [Customizing Feast](how-to-guides/customizing-feast/README.md) * [Adding a custom batch materialization engine](how-to-guides/customizing-feast/creating-a-custom-materialization-engine.md) @@ -87,6 +86,7 @@ * [Trino (contrib)](reference/offline-stores/trino.md) * [Azure Synapse + Azure SQL (contrib)](reference/offline-stores/mssql.md) * [Online stores](reference/online-stores/README.md) + * [Overview](reference/online-stores/overview.md) * [SQLite](reference/online-stores/sqlite.md) * [Snowflake](reference/online-stores/snowflake.md) * [Redis](reference/online-stores/redis.md) @@ -120,6 +120,7 @@ * [Contribution process](project/contributing.md) * [Development guide](project/development-guide.md) +* [Backwards Compatibility Policy](project/compatibility.md) * [Maintainer Docs](project/maintainers.md) * [Versioning policy](project/versioning-policy.md) * [Release process](project/release-process.md) diff --git a/docs/community.md b/docs/community.md index dc1cc8a0fe8..bd01b2a64b8 100644 --- a/docs/community.md +++ b/docs/community.md @@ -1,17 +1,19 @@ -# Community +# Community & Getting Help ## Links & Resources -* [Slack](https://slack.feast.dev): Feel free to ask questions or say hello! +* [GitHub Repository](https://github.com/feast-dev/feast/): Find the complete Feast codebase on GitHub. +* [Slack](https://slack.feast.dev): Feel free to ask questions or say hello! This is the main place where maintainers and contributors brainstorm and where users ask questions or discuss best practices. + * Feast users should join `#feast-general` or `#feast-beginners` to ask questions + * Feast developers / contributors should join `#feast-development` * [Mailing list](https://groups.google.com/d/forum/feast-dev): We have both a user and developer mailing list. * Feast users should join [feast-discuss@googlegroups.com](mailto:feast-discuss@googlegroups.com) group by clicking [here](https://groups.google.com/g/feast-discuss). - * Feast developers should join [feast-dev@googlegroups.com](mailto:feast-dev@googlegroups.com) group by clicking [here](https://groups.google.com/d/forum/feast-dev). + * Feast developers / contributors should join [feast-dev@googlegroups.com](mailto:feast-dev@googlegroups.com) group by clicking [here](https://groups.google.com/d/forum/feast-dev). * [Community Calendar](https://calendar.google.com/calendar/u/0?cid=ZTFsZHVhdGM3MDU3YTJucTBwMzNqNW5rajBAZ3JvdXAuY2FsZW5kYXIuZ29vZ2xlLmNvbQ): Includes community calls and design meetings. * [Google Folder](https://drive.google.com/drive/u/0/folders/1jgMHOPDT2DvBlJeO9LCM79DP4lm4eOrR): This folder is used as a central repository for all Feast resources. For example: * Design proposals in the form of Request for Comments (RFC). * User surveys and meeting minutes. * Slide decks of conferences our contributors have spoken at. -* [Feast GitHub Repository](https://github.com/feast-dev/feast/): Find the complete Feast codebase on GitHub. * [Feast Linux Foundation Wiki](https://wiki.lfaidata.foundation/display/FEAST/Feast+Home): Our LFAI wiki page contains links to resources for contributors and maintainers. ## How can I get help? @@ -22,17 +24,30 @@ ## Community Calls +### General community call (biweekly) We have a user and contributor community call every two weeks (US & EU friendly). {% hint style="info" %} Please join the above Feast user groups in order to see calendar invites to the community calls {% endhint %} -### Frequency (every 2 weeks) +#### Frequency (every 2 weeks) * Tuesday 10:00 am to 10:30 am PST -### Links +#### Links * Zoom: [https://zoom.us/j/6325193230](https://zoom.us/j/6325193230) * Meeting notes (incl recordings): [https://bit.ly/feast-notes](https://bit.ly/feast-notes) + +### Developers call (biweekly) +We also have a `#feast-development` community call every two weeks, where we discuss contributions + brainstorm best practices. + +#### Frequency (every 2 weeks) + +* Tuesday 8:00 am to 8:30 am PST + +#### Links + +* Meeting notes (incl recordings): [Feast Development Biweekly](https://docs.google.com/document/d/1zUbIWFWjaBEVlToOdupnmKQwgAtFYx41sPoEEEdd2io/edit#) +* Zoom: [https://zoom.us/j/93657748160?pwd=K3ZpdzhqejgrcXNhc3BlSjFMdzUxdz09](https://zoom.us/j/93657748160?pwd=K3ZpdzhqejgrcXNhc3BlSjFMdzUxdz09) diff --git a/docs/getting-started/architecture-and-components/online-store.md b/docs/getting-started/architecture-and-components/online-store.md index 21b4dbcb9c7..980089c4fe8 100644 --- a/docs/getting-started/architecture-and-components/online-store.md +++ b/docs/getting-started/architecture-and-components/online-store.md @@ -1,15 +1,18 @@ # Online store -The Feast online store is used for low-latency online feature value lookups. Feature values are loaded into the online store from data sources in feature views using the `materialize` command. +Feast uses online stores to serve features at low latency. +Feature values are loaded from data sources into the online store through _materialization_, which can be triggered through the `materialize` command. -The storage schema of features within the online store mirrors that of the data source used to populate the online store. One key difference between the online store and data sources is that only the latest feature values are stored per entity key. No historical values are stored. +The storage schema of features within the online store mirrors that of the original data source. +One key difference is that for each [entity key](../concepts/entity.md), only the latest feature values are stored. +No historical values are stored. -Example batch data source +Here is an example batch data source: ![](../../.gitbook/assets/image%20%286%29.png) -Once the above data source is materialized into Feast \(using `feast materialize`\), the feature values will be stored as follows: +Once the above data source is materialized into Feast (using `feast materialize`), the feature values will be stored as follows: ![](../../.gitbook/assets/image%20%285%29.png) -Features can also be written to the online store via [push sources](../../reference/data-sources/push.md) \ No newline at end of file +Features can also be written directly to the online store via [push sources](../../reference/data-sources/push.md) . \ No newline at end of file diff --git a/docs/getting-started/feast-workshop.md b/docs/getting-started/feast-workshop.md deleted file mode 100644 index 0d648452223..00000000000 --- a/docs/getting-started/feast-workshop.md +++ /dev/null @@ -1,44 +0,0 @@ -# Learning by example - -This workshop aims to teach users about Feast. - -We explain concepts & best practices by example, and also showcase how to address common use cases. - -### Pre-requisites - -This workshop assumes you have the following installed: - -* A local development environment that supports running Jupyter notebooks (e.g. VSCode with Jupyter plugin) -* Python 3.7+ -* Java 11 (for Spark, e.g. `brew install java11`) -* pip -* Docker & Docker Compose (e.g. `brew install docker docker-compose`) -* Terraform ([docs](https://learn.hashicorp.com/tutorials/terraform/install-cli#install-terraform)) -* AWS CLI -* An AWS account setup with credentials via `aws configure` (e.g see [AWS credentials quickstart](https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-quickstart.html#cli-configure-quickstart-creds)) - -Since we'll be learning how to leverage Feast in CI/CD, you'll also need to fork this workshop repository. - -#### **Caveats** - -* M1 Macbook development is untested with this flow. See also [How to run / develop for Feast on M1 Macs](https://github.com/feast-dev/feast/issues/2105). -* Windows development has only been tested with WSL. You will need to follow this [guide](https://docs.docker.com/desktop/windows/wsl/) to have Docker play nicely. - -### Modules - -_See also:_ [_Feast quickstart_](https://docs.feast.dev/getting-started/quickstart)_,_ [_Feast x Great Expectations tutorial_](https://docs.feast.dev/tutorials/validating-historical-features) - -These are meant mostly to be done in order, with examples building on previous concepts. - -See [https://github.com/feast-dev/feast-workshop](https://github.com/feast-dev/feast-workshop) - -| Time (min) | Description | Module | -| :--------: | ----------------------------------------------------------------------- | -------- | -| 30-45 | Setting up Feast projects & CI/CD + powering batch predictions | Module 0 | -| 15-20 | Streaming ingestion & online feature retrieval with Kafka, Spark, Redis | Module 1 | -| 10-15 | Real-time feature engineering with on demand transformations | Module 2 | -| TBD | Feature server deployment (embed, as a service, AWS Lambda) | TBD | -| TBD | Versioning features / models in Feast | TBD | -| TBD | Data quality monitoring in Feast | TBD | -| TBD | Batch transformations | TBD | -| TBD | Stream transformations | TBD | diff --git a/docs/how-to-guides/feast-snowflake-gcp-aws/README.md b/docs/how-to-guides/feast-snowflake-gcp-aws/README.md index 753650080b0..0f6d099349c 100644 --- a/docs/how-to-guides/feast-snowflake-gcp-aws/README.md +++ b/docs/how-to-guides/feast-snowflake-gcp-aws/README.md @@ -12,3 +12,7 @@ {% page-ref page="read-features-from-the-online-store.md" %} +{% page-ref page="../scaling-feast.md" %} + +{% page-ref page="../structuring-repos.md" %} + diff --git a/docs/how-to-guides/production-spark-bytewax.png b/docs/how-to-guides/production-spark-bytewax.png new file mode 100644 index 00000000000..ce3f7017cdf Binary files /dev/null and b/docs/how-to-guides/production-spark-bytewax.png differ diff --git a/docs/how-to-guides/running-feast-in-production.md b/docs/how-to-guides/running-feast-in-production.md index 61b7b1fe40a..95f3e99581e 100644 --- a/docs/how-to-guides/running-feast-in-production.md +++ b/docs/how-to-guides/running-feast-in-production.md @@ -1,40 +1,47 @@ -# Running Feast in production +# Running Feast in production (e.g. on Kubernetes) ## Overview -After learning about Feast concepts and playing with Feast locally, you're now ready to use Feast in production. This guide aims to help with the transition from a sandbox project to production-grade deployment in the cloud or on-premise. +After learning about Feast concepts and playing with Feast locally, you're now ready to use Feast in production. This guide aims to help with the transition from a sandbox project to production-grade deployment in the cloud or on-premise (e.g. on Kubernetes). -Overview of typical production configuration is given below: +A typical production architecture looks like: ![Overview](production-simple.png) {% hint style="success" %} -**Important note:** Feast is highly customizable and modular. Most Feast blocks are loosely connected and can be used independently. Hence, you are free to build your own production configuration. +**Important note:** Feast is highly customizable and modular. + +Most Feast blocks are loosely connected and can be used independently. Hence, you are free to build your own production configuration. For example, you might not have a stream source and, thus, no need to write features in real-time to an online store. Or you might not need to retrieve online features. Feast also often provides multiple options to achieve the same goal. We discuss tradeoffs below. + +Additionally, please check the how-to guide for some specific recommendations on [how to scale Feast](./scaling-feast.md). {% endhint %} In this guide we will show you how to: 1. Deploy your feature store and keep your infrastructure in sync with your feature repository -2. Keep the data in your online store up to date +2. Keep the data in your online store up to date (from batch and stream sources) 3. Use Feast for model training and serving -4. Ingest features from a stream source -5. Monitor your production deployment ## 1. Automatically deploying changes to your feature definitions -### Setting up a feature repository +### 1.1 Setting up a feature repository The first step to setting up a deployment of Feast is to create a Git repository that contains your feature definitions. The recommended way to version and track your feature definitions is by committing them to a repository and tracking changes through commits. If you recall, running `feast apply` commits feature definitions to a **registry**, which users can then read elsewhere. -### Setting up CI/CD to automatically update the registry +### 1.2 Setting up a database-backed registry + +Out of the box, Feast serializes all of its state into a file-based registry. When running Feast in production, we recommend using the more scalable SQL-based registry that is backed by a database. Details are available [here](./scaling-feast.md#scaling-feast-registry). + +### 1.3 Setting up CI/CD to automatically update the registry We recommend typically setting up CI/CD to automatically run `feast plan` and `feast apply` when pull requests are opened / merged. -### Setting up multiple environments +### 1.4 Setting up multiple environments A common scenario when using Feast in production is to want to test changes to Feast object definitions. For this, we recommend setting up a _staging_ environment for your offline and online stores, which mirrors _production_ (with potentially a smaller data set). + Having this separate environment allows users to test changes by first applying them to staging, and then promoting the changes to production after verifying the changes on staging. Different options are presented in the [how-to guide](structuring-repos.md). @@ -43,101 +50,136 @@ Different options are presented in the [how-to guide](structuring-repos.md). To keep your online store up to date, you need to run a job that loads feature data from your feature view sources into your online store. In Feast, this loading operation is called materialization. -### 2.1. Manual materializations - -The simplest way to schedule materialization is to run an **incremental** materialization using the Feast CLI: +### 2.1 Scalable Materialization -``` -feast materialize-incremental 2022-01-01T00:00:00 -``` +Out of the box, Feast's materialization process uses an in-process materialization engine. This engine loads all the data being materialized into memory from the offline store, and writes it into the online store. -The above command will load all feature values from all feature view sources into the online store up to the time `2022-01-01T00:00:00`. +This approach may not scale to large amounts of data, which users of Feast may be dealing with in production. +In this case, we recommend using one of the more [scalable materialization engines](./scaling-feast.md#scaling-materialization), such as the [Bytewax Materialization Engine](../reference/batch-materialization/bytewax.md), or the [Snowflake Materialization Engine](../reference/batch-materialization/snowflake.md). +Users may also need to [write a custom materialization engine](../how-to-guides/customizing-feast/creating-a-custom-materialization-engine.md) to work on their existing infrastructure. -A timestamp is required to set the end date for materialization. If your source is fully up to date then the end date would be the current time. However, if you are querying a source where data is not yet available, then you do not want to set the timestamp to the current time. You would want to use a timestamp that ends at a date for which data is available. The next time `materialize-incremental` is run, Feast will load data that starts from the previous end date, so it is important to ensure that the materialization interval does not overlap with time periods for which data has not been made available. This is commonly the case when your source is an ETL pipeline that is scheduled on a daily basis. +The Bytewax materialization engine can run materialization on an existing Kubernetes cluster. An example configuration of this in a `feature_store.yaml` is as follows: -An alternative approach to incremental materialization (where Feast tracks the intervals of data that need to be ingested), is to call Feast directly from your scheduler like Airflow. In this case, Airflow is the system that tracks the intervals that have been ingested. - -``` -feast materialize -v driver_hourly_stats 2020-01-01T00:00:00 2020-01-02T00:00:00 +```yaml +batch_engine: + type: bytewax + namespace: bytewax + image: bytewax/bytewax-feast:latest + env: + - name: AWS_ACCESS_KEY_ID + valueFrom: + secretKeyRef: + name: aws-credentials + key: aws-access-key-id + - name: AWS_SECRET_ACCESS_KEY + valueFrom: + secretKeyRef: + name: aws-credentials + key: aws-secret-access-key ``` -In the above example we are materializing the source data from the `driver_hourly_stats` feature view over a day. This command can be scheduled as the final operation in your Airflow ETL, which runs after you have computed your features and stored them in the source location. Feast will then load your feature data into your online store. +### 2.2 Scheduled materialization + +> See also [data ingestion](../getting-started/concepts/data-ingestion.md#batch-data-ingestion) for code snippets -The timestamps above should match the interval of data that has been computed by the data transformation system. +It is up to you to orchestrate and schedule runs of materialization. -### 2.2. Automate periodic materializations +Feast keeps the history of materialization in its registry so that the choice could be as simple as a [unix cron util](https://en.wikipedia.org/wiki/Cron). Cron util should be sufficient when you have just a few materialization jobs (it's usually one materialization job per feature view) triggered infrequently. -It is up to you which orchestration/scheduler to use to periodically run `$ feast materialize`. Feast keeps the history of materialization in its registry so that the choice could be as simple as a [unix cron util](https://en.wikipedia.org/wiki/Cron). Cron util should be sufficient when you have just a few materialization jobs (it's usually one materialization job per feature view) triggered infrequently. However, the amount of work can quickly outgrow the resources of a single machine. That happens because the materialization job needs to repackage all rows before writing them to an online store. That leads to high utilization of CPU and memory. In this case, you might want to use a job orchestrator to run multiple jobs in parallel using several workers. Kubernetes Jobs or Airflow are good choices for more comprehensive job orchestration. +However, the amount of work can quickly outgrow the resources of a single machine. That happens because the materialization job needs to repackage all rows before writing them to an online store. That leads to high utilization of CPU and memory. In this case, you might want to use a job orchestrator to run multiple jobs in parallel using several workers. Kubernetes Jobs or Airflow are good choices for more comprehensive job orchestration. -If you are using Airflow as a scheduler, Feast can be invoked through the [BashOperator](https://airflow.apache.org/docs/apache-airflow/stable/howto/operator/bash.html) after the [Python SDK](https://pypi.org/project/feast/) has been installed into a virtual environment and your feature repo has been synced: +If you are using Airflow as a scheduler, Feast can be invoked through a [PythonOperator](https://airflow.apache.org/docs/apache-airflow/stable/howto/operator/python.html) after the [Python SDK](https://pypi.org/project/feast/) has been installed into a virtual environment and your feature repo has been synced: ```python -materialize = BashOperator( - task_id='materialize', - bash_command=f'feast materialize-incremental {datetime.datetime.now().replace(microsecond=0).isoformat()}', +import datetime +from airflow.operators.python_operator import PythonOperator +from feast import RepoConfig, FeatureStore +from feast.infra.online_stores.dynamodb import DynamoDBOnlineStoreConfig +from feast.repo_config import RegistryConfig + +# Define Python callable +def materialize(): + repo_config = RepoConfig( + registry=RegistryConfig(path="s3://[YOUR BUCKET]/registry.pb"), + project="feast_demo_aws", + provider="aws", + offline_store="file", + online_store=DynamoDBOnlineStoreConfig(region="us-west-2") + ) + store = FeatureStore(config=repo_config) + # Option 1: materialize just one feature view + # store.materialize_incremental(datetime.datetime.now(), feature_views=["my_fv_name"]) + # Option 2: materialize all feature views incrementally + store.materialize_incremental(datetime.datetime.now()) + +# Use Airflow PythonOperator +materialize_python = PythonOperator( + task_id='materialize_python', + python_callable=materialize, ) ``` {% hint style="success" %} -Important note: Airflow worker must have read and write permissions to the registry file on GS / S3 since it pulls configuration and updates materialization history. +Important note: Airflow worker must have read and write permissions to the registry file on GCS / S3 since it pulls configuration and updates materialization history. {% endhint %} -## 3. How to use Feast for model training +### 2.3 Stream feature ingestion +See more details at [data ingestion](../getting-started/concepts/data-ingestion.md), which shows how to ingest streaming features or 3rd party feature data via a push API. -After we've defined our features and data sources in the repository, we can generate training datasets. +This supports pushing feature values into Feast to both online or offline stores. -The first thing we need to do in our training code is to create a `FeatureStore` object with a path to the registry. -One way to ensure your production clients have access to the feature store is to provide a copy of the `feature_store.yaml` to those pipelines. This `feature_store.yaml` file will have a reference to the feature store registry, which allows clients to retrieve features from offline or online stores. +## 3. How to use Feast for model training -```python -fs = FeatureStore(repo_path="production/") -``` +### 3.1. Generating training data +After we've defined our features and data sources in the repository, we can generate training datasets. We highly recommend you use a `FeatureService` to version the features that go into a specific model version. -Then, training data can be retrieved as follows: +1. The first thing we need to do in our training code is to create a `FeatureStore` object with a path to the registry. + - One way to ensure your production clients have access to the feature store is to provide a copy of the `feature_store.yaml` to those pipelines. This `feature_store.yaml` file will have a reference to the feature store registry, which allows clients to retrieve features from offline or online stores. -```python -feature_refs = [ - 'driver_hourly_stats:conv_rate', - 'driver_hourly_stats:acc_rate', - 'driver_hourly_stats:avg_daily_trips' -] - -training_df = fs.get_historical_features( - entity_df=entity_df, - features=feature_refs, -).to_df() + ```python + from feast import FeatureStore -model = ml.fit(training_df) -``` + fs = FeatureStore(repo_path="production/") + ``` +2. Then, you need to generate an **entity dataframe**. You have two options + - Create an entity dataframe manually and pass it in + - Use a SQL query to dynamically generate lists of entities (e.g. all entities within a time range) and timestamps to pass into Feast +3. Then, training data can be retrieved as follows: -The most common way to productionize ML models is by storing and versioning models in a "model store", and then deploying these models into production. When using Feast, it is recommended that the list of feature references also be saved alongside the model. This ensures that models and the features they are trained on are paired together when being shipped into production: + ```python + training_retrieval_job = fs.get_historical_features( + entity_df=entity_df_or_sql_string, + features=fs.get_feature_service("driver_activity_v1"), + ) -```python -# Save model -model.save('my_model.bin') + # Option 1: In memory model training + model = ml.fit(training_retrieval_job.to_df()) -# Save features -open('feature_refs.json', 'w') as f: - json.dump(feature_refs, f) -``` + # Option 2: Unloading to blob storage. Further post-processing can occur before kicking off distributed training. + training_retrieval_job.to_remote_storage() + ``` -To test your model locally, you can simply create a `FeatureStore` object, fetch online features, and then make a prediction: +### 3.2 Versioning features that power ML models +The most common way to productionize ML models is by storing and versioning models in a "model store", and then deploying these models into production. When using Feast, it is recommended that the feature service name and the model versions have some established convention. + +For example, in MLflow: ```python -# Load model -model = ml.load('my_model.bin') +import mlflow.pyfunc -# Load feature references -with open('feature_refs.json', 'r') as f: - feature_refs = json.load(f) +# Load model from MLflow +model_name = "my-model" +model_version = 1 +model = mlflow.pyfunc.load_model( + model_uri=f"models:/{model_name}/{model_version}" +) -# Create feature store object fs = FeatureStore(repo_path="production/") -# Read online features +# Read online features using the same model name and model version feature_vector = fs.get_online_features( - features=feature_refs, + features=fs.get_feature_service(f"{model_name}_v{model_version}"), entity_rows=[{"driver_id": 1001}] ).to_dict() @@ -151,7 +193,7 @@ It is important to note that both the training pipeline and model serving servic ## 4. Retrieving online features for prediction -Once you have successfully loaded (or in Feast terminology materialized) your data from batch sources into the online store, you can start consuming features for model inference. There are three approaches for that purpose sorted from the most simple one (in an operational sense) to the most performant (benchmarks to be published soon): +Once you have successfully loaded data from batch / streaming sources into the online store, you can start consuming features for model inference. ### 4.1. Use the Python SDK within an existing Python service @@ -174,18 +216,11 @@ feature_vector = fs.get_online_features( ).to_dict() ``` -### 4.2. Consume features via HTTP API from Serverless Feature Server - -If you don't want to add the Feast Python SDK as a dependency, or your feature retrieval service is written in a non-Python language, Feast can deploy a simple feature server on serverless infrastructure (eg, AWS Lambda, Google Cloud Run) for you. This service will provide an HTTP API with JSON I/O, which can be easily used with any programming language. - -[Read more about this feature](../reference/feature-servers/alpha-aws-lambda-feature-server.md) +### 4.2. Deploy Feast feature servers on Kubernetes -### 4.3. Go feature server deployed on Kubernetes - -For users with very latency-sensitive and high QPS use-cases, Feast offers a high-performance [Go feature server](../reference/feature-servers/go-feature-server.md). It can use either HTTP or gRPC. - -The Go feature server can be deployed to a Kubernetes cluster via Helm charts in a few simple steps: +To deploy a Feast feature server on Kubernetes, you can use the included [helm chart + tutorial](https://github.com/feast-dev/feast/tree/master/infra/charts/feast-feature-server) (which also has detailed instructions and an example tutorial). +**Basic steps** 1. Install [kubectl](https://kubernetes.io/docs/tasks/tools/install-kubectl/) and [helm 3](https://helm.sh/) 2. Add the Feast Helm repository and download the latest charts: @@ -194,51 +229,16 @@ helm repo add feast-charts https://feast-helm-charts.storage.googleapis.com helm repo update ``` -1. Run Helm Install +3. Run Helm Install ``` helm install feast-release feast-charts/feast-feature-server \ - --set global.registry.path=s3://feast/registries/prod \ - --set global.project= -``` - -This chart will deploy a single service. The service must have read access to the registry file on cloud storage. It will keep a copy of the registry in their memory and periodically refresh it, so expect some delays in update propagation in exchange for better performance. In order for the Go feature server to be enabled, you should set `go_feature_serving: True` in the `feature_store.yaml`. - -## 5. Ingesting features from a stream source - -Recently Feast added functionality for [stream ingestion](../reference/data-sources/push.md). Please note that this is still in an early phase and new incompatible changes may be introduced. - -### 5.1. Using Python SDK in your Apache Spark / Beam pipeline - -The default option to write features from a stream is to add the Python SDK into your existing PySpark / Beam pipeline. Feast SDK provides writer implementation that can be called from `foreachBatch` stream writer in PySpark like this: - -```python -from feast import FeatureStore - -store = FeatureStore(...) - -def feast_writer(spark_df): - pandas_df = spark_df.to_pandas() - store.push("driver_hourly_stats", pandas_df) - -streamingDF.writeStream.foreachBatch(feast_writer).start() + --set feature_store_yaml_base64=$(base64 feature_store.yaml) ``` -### 5.2. Push Service (Alpha) +This will deploy a single service. The service must have read access to the registry file on cloud storage. It will keep a copy of the registry in their memory and periodically refresh it, so expect some delays in update propagation in exchange for better performance. -Alternatively, if you want to ingest features directly from a broker (eg, Kafka or Kinesis), you can use the "push service", which will write to an online store and/or offline store. This service will expose an HTTP API or when deployed on Serverless platforms like AWS Lambda or Google Cloud Run, this service can be directly connected to Kinesis or PubSub. - -If you are using Kafka, [HTTP Sink](https://docs.confluent.io/kafka-connect-http/current/overview.html) could be utilized as a middleware. In this case, the "push service" can be deployed on Kubernetes or as a Serverless function. - -## 6. Monitoring - -Feast services can report their metrics to a StatsD-compatible collector. To activate this function, you'll need to provide a StatsD IP address and a port when deploying the helm chart (in future, this will be added to `feature_store.yaml`). - -We use an [InfluxDB-style extension](https://github.com/prometheus/statsd\_exporter#tagging-extensions) for StatsD format to be able to send tags along with metrics. Keep that in mind while selecting the collector ([telegraph](https://www.influxdata.com/blog/getting-started-with-sending-statsd-metrics-to-telegraf-influxdb/#introducing-influx-statsd) will work for sure). - -We chose StatsD since it's a de-facto standard with various implementations (eg, [1](https://github.com/prometheus/statsd\_exporter), [2](https://github.com/influxdata/telegraf/blob/master/plugins/inputs/statsd/README.md)) and metrics can be easily exported to Prometheus, InfluxDB, AWS CloudWatch, etc. - -## 7. Using environment variables in your yaml configuration +## 5. Using environment variables in your yaml configuration You might want to dynamically set parts of your configuration from your environment. For instance to deploy Feast to production and development with the same configuration, but a different server. Or to inject secrets without exposing them in your git repo. To do this, it is possible to use the `${ENV_VAR}` syntax in your `feature_store.yaml` file. For instance: @@ -266,32 +266,17 @@ online_store: ## Summary -Summarizing it all together we want to show several options of architecture that will be most frequently used in production: +In summary, the overall architecture in production may look like: -### Option #1 (currently preferred) +* Feast SDK is being triggered by CI (eg, Github Actions). It applies the latest changes from the feature repo to the Feast database-backed registry +* Data ingestion + * **Batch data**: Airflow manages materialization jobs to ingest batch data from DWH to the online store periodically. When working with large datasets to materialize, we recommend using a batch materialization engine + * If your offline and online workloads are in Snowflake, the Snowflake materialization engine is likely the best option. + * If your offline and online workloads are not using Snowflake, but using Kubernetes is an option, the Bytewax materialization engine is likely the best option. + * If none of these engines suite your needs, you may continue using the in-process engine, or write a custom engine (e.g with Spark or Ray). + * **Stream data**: The Feast Push API is used within existing Spark / Beam pipelines to push feature values to offline / online stores -* Feast SDK is being triggered by CI (eg, Github Actions). It applies the latest changes from the feature repo to the Feast registry -* Airflow manages materialization jobs to ingest data from DWH to the online store periodically -* For the stream ingestion Feast Python SDK is used in the existing Spark / Beam pipeline -* Online features are served via either a Python feature server or a high performance Go feature server - * The Go feature server can be deployed on a Kubernetes cluster (via Helm charts) +* Online features are served via the Python feature server over HTTP, or consumed using the Feast Python SDK. * Feast Python SDK is called locally to generate a training dataset -![From Repository to Production: Feast Production Architecture](production-spark.png) - -### Option #2 _(still in development)_ - -Same as Option #1, except: - -* Push service is deployed as AWS Lambda / Google Cloud Run and is configured as a sink for Kinesis or PubSub to ingest features directly from a stream broker. Lambda / Cloud Run is being managed by Feast SDK (from CI environment) -* Materialization jobs are managed inside Kubernetes via Kubernetes Job (currently not managed by Helm) - -![With Push Service as Lambda](production-lambda.png) - -### Option #3 _(still in development)_ - -Same as Option #2, except: - -* Push service is deployed on Kubernetes cluster and exposes an HTTP API that can be used as a sink for Kafka (via kafka-http connector) or accessed directly. - -![With Push Service in Kubernetes](production-kube.png) +![From Repository to Production: Feast Production Architecture](production-spark-bytewax.png) diff --git a/docs/project/compatibility.md b/docs/project/compatibility.md new file mode 100644 index 00000000000..9db4d3a14a4 --- /dev/null +++ b/docs/project/compatibility.md @@ -0,0 +1,13 @@ +# API Compatibility in Feast + +Feast follows [semantic versions](./versioning-policy.md). Being pre-1.0, Feast is still considered in active initial development as per the [SemVer spec](https://semver.org/#faq). + +That being said, Feast takes backwards compatibility seriously, to ensure that introducing new functionality across minor versions does not break current users. + +When possible, API changes should always be made in a backwards compatible way. +If this is not possible, the maintainers introduce new APIs alongside existing, now-deprecated APIs, with the intention of supporting the existing APIs for at least 3 minor versions, before deprecating and removing them. +In some cases, the deprecated APIs may be supported for longer than 3 minor versions, if necessary to give users a longer time for migrations. + +When deprecating existing APIs, deprecation warnings should be introduced early, with the expected version at which the deprecated API would be removed. + +At this point, core functionality in Feast is considered "stable" and ready for usage. However, there are still some components that are considered "Alpha". Please check the [roadmap](../roadmap.md) for a full list of all capabilities and their status. diff --git a/docs/project/development-guide.md b/docs/project/development-guide.md index 5aae0628f67..d06ace327cb 100644 --- a/docs/project/development-guide.md +++ b/docs/project/development-guide.md @@ -1,72 +1,438 @@ -# Development guide +# Development Guide: Main Feast Repository -## Overview +## Table of Contents -This guide is targeted at developers looking to contribute to Feast: +- [Overview](#overview) +- [Community](#community) +- [Making a pull request](#making-a-pull-request) + - [Pull request checklist](#pull-request-checklist) + - [Good Practices](#good-practices-to-keep-in-mind) + - [Forking the repo](#forking-the-repo) + - [Pre-commit Hooks](#pre-commit-hooks) + - [Signing off commits](#signing-off-commits) + - [Incorporating upstream changes from master](#incorporating-upstream-changes-from-master) +- [Feast Python SDK / CLI](#feast-python-sdk--cli) + - [Environment Setup](#environment-setup) + - [Code Style & Linting](#code-style--linting) + - [Unit Tests](#unit-tests) + - [Integration Tests](#integration-tests) + - [Local integration tests](#local-integration-tests) + - [(Advanced) Full integration tests](#advanced-full-integration-tests) + - [(Advanced) Running specific provider tests or running your test against specific online or offline stores](#advanced-running-specific-provider-tests-or-running-your-test-against-specific-online-or-offline-stores) + - [(Experimental) Run full integration tests against containerized services](#experimental-run-full-integration-tests-against-containerized-services) + - [Contrib integration tests](#contrib-integration-tests) + - [(Contrib) Running tests for Spark offline store](#contrib-running-tests-for-spark-offline-store) + - [(Contrib) Running tests for Trino offline store](#contrib-running-tests-for-trino-offline-store) + - [(Contrib) Running tests for Postgres offline store](#contrib-running-tests-for-postgres-offline-store) + - [(Contrib) Running tests for Postgres online store](#contrib-running-tests-for-postgres-online-store) + - [(Contrib) Running tests for HBase online store](#contrib-running-tests-for-hbase-online-store) +- [(Experimental) Feast UI](#experimental-feast-ui) +- [Feast Java Serving](#feast-java-serving) +- [Developing the Feast Helm charts](#developing-the-feast-helm-charts) + - [Feast Java Feature Server Helm Chart](#feast-java-feature-server-helm-chart) + - [Feast Python / Go Feature Server Helm Chart](#feast-python--go-feature-server-helm-chart) +- [Feast Go Client](#feast-go-client) + - [Environment Setup](#go-environment-setup) + - [Building](#building-go) + - [Code Style & Linting](#go-code-style--linting) + - [Unit Tests](#go-unit-tests) + - [Testing with GitHub Actions workflows](#testing-with-github-actions-workflows) +- [Data Storage Formats](#feast-data-storage-format) +## Overview +This guide is targeted at developers looking to contribute to Feast components in +the main Feast repository: +- [Feast Python SDK / CLI](#feast-python-sdk--cli) +- [Feast Java Serving](#feast-java-serving) +- [Feast Go Client](#feast-go-client) -* [Project Structure](development-guide.md#repository-structure) -* [Making a Pull Request](development-guide.md#making-a-pull-request) -* [Feast Data Storage Format](development-guide.md#feast-data-storage-format) -* [Feast Protobuf API](development-guide.md#feast-protobuf-api) -* [Maintainer Guide](./maintainers.md) +Please see [this page](../reference/codebase-structure.md) for more details on the structure of the entire codebase. -> Learn How the Feast [Contributing Process](contributing.md) works. +## Compatibility -## Making a Pull Request +The compatibility policy for Feast can be found [here](compatibility.md), and should be followed for all changes proposed, by maintainers or contributors. -{% hint style="info" %} -See also the CONTRIBUTING.md in the corresponding GitHub repository \(e.g. [main repo doc](https://github.com/feast-dev/feast/blob/master/CONTRIBUTING.md)\) -{% endhint %} +## Community +See [Contribution process](./contributing.md) and [Community](../community.md) for details on how to get more involved in the community. -### Incorporating upstream changes from master +A quick few highlights: +- [RFCs](https://drive.google.com/drive/u/0/folders/0AAe8j7ZK3sxSUk9PVA) +- [Community Slack](https://slack.feast.dev/) +- [Feast Dev Mailing List](https://groups.google.com/g/feast-dev) +- [Community Calendar](https://calendar.google.com/calendar/u/0?cid=ZTFsZHVhdGM3MDU3YTJucTBwMzNqNW5rajBAZ3JvdXAuY2FsZW5kYXIuZ29vZ2xlLmNvbQ) + - Includes biweekly community calls at 10AM PST -Our preference is the use of `git rebase` instead of `git merge` : `git pull -r` +## Making a pull request +We use the convention that the assignee of a PR is the person with the next action. -### Signing commits +If the assignee is empty it means that no reviewer has been found yet. +If a reviewer has been found, they should also be the assigned the PR. +Finally, if there are comments to be addressed, the PR author should be the one assigned the PR. -Commits have to be signed before they are allowed to be merged into the Feast codebase: - -```bash -# Include -s flag to signoff -git commit -s -m "My first commit" -``` +### Pull request checklist +A quick list of things to keep in mind as you're making changes: +- As you make changes + - Make your changes in a [forked repo](#forking-the-repo) (instead of making a branch on the main Feast repo) + - [Sign your commits](#signing-off-commits) as you go (to avoid DCO checks failing) + - [Rebase from master](#incorporating-upstream-changes-from-master) instead of using `git pull` on your PR branch + - Install [pre-commit hooks](#pre-commit-hooks) to ensure all the default linters / formatters are run when you push. +- When you make the PR + - Make a pull request from the forked repo you made + - Ensure you add a GitHub **label** (i.e. a kind tag to the PR (e.g. `kind/bug` or `kind/housekeeping`)) or else checks will fail. + - Ensure you leave a release note for any user facing changes in the PR. There is a field automatically generated in the PR request. You can write `NONE` in that field if there are no user facing changes. + - Please run tests locally before submitting a PR (e.g. for Python, the [local integration tests](#local-integration-tests)) + - Try to keep PRs smaller. This makes them easier to review. ### Good practices to keep in mind - * Fill in the description based on the default template configured when you first open the PR * What this PR does/why we need it * Which issue\(s\) this PR fixes * Does this PR introduce a user-facing change -* Include `kind` label when opening the PR * Add `WIP:` to PR name if more work needs to be done prior to review -* Avoid `force-pushing` as it makes reviewing difficult -**Managing CI-test failures** -* GitHub runner tests - * Click `checks` tab to analyse failed tests -* Prow tests - * Visit [Prow status page ](http://prow.feast.ai/)to analyse failed tests +### Forking the repo +Fork the Feast Github repo and clone your fork locally. Then make changes to a local branch to the fork. -## Feast Data Storage Format +See [Creating a pull request from a fork](https://docs.github.com/en/github/collaborating-with-pull-requests/proposing-changes-to-your-work-with-pull-requests/creating-a-pull-request-from-a-fork) -Feast data storage contracts are documented in the following locations: +### Pre-commit Hooks +Setup [`pre-commit`](https://pre-commit.com/) to automatically lint and format the codebase on commit: +1. Ensure that you have Python (3.7 and above) with `pip`, installed. +2. Install `pre-commit` with `pip` & install pre-push hooks +```sh +pip install pre-commit +pre-commit install --hook-type pre-commit --hook-type pre-push +``` +3. On push, the pre-commit hook will run. This runs `make format` and `make lint`. -* [Feast Offline Storage Format](https://github.com/feast-dev/feast/blob/master/docs/specs/offline_store_format.md): Used by BigQuery, Snowflake \(Future\), Redshift \(Future\). -* [Feast Online Storage Format](https://github.com/feast-dev/feast/blob/master/docs/specs/online_store_format.md): Used by Redis, Google Datastore. +### Signing off commits +> :warning: Warning: using the default integrations with IDEs like VSCode or IntelliJ will not sign commits. +> When you submit a PR, you'll have to re-sign commits to pass the DCO check. + +Use git signoffs to sign your commits. See +https://docs.github.com/en/github/authenticating-to-github/managing-commit-signature-verification for details + +Then, you can sign off commits with the `-s` flag: +``` +git commit -s -m "My first commit" +``` + +GPG-signing commits with `-S` is optional. + +### Incorporating upstream changes from master +Our preference is the use of `git rebase [master]` instead of `git merge` : `git pull -r`. + +Note that this means if you are midway through working through a PR and rebase, you'll have to force push: +`git push --force-with-lease origin [branch name]` + +## Feast Python SDK / CLI +### Environment Setup +Setting up your development environment for Feast Python SDK / CLI: +1. Ensure that you have Docker installed in your environment. Docker is used to provision service dependencies during testing, and build images for feature servers and other components. + 1. Please note that we use [Docker with BuiltKit](https://docs.docker.com/develop/develop-images/build_enhancements/). +2. Ensure that you have `make`, Python (3.8 and above) with `pip`, installed. +3. _Recommended:_ Create a virtual environment to isolate development dependencies to be installed + ```sh + # create & activate a virtual environment + python -m venv venv/ + source venv/bin/activate + ``` +4. Upgrade `pip` if outdated + ```sh + pip install --upgrade pip + ``` + +5. (Optional): Install Node & Yarn. Then run the following to build Feast UI artifacts for use in `feast ui` +``` +make build-ui +``` + +6. Install development dependencies for Feast Python SDK / CLI +```sh +pip install -e ".[dev]" +``` + +This will allow the installed feast version to automatically reflect changes to your local development version of Feast without needing to reinstall everytime you make code changes. + +### Code Style & Linting +Feast Python SDK / CLI codebase: +- Conforms to [Black code style](https://black.readthedocs.io/en/stable/the_black_code_style.html) +- Has type annotations as enforced by `mypy` +- Has imports sorted by `isort` +- Is lintable by `flake8` + +To ensure your Python code conforms to Feast Python code standards: +- Autoformat your code to conform to the code style: +```sh +make format-python +``` + +- Lint your Python code before submitting it for review: +```sh +make lint-python +``` + +> Setup [pre-commit hooks](#pre-commit-hooks) to automatically format and lint on commit. + +### Unit Tests +Unit tests (`pytest`) for the Feast Python SDK / CLI can run as follows: +```sh +make test-python +``` + +> :warning: Local configuration can interfere with Unit tests and cause them to fail: +> - Ensure [no AWS configuration is present](https://boto3.amazonaws.com/v1/documentation/api/latest/guide/configuration.html) + > and [no AWS credentials can be accessed](https://boto3.amazonaws.com/v1/documentation/api/latest/guide/credentials.html#configuring-credentials) by `boto3` +> - Ensure Feast Python SDK / CLI is not configured with configuration overrides (ie `~/.feast/config` should be empty). + +### Integration Tests +There are two sets of tests you can run: +1. Local integration tests (for faster development, tests file offline store & key online stores) +2. Full integration tests (requires cloud environment setups) + +#### Local integration tests +For this approach of running tests, you'll need to have docker set up locally: [Get Docker](https://docs.docker.com/get-docker/) + +It leverages a file based offline store to test against emulated versions of Datastore, DynamoDB, and Redis, using ephemeral containers. + +These tests create new temporary tables / datasets locally only, and they are cleaned up. when the containers are torn down. + +```sh +make test-python-integration-local +``` + +#### (Advanced) Full integration tests +To test across clouds, on top of setting up Redis, you also need GCP / AWS / Snowflake setup. + +> Note: you can manually control what tests are run today by inspecting +> [RepoConfiguration](https://github.com/feast-dev/feast/blob/master/sdk/python/tests/integration/feature_repos/repo_configuration.py) +> and commenting out tests that are added to `DEFAULT_FULL_REPO_CONFIGS` + +**GCP** +1. You can get free credits [here](https://cloud.google.com/free/docs/free-cloud-features#free-trial). +2. You will need to setup a service account, enable the BigQuery API, and create a staging location for a bucket. + +* Setup your service account and project using steps 1-5 [here](https://codelabs.developers.google.com/codelabs/cloud-bigquery-python#0). + * Remember to save your `PROJECT_ID` and your `key.json`. These will be your secrets that you will need to configure in Github actions. Namely, `secrets.GCP_PROJECT_ID` and `secrets.GCP_SA_KEY`. The `GCP_SA_KEY` value is the contents of your `key.json` file. +* Follow these [instructions](https://cloud.google.com/storage/docs/creating-buckets) in your project to create a bucket for running GCP tests and remember to save the bucket name. + * Make sure to add the service account email that you created in the previous step to the users that can access your bucket. Then, make sure to give the account the correct access roles, namely `objectCreator`, `objectViewer`, `objectAdmin`, and `admin`, so that your tests can use the bucket. + +3. Install the [Cloud SDK](https://cloud.google.com/sdk/docs/install). +4. Login to gcloud if you haven't already: + ``` + gcloud auth login + gcloud auth application-default login + ``` +- When you run `gcloud auth application-default login`, you should see some output of the form: + ``` + Credentials saved to file: [$HOME/.config/gcloud/application_default_credentials.json] + ``` +- You should run `export GOOGLE_APPLICATION_CREDENTIALS="$HOME/.config/gcloud/application_default_credentials.json”` to add the application credentials to your .zshrc or .bashrc. +5. Run `export GCLOUD_PROJECT=[your project id from step 2]` to your .zshrc or .bashrc. +6. Running `gcloud config list` should give you something like this: + ```sh + $ gcloud config list + [core] + account = [your email] + disable_usage_reporting = True + project = [your project id] + + Your active configuration is: [default] + ``` +7. Export GCP specific environment variables in your workflow. Namely, + ```sh + export GCS_REGION='[your gcs region e.g US]' + export GCS_STAGING_LOCATION='[your gcs staging location]' + ``` +**NOTE**: Your `GCS_STAGING_LOCATION` should be in the form `gs://` where the bucket name is from step 2. + +8. Once authenticated, you should be able to run the integration tests for BigQuery without any failures. -## Feast Protobuf API +**AWS** +1. Setup AWS by creating an account, database, and cluster. You will need to enable Redshift and Dynamo. +* You can get free credits [here](https://aws.amazon.com/free/?all-free-tier.sort-by=item.additionalFields.SortRank&al[…]f.Free%20Tier%20Types=*all&awsf.Free%20Tier%20Categories=*all). +2. To run the AWS Redshift and Dynamo integration tests you will have to export your own AWS credentials. Namely, -Feast Protobuf API defines the common API used by Feast's Components: +```sh +export AWS_REGION='[your aws region]' +export AWS_CLUSTER_ID='[your aws cluster id]' +export AWS_USER='[your aws user]' +export AWS_DB='[your aws database]' +export AWS_STAGING_LOCATION='[your s3 staging location uri]' +export AWS_IAM_ROLE='[redshift and s3 access role]' +export AWS_LAMBDA_ROLE='[your aws lambda execution role]' +export AWS_REGISTRY_PATH='[your aws registry path]' +``` + +**Snowflake** +1. See https://signup.snowflake.com/ to setup a trial. +2. Setup your account and if you are not an `ACCOUNTADMIN` (if you created your own account, you should be), give yourself the `SYSADMIN` role. + ```sql + grant role accountadmin, sysadmin to user user2; + ``` +* Also remember to save your [account name](https://docs.snowflake.com/en/user-guide/admin-account-identifier.html#:~:text=organization_name%20is%20the%20name%20of,your%20account%20within%20your%20organization), username, and role. +* Your account name can be found under +3. Create Dashboard and add a Tile. +4. Create a warehouse and database named `FEAST` with the schemas `OFFLINE` and `ONLINE`. + ```sql + create or replace warehouse feast_tests_wh with + warehouse_size='MEDIUM' --set your warehouse size to whatever your budget allows-- + auto_suspend = 180 + auto_resume = true + initially_suspended=true; + + create or replace database FEAST; + use database FEAST; + create schema OFFLINE; + create schema ONLINE; + ``` +5. You will need to create a data unloading location(either on S3, GCP, or Azure). Detailed instructions [here](https://docs.snowflake.com/en/user-guide/data-unload-overview.html). You will need to save the storage export location and the storage export name. You will need to create a [storage integration ](https://docs.snowflake.com/en/sql-reference/sql/create-storage-integration.html) in your warehouse to make this work. Name this storage integration `FEAST_S3`. +6. Then to run successfully, you'll need some environment variables setup: + ```sh + export SNOWFLAKE_CI_DEPLOYMENT='[your snowflake account name]' + export SNOWFLAKE_CI_USER='[your snowflake username]' + export SNOWFLAKE_CI_PASSWORD='[your snowflake pw]' + export SNOWFLAKE_CI_ROLE='[your CI role e.g. SYSADMIN]' + export SNOWFLAKE_CI_WAREHOUSE='[your warehouse]' + export BLOB_EXPORT_STORAGE_NAME='[your data unloading storage name]' + export BLOB_EXPORT_URI='[your data unloading blob uri]` + ``` +7. Once everything is setup, running snowflake integration tests should pass without failures. + +Note that for Snowflake / GCP / AWS, running `make test-python-integration` will create new temporary tables / datasets in your cloud storage tables. + +#### (Advanced) Running specific provider tests or running your test against specific online or offline stores + +1. If you don't need to have your test run against all of the providers(`gcp`, `aws`, and `snowflake`) or don't need to run against all of the online stores, you can tag your test with specific providers or stores that you need(`@pytest.mark.universal_online_stores` or `@pytest.mark.universal_online_stores` with the `only` parameter). The `only` parameter selects specific offline providers and online stores that your test will test against. Example: + +```python +# Only parametrizes this test with the sqlite online store +@pytest.mark.universal_online_stores(only=["sqlite"]) +def test_feature_get_online_features_types_match(): +``` + +2. You can also filter tests to run by using pytest's cli filtering. Instead of using the make commands to test Feast, you can filter tests by name with the `-k` parameter. The parametrized integration tests are all uniquely identified by their provider and online store so the `-k` option can select only the tests that you need to run. For example, to run only Redshift related tests, you can use the following command: + +```sh +python -m pytest -n 8 --integration -k Redshift sdk/python/tests +``` + +#### (Experimental) Run full integration tests against containerized services +Test across clouds requires existing accounts on GCP / AWS / Snowflake, and may incur costs when using these services. + +For this approach of running tests, you'll need to have docker set up locally: [Get Docker](https://docs.docker.com/get-docker/) + +It's possible to run some integration tests against emulated local versions of these services, using ephemeral containers. +These tests create new temporary tables / datasets locally only, and they are cleaned up. when the containers are torn down. + +The services with containerized replacements currently implemented are: +- Datastore +- DynamoDB +- Redis +- Trino +- HBase +- Postgres +- Cassandra + +You can run `make test-python-integration-container` to run tests against the containerized versions of dependencies. + +### Contrib integration tests +#### (Contrib) Running tests for Spark offline store +You can run `make test-python-universal-spark` to run all tests against the Spark offline store. (Note: you'll have to run `pip install -e ".[dev]"` first). + +Not all tests are passing yet + +#### (Contrib) Running tests for Trino offline store +You can run `make test-python-universal-trino` to run all tests against the Trino offline store. (Note: you'll have to run `pip install -e ".[dev]"` first) + +#### (Contrib) Running tests for Postgres offline store +You can run `test-python-universal-postgres-offline` to run all tests against the Postgres offline store. (Note: you'll have to run `pip install -e ".[dev]"` first) + +#### (Contrib) Running tests for Postgres online store +You can run `test-python-universal-postgres-online` to run all tests against the Postgres offline store. (Note: you'll have to run `pip install -e ".[dev]"` first) + +#### (Contrib) Running tests for HBase online store +TODO + +## (Experimental) Feast UI +See [Feast contributing guide](ui/CONTRIBUTING.md) + +## Feast Java Serving +See [Java contributing guide](java/CONTRIBUTING.md) + +See also development instructions related to the helm chart below at [Developing the Feast Helm charts](#developing-the-feast-helm-charts) -* Feast Protobuf API specifications are written in [proto3](https://developers.google.com/protocol-buffers/docs/proto3) in the Main Feast Repository. -* Changes to the API should be proposed via a [GitHub Issue](https://github.com/feast-dev/feast/issues/new/choose) for discussion first. +## Developing the Feast Helm charts +There are 3 helm charts: +- Feast Java feature server +- Feast Python / Go feature server +- (deprecated) Feast Python feature server -### Generating Language Bindings +Generally, you can override the images in the helm charts with locally built Docker images, and install the local helm +chart. -The language specific bindings have to be regenerated when changes are made to the Feast Protobuf API: +All README's for helm charts are generated using [helm-docs](https://github.com/norwoodj/helm-docs). You can install it +(e.g. with `brew install norwoodj/tap/helm-docs`) and then run `make build-helm-docs`. + +### Feast Java Feature Server Helm Chart +See the Java demo example (it has development instructions too using minikube) [here](examples/java-demo/README.md) + +It will: +- run `make build-java-docker-dev` to build local Java feature server binaries +- configure the included `application-override.yaml` to override the image tag to use the locally built dev images. +- install the local chart with `helm install feast-release ../../../infra/charts/feast --values application-override.yaml` + +### Feast Python / Go Feature Server Helm Chart +See the Python demo example (it has development instructions too using minikube) [here](examples/python-helm-demo/README.md) + +It will: +- run `make build-feature-server-dev` to build a local python feature server binary +- install the local chart with `helm install feast-release ../../../infra/charts/feast-feature-server --set image.tag=dev --set feature_store_yaml_base64=$(base64 feature_store.yaml)` + +## Feast Go Client +### Go Environment Setup +Setting up your development environment for Feast Go SDK: + +- Install Golang, [`protoc` with the Golang & grpc plugins](https://developers.google.com/protocol-buffers/docs/gotutorial#compiling-your-protocol-buffers) + +### Building Go +Build the Feast Go Client with the `go` toolchain: +```sh +make compile-go-lib +``` + +### Go Code Style & Linting +Feast Go Client codebase: +- Conforms to the code style enforced by `go fmt`. +- Is lintable by `go vet`. + +Autoformat your Go code to satisfy the Code Style standard: +```sh +go fmt +``` + +Lint your Go code: +```sh +go vet +``` + +> Setup [pre-commit hooks](#pre-commit-hooks) to automatically format and lint on commit. + +### Go Unit Tests +Unit tests for the Feast Go Client can be run as follows: +```sh +make test-go +``` + +### Testing with Github Actions workflows + +Please refer to the maintainers [doc](./docs/project/maintainers.md) if you would like to locally test out the github actions workflow changes. +This document will help you setup your fork to test the ci integration tests and other workflows without needing to make a pull request against feast-dev master. + +## Feast Data Storage Format + +Feast data storage contracts are documented in the following locations: + +* [Feast Offline Storage Format](https://github.com/feast-dev/feast/blob/master/docs/specs/offline_store_format.md): Used by BigQuery, Snowflake \(Future\), Redshift \(Future\). +* [Feast Online Storage Format](https://github.com/feast-dev/feast/blob/master/docs/specs/online_store_format.md): Used by Redis, Google Datastore. -| Repository | Language | Regenerating Language Bindings | -| :--- | :--- | :--- | -| [Main Feast Repository](https://github.com/feast-dev/feast) | Python | Run `make compile-protos-python` to generate bindings | -| [Main Feast Repository](https://github.com/feast-dev/feast) | Golang | Run `make compile-protos-go` to generate bindings | diff --git a/docs/reference/codebase-structure.md b/docs/reference/codebase-structure.md index b75227860bf..8eb55726793 100644 --- a/docs/reference/codebase-structure.md +++ b/docs/reference/codebase-structure.md @@ -125,6 +125,11 @@ Within `go/`, the `internal/feast/` directory contains most of the core logic: Feast uses [protobuf](https://github.com/protocolbuffers/protobuf) to store serialized versions of the core Feast objects. The protobuf definitions are stored in `protos/feast`. +The [registry](../getting-started/concepts/registry.md) consists of the serialized representations of the Feast objects. + +Typically, changes being made to the Feast objects require changes to their corresponding protobuf representations. +The usual best practices for making changes to protobufs should be followed ensure backwards and forwards compatibility. + ## Web UI The `ui/` directory contains the Web UI. diff --git a/docs/reference/data-sources/push.md b/docs/reference/data-sources/push.md index 035ee583604..6dad690c16a 100644 --- a/docs/reference/data-sources/push.md +++ b/docs/reference/data-sources/push.md @@ -1,7 +1,5 @@ # Push source -**Warning**: This is an _experimental_ feature. It's intended for early testing and feedback, and could change without warnings in future releases. - ## Description Push sources allow feature values to be pushed to the online store and offline store in real time. This allows fresh feature values to be made available to applications. Push sources supercede the @@ -22,7 +20,7 @@ Streaming data sources are important sources of feature values. A typical setup Feast allows users to push features previously registered in a feature view to the online store for fresher features. It also allows users to push batches of stream data to the offline store by specifying that the push be directed to the offline store. This will push the data to the offline store declared in the repository configuration used to initialize the feature store. -## Example +## Example (basic) ### Defining a push source Note that the push schema needs to also include the entity. @@ -59,3 +57,24 @@ fs.push("push_source_name", feature_data_frame, to=PushMode.ONLINE_AND_OFFLINE) See also [Python feature server](../feature-servers/python-feature-server.md) for instructions on how to push data to a deployed feature server. +## Example (Spark Streaming) + +The default option to write features from a stream is to add the Python SDK into your existing PySpark pipeline. + +```python +from feast import FeatureStore + +store = FeatureStore(...) + +spark = SparkSession.builder.getOrCreate() + +streamingDF = spark.readStream.format(...).load() + +def feast_writer(spark_df): + pandas_df = spark_df.to_pandas() + store.push("driver_hourly_stats", pandas_df) + +streamingDF.writeStream.foreachBatch(feast_writer).start() +``` + +This can also be used under the hood by a contrib stream processor (see [Tutorial: Building streaming features](../../tutorials/building-streaming-features.md)) \ No newline at end of file diff --git a/docs/reference/online-stores/README.md b/docs/reference/online-stores/README.md index 6d616b46f25..58c8705e9ee 100644 --- a/docs/reference/online-stores/README.md +++ b/docs/reference/online-stores/README.md @@ -2,6 +2,10 @@ Please see [Online Store](../../getting-started/architecture-and-components/online-store.md) for an explanation of online stores. +{% content-ref url="overview.md" %} +[overview.md](overview.md) +{% endcontent-ref %} + {% content-ref url="sqlite.md" %} [sqlite.md](sqlite.md) {% endcontent-ref %} diff --git a/docs/reference/online-stores/cassandra.md b/docs/reference/online-stores/cassandra.md index 3355c3728ce..48b7b73f439 100644 --- a/docs/reference/online-stores/cassandra.md +++ b/docs/reference/online-stores/cassandra.md @@ -55,7 +55,34 @@ online_store: ``` {% endcode %} +The full set of configuration options is available in [CassandraOnlineStoreConfig](https://rtd.feast.dev/en/master/#feast.infra.online_stores.contrib.cassandra_online_store.cassandra_online_store.CassandraOnlineStoreConfig). For a full explanation of configuration options please look at file `sdk/python/feast/infra/online_stores/contrib/cassandra_online_store/README.md`. -Storage specifications can be found at `docs/specs/online_store_format.md`. \ No newline at end of file +Storage specifications can be found at `docs/specs/online_store_format.md`. + +## Functionality Matrix + +The set of functionality supported by online stores is described in detail [here](overview.md#functionality). +Below is a matrix indicating which functionality is supported by the Cassandra online store. + +| | Cassandra | +| :-------------------------------------------------------- | :-- | +| write feature values to the online store | yes | +| read feature values from the online store | yes | +| update infrastructure (e.g. tables) in the online store | yes | +| teardown infrastructure (e.g. tables) in the online store | yes | +| generate a plan of infrastructure changes | yes | +| support for on-demand transforms | yes | +| readable by Python SDK | yes | +| readable by Java | no | +| readable by Go | no | +| support for entityless feature views | yes | +| support for concurrent writing to the same key | no | +| support for ttl (time to live) at retrieval | no | +| support for deleting expired data | no | +| collocated by feature view | yes | +| collocated by feature service | no | +| collocated by entity key | no | + +To compare this set of functionality against other online stores, please see the full [functionality matrix](overview.md#functionality-matrix). diff --git a/docs/reference/online-stores/datastore.md b/docs/reference/online-stores/datastore.md index ed1425abb68..0867853f15d 100644 --- a/docs/reference/online-stores/datastore.md +++ b/docs/reference/online-stores/datastore.md @@ -18,4 +18,30 @@ online_store: ``` {% endcode %} -Configuration options are available [here](https://rtd.feast.dev/en/latest/#feast.repo_config.DatastoreOnlineStoreConfig). +The full set of configuration options is available in [DatastoreOnlineStoreConfig](https://rtd.feast.dev/en/latest/#feast.infra.online_stores.datastore.DatastoreOnlineStoreConfig). + +## Functionality Matrix + +The set of functionality supported by online stores is described in detail [here](overview.md#functionality). +Below is a matrix indicating which functionality is supported by the Datastore online store. + +| | Datastore | +| :-------------------------------------------------------- | :-- | +| write feature values to the online store | yes | +| read feature values from the online store | yes | +| update infrastructure (e.g. tables) in the online store | yes | +| teardown infrastructure (e.g. tables) in the online store | yes | +| generate a plan of infrastructure changes | no | +| support for on-demand transforms | yes | +| readable by Python SDK | yes | +| readable by Java | no | +| readable by Go | no | +| support for entityless feature views | yes | +| support for concurrent writing to the same key | no | +| support for ttl (time to live) at retrieval | no | +| support for deleting expired data | no | +| collocated by feature view | yes | +| collocated by feature service | no | +| collocated by entity key | no | + +To compare this set of functionality against other online stores, please see the full [functionality matrix](overview.md#functionality-matrix). diff --git a/docs/reference/online-stores/dynamodb.md b/docs/reference/online-stores/dynamodb.md index f9f8b4339d7..2f94c768199 100644 --- a/docs/reference/online-stores/dynamodb.md +++ b/docs/reference/online-stores/dynamodb.md @@ -17,7 +17,7 @@ online_store: ``` {% endcode %} -Configuration options are available [here](https://github.com/feast-dev/feast/blob/17bfa6118d6658d2bff53d7de8e2ccef5681714d/sdk/python/feast/infra/online_stores/dynamodb.py#L36). +The full set of configuration options is available in [DynamoDBOnlineStoreConfig](https://rtd.feast.dev/en/master/#feast.infra.online_stores.dynamodb.DynamoDBOnlineStoreConfig). ## Permissions @@ -53,3 +53,29 @@ The following inline policy can be used to grant Feast the necessary permissions ``` Lastly, this IAM role needs to be associated with the desired Redshift cluster. Please follow the official AWS guide for the necessary steps [here](https://docs.aws.amazon.com/redshift/latest/dg/c-getting-started-using-spectrum-add-role.html). + +## Functionality Matrix + +The set of functionality supported by online stores is described in detail [here](overview.md#functionality). +Below is a matrix indicating which functionality is supported by the DynamoDB online store. + +| | DynamoDB | +| :-------------------------------------------------------- | :-- | +| write feature values to the online store | yes | +| read feature values from the online store | yes | +| update infrastructure (e.g. tables) in the online store | yes | +| teardown infrastructure (e.g. tables) in the online store | yes | +| generate a plan of infrastructure changes | no | +| support for on-demand transforms | yes | +| readable by Python SDK | yes | +| readable by Java | no | +| readable by Go | no | +| support for entityless feature views | yes | +| support for concurrent writing to the same key | no | +| support for ttl (time to live) at retrieval | no | +| support for deleting expired data | no | +| collocated by feature view | yes | +| collocated by feature service | no | +| collocated by entity key | no | + +To compare this set of functionality against other online stores, please see the full [functionality matrix](overview.md#functionality-matrix). diff --git a/docs/reference/online-stores/overview.md b/docs/reference/online-stores/overview.md new file mode 100644 index 00000000000..981a1aeeed0 --- /dev/null +++ b/docs/reference/online-stores/overview.md @@ -0,0 +1,54 @@ +# Overview + +## Functionality + +Here are the methods exposed by the `OnlineStore` interface, along with the core functionality supported by the method: +* `online_write_batch`: write feature values to the online store +* `online_read`: read feature values from the online store +* `update`: update infrastructure (e.g. tables) in the online store +* `teardown`: teardown infrastructure (e.g. tables) in the online store +* `plan`: generate a plan of infrastructure changes based on feature repo changes + +There is also additional functionality not properly captured by these interface methods: +* support for on-demand transforms +* readable by Python SDK +* readable by Java +* readable by Go +* support for entityless feature views +* support for concurrent writing to the same key +* support for ttl (time to live) at retrieval +* support for deleting expired data + +Finally, there are multiple data models for storing the features in the online store. For example, features could be: +* collocated by feature view +* collocated by feature service +* collocated by entity key + +See this [issue](https://github.com/feast-dev/feast/issues/2254) for a discussion around the tradeoffs of each of these data models. + +## Functionality Matrix + +There are currently five core online store implementations: `SqliteOnlineStore`, `RedisOnlineStore`, `DynamoDBOnlineStore`, `SnowflakeOnlineStore`, and `DatastoreOnlineStore`. +There are several additional implementations contributed by the Feast community (`PostgreSQLOnlineStore`, `HbaseOnlineStore`, and `CassandraOnlineStore`), which are not guaranteed to be stable or to match the functionality of the core implementations. +Details for each specific online store, such as how to configure it in a `feature_store.yaml`, can be found [here](README.md). + +Below is a matrix indicating which online stores support what functionality. + +| | Sqlite | Redis | DynamoDB | Snowflake | Datastore | Postgres | Hbase | Cassandra | +| :-------------------------------------------------------- | :-- | :-- | :-- | :-- | :-- | :-- | :-- | :-- | +| write feature values to the online store | yes | yes | yes | yes | yes | yes | yes | yes | +| read feature values from the online store | yes | yes | yes | yes | yes | yes | yes | yes | +| update infrastructure (e.g. tables) in the online store | yes | yes | yes | yes | yes | yes | yes | yes | +| teardown infrastructure (e.g. tables) in the online store | yes | yes | yes | yes | yes | yes | yes | yes | +| generate a plan of infrastructure changes | yes | no | no | no | no | no | no | yes | +| support for on-demand transforms | yes | yes | yes | yes | yes | yes | yes | yes | +| readable by Python SDK | yes | yes | yes | yes | yes | yes | yes | yes | +| readable by Java | no | yes | no | no | no | no | no | no | +| readable by Go | yes | yes | no | no | no | no | no | no | +| support for entityless feature views | yes | yes | yes | yes | yes | yes | yes | yes | +| support for concurrent writing to the same key | no | yes | no | no | no | no | no | no | +| support for ttl (time to live) at retrieval | no | yes | no | no | no | no | no | no | +| support for deleting expired data | no | yes | no | no | no | no | no | no | +| collocated by feature view | yes | no | yes | yes | yes | yes | yes | yes | +| collocated by feature service | no | no | no | no | no | no | no | no | +| collocated by entity key | no | yes | no | no | no | no | no | no | diff --git a/docs/reference/online-stores/postgres.md b/docs/reference/online-stores/postgres.md index 4f51dff6172..083c0006359 100644 --- a/docs/reference/online-stores/postgres.md +++ b/docs/reference/online-stores/postgres.md @@ -30,4 +30,30 @@ online_store: ``` {% endcode %} -Configuration options are available [here](https://rtd.feast.dev/en/latest/feast.infra.utils.postgres.html#module-feast.infra.utils.postgres.postgres_config). +The full set of configuration options is available in [PostgreSQLOnlineStoreConfig](https://rtd.feast.dev/en/master/#feast.infra.online_stores.contrib.postgres.PostgreSQLOnlineStoreConfig). + +## Functionality Matrix + +The set of functionality supported by online stores is described in detail [here](overview.md#functionality). +Below is a matrix indicating which functionality is supported by the Postgres online store. + +| | Postgres | +| :-------------------------------------------------------- | :-- | +| write feature values to the online store | yes | +| read feature values from the online store | yes | +| update infrastructure (e.g. tables) in the online store | yes | +| teardown infrastructure (e.g. tables) in the online store | yes | +| generate a plan of infrastructure changes | no | +| support for on-demand transforms | yes | +| readable by Python SDK | yes | +| readable by Java | no | +| readable by Go | no | +| support for entityless feature views | yes | +| support for concurrent writing to the same key | no | +| support for ttl (time to live) at retrieval | no | +| support for deleting expired data | no | +| collocated by feature view | yes | +| collocated by feature service | no | +| collocated by entity key | no | + +To compare this set of functionality against other online stores, please see the full [functionality matrix](overview.md#functionality-matrix). diff --git a/docs/reference/online-stores/redis.md b/docs/reference/online-stores/redis.md index 4388ccfa0ab..80e90348c55 100644 --- a/docs/reference/online-stores/redis.md +++ b/docs/reference/online-stores/redis.md @@ -4,12 +4,12 @@ The [Redis](https://redis.io) online store provides support for materializing feature values into Redis. -* Both Redis and Redis Cluster are supported +* Both Redis and Redis Cluster are supported. * The data model used to store feature values in Redis is described in more detail [here](../../specs/online\_store\_format.md). ## Examples -Connecting to a single Redis instance +Connecting to a single Redis instance: {% code title="feature_store.yaml" %} ```yaml @@ -22,7 +22,7 @@ online_store: ``` {% endcode %} -Connecting to a Redis Cluster with SSL enabled and password authentication +Connecting to a Redis Cluster with SSL enabled and password authentication: {% code title="feature_store.yaml" %} ```yaml @@ -36,4 +36,30 @@ online_store: ``` {% endcode %} -Configuration options are available [here](https://rtd.feast.dev/en/master/#feast.infra.online\_stores.redis.RedisOnlineStoreConfig). +The full set of configuration options is available in [RedisOnlineStoreConfig](https://rtd.feast.dev/en/latest/#feast.infra.online_stores.redis.RedisOnlineStoreConfig). + +## Functionality Matrix + +The set of functionality supported by online stores is described in detail [here](overview.md#functionality). +Below is a matrix indicating which functionality is supported by the Redis online store. + +| | Redis | +| :-------------------------------------------------------- | :-- | +| write feature values to the online store | yes | +| read feature values from the online store | yes | +| update infrastructure (e.g. tables) in the online store | yes | +| teardown infrastructure (e.g. tables) in the online store | yes | +| generate a plan of infrastructure changes | no | +| support for on-demand transforms | yes | +| readable by Python SDK | yes | +| readable by Java | yes | +| readable by Go | yes | +| support for entityless feature views | yes | +| support for concurrent writing to the same key | yes | +| support for ttl (time to live) at retrieval | yes | +| support for deleting expired data | yes | +| collocated by feature view | no | +| collocated by feature service | no | +| collocated by entity key | yes | + +To compare this set of functionality against other online stores, please see the full [functionality matrix](overview.md#functionality-matrix). diff --git a/docs/reference/online-stores/snowflake.md b/docs/reference/online-stores/snowflake.md index bf975fa7eab..7db45dd7bd8 100644 --- a/docs/reference/online-stores/snowflake.md +++ b/docs/reference/online-stores/snowflake.md @@ -33,3 +33,31 @@ online_store: database: SNOWFLAKE_DATABASE ``` {% endcode %} + +The full set of configuration options is available in [SnowflakeOnlineStoreConfig](https://rtd.feast.dev/en/latest/#feast.infra.online_stores.snowflake.SnowflakeOnlineStoreConfig). + +## Functionality Matrix + +The set of functionality supported by online stores is described in detail [here](overview.md#functionality). +Below is a matrix indicating which functionality is supported by the Snowflake online store. + +| | Snowflake | +| :-------------------------------------------------------- | :-- | +| write feature values to the online store | yes | +| read feature values from the online store | yes | +| update infrastructure (e.g. tables) in the online store | yes | +| teardown infrastructure (e.g. tables) in the online store | yes | +| generate a plan of infrastructure changes | no | +| support for on-demand transforms | yes | +| readable by Python SDK | yes | +| readable by Java | no | +| readable by Go | no | +| support for entityless feature views | yes | +| support for concurrent writing to the same key | no | +| support for ttl (time to live) at retrieval | no | +| support for deleting expired data | no | +| collocated by feature view | yes | +| collocated by feature service | no | +| collocated by entity key | no | + +To compare this set of functionality against other online stores, please see the full [functionality matrix](overview.md#functionality-matrix). diff --git a/docs/reference/online-stores/sqlite.md b/docs/reference/online-stores/sqlite.md index 668e6024e3c..859702c07f1 100644 --- a/docs/reference/online-stores/sqlite.md +++ b/docs/reference/online-stores/sqlite.md @@ -20,4 +20,30 @@ online_store: ``` {% endcode %} -Configuration options are available [here](https://rtd.feast.dev/en/latest/#feast.repo_config.SqliteOnlineStoreConfig). +The full set of configuration options is available in [SqliteOnlineStoreConfig](https://rtd.feast.dev/en/latest/#feast.infra.online_stores.sqlite.SqliteOnlineStoreConfig). + +## Functionality Matrix + +The set of functionality supported by online stores is described in detail [here](overview.md#functionality). +Below is a matrix indicating which functionality is supported by the Sqlite online store. + +| | Sqlite | +| :-------------------------------------------------------- | :-- | +| write feature values to the online store | yes | +| read feature values from the online store | yes | +| update infrastructure (e.g. tables) in the online store | yes | +| teardown infrastructure (e.g. tables) in the online store | yes | +| generate a plan of infrastructure changes | yes | +| support for on-demand transforms | yes | +| readable by Python SDK | yes | +| readable by Java | no | +| readable by Go | yes | +| support for entityless feature views | yes | +| support for concurrent writing to the same key | no | +| support for ttl (time to live) at retrieval | no | +| support for deleting expired data | no | +| collocated by feature view | yes | +| collocated by feature service | no | +| collocated by entity key | no | + +To compare this set of functionality against other online stores, please see the full [functionality matrix](overview.md#functionality-matrix). diff --git a/docs/roadmap.md b/docs/roadmap.md index dc1d9ae1ab8..30f4317054b 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -43,15 +43,16 @@ The list below contains the functionality that contributors are planning to deve * [ ] Batch transformation (In progress. See [RFC](https://docs.google.com/document/d/1964OkzuBljifDvkV-0fakp2uaijnVzdwWNGdz7Vz50A/edit)) * **Streaming** * [x] [Custom streaming ingestion job support](https://docs.feast.dev/how-to-guides/creating-a-custom-provider) - * [x] [Push based streaming data ingestion to online store (Alpha)](https://docs.feast.dev/reference/data-sources/push) - * [x] [Push based streaming data ingestion to offline store (Alpha)](https://docs.feast.dev/reference/data-sources/push) + * [x] [Push based streaming data ingestion to online store](https://docs.feast.dev/reference/data-sources/push) + * [x] [Push based streaming data ingestion to offline store](https://docs.feast.dev/reference/data-sources/push) * **Deployments** * [x] AWS Lambda (Alpha release. See [RFC](https://docs.google.com/document/d/1eZWKWzfBif66LDN32IajpaG-j82LSHCCOzY6R7Ax7MI/edit)) * [x] Kubernetes (See [guide](https://docs.feast.dev/how-to-guides/running-feast-in-production#4.3.-java-based-feature-server-deployed-on-kubernetes)) * **Feature Serving** * [x] Python Client * [x] [Python feature server](https://docs.feast.dev/reference/feature-servers/python-feature-server) - * [x] [Go feature server](https://docs.feast.dev/reference/feature-servers/go-feature-server) + * [x] [Java feature server (alpha)](https://github.com/feast-dev/feast/blob/master/infra/charts/feast/README.md) + * [x] [Go feature server (alpha)](https://docs.feast.dev/reference/feature-servers/go-feature-server) * **Data Quality Management (See [RFC](https://docs.google.com/document/d/110F72d4NTv80p35wDSONxhhPBqWRwbZXG4f9mNEMd98/edit))** * [x] Data profiling and validation (Great Expectations) * **Feature Discovery and Governance** @@ -60,4 +61,4 @@ The list below contains the functionality that contributors are planning to deve * [x] Model-centric feature tracking (feature services) * [x] Amundsen integration (see [Feast extractor](https://github.com/amundsen-io/amundsen/blob/main/databuilder/databuilder/extractor/feast_extractor.py)) * [x] DataHub integration (see [DataHub Feast docs](https://datahubproject.io/docs/generated/ingestion/sources/feast/)) - * [x] Feast Web UI (Alpha release. See [docs](https://docs.feast.dev/reference/alpha-web-ui)) + * [x] Feast Web UI (Beta release. See [docs](https://docs.feast.dev/reference/alpha-web-ui)) diff --git a/go/embedded/online_features.go b/go/embedded/online_features.go index 7fd34d16e40..3c470e4b244 100644 --- a/go/embedded/online_features.go +++ b/go/embedded/online_features.go @@ -378,12 +378,12 @@ func (s *OnlineFeatureService) StopGrpcServer() { } /* - Read Record Batch from memory managed by Python caller. - Python part uses C ABI interface to export this record into C Data Interface, - and then it provides pointers (dataPtr & schemaPtr) to the Go part. - Here we import this data from given pointers and wrap the underlying values - into Go Arrow Interface (array.Record). - See export code here https://github.com/feast-dev/feast/blob/master/sdk/python/feast/embedded_go/online_features_service.py +Read Record Batch from memory managed by Python caller. +Python part uses C ABI interface to export this record into C Data Interface, +and then it provides pointers (dataPtr & schemaPtr) to the Go part. +Here we import this data from given pointers and wrap the underlying values +into Go Arrow Interface (array.Record). +See export code here https://github.com/feast-dev/feast/blob/master/sdk/python/feast/embedded_go/online_features_service.py */ func readArrowRecord(data DataTable) (arrow.Record, error) { return cdata.ImportCRecordBatch( diff --git a/go/internal/feast/onlineserving/serving.go b/go/internal/feast/onlineserving/serving.go index 3c6f5451537..131cc6d1f14 100644 --- a/go/internal/feast/onlineserving/serving.go +++ b/go/internal/feast/onlineserving/serving.go @@ -21,11 +21,11 @@ import ( ) /* - FeatureVector type represent result of retrieving single feature for multiple rows. - It can be imagined as a column in output dataframe / table. - It contains of feature name, list of values (across all rows), - list of statuses and list of timestamp. All these lists have equal length. - And this length is also equal to number of entity rows received in request. +FeatureVector type represent result of retrieving single feature for multiple rows. +It can be imagined as a column in output dataframe / table. +It contains of feature name, list of values (across all rows), +list of statuses and list of timestamp. All these lists have equal length. +And this length is also equal to number of entity rows received in request. */ type FeatureVector struct { Name string @@ -40,11 +40,11 @@ type FeatureViewAndRefs struct { } /* - We group all features from a single request by entities they attached to. - Thus, we will be able to call online retrieval per entity and not per each feature View. - In this struct we collect all features and views that belongs to a group. - We also store here projected entity keys (only ones that needed to retrieve these features) - and indexes to map result of retrieval into output response. +We group all features from a single request by entities they attached to. +Thus, we will be able to call online retrieval per entity and not per each feature View. +In this struct we collect all features and views that belongs to a group. +We also store here projected entity keys (only ones that needed to retrieve these features) +and indexes to map result of retrieval into output response. */ type GroupedFeaturesPerEntitySet struct { // A list of requested feature references of the form featureViewName:featureName that share this entity set @@ -59,11 +59,12 @@ type GroupedFeaturesPerEntitySet struct { } /* - Return - (1) requested feature views and features grouped per View - (2) requested on demand feature views - existed in the registry +Return + (1) requested feature views and features grouped per View + (2) requested on demand feature views + +existed in the registry */ func GetFeatureViewsToUseByService( featureService *model.FeatureService, @@ -124,10 +125,12 @@ func GetFeatureViewsToUseByService( } /* - Return - (1) requested feature views and features grouped per View - (2) requested on demand feature views - existed in the registry +Return + + (1) requested feature views and features grouped per View + (2) requested on demand feature views + +existed in the registry */ func GetFeatureViewsToUseByFeatureRefs( features []string, diff --git a/go/internal/feast/transformation/transformation.go b/go/internal/feast/transformation/transformation.go index 810b4e9bcbd..1cf1dd3311b 100644 --- a/go/internal/feast/transformation/transformation.go +++ b/go/internal/feast/transformation/transformation.go @@ -20,10 +20,10 @@ import ( ) /* - TransformationCallback is a Python callback function's expected signature. - The function should accept name of the on demand feature view and pointers to input & output record batches. - Each record batch is being passed as two pointers: pointer to array (data) and pointer to schema. - Python function is expected to return number of rows added to the output record batch. +TransformationCallback is a Python callback function's expected signature. +The function should accept name of the on demand feature view and pointers to input & output record batches. +Each record batch is being passed as two pointers: pointer to array (data) and pointer to schema. +Python function is expected to return number of rows added to the output record batch. */ type TransformationCallback func(ODFVName string, inputArrPtr, inputSchemaPtr, outArrPtr, outSchemaPtr uintptr, fullFeatureNames bool) int diff --git a/infra/charts/feast-feature-server/Chart.yaml b/infra/charts/feast-feature-server/Chart.yaml index 81970bc1a84..fc3264b0563 100644 --- a/infra/charts/feast-feature-server/Chart.yaml +++ b/infra/charts/feast-feature-server/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: feast-feature-server description: Feast Feature Server in Go or Python type: application -version: 0.24.0 +version: 0.24.1 keywords: - machine learning - big data diff --git a/infra/charts/feast-feature-server/README.md b/infra/charts/feast-feature-server/README.md index 1ee114d9c88..24e45712e02 100644 --- a/infra/charts/feast-feature-server/README.md +++ b/infra/charts/feast-feature-server/README.md @@ -1,6 +1,6 @@ # Feast Python / Go Feature Server Helm Charts -Current chart version is `0.24.0` +Current chart version is `0.24.1` ## Installation @@ -11,13 +11,16 @@ helm repo add feast-charts https://feast-helm-charts.storage.googleapis.com helm repo update ``` -Install Feast +Install Feast Feature Server on Kubernetes A base64 encoded version of the `feature_store.yaml` file is needed. Helm install example: ``` helm install feast-feature-server feast-charts/feast-feature-server --set feature_store_yaml_base64=$(base64 feature_store.yaml) ``` +## Tutorial +See [here](https://github.com/feast-dev/feast/tree/master/examples/python-helm-demo) for a sample tutorial on testing this helm chart with a demo feature repository and a local Redis instance. + ## Values | Key | Type | Default | Description | @@ -27,7 +30,7 @@ helm install feast-feature-server feast-charts/feast-feature-server --set featur | fullnameOverride | string | `""` | | | image.pullPolicy | string | `"IfNotPresent"` | | | image.repository | string | `"feastdev/feature-server"` | Docker image for Feature Server repository | -| image.tag | string | `"0.23.0"` | The Docker image tag (can be overwritten if custom feature server deps are needed for on demand transforms) | +| image.tag | string | `"0.24.1"` | The Docker image tag (can be overwritten if custom feature server deps are needed for on demand transforms) | | imagePullSecrets | list | `[]` | | | livenessProbe.initialDelaySeconds | int | `30` | | | livenessProbe.periodSeconds | int | `30` | | diff --git a/infra/charts/feast-feature-server/README.md.gotmpl b/infra/charts/feast-feature-server/README.md.gotmpl index 75f28274663..fb877208e06 100644 --- a/infra/charts/feast-feature-server/README.md.gotmpl +++ b/infra/charts/feast-feature-server/README.md.gotmpl @@ -11,13 +11,16 @@ helm repo add feast-charts https://feast-helm-charts.storage.googleapis.com helm repo update ``` -Install Feast +Install Feast Feature Server on Kubernetes A base64 encoded version of the `feature_store.yaml` file is needed. Helm install example: ``` helm install feast-feature-server feast-charts/feast-feature-server --set feature_store_yaml_base64=$(base64 feature_store.yaml) ``` +## Tutorial +See [here](https://github.com/feast-dev/feast/tree/master/examples/python-helm-demo) for a sample tutorial on testing this helm chart with a demo feature repository and a local Redis instance. + {{ template "chart.requirementsSection" . }} {{ template "chart.valuesSection" . }} \ No newline at end of file diff --git a/infra/charts/feast-feature-server/values.yaml b/infra/charts/feast-feature-server/values.yaml index 257cf03bfa2..5667315f48d 100644 --- a/infra/charts/feast-feature-server/values.yaml +++ b/infra/charts/feast-feature-server/values.yaml @@ -9,7 +9,7 @@ image: repository: feastdev/feature-server pullPolicy: IfNotPresent # image.tag -- The Docker image tag (can be overwritten if custom feature server deps are needed for on demand transforms) - tag: 0.24.0 + tag: 0.24.1 imagePullSecrets: [] nameOverride: "" diff --git a/infra/charts/feast-python-server/Chart.yaml b/infra/charts/feast-python-server/Chart.yaml index d2b45ee8b60..9a8f2f011a5 100644 --- a/infra/charts/feast-python-server/Chart.yaml +++ b/infra/charts/feast-python-server/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: feast-python-server description: Feast Feature Server in Python type: application -version: 0.24.0 +version: 0.24.1 keywords: - machine learning - big data diff --git a/infra/charts/feast-python-server/README.md b/infra/charts/feast-python-server/README.md index acdf527531b..bd1219527e0 100644 --- a/infra/charts/feast-python-server/README.md +++ b/infra/charts/feast-python-server/README.md @@ -2,7 +2,7 @@ > Note: this helm chart is deprecated in favor of [feast-feature-server](../feast-feature-server/README.md) -Current chart version is `0.24.0` +Current chart version is `0.24.1` ## Installation Docker repository and tag are required. Helm install example: diff --git a/infra/charts/feast/Chart.yaml b/infra/charts/feast/Chart.yaml index a657298b52f..da5ec639067 100644 --- a/infra/charts/feast/Chart.yaml +++ b/infra/charts/feast/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v1 description: Feature store for machine learning name: feast -version: 0.24.0 +version: 0.24.1 keywords: - machine learning - big data diff --git a/infra/charts/feast/README.md b/infra/charts/feast/README.md index 7a0f5f77aa1..218f18f197e 100644 --- a/infra/charts/feast/README.md +++ b/infra/charts/feast/README.md @@ -1,6 +1,6 @@ -# Feast Helm Charts +# Feast Java Helm Charts (alpha) -This repo contains Helm charts for Feast components that are being installed on Kubernetes: +This repo contains Helm charts for Feast Java components that are being installed on Kubernetes: * Feast (root chart): The complete Helm chart containing all Feast components and dependencies. Most users will use this chart, but can selectively enable/disable subcharts using the values.yaml file. * [Feature Server](charts/feature-server): High performant JVM-based implementation of feature server. * [Transformation Service](charts/transformation-service): Transformation server for calculating on-demand features @@ -8,7 +8,7 @@ This repo contains Helm charts for Feast components that are being installed on ## Chart: Feast -Feature store for machine learning Current chart version is `0.24.0` +Feature store for machine learning Current chart version is `0.24.1` ## Installation @@ -50,13 +50,16 @@ For the default configuration, please see the [Feature Server Configuration](htt For more details, please see: https://docs.feast.dev/how-to-guides/running-feast-in-production +## Example +See [here](https://github.com/feast-dev/feast/tree/master/examples/java-demo) for a sample tutorial on testing this helm chart with a demo feature repository and a local Redis instance. + ## Requirements | Repository | Name | Version | |------------|------|---------| | https://charts.helm.sh/stable | redis | 10.5.6 | -| https://feast-helm-charts.storage.googleapis.com | feature-server(feature-server) | 0.24.0 | -| https://feast-helm-charts.storage.googleapis.com | transformation-service(transformation-service) | 0.24.0 | +| https://feast-helm-charts.storage.googleapis.com | feature-server(feature-server) | 0.24.1 | +| https://feast-helm-charts.storage.googleapis.com | transformation-service(transformation-service) | 0.24.1 | ## Values diff --git a/infra/charts/feast/README.md.gotmpl b/infra/charts/feast/README.md.gotmpl index e215858fe01..362f921d7ef 100644 --- a/infra/charts/feast/README.md.gotmpl +++ b/infra/charts/feast/README.md.gotmpl @@ -1,6 +1,6 @@ -# Feast Helm Charts +# Feast Java Helm Charts (alpha) -This repo contains Helm charts for Feast components that are being installed on Kubernetes: +This repo contains Helm charts for Feast Java components that are being installed on Kubernetes: * Feast (root chart): The complete Helm chart containing all Feast components and dependencies. Most users will use this chart, but can selectively enable/disable subcharts using the values.yaml file. * [Feature Server](charts/feature-server): High performant JVM-based implementation of feature server. * [Transformation Service](charts/transformation-service): Transformation server for calculating on-demand features @@ -50,6 +50,10 @@ For the default configuration, please see the [Feature Server Configuration](htt For more details, please see: https://docs.feast.dev/how-to-guides/running-feast-in-production +## Example +See [here](https://github.com/feast-dev/feast/tree/master/examples/java-demo) for a sample tutorial on testing this helm chart with a demo feature repository and a local Redis instance. + + {{ template "chart.requirementsSection" . }} {{ template "chart.valuesSection" . }} \ No newline at end of file diff --git a/infra/charts/feast/charts/feature-server/Chart.yaml b/infra/charts/feast/charts/feature-server/Chart.yaml index f238b6aee4b..2f0b385a615 100644 --- a/infra/charts/feast/charts/feature-server/Chart.yaml +++ b/infra/charts/feast/charts/feature-server/Chart.yaml @@ -1,8 +1,8 @@ apiVersion: v1 description: "Feast Feature Server: Online feature serving service for Feast" name: feature-server -version: 0.24.0 -appVersion: v0.24.0 +version: 0.24.1 +appVersion: v0.24.1 keywords: - machine learning - big data diff --git a/infra/charts/feast/charts/feature-server/README.md b/infra/charts/feast/charts/feature-server/README.md index 465665fb3b0..cd0da20427b 100644 --- a/infra/charts/feast/charts/feature-server/README.md +++ b/infra/charts/feast/charts/feature-server/README.md @@ -1,6 +1,6 @@ # feature-server -![Version: 0.24.0](https://img.shields.io/badge/Version-0.24.0-informational?style=flat-square) ![AppVersion: v0.24.0](https://img.shields.io/badge/AppVersion-v0.24.0-informational?style=flat-square) +![Version: 0.24.1](https://img.shields.io/badge/Version-0.24.1-informational?style=flat-square) ![AppVersion: v0.24.1](https://img.shields.io/badge/AppVersion-v0.24.1-informational?style=flat-square) Feast Feature Server: Online feature serving service for Feast @@ -17,7 +17,7 @@ Feast Feature Server: Online feature serving service for Feast | envOverrides | object | `{}` | Extra environment variables to set | | image.pullPolicy | string | `"IfNotPresent"` | Image pull policy | | image.repository | string | `"feastdev/feature-server-java"` | Docker image for Feature Server repository | -| image.tag | string | `"0.24.0"` | Image tag | +| image.tag | string | `"0.24.1"` | Image tag | | ingress.grpc.annotations | object | `{}` | Extra annotations for the ingress | | ingress.grpc.auth.enabled | bool | `false` | Flag to enable auth | | ingress.grpc.class | string | `"nginx"` | Which ingress controller to use | diff --git a/infra/charts/feast/charts/feature-server/values.yaml b/infra/charts/feast/charts/feature-server/values.yaml index b014d8cee79..fe9e3bee54c 100644 --- a/infra/charts/feast/charts/feature-server/values.yaml +++ b/infra/charts/feast/charts/feature-server/values.yaml @@ -5,7 +5,7 @@ image: # image.repository -- Docker image for Feature Server repository repository: feastdev/feature-server-java # image.tag -- Image tag - tag: 0.24.0 + tag: 0.24.1 # image.pullPolicy -- Image pull policy pullPolicy: IfNotPresent diff --git a/infra/charts/feast/charts/transformation-service/Chart.yaml b/infra/charts/feast/charts/transformation-service/Chart.yaml index 4c650544f58..02f26f1ed95 100644 --- a/infra/charts/feast/charts/transformation-service/Chart.yaml +++ b/infra/charts/feast/charts/transformation-service/Chart.yaml @@ -1,8 +1,8 @@ apiVersion: v1 description: "Transformation service: to compute on-demand features" name: transformation-service -version: 0.24.0 -appVersion: v0.24.0 +version: 0.24.1 +appVersion: v0.24.1 keywords: - machine learning - big data diff --git a/infra/charts/feast/charts/transformation-service/README.md b/infra/charts/feast/charts/transformation-service/README.md index 7b55e1a10c6..39ad9154032 100644 --- a/infra/charts/feast/charts/transformation-service/README.md +++ b/infra/charts/feast/charts/transformation-service/README.md @@ -1,6 +1,6 @@ # transformation-service -![Version: 0.24.0](https://img.shields.io/badge/Version-0.24.0-informational?style=flat-square) ![AppVersion: v0.24.0](https://img.shields.io/badge/AppVersion-v0.24.0-informational?style=flat-square) +![Version: 0.24.1](https://img.shields.io/badge/Version-0.24.1-informational?style=flat-square) ![AppVersion: v0.24.1](https://img.shields.io/badge/AppVersion-v0.24.1-informational?style=flat-square) Transformation service: to compute on-demand features @@ -13,7 +13,7 @@ Transformation service: to compute on-demand features | envOverrides | object | `{}` | Extra environment variables to set | | image.pullPolicy | string | `"IfNotPresent"` | Image pull policy | | image.repository | string | `"feastdev/feature-transformation-server"` | Docker image for Transformation Server repository | -| image.tag | string | `"0.24.0"` | Image tag | +| image.tag | string | `"0.24.1"` | Image tag | | nodeSelector | object | `{}` | Node labels for pod assignment | | podLabels | object | `{}` | Labels to be added to Feast Serving pods | | replicaCount | int | `1` | Number of pods that will be created | diff --git a/infra/charts/feast/charts/transformation-service/values.yaml b/infra/charts/feast/charts/transformation-service/values.yaml index 149d613e9fb..d0584b113ed 100644 --- a/infra/charts/feast/charts/transformation-service/values.yaml +++ b/infra/charts/feast/charts/transformation-service/values.yaml @@ -5,7 +5,7 @@ image: # image.repository -- Docker image for Transformation Server repository repository: feastdev/feature-transformation-server # image.tag -- Image tag - tag: 0.24.0 + tag: 0.24.1 # image.pullPolicy -- Image pull policy pullPolicy: IfNotPresent diff --git a/infra/charts/feast/requirements.yaml b/infra/charts/feast/requirements.yaml index 5dd4a4bce1b..59079114dae 100644 --- a/infra/charts/feast/requirements.yaml +++ b/infra/charts/feast/requirements.yaml @@ -1,12 +1,12 @@ dependencies: - name: feature-server alias: feature-server - version: 0.24.0 + version: 0.24.1 condition: feature-server.enabled repository: https://feast-helm-charts.storage.googleapis.com - name: transformation-service alias: transformation-service - version: 0.24.0 + version: 0.24.1 condition: transformation-service.enabled repository: https://feast-helm-charts.storage.googleapis.com - name: redis diff --git a/infra/scripts/helm/validate-helm-chart-versions.sh b/infra/scripts/helm/validate-helm-chart-versions.sh index aac79d93154..cd8317222bd 100755 --- a/infra/scripts/helm/validate-helm-chart-versions.sh +++ b/infra/scripts/helm/validate-helm-chart-versions.sh @@ -3,7 +3,7 @@ set -e # Amount of file locations that need to be bumped in unison when versions increment -UNIQUE_VERSIONS_COUNT=20 +UNIQUE_VERSIONS_COUNT=22 # Change in release 0.24.0 if [ $# -ne 1 ]; then echo "Please provide a single semver version (without a \"v\" prefix) to test the repository against, e.g 0.99.0" diff --git a/infra/scripts/release/files_to_bump.txt b/infra/scripts/release/files_to_bump.txt index e94ec88db0a..e43d637ede2 100644 --- a/infra/scripts/release/files_to_bump.txt +++ b/infra/scripts/release/files_to_bump.txt @@ -6,7 +6,7 @@ infra/charts/feast/charts/transformation-service/values.yaml 8 infra/charts/feast/charts/feature-server/Chart.yaml 4 5 infra/charts/feast/charts/feature-server/README.md 3 20 infra/charts/feast/charts/feature-server/values.yaml 8 -infra/charts/feast/README.md 11 58 59 +infra/charts/feast/README.md 11 61 62 infra/charts/feast-python-server/Chart.yaml 5 infra/charts/feast-python-server/README.md 5 infra/charts/feast-feature-server/Chart.yaml 5 diff --git a/java/infra/docker/feature-server/Dockerfile b/java/infra/docker/feature-server/Dockerfile index a728340d6b4..bf4e172f763 100644 --- a/java/infra/docker/feature-server/Dockerfile +++ b/java/infra/docker/feature-server/Dockerfile @@ -2,7 +2,7 @@ # Build stage 1: Builder # ============================================================ -FROM maven:3.6-jdk-11 as builder +FROM maven:3-jdk-11 as builder WORKDIR /build diff --git a/java/pom.xml b/java/pom.xml index 9cff26daa6e..77fd3fa16e8 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -35,7 +35,7 @@ - 0.24.0 + 0.24.1 https://github.com/feast-dev/feast UTF-8 diff --git a/java/serving/src/main/java/feast/serving/service/config/ApplicationProperties.java b/java/serving/src/main/java/feast/serving/service/config/ApplicationProperties.java index e4c33434a10..7cef10e61a8 100644 --- a/java/serving/src/main/java/feast/serving/service/config/ApplicationProperties.java +++ b/java/serving/src/main/java/feast/serving/service/config/ApplicationProperties.java @@ -38,7 +38,6 @@ public class ApplicationProperties { private static final Logger log = org.slf4j.LoggerFactory.getLogger(ApplicationProperties.class); private FeastProperties feast; private GrpcServer grpc; - private RestServer rest; public FeastProperties getFeast() { return feast; @@ -331,18 +330,6 @@ public void setServer(Server server) { } } - public static class RestServer { - private Server server; - - public Server getServer() { - return server; - } - - public void setServer(Server server) { - this.server = server; - } - } - /** Trace metric collection properties */ public static class TracingProperties { diff --git a/java/serving/src/main/java/feast/serving/service/config/ApplicationPropertiesModule.java b/java/serving/src/main/java/feast/serving/service/config/ApplicationPropertiesModule.java index 35757330736..588a17d269c 100644 --- a/java/serving/src/main/java/feast/serving/service/config/ApplicationPropertiesModule.java +++ b/java/serving/src/main/java/feast/serving/service/config/ApplicationPropertiesModule.java @@ -16,6 +16,7 @@ */ package feast.serving.service.config; +import com.fasterxml.jackson.databind.DeserializationFeature; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectReader; import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; @@ -40,6 +41,7 @@ public ApplicationProperties provideApplicationProperties() throws IOException { ObjectMapper mapper = new ObjectMapper(new YAMLFactory()); mapper.findAndRegisterModules(); mapper.setDefaultMergeable(Boolean.TRUE); + mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); ApplicationProperties properties = new ApplicationProperties(); ObjectReader objectReader = mapper.readerForUpdating(properties); diff --git a/sdk/python/docs/index.rst b/sdk/python/docs/index.rst index beca384137d..ca96782db8a 100644 --- a/sdk/python/docs/index.rst +++ b/sdk/python/docs/index.rst @@ -353,6 +353,15 @@ Redis Online Store .. autoclass:: feast.infra.online_stores.redis.RedisOnlineStoreConfig :members: +Snowflake Online Store +------------------ + +.. autoclass:: feast.infra.online_stores.snowflake.SnowflakeOnlineStore + :members: + +.. autoclass:: feast.infra.online_stores.snowflake.SnowflakeOnlineStoreConfig + :members: + PostgreSQL Online Store ----------------------- diff --git a/sdk/python/docs/source/conf.py b/sdk/python/docs/source/conf.py new file mode 100644 index 00000000000..5e8fd11d161 --- /dev/null +++ b/sdk/python/docs/source/conf.py @@ -0,0 +1,201 @@ +# -*- coding: utf-8 -*- +# +# Feast documentation build configuration file, created by +# sphinx-quickstart on Sat Nov 30 15:06:53 2019. +# +# This file is execfile()d with the current directory set to its +# containing dir. +# +# Note that not all possible configuration values are present in this +# autogenerated file. +# +# All configuration values have a default; values that are commented out +# serve to show the default. + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +# +import os +import sys + +import sphinx_rtd_theme + +sys.path.insert(0, os.path.abspath("../../feast")) +sys.path.insert(0, os.path.abspath("../..")) + + +# -- Build protos --------------------------------------------------------- + +# For an unknown reason, the Python protos stopped being built correctly. +# See https://readthedocs.org/projects/feast/builds/17686555/ for an +# example where the Python protos did not build, which subsequently broke +# the RTD build. In order to fix this, we manually compile the protos. +import subprocess + +from pathlib import Path + +# cwd will be feast/sdk/python/docs/source +cwd = Path(os.getcwd()) + +# Change to feast/ +os.chdir(cwd.parent.parent.parent.parent) + +# Compile Python protos +result = subprocess.run(["python", "setup.py", "build_python_protos", "--inplace"], capture_output=True) +stdout = result.stdout.decode("utf-8") +stderr = result.stderr.decode("utf-8") +print(f"Apply stdout:\n{stdout}") +print(f"Apply stderr:\n{stderr}") + +# -- General configuration ------------------------------------------------ + +# If your documentation needs a minimal Sphinx version, state it here. +# +# needs_sphinx = '1.0' + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = [ + "sphinx.ext.doctest", + "sphinx.ext.intersphinx", + "sphinx.ext.todo", + "sphinx.ext.coverage", + "sphinx.ext.mathjax", + "sphinx.ext.ifconfig", + "sphinx.ext.viewcode", + "sphinx.ext.githubpages", + "sphinx.ext.napoleon", + "sphinx.ext.autodoc", + "sphinx_rtd_theme", +] + +# Add any paths that contain templates here, relative to this directory. +templates_path = ["_templates"] + +# The suffix(es) of source filenames. +# You can specify multiple suffix as a list of string: +# +# source_suffix = ['.rst', '.md'] +source_suffix = ".rst" + +# The master toctree document. +master_doc = "index" + +# General information about the project. +project = "Feast" +copyright = "2021, Feast Authors" +author = "Feast Authors" + +# The version info for the project you're documenting, acts as replacement for +# |version| and |release|, also used in various other places throughout the +# built documents. +# +# The short X.Y version. + +# TODO: Add the below versions back to documentation building. +# version = ( +# os.popen("git describe --tags $(git rev-list --tags --max-count=1)").read().strip() +# ) +# The full version, including alpha/beta/rc tags. +# release = ( +# os.popen("git describe --tags $(git rev-list --tags --max-count=1)").read().strip() +# ) + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +# +# This is also used if you do content translation via gettext catalogs. +# Usually you set "language" from the command line for these cases. +language = None + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +# This patterns also effect to html_static_path and html_extra_path +exclude_patterns = [] + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = "sphinx" + +# If true, `todo` and `todoList` produce output, else they produce nothing. +todo_include_todos = True + + +# -- Options for HTML output ---------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +# +html_theme = "sphinx_rtd_theme" + +# Theme options are theme-specific and customize the look and feel of a theme +# further. For a list of options available for each theme, see the +# documentation. +# +html_theme_options = {} + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ["_static"] + + +# -- Options for HTMLHelp output ------------------------------------------ + +# Output file base name for HTML help builder. +htmlhelp_basename = "Feastdoc" + + +# -- Options for LaTeX output --------------------------------------------- + +latex_elements = { + # The paper size ('letterpaper' or 'a4paper'). + # + # 'papersize': 'letterpaper', + # The font size ('10pt', '11pt' or '12pt'). + # + # 'pointsize': '10pt', + # Additional stuff for the LaTeX preamble. + # + # 'preamble': '', + # Latex figure (float) alignment + # + # 'figure_align': 'htbp', +} + +# Grouping the document tree into LaTeX files. List of tuples +# (source start file, target name, title, +# author, documentclass [howto, manual, or own class]). +latex_documents = [ + (master_doc, "Feast.tex", "Feast Documentation", "Feast Authors", "manual") +] + + +# -- Options for manual page output --------------------------------------- + +# One entry per manual page. List of tuples +# (source start file, name, description, authors, manual section). +man_pages = [(master_doc, "feast", "Feast Documentation", [author], 1)] + + +# -- Options for Texinfo output ------------------------------------------- + +# Grouping the document tree into Texinfo files. List of tuples +# (source start file, target name, title, author, +# dir menu entry, description, category) +texinfo_documents = [ + ( + master_doc, + "Feast", + "Feast Documentation", + author, + "Feast", + "One line description of project.", + "Miscellaneous", + ) +] + + +# Example configuration for intersphinx: refer to the Python standard library. +intersphinx_mapping = {"https://docs.python.org/": None} diff --git a/sdk/python/docs/source/feast.diff.rst b/sdk/python/docs/source/feast.diff.rst new file mode 100644 index 00000000000..e4142171711 --- /dev/null +++ b/sdk/python/docs/source/feast.diff.rst @@ -0,0 +1,37 @@ +feast.diff package +================== + +Submodules +---------- + +feast.diff.infra\_diff module +----------------------------- + +.. automodule:: feast.diff.infra_diff + :members: + :undoc-members: + :show-inheritance: + +feast.diff.property\_diff module +-------------------------------- + +.. automodule:: feast.diff.property_diff + :members: + :undoc-members: + :show-inheritance: + +feast.diff.registry\_diff module +-------------------------------- + +.. automodule:: feast.diff.registry_diff + :members: + :undoc-members: + :show-inheritance: + +Module contents +--------------- + +.. automodule:: feast.diff + :members: + :undoc-members: + :show-inheritance: diff --git a/sdk/python/docs/source/feast.dqm.profilers.rst b/sdk/python/docs/source/feast.dqm.profilers.rst new file mode 100644 index 00000000000..24f452ada8f --- /dev/null +++ b/sdk/python/docs/source/feast.dqm.profilers.rst @@ -0,0 +1,29 @@ +feast.dqm.profilers package +=========================== + +Submodules +---------- + +feast.dqm.profilers.ge\_profiler module +--------------------------------------- + +.. automodule:: feast.dqm.profilers.ge_profiler + :members: + :undoc-members: + :show-inheritance: + +feast.dqm.profilers.profiler module +----------------------------------- + +.. automodule:: feast.dqm.profilers.profiler + :members: + :undoc-members: + :show-inheritance: + +Module contents +--------------- + +.. automodule:: feast.dqm.profilers + :members: + :undoc-members: + :show-inheritance: diff --git a/sdk/python/docs/source/feast.dqm.rst b/sdk/python/docs/source/feast.dqm.rst new file mode 100644 index 00000000000..0c1b82f0fa2 --- /dev/null +++ b/sdk/python/docs/source/feast.dqm.rst @@ -0,0 +1,29 @@ +feast.dqm package +================= + +Subpackages +----------- + +.. toctree:: + :maxdepth: 4 + + feast.dqm.profilers + +Submodules +---------- + +feast.dqm.errors module +----------------------- + +.. automodule:: feast.dqm.errors + :members: + :undoc-members: + :show-inheritance: + +Module contents +--------------- + +.. automodule:: feast.dqm + :members: + :undoc-members: + :show-inheritance: diff --git a/sdk/python/docs/source/feast.infra.materialization.contrib.bytewax.rst b/sdk/python/docs/source/feast.infra.materialization.contrib.bytewax.rst new file mode 100644 index 00000000000..86fbaa61515 --- /dev/null +++ b/sdk/python/docs/source/feast.infra.materialization.contrib.bytewax.rst @@ -0,0 +1,29 @@ +feast.infra.materialization.contrib.bytewax package +================================================================= + +Submodules +---------- + +feast.infra.materialization.contrib.bytewax.bytewax\_materialization\_engine +---------------------------------------------------------------------- + +.. automodule:: feast.infra.materialization.contrib.bytewax.bytewax_materialization_engine + :members: + :undoc-members: + :show-inheritance: + +feast.infra.materialization.contrib.bytewax.bytewax\_materialization\_job +---------------------------------------------------------------------- + +.. automodule:: feast.infra.materialization.contrib.bytewax.bytewax_materialization_job + :members: + :undoc-members: + :show-inheritance: + +Module contents +--------------- + +.. automodule:: feast.infra.materialization.contrib.bytewax + :members: + :undoc-members: + :show-inheritance: diff --git a/sdk/python/docs/source/feast.infra.materialization.contrib.rst b/sdk/python/docs/source/feast.infra.materialization.contrib.rst new file mode 100644 index 00000000000..f9d77006610 --- /dev/null +++ b/sdk/python/docs/source/feast.infra.materialization.contrib.rst @@ -0,0 +1,10 @@ +feast.infra.materialization.contrib package +========================================== + +Subpackages +----------- + +.. toctree:: + :maxdepth: 4 + + feast.infra.materialization.contrib.bytewax diff --git a/sdk/python/docs/source/feast.infra.materialization.lambda.rst b/sdk/python/docs/source/feast.infra.materialization.lambda.rst new file mode 100644 index 00000000000..7ca1d44314a --- /dev/null +++ b/sdk/python/docs/source/feast.infra.materialization.lambda.rst @@ -0,0 +1,29 @@ +feast.infra.materialization.lambda package +========================================== + +Submodules +---------- + +feast.infra.materialization.lambda.app module +--------------------------------------------- + +.. automodule:: feast.infra.materialization.lambda.app + :members: + :undoc-members: + :show-inheritance: + +feast.infra.materialization.lambda.lambda\_engine module +-------------------------------------------------------- + +.. automodule:: feast.infra.materialization.lambda.lambda_engine + :members: + :undoc-members: + :show-inheritance: + +Module contents +--------------- + +.. automodule:: feast.infra.materialization.lambda + :members: + :undoc-members: + :show-inheritance: diff --git a/sdk/python/docs/source/feast.infra.materialization.rst b/sdk/python/docs/source/feast.infra.materialization.rst new file mode 100644 index 00000000000..6e526c367cd --- /dev/null +++ b/sdk/python/docs/source/feast.infra.materialization.rst @@ -0,0 +1,45 @@ +feast.infra.materialization package +=================================== + +Subpackages +----------- + +.. toctree:: + :maxdepth: 4 + + feast.infra.materialization.lambda + +Submodules +---------- + +feast.infra.materialization.batch\_materialization\_engine module +----------------------------------------------------------------- + +.. automodule:: feast.infra.materialization.batch_materialization_engine + :members: + :undoc-members: + :show-inheritance: + +feast.infra.materialization.local\_engine module +------------------------------------------------ + +.. automodule:: feast.infra.materialization.local_engine + :members: + :undoc-members: + :show-inheritance: + +feast.infra.materialization.snowflake\_engine module +---------------------------------------------------- + +.. automodule:: feast.infra.materialization.snowflake_engine + :members: + :undoc-members: + :show-inheritance: + +Module contents +--------------- + +.. automodule:: feast.infra.materialization + :members: + :undoc-members: + :show-inheritance: diff --git a/sdk/python/docs/source/feast.infra.offline_stores.contrib.athena_offline_store.rst b/sdk/python/docs/source/feast.infra.offline_stores.contrib.athena_offline_store.rst new file mode 100644 index 00000000000..d2275b2b393 --- /dev/null +++ b/sdk/python/docs/source/feast.infra.offline_stores.contrib.athena_offline_store.rst @@ -0,0 +1,37 @@ +feast.infra.offline\_stores.contrib.athena\_offline\_store package +================================================================== + +Subpackages +----------- + +.. toctree:: + :maxdepth: 4 + + feast.infra.offline_stores.contrib.athena_offline_store.tests + +Submodules +---------- + +feast.infra.offline\_stores.contrib.athena\_offline\_store.athena module +------------------------------------------------------------------------ + +.. automodule:: feast.infra.offline_stores.contrib.athena_offline_store.athena + :members: + :undoc-members: + :show-inheritance: + +feast.infra.offline\_stores.contrib.athena\_offline\_store.athena\_source module +-------------------------------------------------------------------------------- + +.. automodule:: feast.infra.offline_stores.contrib.athena_offline_store.athena_source + :members: + :undoc-members: + :show-inheritance: + +Module contents +--------------- + +.. automodule:: feast.infra.offline_stores.contrib.athena_offline_store + :members: + :undoc-members: + :show-inheritance: diff --git a/sdk/python/docs/source/feast.infra.offline_stores.contrib.athena_offline_store.tests.rst b/sdk/python/docs/source/feast.infra.offline_stores.contrib.athena_offline_store.tests.rst new file mode 100644 index 00000000000..47a8f83e2b7 --- /dev/null +++ b/sdk/python/docs/source/feast.infra.offline_stores.contrib.athena_offline_store.tests.rst @@ -0,0 +1,21 @@ +feast.infra.offline\_stores.contrib.athena\_offline\_store.tests package +======================================================================== + +Submodules +---------- + +feast.infra.offline\_stores.contrib.athena\_offline\_store.tests.data\_source module +------------------------------------------------------------------------------------ + +.. automodule:: feast.infra.offline_stores.contrib.athena_offline_store.tests.data_source + :members: + :undoc-members: + :show-inheritance: + +Module contents +--------------- + +.. automodule:: feast.infra.offline_stores.contrib.athena_offline_store.tests + :members: + :undoc-members: + :show-inheritance: diff --git a/sdk/python/docs/source/feast.infra.offline_stores.contrib.mssql_offline_store.rst b/sdk/python/docs/source/feast.infra.offline_stores.contrib.mssql_offline_store.rst new file mode 100644 index 00000000000..8fb0b966bf8 --- /dev/null +++ b/sdk/python/docs/source/feast.infra.offline_stores.contrib.mssql_offline_store.rst @@ -0,0 +1,37 @@ +feast.infra.offline\_stores.contrib.mssql\_offline\_store package +================================================================= + +Subpackages +----------- + +.. toctree:: + :maxdepth: 4 + + feast.infra.offline_stores.contrib.mssql_offline_store.tests + +Submodules +---------- + +feast.infra.offline\_stores.contrib.mssql\_offline\_store.mssql module +---------------------------------------------------------------------- + +.. automodule:: feast.infra.offline_stores.contrib.mssql_offline_store.mssql + :members: + :undoc-members: + :show-inheritance: + +feast.infra.offline\_stores.contrib.mssql\_offline\_store.mssqlserver\_source module +------------------------------------------------------------------------------------ + +.. automodule:: feast.infra.offline_stores.contrib.mssql_offline_store.mssqlserver_source + :members: + :undoc-members: + :show-inheritance: + +Module contents +--------------- + +.. automodule:: feast.infra.offline_stores.contrib.mssql_offline_store + :members: + :undoc-members: + :show-inheritance: diff --git a/sdk/python/docs/source/feast.infra.offline_stores.contrib.mssql_offline_store.tests.rst b/sdk/python/docs/source/feast.infra.offline_stores.contrib.mssql_offline_store.tests.rst new file mode 100644 index 00000000000..2f01ddd091a --- /dev/null +++ b/sdk/python/docs/source/feast.infra.offline_stores.contrib.mssql_offline_store.tests.rst @@ -0,0 +1,21 @@ +feast.infra.offline\_stores.contrib.mssql\_offline\_store.tests package +======================================================================= + +Submodules +---------- + +feast.infra.offline\_stores.contrib.mssql\_offline\_store.tests.data\_source module +----------------------------------------------------------------------------------- + +.. automodule:: feast.infra.offline_stores.contrib.mssql_offline_store.tests.data_source + :members: + :undoc-members: + :show-inheritance: + +Module contents +--------------- + +.. automodule:: feast.infra.offline_stores.contrib.mssql_offline_store.tests + :members: + :undoc-members: + :show-inheritance: diff --git a/sdk/python/docs/source/feast.infra.offline_stores.contrib.postgres_offline_store.rst b/sdk/python/docs/source/feast.infra.offline_stores.contrib.postgres_offline_store.rst new file mode 100644 index 00000000000..a80690fe859 --- /dev/null +++ b/sdk/python/docs/source/feast.infra.offline_stores.contrib.postgres_offline_store.rst @@ -0,0 +1,37 @@ +feast.infra.offline\_stores.contrib.postgres\_offline\_store package +==================================================================== + +Subpackages +----------- + +.. toctree:: + :maxdepth: 4 + + feast.infra.offline_stores.contrib.postgres_offline_store.tests + +Submodules +---------- + +feast.infra.offline\_stores.contrib.postgres\_offline\_store.postgres module +---------------------------------------------------------------------------- + +.. automodule:: feast.infra.offline_stores.contrib.postgres_offline_store.postgres + :members: + :undoc-members: + :show-inheritance: + +feast.infra.offline\_stores.contrib.postgres\_offline\_store.postgres\_source module +------------------------------------------------------------------------------------ + +.. automodule:: feast.infra.offline_stores.contrib.postgres_offline_store.postgres_source + :members: + :undoc-members: + :show-inheritance: + +Module contents +--------------- + +.. automodule:: feast.infra.offline_stores.contrib.postgres_offline_store + :members: + :undoc-members: + :show-inheritance: diff --git a/sdk/python/docs/source/feast.infra.offline_stores.contrib.postgres_offline_store.tests.rst b/sdk/python/docs/source/feast.infra.offline_stores.contrib.postgres_offline_store.tests.rst new file mode 100644 index 00000000000..35e60d2998b --- /dev/null +++ b/sdk/python/docs/source/feast.infra.offline_stores.contrib.postgres_offline_store.tests.rst @@ -0,0 +1,21 @@ +feast.infra.offline\_stores.contrib.postgres\_offline\_store.tests package +========================================================================== + +Submodules +---------- + +feast.infra.offline\_stores.contrib.postgres\_offline\_store.tests.data\_source module +-------------------------------------------------------------------------------------- + +.. automodule:: feast.infra.offline_stores.contrib.postgres_offline_store.tests.data_source + :members: + :undoc-members: + :show-inheritance: + +Module contents +--------------- + +.. automodule:: feast.infra.offline_stores.contrib.postgres_offline_store.tests + :members: + :undoc-members: + :show-inheritance: diff --git a/sdk/python/docs/source/feast.infra.offline_stores.contrib.rst b/sdk/python/docs/source/feast.infra.offline_stores.contrib.rst new file mode 100644 index 00000000000..ec74ddab05c --- /dev/null +++ b/sdk/python/docs/source/feast.infra.offline_stores.contrib.rst @@ -0,0 +1,65 @@ +feast.infra.offline\_stores.contrib package +=========================================== + +Subpackages +----------- + +.. toctree:: + :maxdepth: 4 + + feast.infra.offline_stores.contrib.athena_offline_store + feast.infra.offline_stores.contrib.mssql_offline_store + feast.infra.offline_stores.contrib.postgres_offline_store + feast.infra.offline_stores.contrib.spark_offline_store + feast.infra.offline_stores.contrib.trino_offline_store + +Submodules +---------- + +feast.infra.offline\_stores.contrib.athena\_repo\_configuration module +---------------------------------------------------------------------- + +.. automodule:: feast.infra.offline_stores.contrib.athena_repo_configuration + :members: + :undoc-members: + :show-inheritance: + +feast.infra.offline\_stores.contrib.mssql\_repo\_configuration module +--------------------------------------------------------------------- + +.. automodule:: feast.infra.offline_stores.contrib.mssql_repo_configuration + :members: + :undoc-members: + :show-inheritance: + +feast.infra.offline\_stores.contrib.postgres\_repo\_configuration module +------------------------------------------------------------------------ + +.. automodule:: feast.infra.offline_stores.contrib.postgres_repo_configuration + :members: + :undoc-members: + :show-inheritance: + +feast.infra.offline\_stores.contrib.spark\_repo\_configuration module +--------------------------------------------------------------------- + +.. automodule:: feast.infra.offline_stores.contrib.spark_repo_configuration + :members: + :undoc-members: + :show-inheritance: + +feast.infra.offline\_stores.contrib.trino\_repo\_configuration module +--------------------------------------------------------------------- + +.. automodule:: feast.infra.offline_stores.contrib.trino_repo_configuration + :members: + :undoc-members: + :show-inheritance: + +Module contents +--------------- + +.. automodule:: feast.infra.offline_stores.contrib + :members: + :undoc-members: + :show-inheritance: diff --git a/sdk/python/docs/source/feast.infra.offline_stores.contrib.spark_offline_store.rst b/sdk/python/docs/source/feast.infra.offline_stores.contrib.spark_offline_store.rst new file mode 100644 index 00000000000..b8b79bb48e8 --- /dev/null +++ b/sdk/python/docs/source/feast.infra.offline_stores.contrib.spark_offline_store.rst @@ -0,0 +1,37 @@ +feast.infra.offline\_stores.contrib.spark\_offline\_store package +================================================================= + +Subpackages +----------- + +.. toctree:: + :maxdepth: 4 + + feast.infra.offline_stores.contrib.spark_offline_store.tests + +Submodules +---------- + +feast.infra.offline\_stores.contrib.spark\_offline\_store.spark module +---------------------------------------------------------------------- + +.. automodule:: feast.infra.offline_stores.contrib.spark_offline_store.spark + :members: + :undoc-members: + :show-inheritance: + +feast.infra.offline\_stores.contrib.spark\_offline\_store.spark\_source module +------------------------------------------------------------------------------ + +.. automodule:: feast.infra.offline_stores.contrib.spark_offline_store.spark_source + :members: + :undoc-members: + :show-inheritance: + +Module contents +--------------- + +.. automodule:: feast.infra.offline_stores.contrib.spark_offline_store + :members: + :undoc-members: + :show-inheritance: diff --git a/sdk/python/docs/source/feast.infra.offline_stores.contrib.spark_offline_store.tests.rst b/sdk/python/docs/source/feast.infra.offline_stores.contrib.spark_offline_store.tests.rst new file mode 100644 index 00000000000..8b0f9bd88b3 --- /dev/null +++ b/sdk/python/docs/source/feast.infra.offline_stores.contrib.spark_offline_store.tests.rst @@ -0,0 +1,21 @@ +feast.infra.offline\_stores.contrib.spark\_offline\_store.tests package +======================================================================= + +Submodules +---------- + +feast.infra.offline\_stores.contrib.spark\_offline\_store.tests.data\_source module +----------------------------------------------------------------------------------- + +.. automodule:: feast.infra.offline_stores.contrib.spark_offline_store.tests.data_source + :members: + :undoc-members: + :show-inheritance: + +Module contents +--------------- + +.. automodule:: feast.infra.offline_stores.contrib.spark_offline_store.tests + :members: + :undoc-members: + :show-inheritance: diff --git a/sdk/python/docs/source/feast.infra.offline_stores.contrib.trino_offline_store.connectors.rst b/sdk/python/docs/source/feast.infra.offline_stores.contrib.trino_offline_store.connectors.rst new file mode 100644 index 00000000000..a0ee8dceabf --- /dev/null +++ b/sdk/python/docs/source/feast.infra.offline_stores.contrib.trino_offline_store.connectors.rst @@ -0,0 +1,21 @@ +feast.infra.offline\_stores.contrib.trino\_offline\_store.connectors package +============================================================================ + +Submodules +---------- + +feast.infra.offline\_stores.contrib.trino\_offline\_store.connectors.upload module +---------------------------------------------------------------------------------- + +.. automodule:: feast.infra.offline_stores.contrib.trino_offline_store.connectors.upload + :members: + :undoc-members: + :show-inheritance: + +Module contents +--------------- + +.. automodule:: feast.infra.offline_stores.contrib.trino_offline_store.connectors + :members: + :undoc-members: + :show-inheritance: diff --git a/sdk/python/docs/source/feast.infra.offline_stores.contrib.trino_offline_store.rst b/sdk/python/docs/source/feast.infra.offline_stores.contrib.trino_offline_store.rst new file mode 100644 index 00000000000..857326003f3 --- /dev/null +++ b/sdk/python/docs/source/feast.infra.offline_stores.contrib.trino_offline_store.rst @@ -0,0 +1,55 @@ +feast.infra.offline\_stores.contrib.trino\_offline\_store package +================================================================= + +Subpackages +----------- + +.. toctree:: + :maxdepth: 4 + + feast.infra.offline_stores.contrib.trino_offline_store.connectors + feast.infra.offline_stores.contrib.trino_offline_store.test_config + feast.infra.offline_stores.contrib.trino_offline_store.tests + +Submodules +---------- + +feast.infra.offline\_stores.contrib.trino\_offline\_store.trino module +---------------------------------------------------------------------- + +.. automodule:: feast.infra.offline_stores.contrib.trino_offline_store.trino + :members: + :undoc-members: + :show-inheritance: + +feast.infra.offline\_stores.contrib.trino\_offline\_store.trino\_queries module +------------------------------------------------------------------------------- + +.. automodule:: feast.infra.offline_stores.contrib.trino_offline_store.trino_queries + :members: + :undoc-members: + :show-inheritance: + +feast.infra.offline\_stores.contrib.trino\_offline\_store.trino\_source module +------------------------------------------------------------------------------ + +.. automodule:: feast.infra.offline_stores.contrib.trino_offline_store.trino_source + :members: + :undoc-members: + :show-inheritance: + +feast.infra.offline\_stores.contrib.trino\_offline\_store.trino\_type\_map module +--------------------------------------------------------------------------------- + +.. automodule:: feast.infra.offline_stores.contrib.trino_offline_store.trino_type_map + :members: + :undoc-members: + :show-inheritance: + +Module contents +--------------- + +.. automodule:: feast.infra.offline_stores.contrib.trino_offline_store + :members: + :undoc-members: + :show-inheritance: diff --git a/sdk/python/docs/source/feast.infra.offline_stores.contrib.trino_offline_store.test_config.rst b/sdk/python/docs/source/feast.infra.offline_stores.contrib.trino_offline_store.test_config.rst new file mode 100644 index 00000000000..ef43a191d0a --- /dev/null +++ b/sdk/python/docs/source/feast.infra.offline_stores.contrib.trino_offline_store.test_config.rst @@ -0,0 +1,21 @@ +feast.infra.offline\_stores.contrib.trino\_offline\_store.test\_config package +============================================================================== + +Submodules +---------- + +feast.infra.offline\_stores.contrib.trino\_offline\_store.test\_config.manual\_tests module +------------------------------------------------------------------------------------------- + +.. automodule:: feast.infra.offline_stores.contrib.trino_offline_store.test_config.manual_tests + :members: + :undoc-members: + :show-inheritance: + +Module contents +--------------- + +.. automodule:: feast.infra.offline_stores.contrib.trino_offline_store.test_config + :members: + :undoc-members: + :show-inheritance: diff --git a/sdk/python/docs/source/feast.infra.offline_stores.contrib.trino_offline_store.tests.rst b/sdk/python/docs/source/feast.infra.offline_stores.contrib.trino_offline_store.tests.rst new file mode 100644 index 00000000000..9102f1f8d64 --- /dev/null +++ b/sdk/python/docs/source/feast.infra.offline_stores.contrib.trino_offline_store.tests.rst @@ -0,0 +1,21 @@ +feast.infra.offline\_stores.contrib.trino\_offline\_store.tests package +======================================================================= + +Submodules +---------- + +feast.infra.offline\_stores.contrib.trino\_offline\_store.tests.data\_source module +----------------------------------------------------------------------------------- + +.. automodule:: feast.infra.offline_stores.contrib.trino_offline_store.tests.data_source + :members: + :undoc-members: + :show-inheritance: + +Module contents +--------------- + +.. automodule:: feast.infra.offline_stores.contrib.trino_offline_store.tests + :members: + :undoc-members: + :show-inheritance: diff --git a/sdk/python/docs/source/feast.infra.offline_stores.rst b/sdk/python/docs/source/feast.infra.offline_stores.rst new file mode 100644 index 00000000000..7949c9efb32 --- /dev/null +++ b/sdk/python/docs/source/feast.infra.offline_stores.rst @@ -0,0 +1,101 @@ +feast.infra.offline\_stores package +=================================== + +Subpackages +----------- + +.. toctree:: + :maxdepth: 4 + + feast.infra.offline_stores.contrib + +Submodules +---------- + +feast.infra.offline\_stores.bigquery module +------------------------------------------- + +.. automodule:: feast.infra.offline_stores.bigquery + :members: + :undoc-members: + :show-inheritance: + +feast.infra.offline\_stores.bigquery\_source module +--------------------------------------------------- + +.. automodule:: feast.infra.offline_stores.bigquery_source + :members: + :undoc-members: + :show-inheritance: + +feast.infra.offline\_stores.file module +--------------------------------------- + +.. automodule:: feast.infra.offline_stores.file + :members: + :undoc-members: + :show-inheritance: + +feast.infra.offline\_stores.file\_source module +----------------------------------------------- + +.. automodule:: feast.infra.offline_stores.file_source + :members: + :undoc-members: + :show-inheritance: + +feast.infra.offline\_stores.offline\_store module +------------------------------------------------- + +.. automodule:: feast.infra.offline_stores.offline_store + :members: + :undoc-members: + :show-inheritance: + +feast.infra.offline\_stores.offline\_utils module +------------------------------------------------- + +.. automodule:: feast.infra.offline_stores.offline_utils + :members: + :undoc-members: + :show-inheritance: + +feast.infra.offline\_stores.redshift module +------------------------------------------- + +.. automodule:: feast.infra.offline_stores.redshift + :members: + :undoc-members: + :show-inheritance: + +feast.infra.offline\_stores.redshift\_source module +--------------------------------------------------- + +.. automodule:: feast.infra.offline_stores.redshift_source + :members: + :undoc-members: + :show-inheritance: + +feast.infra.offline\_stores.snowflake module +-------------------------------------------- + +.. automodule:: feast.infra.offline_stores.snowflake + :members: + :undoc-members: + :show-inheritance: + +feast.infra.offline\_stores.snowflake\_source module +---------------------------------------------------- + +.. automodule:: feast.infra.offline_stores.snowflake_source + :members: + :undoc-members: + :show-inheritance: + +Module contents +--------------- + +.. automodule:: feast.infra.offline_stores + :members: + :undoc-members: + :show-inheritance: diff --git a/sdk/python/docs/source/feast.infra.online_stores.contrib.cassandra_online_store.rst b/sdk/python/docs/source/feast.infra.online_stores.contrib.cassandra_online_store.rst new file mode 100644 index 00000000000..3770cc8af70 --- /dev/null +++ b/sdk/python/docs/source/feast.infra.online_stores.contrib.cassandra_online_store.rst @@ -0,0 +1,21 @@ +feast.infra.online\_stores.contrib.cassandra\_online\_store package +=================================================================== + +Submodules +---------- + +feast.infra.online\_stores.contrib.cassandra\_online\_store.cassandra\_online\_store module +------------------------------------------------------------------------------------------- + +.. automodule:: feast.infra.online_stores.contrib.cassandra_online_store.cassandra_online_store + :members: + :undoc-members: + :show-inheritance: + +Module contents +--------------- + +.. automodule:: feast.infra.online_stores.contrib.cassandra_online_store + :members: + :undoc-members: + :show-inheritance: diff --git a/sdk/python/docs/source/feast.infra.online_stores.contrib.hbase_online_store.rst b/sdk/python/docs/source/feast.infra.online_stores.contrib.hbase_online_store.rst new file mode 100644 index 00000000000..ce249023049 --- /dev/null +++ b/sdk/python/docs/source/feast.infra.online_stores.contrib.hbase_online_store.rst @@ -0,0 +1,21 @@ +feast.infra.online\_stores.contrib.hbase\_online\_store package +=============================================================== + +Submodules +---------- + +feast.infra.online\_stores.contrib.hbase\_online\_store.hbase module +-------------------------------------------------------------------- + +.. automodule:: feast.infra.online_stores.contrib.hbase_online_store.hbase + :members: + :undoc-members: + :show-inheritance: + +Module contents +--------------- + +.. automodule:: feast.infra.online_stores.contrib.hbase_online_store + :members: + :undoc-members: + :show-inheritance: diff --git a/sdk/python/docs/source/feast.infra.online_stores.contrib.rst b/sdk/python/docs/source/feast.infra.online_stores.contrib.rst new file mode 100644 index 00000000000..6afe9071ace --- /dev/null +++ b/sdk/python/docs/source/feast.infra.online_stores.contrib.rst @@ -0,0 +1,54 @@ +feast.infra.online\_stores.contrib package +========================================== + +Subpackages +----------- + +.. toctree:: + :maxdepth: 4 + + feast.infra.online_stores.contrib.cassandra_online_store + feast.infra.online_stores.contrib.hbase_online_store + +Submodules +---------- + +feast.infra.online\_stores.contrib.cassandra\_repo\_configuration module +------------------------------------------------------------------------ + +.. automodule:: feast.infra.online_stores.contrib.cassandra_repo_configuration + :members: + :undoc-members: + :show-inheritance: + +feast.infra.online\_stores.contrib.hbase\_repo\_configuration module +-------------------------------------------------------------------- + +.. automodule:: feast.infra.online_stores.contrib.hbase_repo_configuration + :members: + :undoc-members: + :show-inheritance: + +feast.infra.online\_stores.contrib.postgres module +-------------------------------------------------- + +.. automodule:: feast.infra.online_stores.contrib.postgres + :members: + :undoc-members: + :show-inheritance: + +feast.infra.online\_stores.contrib.postgres\_repo\_configuration module +----------------------------------------------------------------------- + +.. automodule:: feast.infra.online_stores.contrib.postgres_repo_configuration + :members: + :undoc-members: + :show-inheritance: + +Module contents +--------------- + +.. automodule:: feast.infra.online_stores.contrib + :members: + :undoc-members: + :show-inheritance: diff --git a/sdk/python/docs/source/feast.infra.online_stores.rst b/sdk/python/docs/source/feast.infra.online_stores.rst new file mode 100644 index 00000000000..65758c409c0 --- /dev/null +++ b/sdk/python/docs/source/feast.infra.online_stores.rst @@ -0,0 +1,77 @@ +feast.infra.online\_stores package +================================== + +Subpackages +----------- + +.. toctree:: + :maxdepth: 4 + + feast.infra.online_stores.contrib + +Submodules +---------- + +feast.infra.online\_stores.datastore module +------------------------------------------- + +.. automodule:: feast.infra.online_stores.datastore + :members: + :undoc-members: + :show-inheritance: + +feast.infra.online\_stores.dynamodb module +------------------------------------------ + +.. automodule:: feast.infra.online_stores.dynamodb + :members: + :undoc-members: + :show-inheritance: + +feast.infra.online\_stores.helpers module +----------------------------------------- + +.. automodule:: feast.infra.online_stores.helpers + :members: + :undoc-members: + :show-inheritance: + +feast.infra.online\_stores.online\_store module +----------------------------------------------- + +.. automodule:: feast.infra.online_stores.online_store + :members: + :undoc-members: + :show-inheritance: + +feast.infra.online\_stores.redis module +--------------------------------------- + +.. automodule:: feast.infra.online_stores.redis + :members: + :undoc-members: + :show-inheritance: + +feast.infra.online\_stores.snowflake module +------------------------------------------- + +.. automodule:: feast.infra.online_stores.snowflake + :members: + :undoc-members: + :show-inheritance: + +feast.infra.online\_stores.sqlite module +---------------------------------------- + +.. automodule:: feast.infra.online_stores.sqlite + :members: + :undoc-members: + :show-inheritance: + +Module contents +--------------- + +.. automodule:: feast.infra.online_stores + :members: + :undoc-members: + :show-inheritance: diff --git a/sdk/python/docs/source/feast.infra.registry.rst b/sdk/python/docs/source/feast.infra.registry.rst new file mode 100644 index 00000000000..7a2d9689975 --- /dev/null +++ b/sdk/python/docs/source/feast.infra.registry.rst @@ -0,0 +1,69 @@ +feast.infra.registry package +============================ + +Submodules +---------- + +feast.infra.registry.base\_registry module +------------------------------------------ + +.. automodule:: feast.infra.registry.base_registry + :members: + :undoc-members: + :show-inheritance: + +feast.infra.registry.file module +-------------------------------- + +.. automodule:: feast.infra.registry.file + :members: + :undoc-members: + :show-inheritance: + +feast.infra.registry.gcs module +------------------------------- + +.. automodule:: feast.infra.registry.gcs + :members: + :undoc-members: + :show-inheritance: + +feast.infra.registry.registry module +------------------------------------ + +.. automodule:: feast.infra.registry.registry + :members: + :undoc-members: + :show-inheritance: + +feast.infra.registry.registry\_store module +------------------------------------------- + +.. automodule:: feast.infra.registry.registry_store + :members: + :undoc-members: + :show-inheritance: + +feast.infra.registry.s3 module +------------------------------ + +.. automodule:: feast.infra.registry.s3 + :members: + :undoc-members: + :show-inheritance: + +feast.infra.registry.sql module +------------------------------- + +.. automodule:: feast.infra.registry.sql + :members: + :undoc-members: + :show-inheritance: + +Module contents +--------------- + +.. automodule:: feast.infra.registry + :members: + :undoc-members: + :show-inheritance: diff --git a/sdk/python/docs/source/feast.infra.registry_stores.rst b/sdk/python/docs/source/feast.infra.registry_stores.rst new file mode 100644 index 00000000000..cff02fa3380 --- /dev/null +++ b/sdk/python/docs/source/feast.infra.registry_stores.rst @@ -0,0 +1,21 @@ +feast.infra.registry\_stores package +==================================== + +Submodules +---------- + +feast.infra.registry\_stores.sql module +--------------------------------------- + +.. automodule:: feast.infra.registry_stores.sql + :members: + :undoc-members: + :show-inheritance: + +Module contents +--------------- + +.. automodule:: feast.infra.registry_stores + :members: + :undoc-members: + :show-inheritance: diff --git a/sdk/python/docs/source/feast.infra.rst b/sdk/python/docs/source/feast.infra.rst new file mode 100644 index 00000000000..50e1f37f1c6 --- /dev/null +++ b/sdk/python/docs/source/feast.infra.rst @@ -0,0 +1,81 @@ +feast.infra package +=================== + +Subpackages +----------- + +.. toctree:: + :maxdepth: 4 + + feast.infra.offline_stores + feast.infra.online_stores + feast.infra.registry + feast.infra.transformation_servers + feast.infra.utils + +Submodules +---------- + +feast.infra.aws module +---------------------- + +.. automodule:: feast.infra.aws + :members: + :undoc-members: + :show-inheritance: + +feast.infra.gcp module +---------------------- + +.. automodule:: feast.infra.gcp + :members: + :undoc-members: + :show-inheritance: + +feast.infra.infra\_object module +-------------------------------- + +.. automodule:: feast.infra.infra_object + :members: + :undoc-members: + :show-inheritance: + +feast.infra.key\_encoding\_utils module +--------------------------------------- + +.. automodule:: feast.infra.key_encoding_utils + :members: + :undoc-members: + :show-inheritance: + +feast.infra.local module +------------------------ + +.. automodule:: feast.infra.local + :members: + :undoc-members: + :show-inheritance: + +feast.infra.passthrough\_provider module +---------------------------------------- + +.. automodule:: feast.infra.passthrough_provider + :members: + :undoc-members: + :show-inheritance: + +feast.infra.provider module +--------------------------- + +.. automodule:: feast.infra.provider + :members: + :undoc-members: + :show-inheritance: + +Module contents +--------------- + +.. automodule:: feast.infra + :members: + :undoc-members: + :show-inheritance: diff --git a/sdk/python/docs/source/feast.infra.transformation_servers.rst b/sdk/python/docs/source/feast.infra.transformation_servers.rst new file mode 100644 index 00000000000..7de2dc79f29 --- /dev/null +++ b/sdk/python/docs/source/feast.infra.transformation_servers.rst @@ -0,0 +1,21 @@ +feast.infra.transformation\_servers package +=========================================== + +Submodules +---------- + +feast.infra.transformation\_servers.app module +---------------------------------------------- + +.. automodule:: feast.infra.transformation_servers.app + :members: + :undoc-members: + :show-inheritance: + +Module contents +--------------- + +.. automodule:: feast.infra.transformation_servers + :members: + :undoc-members: + :show-inheritance: diff --git a/sdk/python/docs/source/feast.infra.utils.postgres.rst b/sdk/python/docs/source/feast.infra.utils.postgres.rst new file mode 100644 index 00000000000..119c8c1dee9 --- /dev/null +++ b/sdk/python/docs/source/feast.infra.utils.postgres.rst @@ -0,0 +1,29 @@ +feast.infra.utils.postgres package +================================== + +Submodules +---------- + +feast.infra.utils.postgres.connection\_utils module +--------------------------------------------------- + +.. automodule:: feast.infra.utils.postgres.connection_utils + :members: + :undoc-members: + :show-inheritance: + +feast.infra.utils.postgres.postgres\_config module +-------------------------------------------------- + +.. automodule:: feast.infra.utils.postgres.postgres_config + :members: + :undoc-members: + :show-inheritance: + +Module contents +--------------- + +.. automodule:: feast.infra.utils.postgres + :members: + :undoc-members: + :show-inheritance: diff --git a/sdk/python/docs/source/feast.infra.utils.rst b/sdk/python/docs/source/feast.infra.utils.rst new file mode 100644 index 00000000000..e4116e7a172 --- /dev/null +++ b/sdk/python/docs/source/feast.infra.utils.rst @@ -0,0 +1,37 @@ +feast.infra.utils package +========================= + +Subpackages +----------- + +.. toctree:: + :maxdepth: 4 + + feast.infra.utils.postgres + +Submodules +---------- + +feast.infra.utils.aws\_utils module +----------------------------------- + +.. automodule:: feast.infra.utils.aws_utils + :members: + :undoc-members: + :show-inheritance: + +feast.infra.utils.hbase\_utils module +------------------------------------- + +.. automodule:: feast.infra.utils.hbase_utils + :members: + :undoc-members: + :show-inheritance: + +Module contents +--------------- + +.. automodule:: feast.infra.utils + :members: + :undoc-members: + :show-inheritance: diff --git a/sdk/python/docs/source/feast.loaders.rst b/sdk/python/docs/source/feast.loaders.rst new file mode 100644 index 00000000000..d4968a29999 --- /dev/null +++ b/sdk/python/docs/source/feast.loaders.rst @@ -0,0 +1,21 @@ +feast.loaders package +===================== + +Submodules +---------- + +feast.loaders.yaml module +------------------------- + +.. automodule:: feast.loaders.yaml + :members: + :undoc-members: + :show-inheritance: + +Module contents +--------------- + +.. automodule:: feast.loaders + :members: + :undoc-members: + :show-inheritance: diff --git a/sdk/python/docs/source/feast.protos.feast.core.rst b/sdk/python/docs/source/feast.protos.feast.core.rst new file mode 100644 index 00000000000..aaed49cd731 --- /dev/null +++ b/sdk/python/docs/source/feast.protos.feast.core.rst @@ -0,0 +1,333 @@ +feast.protos.feast.core package +=============================== + +Submodules +---------- + +feast.protos.feast.core.Aggregation\_pb2 module +----------------------------------------------- + +.. automodule:: feast.protos.feast.core.Aggregation_pb2 + :members: + :undoc-members: + :show-inheritance: + +feast.protos.feast.core.Aggregation\_pb2\_grpc module +----------------------------------------------------- + +.. automodule:: feast.protos.feast.core.Aggregation_pb2_grpc + :members: + :undoc-members: + :show-inheritance: + +feast.protos.feast.core.DataFormat\_pb2 module +---------------------------------------------- + +.. automodule:: feast.protos.feast.core.DataFormat_pb2 + :members: + :undoc-members: + :show-inheritance: + +feast.protos.feast.core.DataFormat\_pb2\_grpc module +---------------------------------------------------- + +.. automodule:: feast.protos.feast.core.DataFormat_pb2_grpc + :members: + :undoc-members: + :show-inheritance: + +feast.protos.feast.core.DataSource\_pb2 module +---------------------------------------------- + +.. automodule:: feast.protos.feast.core.DataSource_pb2 + :members: + :undoc-members: + :show-inheritance: + +feast.protos.feast.core.DataSource\_pb2\_grpc module +---------------------------------------------------- + +.. automodule:: feast.protos.feast.core.DataSource_pb2_grpc + :members: + :undoc-members: + :show-inheritance: + +feast.protos.feast.core.DatastoreTable\_pb2 module +-------------------------------------------------- + +.. automodule:: feast.protos.feast.core.DatastoreTable_pb2 + :members: + :undoc-members: + :show-inheritance: + +feast.protos.feast.core.DatastoreTable\_pb2\_grpc module +-------------------------------------------------------- + +.. automodule:: feast.protos.feast.core.DatastoreTable_pb2_grpc + :members: + :undoc-members: + :show-inheritance: + +feast.protos.feast.core.DynamoDBTable\_pb2 module +------------------------------------------------- + +.. automodule:: feast.protos.feast.core.DynamoDBTable_pb2 + :members: + :undoc-members: + :show-inheritance: + +feast.protos.feast.core.DynamoDBTable\_pb2\_grpc module +------------------------------------------------------- + +.. automodule:: feast.protos.feast.core.DynamoDBTable_pb2_grpc + :members: + :undoc-members: + :show-inheritance: + +feast.protos.feast.core.Entity\_pb2 module +------------------------------------------ + +.. automodule:: feast.protos.feast.core.Entity_pb2 + :members: + :undoc-members: + :show-inheritance: + +feast.protos.feast.core.Entity\_pb2\_grpc module +------------------------------------------------ + +.. automodule:: feast.protos.feast.core.Entity_pb2_grpc + :members: + :undoc-members: + :show-inheritance: + +feast.protos.feast.core.FeatureService\_pb2 module +-------------------------------------------------- + +.. automodule:: feast.protos.feast.core.FeatureService_pb2 + :members: + :undoc-members: + :show-inheritance: + +feast.protos.feast.core.FeatureService\_pb2\_grpc module +-------------------------------------------------------- + +.. automodule:: feast.protos.feast.core.FeatureService_pb2_grpc + :members: + :undoc-members: + :show-inheritance: + +feast.protos.feast.core.FeatureTable\_pb2 module +------------------------------------------------ + +.. automodule:: feast.protos.feast.core.FeatureTable_pb2 + :members: + :undoc-members: + :show-inheritance: + +feast.protos.feast.core.FeatureTable\_pb2\_grpc module +------------------------------------------------------ + +.. automodule:: feast.protos.feast.core.FeatureTable_pb2_grpc + :members: + :undoc-members: + :show-inheritance: + +feast.protos.feast.core.FeatureViewProjection\_pb2 module +--------------------------------------------------------- + +.. automodule:: feast.protos.feast.core.FeatureViewProjection_pb2 + :members: + :undoc-members: + :show-inheritance: + +feast.protos.feast.core.FeatureViewProjection\_pb2\_grpc module +--------------------------------------------------------------- + +.. automodule:: feast.protos.feast.core.FeatureViewProjection_pb2_grpc + :members: + :undoc-members: + :show-inheritance: + +feast.protos.feast.core.FeatureView\_pb2 module +----------------------------------------------- + +.. automodule:: feast.protos.feast.core.FeatureView_pb2 + :members: + :undoc-members: + :show-inheritance: + +feast.protos.feast.core.FeatureView\_pb2\_grpc module +----------------------------------------------------- + +.. automodule:: feast.protos.feast.core.FeatureView_pb2_grpc + :members: + :undoc-members: + :show-inheritance: + +feast.protos.feast.core.Feature\_pb2 module +------------------------------------------- + +.. automodule:: feast.protos.feast.core.Feature_pb2 + :members: + :undoc-members: + :show-inheritance: + +feast.protos.feast.core.Feature\_pb2\_grpc module +------------------------------------------------- + +.. automodule:: feast.protos.feast.core.Feature_pb2_grpc + :members: + :undoc-members: + :show-inheritance: + +feast.protos.feast.core.InfraObject\_pb2 module +----------------------------------------------- + +.. automodule:: feast.protos.feast.core.InfraObject_pb2 + :members: + :undoc-members: + :show-inheritance: + +feast.protos.feast.core.InfraObject\_pb2\_grpc module +----------------------------------------------------- + +.. automodule:: feast.protos.feast.core.InfraObject_pb2_grpc + :members: + :undoc-members: + :show-inheritance: + +feast.protos.feast.core.OnDemandFeatureView\_pb2 module +------------------------------------------------------- + +.. automodule:: feast.protos.feast.core.OnDemandFeatureView_pb2 + :members: + :undoc-members: + :show-inheritance: + +feast.protos.feast.core.OnDemandFeatureView\_pb2\_grpc module +------------------------------------------------------------- + +.. automodule:: feast.protos.feast.core.OnDemandFeatureView_pb2_grpc + :members: + :undoc-members: + :show-inheritance: + +feast.protos.feast.core.Registry\_pb2 module +-------------------------------------------- + +.. automodule:: feast.protos.feast.core.Registry_pb2 + :members: + :undoc-members: + :show-inheritance: + +feast.protos.feast.core.Registry\_pb2\_grpc module +-------------------------------------------------- + +.. automodule:: feast.protos.feast.core.Registry_pb2_grpc + :members: + :undoc-members: + :show-inheritance: + +feast.protos.feast.core.RequestFeatureView\_pb2 module +------------------------------------------------------ + +.. automodule:: feast.protos.feast.core.RequestFeatureView_pb2 + :members: + :undoc-members: + :show-inheritance: + +feast.protos.feast.core.RequestFeatureView\_pb2\_grpc module +------------------------------------------------------------ + +.. automodule:: feast.protos.feast.core.RequestFeatureView_pb2_grpc + :members: + :undoc-members: + :show-inheritance: + +feast.protos.feast.core.SavedDataset\_pb2 module +------------------------------------------------ + +.. automodule:: feast.protos.feast.core.SavedDataset_pb2 + :members: + :undoc-members: + :show-inheritance: + +feast.protos.feast.core.SavedDataset\_pb2\_grpc module +------------------------------------------------------ + +.. automodule:: feast.protos.feast.core.SavedDataset_pb2_grpc + :members: + :undoc-members: + :show-inheritance: + +feast.protos.feast.core.SqliteTable\_pb2 module +----------------------------------------------- + +.. automodule:: feast.protos.feast.core.SqliteTable_pb2 + :members: + :undoc-members: + :show-inheritance: + +feast.protos.feast.core.SqliteTable\_pb2\_grpc module +----------------------------------------------------- + +.. automodule:: feast.protos.feast.core.SqliteTable_pb2_grpc + :members: + :undoc-members: + :show-inheritance: + +feast.protos.feast.core.Store\_pb2 module +----------------------------------------- + +.. automodule:: feast.protos.feast.core.Store_pb2 + :members: + :undoc-members: + :show-inheritance: + +feast.protos.feast.core.Store\_pb2\_grpc module +----------------------------------------------- + +.. automodule:: feast.protos.feast.core.Store_pb2_grpc + :members: + :undoc-members: + :show-inheritance: + +feast.protos.feast.core.StreamFeatureView\_pb2 module +----------------------------------------------------- + +.. automodule:: feast.protos.feast.core.StreamFeatureView_pb2 + :members: + :undoc-members: + :show-inheritance: + +feast.protos.feast.core.StreamFeatureView\_pb2\_grpc module +----------------------------------------------------------- + +.. automodule:: feast.protos.feast.core.StreamFeatureView_pb2_grpc + :members: + :undoc-members: + :show-inheritance: + +feast.protos.feast.core.ValidationProfile\_pb2 module +----------------------------------------------------- + +.. automodule:: feast.protos.feast.core.ValidationProfile_pb2 + :members: + :undoc-members: + :show-inheritance: + +feast.protos.feast.core.ValidationProfile\_pb2\_grpc module +----------------------------------------------------------- + +.. automodule:: feast.protos.feast.core.ValidationProfile_pb2_grpc + :members: + :undoc-members: + :show-inheritance: + +Module contents +--------------- + +.. automodule:: feast.protos.feast.core + :members: + :undoc-members: + :show-inheritance: diff --git a/sdk/python/docs/source/feast.protos.feast.rst b/sdk/python/docs/source/feast.protos.feast.rst new file mode 100644 index 00000000000..f519165db8e --- /dev/null +++ b/sdk/python/docs/source/feast.protos.feast.rst @@ -0,0 +1,21 @@ +feast.protos.feast package +========================== + +Subpackages +----------- + +.. toctree:: + :maxdepth: 4 + + feast.protos.feast.core + feast.protos.feast.serving + feast.protos.feast.storage + feast.protos.feast.types + +Module contents +--------------- + +.. automodule:: feast.protos.feast + :members: + :undoc-members: + :show-inheritance: diff --git a/sdk/python/docs/source/feast.protos.feast.serving.rst b/sdk/python/docs/source/feast.protos.feast.serving.rst new file mode 100644 index 00000000000..792335b189d --- /dev/null +++ b/sdk/python/docs/source/feast.protos.feast.serving.rst @@ -0,0 +1,61 @@ +feast.protos.feast.serving package +================================== + +Submodules +---------- + +feast.protos.feast.serving.Connector\_pb2 module +------------------------------------------------ + +.. automodule:: feast.protos.feast.serving.Connector_pb2 + :members: + :undoc-members: + :show-inheritance: + +feast.protos.feast.serving.Connector\_pb2\_grpc module +------------------------------------------------------ + +.. automodule:: feast.protos.feast.serving.Connector_pb2_grpc + :members: + :undoc-members: + :show-inheritance: + +feast.protos.feast.serving.ServingService\_pb2 module +----------------------------------------------------- + +.. automodule:: feast.protos.feast.serving.ServingService_pb2 + :members: + :undoc-members: + :show-inheritance: + +feast.protos.feast.serving.ServingService\_pb2\_grpc module +----------------------------------------------------------- + +.. automodule:: feast.protos.feast.serving.ServingService_pb2_grpc + :members: + :undoc-members: + :show-inheritance: + +feast.protos.feast.serving.TransformationService\_pb2 module +------------------------------------------------------------ + +.. automodule:: feast.protos.feast.serving.TransformationService_pb2 + :members: + :undoc-members: + :show-inheritance: + +feast.protos.feast.serving.TransformationService\_pb2\_grpc module +------------------------------------------------------------------ + +.. automodule:: feast.protos.feast.serving.TransformationService_pb2_grpc + :members: + :undoc-members: + :show-inheritance: + +Module contents +--------------- + +.. automodule:: feast.protos.feast.serving + :members: + :undoc-members: + :show-inheritance: diff --git a/sdk/python/docs/source/feast.protos.feast.storage.rst b/sdk/python/docs/source/feast.protos.feast.storage.rst new file mode 100644 index 00000000000..90bc1adc9b5 --- /dev/null +++ b/sdk/python/docs/source/feast.protos.feast.storage.rst @@ -0,0 +1,29 @@ +feast.protos.feast.storage package +================================== + +Submodules +---------- + +feast.protos.feast.storage.Redis\_pb2 module +-------------------------------------------- + +.. automodule:: feast.protos.feast.storage.Redis_pb2 + :members: + :undoc-members: + :show-inheritance: + +feast.protos.feast.storage.Redis\_pb2\_grpc module +-------------------------------------------------- + +.. automodule:: feast.protos.feast.storage.Redis_pb2_grpc + :members: + :undoc-members: + :show-inheritance: + +Module contents +--------------- + +.. automodule:: feast.protos.feast.storage + :members: + :undoc-members: + :show-inheritance: diff --git a/sdk/python/docs/source/feast.protos.feast.types.rst b/sdk/python/docs/source/feast.protos.feast.types.rst new file mode 100644 index 00000000000..aeb31bc9ad3 --- /dev/null +++ b/sdk/python/docs/source/feast.protos.feast.types.rst @@ -0,0 +1,61 @@ +feast.protos.feast.types package +================================ + +Submodules +---------- + +feast.protos.feast.types.EntityKey\_pb2 module +---------------------------------------------- + +.. automodule:: feast.protos.feast.types.EntityKey_pb2 + :members: + :undoc-members: + :show-inheritance: + +feast.protos.feast.types.EntityKey\_pb2\_grpc module +---------------------------------------------------- + +.. automodule:: feast.protos.feast.types.EntityKey_pb2_grpc + :members: + :undoc-members: + :show-inheritance: + +feast.protos.feast.types.Field\_pb2 module +------------------------------------------ + +.. automodule:: feast.protos.feast.types.Field_pb2 + :members: + :undoc-members: + :show-inheritance: + +feast.protos.feast.types.Field\_pb2\_grpc module +------------------------------------------------ + +.. automodule:: feast.protos.feast.types.Field_pb2_grpc + :members: + :undoc-members: + :show-inheritance: + +feast.protos.feast.types.Value\_pb2 module +------------------------------------------ + +.. automodule:: feast.protos.feast.types.Value_pb2 + :members: + :undoc-members: + :show-inheritance: + +feast.protos.feast.types.Value\_pb2\_grpc module +------------------------------------------------ + +.. automodule:: feast.protos.feast.types.Value_pb2_grpc + :members: + :undoc-members: + :show-inheritance: + +Module contents +--------------- + +.. automodule:: feast.protos.feast.types + :members: + :undoc-members: + :show-inheritance: diff --git a/sdk/python/docs/source/feast.protos.rst b/sdk/python/docs/source/feast.protos.rst new file mode 100644 index 00000000000..7bec91eb030 --- /dev/null +++ b/sdk/python/docs/source/feast.protos.rst @@ -0,0 +1,18 @@ +feast.protos package +==================== + +Subpackages +----------- + +.. toctree:: + :maxdepth: 4 + + feast.protos.feast + +Module contents +--------------- + +.. automodule:: feast.protos + :members: + :undoc-members: + :show-inheritance: diff --git a/sdk/python/docs/source/feast.rst b/sdk/python/docs/source/feast.rst new file mode 100644 index 00000000000..b0ed92c4cce --- /dev/null +++ b/sdk/python/docs/source/feast.rst @@ -0,0 +1,378 @@ +feast package +============= + +Subpackages +----------- + +.. toctree:: + :maxdepth: 4 + + feast.diff + feast.dqm + feast.infra + feast.loaders + feast.protos + feast.ui + +Submodules +---------- + +feast.aggregation module +------------------------ + +.. automodule:: feast.aggregation + :members: + :undoc-members: + :show-inheritance: + +feast.base\_feature\_view module +-------------------------------- + +.. automodule:: feast.base_feature_view + :members: + :undoc-members: + :show-inheritance: + +feast.batch\_feature\_view module +--------------------------------- + +.. automodule:: feast.batch_feature_view + :members: + :undoc-members: + :show-inheritance: + +feast.cli module +---------------- + +.. automodule:: feast.cli + :members: + :undoc-members: + :show-inheritance: + +feast.constants module +---------------------- + +.. automodule:: feast.constants + :members: + :undoc-members: + :show-inheritance: + +feast.data\_format module +------------------------- + +.. automodule:: feast.data_format + :members: + :undoc-members: + :show-inheritance: + +feast.data\_source module +------------------------- + +.. automodule:: feast.data_source + :members: + :undoc-members: + :show-inheritance: + +feast.driver\_test\_data module +------------------------------- + +.. automodule:: feast.driver_test_data + :members: + :undoc-members: + :show-inheritance: + +feast.entity module +------------------- + +.. automodule:: feast.entity + :members: + :undoc-members: + :show-inheritance: + +feast.errors module +------------------- + +.. automodule:: feast.errors + :members: + :undoc-members: + :show-inheritance: + +feast.feast\_object module +-------------------------- + +.. automodule:: feast.feast_object + :members: + :undoc-members: + :show-inheritance: + +feast.feature module +-------------------- + +.. automodule:: feast.feature + :members: + :undoc-members: + :show-inheritance: + +feast.feature\_logging module +----------------------------- + +.. automodule:: feast.feature_logging + :members: + :undoc-members: + :show-inheritance: + +feast.feature\_server module +---------------------------- + +.. automodule:: feast.feature_server + :members: + :undoc-members: + :show-inheritance: + +feast.feature\_service module +----------------------------- + +.. automodule:: feast.feature_service + :members: + :undoc-members: + :show-inheritance: + +feast.feature\_store module +--------------------------- + +.. automodule:: feast.feature_store + :members: + :undoc-members: + :show-inheritance: + +feast.feature\_view module +-------------------------- + +.. automodule:: feast.feature_view + :members: + :undoc-members: + :show-inheritance: + +feast.feature\_view\_projection module +-------------------------------------- + +.. automodule:: feast.feature_view_projection + :members: + :undoc-members: + :show-inheritance: + +feast.field module +------------------ + +.. automodule:: feast.field + :members: + :undoc-members: + :show-inheritance: + +feast.file\_utils module +------------------------ + +.. automodule:: feast.file_utils + :members: + :undoc-members: + :show-inheritance: + +feast.flags\_helper module +-------------------------- + +.. automodule:: feast.flags_helper + :members: + :undoc-members: + :show-inheritance: + +feast.importer module +--------------------- + +.. automodule:: feast.importer + :members: + :undoc-members: + :show-inheritance: + +feast.inference module +---------------------- + +.. automodule:: feast.inference + :members: + :undoc-members: + :show-inheritance: + +feast.names module +------------------ + +.. automodule:: feast.names + :members: + :undoc-members: + :show-inheritance: + +feast.on\_demand\_feature\_view module +-------------------------------------- + +.. automodule:: feast.on_demand_feature_view + :members: + :undoc-members: + :show-inheritance: + +feast.online\_response module +----------------------------- + +.. automodule:: feast.online_response + :members: + :undoc-members: + :show-inheritance: + +feast.project\_metadata module +------------------------------ + +.. automodule:: feast.project_metadata + :members: + :undoc-members: + :show-inheritance: + +feast.proto\_json module +------------------------ + +.. automodule:: feast.proto_json + :members: + :undoc-members: + :show-inheritance: + +feast.repo\_config module +------------------------- + +.. automodule:: feast.repo_config + :members: + :undoc-members: + :show-inheritance: + +feast.repo\_contents module +--------------------------- + +.. automodule:: feast.repo_contents + :members: + :undoc-members: + :show-inheritance: + +feast.repo\_operations module +----------------------------- + +.. automodule:: feast.repo_operations + :members: + :undoc-members: + :show-inheritance: + +feast.repo\_upgrade module +-------------------------- + +.. automodule:: feast.repo_upgrade + :members: + :undoc-members: + :show-inheritance: + +feast.request\_feature\_view module +----------------------------------- + +.. automodule:: feast.request_feature_view + :members: + :undoc-members: + :show-inheritance: + +feast.saved\_dataset module +--------------------------- + +.. automodule:: feast.saved_dataset + :members: + :undoc-members: + :show-inheritance: + +feast.stream\_feature\_view module +---------------------------------- + +.. automodule:: feast.stream_feature_view + :members: + :undoc-members: + :show-inheritance: + +feast.transformation\_server module +----------------------------------- + +.. automodule:: feast.transformation_server + :members: + :undoc-members: + :show-inheritance: + +feast.type\_map module +---------------------- + +.. automodule:: feast.type_map + :members: + :undoc-members: + :show-inheritance: + +feast.types module +------------------ + +.. automodule:: feast.types + :members: + :undoc-members: + :show-inheritance: + +feast.ui\_server module +----------------------- + +.. automodule:: feast.ui_server + :members: + :undoc-members: + :show-inheritance: + +feast.usage module +------------------ + +.. automodule:: feast.usage + :members: + :undoc-members: + :show-inheritance: + +feast.utils module +------------------ + +.. automodule:: feast.utils + :members: + :undoc-members: + :show-inheritance: + +feast.value\_type module +------------------------ + +.. automodule:: feast.value_type + :members: + :undoc-members: + :show-inheritance: + +feast.version module +-------------------- + +.. automodule:: feast.version + :members: + :undoc-members: + :show-inheritance: + +feast.wait module +----------------- + +.. automodule:: feast.wait + :members: + :undoc-members: + :show-inheritance: + +Module contents +--------------- + +.. automodule:: feast + :members: + :undoc-members: + :show-inheritance: diff --git a/sdk/python/docs/source/feast.ui.rst b/sdk/python/docs/source/feast.ui.rst new file mode 100644 index 00000000000..01b16cb0a6a --- /dev/null +++ b/sdk/python/docs/source/feast.ui.rst @@ -0,0 +1,10 @@ +feast.ui package +================ + +Module contents +--------------- + +.. automodule:: feast.ui + :members: + :undoc-members: + :show-inheritance: diff --git a/sdk/python/docs/source/index.rst b/sdk/python/docs/source/index.rst new file mode 100644 index 00000000000..ca96782db8a --- /dev/null +++ b/sdk/python/docs/source/index.rst @@ -0,0 +1,450 @@ +Feast Python API Documentation +============================== + +.. We prefer 'autoclass' instead of 'autoclass' as 'autoclass' can specify a class, whereas + 'autoclass' will pull in all public classes and methods from that module, which we typically + do not want. + +Feature Store +================== + +.. autoclass:: feast.feature_store.FeatureStore + :members: + +Config +================== + +.. autoclass:: feast.repo_config.RepoConfig + :members: + +.. autoclass:: feast.repo_config.RegistryConfig + :members: + +Data Source +================== + +.. autoclass:: feast.data_source.DataSource + :members: + +File Source +------------------ + +.. autoclass:: feast.infra.offline_stores.file_source.FileSource + :members: + +Snowflake Source +------------------ + +.. autoclass:: feast.infra.offline_stores.snowflake_source.SnowflakeSource + :members: + +BigQuery Source +------------------ + +.. autoclass:: feast.infra.offline_stores.bigquery_source.BigQuerySource + :members: + +Redshift Source +------------------ + +.. autoclass:: feast.infra.offline_stores.redshift_source.RedshiftSource + :members: + +Spark Source +------------------ + +.. autoclass:: feast.infra.offline_stores.contrib.spark_offline_store.spark_source.SparkSource + :members: + +Trino Source +------------------ + +.. autoclass:: feast.infra.offline_stores.contrib.trino_offline_store.trino_source.TrinoSource + :members: + +PostgreSQL Source +------------------ + +.. autoclass:: feast.infra.offline_stores.contrib.postgres_offline_store.postgres_source.PostgreSQLSource + :members: + +Request Source +------------------ + +.. autoclass:: feast.data_source.RequestSource + :members: + +Push Source +------------------ + +.. autoclass:: feast.data_source.PushSource + :members: + +Kafka Source +------------------ + +.. autoclass:: feast.data_source.KafkaSource + :members: + +Kinesis Source +------------------ + +.. autoclass:: feast.data_source.KinesisSource + :members: + +Entity +================== + +.. autoclass:: feast.entity.Entity + :members: + +Feature View +================== + +.. autoclass:: feast.base_feature_view.BaseFeatureView + :members: + +Feature View +---------------------- + +.. autoclass:: feast.feature_view.FeatureView + :members: + +On Demand Feature View +---------------------- + +.. autoclass:: feast.on_demand_feature_view.OnDemandFeatureView + :members: + +Batch Feature View +---------------------- + +.. autoclass:: feast.batch_feature_view.BatchFeatureView + :members: + +Stream Feature View +---------------------- + +.. autoclass:: feast.stream_feature_view.StreamFeatureView + :members: + +Field +================== + +.. autoclass:: feast.field.Field + :members: + +Feature Service +================== + +.. autoclass:: feast.feature_service.FeatureService + :members: + +Registry +================== + +.. autoclass:: feast.infra.registry.base_registry.BaseRegistry + :members: + +Registry +---------------------- + +.. autoclass:: feast.infra.registry.registry.Registry + :members: + +SQL Registry +---------------------- + +.. autoclass:: feast.infra.registry.sql.SqlRegistry + :members: + +Registry Store +================== + +.. autoclass:: feast.infra.registry.registry_store.RegistryStore + :members: + +File Registry Store +----------------------- + +.. autoclass:: feast.infra.registry.file.FileRegistryStore + :members: + +GCS Registry Store +----------------------- + +.. autoclass:: feast.infra.registry.gcs.GCSRegistryStore + :members: + +S3 Registry Store +----------------------- + +.. autoclass:: feast.infra.registry.s3.S3RegistryStore + :members: + +PostgreSQL Registry Store +----------------------- + +.. autoclass:: feast.infra.registry.contrib.postgres.postgres_registry_store.PostgreSQLRegistryStore + :members: + +Provider +================== + +.. autoclass:: feast.infra.provider.Provider + :members: + +Passthrough Provider +-------------------- + +.. autoclass:: feast.infra.passthrough_provider.PassthroughProvider + :members: + +Local Provider +------------------ + +.. autoclass:: feast.infra.local.LocalProvider + :members: + +GCP Provider +------------------ + +.. autoclass:: feast.infra.gcp.GcpProvider + :members: + +AWS Provider +------------------ + +.. autoclass:: feast.infra.aws.AwsProvider + :members: + +Offline Store +================== + +.. autoclass:: feast.infra.offline_stores.offline_store.OfflineStore + :members: + +.. autoclass:: feast.infra.offline_stores.offline_store.RetrievalJob + :members: + +File Offline Store +------------------ + +.. autoclass:: feast.infra.offline_stores.file.FileOfflineStore + :members: + +.. autoclass:: feast.infra.offline_stores.file.FileOfflineStoreConfig + :members: + +.. autoclass:: feast.infra.offline_stores.file.FileRetrievalJob + :members: + +Snowflake Offline Store +----------------------- + +.. autoclass:: feast.infra.offline_stores.snowflake.SnowflakeOfflineStore + :members: + +.. autoclass:: feast.infra.offline_stores.snowflake.SnowflakeOfflineStoreConfig + :members: + +.. autoclass:: feast.infra.offline_stores.snowflake.SnowflakeRetrievalJob + :members: + +BigQuery Offline Store +---------------------- + +.. autoclass:: feast.infra.offline_stores.bigquery.BigQueryOfflineStore + :members: + +.. autoclass:: feast.infra.offline_stores.bigquery.BigQueryOfflineStoreConfig + :members: + +.. autoclass:: feast.infra.offline_stores.bigquery.BigQueryRetrievalJob + :members: + +Redshift Offline Store +---------------------- + +.. autoclass:: feast.infra.offline_stores.redshift.RedshiftOfflineStore + :members: + +.. autoclass:: feast.infra.offline_stores.redshift.RedshiftOfflineStoreConfig + :members: + +.. autoclass:: feast.infra.offline_stores.redshift.RedshiftRetrievalJob + :members: + +Spark Offline Store +------------------- + +.. autoclass:: feast.infra.offline_stores.contrib.spark_offline_store.spark.SparkOfflineStore + :members: + +.. autoclass:: feast.infra.offline_stores.contrib.spark_offline_store.spark.SparkOfflineStoreConfig + :members: + +.. autoclass:: feast.infra.offline_stores.contrib.spark_offline_store.spark.SparkRetrievalJob + :members: + +Trino Offline Store +------------------- + +.. autoclass:: feast.infra.offline_stores.contrib.trino_offline_store.trino.TrinoOfflineStore + :members: + +.. autoclass:: feast.infra.offline_stores.contrib.trino_offline_store.trino.TrinoOfflineStoreConfig + :members: + +.. autoclass:: feast.infra.offline_stores.contrib.trino_offline_store.trino.TrinoRetrievalJob + :members: + +PostgreSQL Offline Store +------------------------ + +.. autoclass:: feast.infra.offline_stores.contrib.postgres_offline_store.postgres.PostgreSQLOfflineStore + :members: + +.. autoclass:: feast.infra.offline_stores.contrib.postgres_offline_store.postgres.PostgreSQLOfflineStoreConfig + :members: + +.. autoclass:: feast.infra.offline_stores.contrib.postgres_offline_store.postgres.PostgreSQLRetrievalJob + :members: + +Online Store +================== + +.. autoclass:: feast.infra.online_stores.online_store.OnlineStore + :members: + +Sqlite Online Store +------------------- + +.. autoclass:: feast.infra.online_stores.sqlite.SqliteOnlineStore + :members: + +.. autoclass:: feast.infra.online_stores.sqlite.SqliteOnlineStoreConfig + :members: + +Datastore Online Store +---------------------- + +.. autoclass:: feast.infra.online_stores.datastore.DatastoreOnlineStore + :members: + +.. autoclass:: feast.infra.online_stores.datastore.DatastoreOnlineStoreConfig + :members: + +DynamoDB Online Store +--------------------- + +.. autoclass:: feast.infra.online_stores.dynamodb.DynamoDBOnlineStore + :members: + +.. autoclass:: feast.infra.online_stores.dynamodb.DynamoDBOnlineStoreConfig + :members: + +Redis Online Store +------------------ + +.. autoclass:: feast.infra.online_stores.redis.RedisOnlineStore + :members: + +.. autoclass:: feast.infra.online_stores.redis.RedisOnlineStoreConfig + :members: + +Snowflake Online Store +------------------ + +.. autoclass:: feast.infra.online_stores.snowflake.SnowflakeOnlineStore + :members: + +.. autoclass:: feast.infra.online_stores.snowflake.SnowflakeOnlineStoreConfig + :members: + +PostgreSQL Online Store +----------------------- + +.. autoclass:: feast.infra.online_stores.contrib.postgres.PostgreSQLOnlineStore + :members: + +.. autoclass:: feast.infra.online_stores.contrib.postgres.PostgreSQLOnlineStoreConfig + :members: + +HBase Online Store +----------------------- + +.. autoclass:: feast.infra.online_stores.contrib.hbase_online_store.hbase.HbaseOnlineStore + :members: + +.. autoclass:: feast.infra.online_stores.contrib.hbase_online_store.hbase.HbaseOnlineStoreConfig + :members: + +Cassandra Online Store +----------------------- + +.. autoclass:: feast.infra.online_stores.contrib.cassandra_online_store.cassandra_online_store.CassandraOnlineStore + :members: + +.. autoclass:: feast.infra.online_stores.contrib.cassandra_online_store.cassandra_online_store.CassandraOnlineStoreConfig + :members: + +Batch Materialization Engine +============================ + +.. autoclass:: feast.infra.materialization.batch_materialization_engine.BatchMaterializationEngine + :members: + +.. autoclass:: feast.infra.materialization.batch_materialization_engine.MaterializationJob + :members: + +.. autoclass:: feast.infra.materialization.batch_materialization_engine.MaterializationTask + :members: + +Local Engine +------------ + +.. autoclass:: feast.infra.materialization.local_engine.LocalMaterializationEngine + :members: + +.. autoclass:: feast.infra.materialization.local_engine.LocalMaterializationEngineConfig + :members: + +.. autoclass:: feast.infra.materialization.local_engine.LocalMaterializationJob + :members: + +Bytewax Engine +--------------------------- + +.. autoclass:: feast.infra.materialization.contrib.bytewax.bytewax_materialization_engine.BytewaxMaterializationEngine + :members: + +.. autoclass:: feast.infra.materialization.contrib.bytewax.bytewax_materialization_engine.BytewaxMaterializationEngineConfig + :members: + +.. autoclass:: feast.infra.materialization.contrib.bytewax.bytewax_materialization_job.BytewaxMaterializationJob + :members: + +Snowflake Engine +--------------------------- + +.. autoclass:: feast.infra.materialization.snowflake_engine.SnowflakeMaterializationEngine + :members: + +.. autoclass:: feast.infra.materialization.snowflake_engine.SnowflakeMaterializationEngineConfig + :members: + +.. autoclass:: feast.infra.materialization.snowflake_engine.SnowflakeMaterializationJob + :members: + +(Alpha) AWS Lambda Engine +--------------------------- + +.. autoclass:: feast.infra.materialization.aws_lambda.lambda_engine.LambdaMaterializationEngine + :members: + +.. autoclass:: feast.infra.materialization.aws_lambda.lambda_engine.LambdaMaterializationEngineConfig + :members: + +.. autoclass:: feast.infra.materialization.aws_lambda.lambda_engine.LambdaMaterializationJob + :members: diff --git a/sdk/python/docs/source/modules.rst b/sdk/python/docs/source/modules.rst new file mode 100644 index 00000000000..3a6f8333abd --- /dev/null +++ b/sdk/python/docs/source/modules.rst @@ -0,0 +1,7 @@ +feast +===== + +.. toctree:: + :maxdepth: 4 + + feast diff --git a/sdk/python/feast/entity.py b/sdk/python/feast/entity.py index 55de68a4041..57d811d9f0f 100644 --- a/sdk/python/feast/entity.py +++ b/sdk/python/feast/entity.py @@ -62,6 +62,7 @@ def __init__( *, name: str, join_keys: Optional[List[str]] = None, + value_type: Optional[ValueType] = None, description: str = "", tags: Optional[Dict[str, str]] = None, owner: str = "", @@ -74,6 +75,8 @@ def __init__( join_keys (optional): A list of properties that uniquely identifies different entities within the collection. This currently only supports a list of size one, but is intended to eventually support multiple join keys. + value_type (optional): The type of the entity, such as string or float. If not specified, + it will be inferred from the schema of the underlying data source. description (optional): A human-readable description. tags (optional): A dictionary of key-value pairs to store arbitrary metadata. owner (optional): The owner of the entity, typically the email of the primary maintainer. @@ -82,7 +85,7 @@ def __init__( ValueError: Parameters are specified incorrectly. """ self.name = name - self.value_type = ValueType.UNKNOWN + self.value_type = value_type or ValueType.UNKNOWN # For now, both the `join_key` and `join_keys` attributes are set correctly, # so both are usable. diff --git a/sdk/python/feast/feature_store.py b/sdk/python/feast/feature_store.py index 23600e7c64f..9350220c21c 100644 --- a/sdk/python/feast/feature_store.py +++ b/sdk/python/feast/feature_store.py @@ -1478,13 +1478,8 @@ def write_to_online_store( feature_view = self.get_feature_view( feature_view_name, allow_registry_cache=allow_registry_cache ) - entities = [] - for entity_name in feature_view.entities: - entities.append( - self.get_entity(entity_name, allow_registry_cache=allow_registry_cache) - ) provider = self._get_provider() - provider.ingest_df(feature_view, entities, df) + provider.ingest_df(feature_view, df) @log_exceptions_and_usage def write_to_offline_store( diff --git a/sdk/python/feast/feature_view.py b/sdk/python/feast/feature_view.py index 41bad5828a0..4c4e6391b35 100644 --- a/sdk/python/feast/feature_view.py +++ b/sdk/python/feast/feature_view.py @@ -35,7 +35,9 @@ from feast.protos.feast.core.FeatureView_pb2 import ( MaterializationInterval as MaterializationIntervalProto, ) +from feast.types import from_value_type from feast.usage import log_exceptions +from feast.value_type import ValueType warnings.simplefilter("once", DeprecationWarning) @@ -115,6 +117,7 @@ def __init__( If a stream source, the source should contain a batch_source for backfills & batch materialization. schema (optional): The schema of the feature view, including feature, timestamp, and entity columns. + # TODO: clarify that schema is only useful here... entities (optional): The list of entities with which this group of features is associated. ttl (optional): The amount of time this group of features lives. A ttl of 0 indicates that this group of features lives forever. Note that large ttl's or a ttl of 0 @@ -160,9 +163,29 @@ def __init__( for entity in entities: join_keys += entity.join_keys + # Ensure that entities have unique join keys. + if len(set(join_keys)) < len(join_keys): + raise ValueError( + "A feature view should not have entities that share a join key." + ) + for field in self.schema: if field.name in join_keys: self.entity_columns.append(field) + + # Confirm that the inferred type matches the specified entity type, if it exists. + matching_entities = ( + [e for e in entities if e.join_key == field.name] + if entities + else [] + ) + assert len(matching_entities) == 1 + entity = matching_entities[0] + if entity.value_type != ValueType.UNKNOWN: + if from_value_type(entity.value_type) != field.dtype: + raise ValueError( + f"Entity {entity.name} has type {entity.value_type}, which does not match the inferred type {field.dtype}." + ) else: features.append(field) @@ -386,6 +409,12 @@ def from_proto(cls, feature_view_proto: FeatureViewProto): for field_proto in feature_view_proto.spec.entity_columns ] + if len(feature_view.entities) != len(feature_view.entity_columns): + warnings.warn( + f"There are some mismatches in your feature view's registered entities. Please check if you have applied your entities correctly." + f"Entities: {feature_view.entities} vs Entity Columns: {feature_view.entity_columns}" + ) + # FeatureViewProjections are not saved in the FeatureView proto. # Create the default projection. feature_view.projection = FeatureViewProjection.from_definition(feature_view) diff --git a/sdk/python/feast/inference.py b/sdk/python/feast/inference.py index 84e8321c122..223eacb66af 100644 --- a/sdk/python/feast/inference.py +++ b/sdk/python/feast/inference.py @@ -139,8 +139,7 @@ def update_feature_views_with_inferred_features_and_entities( if field.name not in [feature.name for feature in fv.features]: fv.features.append(field) - # Since the `value_type` parameter has not yet been fully deprecated for - # entities, we respect the `value_type` attribute if it still exists. + # Respect the `value_type` attribute of the entity, if it is specified. for entity_name in fv.entities: entity = entity_name_to_entity_map[entity_name] if ( diff --git a/sdk/python/feast/infra/offline_stores/contrib/spark_offline_store/spark_source.py b/sdk/python/feast/infra/offline_stores/contrib/spark_offline_store/spark_source.py index 5b9f5621813..a27065fb5ed 100644 --- a/sdk/python/feast/infra/offline_stores/contrib/spark_offline_store/spark_source.py +++ b/sdk/python/feast/infra/offline_stores/contrib/spark_offline_store/spark_source.py @@ -159,10 +159,7 @@ def get_table_column_names_and_types( store_config=config.offline_store ) df = spark_session.sql(f"SELECT * FROM {self.get_table_query_string()}") - return ( - (fields["name"], fields["type"]) - for fields in df.schema.jsonValue()["fields"] - ) + return ((field.name, field.dataType.simpleString()) for field in df.schema) def get_table_query_string(self) -> str: """Returns a string that can directly be used to reference this table in SQL""" diff --git a/sdk/python/feast/infra/passthrough_provider.py b/sdk/python/feast/infra/passthrough_provider.py index bb5cd38a835..28b10c12595 100644 --- a/sdk/python/feast/infra/passthrough_provider.py +++ b/sdk/python/feast/infra/passthrough_provider.py @@ -193,7 +193,6 @@ def online_read( def ingest_df( self, feature_view: FeatureView, - entities: List[Entity], df: pd.DataFrame, ): set_usage_attribute("provider", self.__class__.__name__) @@ -204,7 +203,10 @@ def ingest_df( table, feature_view.batch_source.field_mapping ) - join_keys = {entity.join_key: entity.value_type for entity in entities} + join_keys = { + entity.name: entity.dtype.to_value_type() + for entity in feature_view.entity_columns + } rows_to_write = _convert_arrow_to_proto(table, feature_view, join_keys) self.online_write_batch( diff --git a/sdk/python/feast/infra/provider.py b/sdk/python/feast/infra/provider.py index 7d3c37e4c2e..82879b264af 100644 --- a/sdk/python/feast/infra/provider.py +++ b/sdk/python/feast/infra/provider.py @@ -123,7 +123,6 @@ def online_write_batch( def ingest_df( self, feature_view: FeatureView, - entities: List[Entity], df: pd.DataFrame, ): """ @@ -131,7 +130,6 @@ def ingest_df( Args: feature_view: The feature view to which the dataframe corresponds. - entities: The entities that are referenced by the dataframe. df: The dataframe to be persisted. """ pass diff --git a/sdk/python/feast/infra/utils/snowflake/snowpark/snowflake_udfs.py b/sdk/python/feast/infra/utils/snowflake/snowpark/snowflake_udfs.py index 7fde4dd3a1c..02311ca55d6 100644 --- a/sdk/python/feast/infra/utils/snowflake/snowpark/snowflake_udfs.py +++ b/sdk/python/feast/infra/utils/snowflake/snowpark/snowflake_udfs.py @@ -1,3 +1,4 @@ +import sys from binascii import unhexlify import pandas @@ -24,6 +25,8 @@ # ValueType.BYTES = 1 @vectorized(input=pandas.DataFrame) def feast_snowflake_binary_to_bytes_proto(df): + sys._xoptions["snowflake_partner_attribution"].append("feast") + df = list( map( ValueProto.SerializeToString, @@ -45,6 +48,8 @@ def feast_snowflake_binary_to_bytes_proto(df): # ValueType.STRING = 2 @vectorized(input=pandas.DataFrame) def feast_snowflake_varchar_to_string_proto(df): + sys._xoptions["snowflake_partner_attribution"].append("feast") + df = list( map( ValueProto.SerializeToString, @@ -66,6 +71,8 @@ def feast_snowflake_varchar_to_string_proto(df): # ValueType.INT32 = 3 @vectorized(input=pandas.DataFrame) def feast_snowflake_number_to_int32_proto(df): + sys._xoptions["snowflake_partner_attribution"].append("feast") + df = list( map( ValueProto.SerializeToString, @@ -87,6 +94,8 @@ def feast_snowflake_number_to_int32_proto(df): # ValueType.INT64 = 4 @vectorized(input=pandas.DataFrame) def feast_snowflake_number_to_int64_proto(df): + sys._xoptions["snowflake_partner_attribution"].append("feast") + df = list( map( ValueProto.SerializeToString, @@ -110,6 +119,8 @@ def feast_snowflake_number_to_int64_proto(df): # ValueType.FLOAT = 5 & ValueType.DOUBLE = 6 @vectorized(input=pandas.DataFrame) def feast_snowflake_float_to_double_proto(df): + sys._xoptions["snowflake_partner_attribution"].append("feast") + df = list( map( ValueProto.SerializeToString, @@ -131,6 +142,8 @@ def feast_snowflake_float_to_double_proto(df): # ValueType.BOOL = 7 @vectorized(input=pandas.DataFrame) def feast_snowflake_boolean_to_bool_boolean_proto(df): + sys._xoptions["snowflake_partner_attribution"].append("feast") + df = list( map( ValueProto.SerializeToString, @@ -152,6 +165,7 @@ def feast_snowflake_boolean_to_bool_boolean_proto(df): # ValueType.UNIX_TIMESTAMP = 8 @vectorized(input=pandas.DataFrame) def feast_snowflake_timestamp_to_unix_timestamp_proto(df): + sys._xoptions["snowflake_partner_attribution"].append("feast") df = list( map( @@ -177,6 +191,8 @@ def feast_snowflake_timestamp_to_unix_timestamp_proto(df): # converts 1 to n many entity keys to a single binary for lookups @vectorized(input=pandas.DataFrame) def feast_serialize_entity_keys(df): + sys._xoptions["snowflake_partner_attribution"].append("feast") + join_keys = create_entity_dict(df[0].values[0], df[2].values[0]) df = pandas.DataFrame.from_dict( @@ -222,6 +238,8 @@ def feast_serialize_entity_keys(df): # converts 1 to n many entity keys to a single binary for lookups @vectorized(input=pandas.DataFrame) def feast_entity_key_proto_to_string(df): + sys._xoptions["snowflake_partner_attribution"].append("feast") + join_keys = create_entity_dict(df[0].values[0], df[2].values[0]) df = pandas.DataFrame.from_dict( diff --git a/sdk/python/feast/templates/aws/feature_repo/example_repo.py b/sdk/python/feast/templates/aws/feature_repo/example_repo.py index eaa1a1bfd4c..dd5a9c925b7 100644 --- a/sdk/python/feast/templates/aws/feature_repo/example_repo.py +++ b/sdk/python/feast/templates/aws/feature_repo/example_repo.py @@ -59,12 +59,6 @@ tags={"team": "driver_performance"}, ) -# Defines a way to push data (to be available offline, online or both) into Feast. -driver_stats_push_source = PushSource( - name="driver_stats_push_source", - batch_source=driver_stats_source, -) - # Define a request data source which encodes features / information only # available at request time (e.g. part of the user initiated HTTP request) input_request = RequestSource( @@ -103,3 +97,48 @@ def transformed_conv_rate(inputs: pd.DataFrame) -> pd.DataFrame: driver_activity_v2 = FeatureService( name="driver_activity_v2", features=[driver_stats_fv, transformed_conv_rate] ) + +# Defines a way to push data (to be available offline, online or both) into Feast. +driver_stats_push_source = PushSource( + name="driver_stats_push_source", + batch_source=driver_stats_source, +) + +# Defines a slightly modified version of the feature view from above, where the source +# has been changed to the push source. This allows fresh features to be directly pushed +# to the online store for this feature view. +driver_stats_fresh_fv = FeatureView( + name="driver_hourly_stats_fresh", + entities=[driver], + ttl=timedelta(days=1), + schema=[ + Field(name="conv_rate", dtype=Float32), + Field(name="acc_rate", dtype=Float32), + Field(name="avg_daily_trips", dtype=Int64), + ], + online=True, + source=driver_stats_push_source, # Changed from above + tags={"team": "driver_performance"}, +) + + +# Define an on demand feature view which can generate new features based on +# existing feature views and RequestSource features +@on_demand_feature_view( + sources=[driver_stats_fresh_fv, input_request], # relies on fresh version of FV + schema=[ + Field(name="conv_rate_plus_val1", dtype=Float64), + Field(name="conv_rate_plus_val2", dtype=Float64), + ], +) +def transformed_conv_rate_fresh(inputs: pd.DataFrame) -> pd.DataFrame: + df = pd.DataFrame() + df["conv_rate_plus_val1"] = inputs["conv_rate"] + inputs["val_to_add"] + df["conv_rate_plus_val2"] = inputs["conv_rate"] + inputs["val_to_add_2"] + return df + + +driver_activity_v3 = FeatureService( + name="driver_activity_v3", + features=[driver_stats_fresh_fv, transformed_conv_rate_fresh], +) diff --git a/sdk/python/feast/templates/aws/feature_repo/test_workflow.py b/sdk/python/feast/templates/aws/feature_repo/test_workflow.py index 0d5b2714d9b..494b1914f92 100644 --- a/sdk/python/feast/templates/aws/feature_repo/test_workflow.py +++ b/sdk/python/feast/templates/aws/feature_repo/test_workflow.py @@ -22,20 +22,25 @@ def run_demo(): store.materialize_incremental(end_date=datetime.now()) print("\n--- Online features ---") - fetch_online_features(store, use_feature_service=False) + fetch_online_features(store) print("\n--- Online features retrieved (instead) through a feature service---") - fetch_online_features(store, use_feature_service=True) + fetch_online_features(store, source="feature_service") + + print( + "\n--- Online features retrieved (using feature service v3, which uses a feature view with a push source---" + ) + fetch_online_features(store, source="push") print("\n--- Simulate a stream event ingestion of the hourly stats df ---") event_df = pd.DataFrame.from_dict( { "driver_id": [1001], "event_timestamp": [ - datetime(2021, 5, 13, 10, 59, 42), + datetime.now(), ], "created": [ - datetime(2021, 5, 13, 10, 59, 42), + datetime.now(), ], "conv_rate": [1.0], "acc_rate": [1.0], @@ -46,7 +51,7 @@ def run_demo(): store.push("driver_stats_push_source", event_df, to=PushMode.ONLINE_AND_OFFLINE) print("\n--- Online features again with updated values from a stream push---") - fetch_online_features(store, use_feature_service=True) + fetch_online_features(store, source="push") print("\n--- Run feast teardown ---") subprocess.run(["feast", "teardown"]) @@ -89,7 +94,7 @@ def fetch_historical_features_entity_df(store: FeatureStore, for_batch_scoring: print(training_df.head()) -def fetch_online_features(store, use_feature_service: bool): +def fetch_online_features(store, source: str = ""): entity_rows = [ # {join_key: entity_value} { @@ -103,12 +108,13 @@ def fetch_online_features(store, use_feature_service: bool): "val_to_add_2": 2002, }, ] - if use_feature_service: + if source == "feature_service": features_to_fetch = store.get_feature_service("driver_activity_v1") + elif source == "push": + features_to_fetch = store.get_feature_service("driver_activity_v3") else: features_to_fetch = [ "driver_hourly_stats:acc_rate", - "driver_hourly_stats:avg_daily_trips", "transformed_conv_rate:conv_rate_plus_val1", "transformed_conv_rate:conv_rate_plus_val2", ] diff --git a/sdk/python/feast/templates/cassandra/feature_repo/example_repo.py b/sdk/python/feast/templates/cassandra/feature_repo/example_repo.py index b3c71154825..131f1bcaa61 100644 --- a/sdk/python/feast/templates/cassandra/feature_repo/example_repo.py +++ b/sdk/python/feast/templates/cassandra/feature_repo/example_repo.py @@ -54,12 +54,6 @@ tags={"team": "driver_performance"}, ) -# Defines a way to push data (to be available offline, online or both) into Feast. -driver_stats_push_source = PushSource( - name="driver_stats_push_source", - batch_source=driver_stats_source, -) - # Define a request data source which encodes features / information only # available at request time (e.g. part of the user initiated HTTP request) input_request = RequestSource( @@ -98,3 +92,48 @@ def transformed_conv_rate(inputs: pd.DataFrame) -> pd.DataFrame: driver_activity_v2 = FeatureService( name="driver_activity_v2", features=[driver_stats_fv, transformed_conv_rate] ) + +# Defines a way to push data (to be available offline, online or both) into Feast. +driver_stats_push_source = PushSource( + name="driver_stats_push_source", + batch_source=driver_stats_source, +) + +# Defines a slightly modified version of the feature view from above, where the source +# has been changed to the push source. This allows fresh features to be directly pushed +# to the online store for this feature view. +driver_stats_fresh_fv = FeatureView( + name="driver_hourly_stats_fresh", + entities=[driver], + ttl=timedelta(days=1), + schema=[ + Field(name="conv_rate", dtype=Float32), + Field(name="acc_rate", dtype=Float32), + Field(name="avg_daily_trips", dtype=Int64), + ], + online=True, + source=driver_stats_push_source, # Changed from above + tags={"team": "driver_performance"}, +) + + +# Define an on demand feature view which can generate new features based on +# existing feature views and RequestSource features +@on_demand_feature_view( + sources=[driver_stats_fresh_fv, input_request], # relies on fresh version of FV + schema=[ + Field(name="conv_rate_plus_val1", dtype=Float64), + Field(name="conv_rate_plus_val2", dtype=Float64), + ], +) +def transformed_conv_rate_fresh(inputs: pd.DataFrame) -> pd.DataFrame: + df = pd.DataFrame() + df["conv_rate_plus_val1"] = inputs["conv_rate"] + inputs["val_to_add"] + df["conv_rate_plus_val2"] = inputs["conv_rate"] + inputs["val_to_add_2"] + return df + + +driver_activity_v3 = FeatureService( + name="driver_activity_v3", + features=[driver_stats_fresh_fv, transformed_conv_rate_fresh], +) diff --git a/sdk/python/feast/templates/cassandra/feature_repo/test_workflow.py b/sdk/python/feast/templates/cassandra/feature_repo/test_workflow.py index 2c388deea9e..eebeb113115 100644 --- a/sdk/python/feast/templates/cassandra/feature_repo/test_workflow.py +++ b/sdk/python/feast/templates/cassandra/feature_repo/test_workflow.py @@ -22,17 +22,25 @@ def run_demo(): store.materialize_incremental(end_date=datetime.now()) print("\n--- Online features ---") - fetch_online_features(store, use_feature_service=False) + fetch_online_features(store) + + print("\n--- Online features retrieved (instead) through a feature service---") + fetch_online_features(store, source="feature_service") + + print( + "\n--- Online features retrieved (using feature service v3, which uses a feature view with a push source---" + ) + fetch_online_features(store, source="push") print("\n--- Simulate a stream event ingestion of the hourly stats df ---") event_df = pd.DataFrame.from_dict( { "driver_id": [1001], "event_timestamp": [ - datetime(2021, 5, 13, 10, 59, 42), + datetime.now(), ], "created": [ - datetime(2021, 5, 13, 10, 59, 42), + datetime.now(), ], "conv_rate": [1.0], "acc_rate": [1.0], @@ -43,10 +51,7 @@ def run_demo(): store.push("driver_stats_push_source", event_df, to=PushMode.ONLINE_AND_OFFLINE) print("\n--- Online features again with updated values from a stream push---") - fetch_online_features(store, use_feature_service=True) - - print("\n--- Online features retrieved (instead) through a feature service---") - fetch_online_features(store, use_feature_service=True) + fetch_online_features(store, source="push") print("\n--- Run feast teardown ---") subprocess.run(["feast", "teardown"]) @@ -89,7 +94,7 @@ def fetch_historical_features_entity_df(store: FeatureStore, for_batch_scoring: print(training_df.head()) -def fetch_online_features(store, use_feature_service: bool): +def fetch_online_features(store, source: str = ""): entity_rows = [ # {join_key: entity_value} { @@ -103,12 +108,13 @@ def fetch_online_features(store, use_feature_service: bool): "val_to_add_2": 2002, }, ] - if use_feature_service: + if source == "feature_service": features_to_fetch = store.get_feature_service("driver_activity_v1") + elif source == "push": + features_to_fetch = store.get_feature_service("driver_activity_v3") else: features_to_fetch = [ "driver_hourly_stats:acc_rate", - "driver_hourly_stats:avg_daily_trips", "transformed_conv_rate:conv_rate_plus_val1", "transformed_conv_rate:conv_rate_plus_val2", ] diff --git a/sdk/python/feast/templates/gcp/feature_repo/example_repo.py b/sdk/python/feast/templates/gcp/feature_repo/example_repo.py index ab2f696ef23..81e06c72018 100644 --- a/sdk/python/feast/templates/gcp/feature_repo/example_repo.py +++ b/sdk/python/feast/templates/gcp/feature_repo/example_repo.py @@ -63,12 +63,6 @@ tags={"team": "driver_performance"}, ) -# Defines a way to push data (to be available offline, online or both) into Feast. -driver_stats_push_source = PushSource( - name="driver_stats_push_source", - batch_source=driver_stats_source, -) - # Define a request data source which encodes features / information only # available at request time (e.g. part of the user initiated HTTP request) input_request = RequestSource( @@ -107,3 +101,48 @@ def transformed_conv_rate(inputs: pd.DataFrame) -> pd.DataFrame: driver_activity_v2 = FeatureService( name="driver_activity_v2", features=[driver_stats_fv, transformed_conv_rate] ) + +# Defines a way to push data (to be available offline, online or both) into Feast. +driver_stats_push_source = PushSource( + name="driver_stats_push_source", + batch_source=driver_stats_source, +) + +# Defines a slightly modified version of the feature view from above, where the source +# has been changed to the push source. This allows fresh features to be directly pushed +# to the online store for this feature view. +driver_stats_fresh_fv = FeatureView( + name="driver_hourly_stats_fresh", + entities=[driver], + ttl=timedelta(weeks=52 * 10), # Set to be very long for example purposes only + schema=[ + Field(name="conv_rate", dtype=Float32), + Field(name="acc_rate", dtype=Float32), + Field(name="avg_daily_trips", dtype=Int64), + ], + online=True, + source=driver_stats_push_source, # Changed from above + tags={"team": "driver_performance"}, +) + + +# Define an on demand feature view which can generate new features based on +# existing feature views and RequestSource features +@on_demand_feature_view( + sources=[driver_stats_fresh_fv, input_request], # relies on fresh version of FV + schema=[ + Field(name="conv_rate_plus_val1", dtype=Float64), + Field(name="conv_rate_plus_val2", dtype=Float64), + ], +) +def transformed_conv_rate_fresh(inputs: pd.DataFrame) -> pd.DataFrame: + df = pd.DataFrame() + df["conv_rate_plus_val1"] = inputs["conv_rate"] + inputs["val_to_add"] + df["conv_rate_plus_val2"] = inputs["conv_rate"] + inputs["val_to_add_2"] + return df + + +driver_activity_v3 = FeatureService( + name="driver_activity_v3", + features=[driver_stats_fresh_fv, transformed_conv_rate_fresh], +) diff --git a/sdk/python/feast/templates/gcp/feature_repo/test_workflow.py b/sdk/python/feast/templates/gcp/feature_repo/test_workflow.py index 0f8d8894772..51ce1eb05be 100644 --- a/sdk/python/feast/templates/gcp/feature_repo/test_workflow.py +++ b/sdk/python/feast/templates/gcp/feature_repo/test_workflow.py @@ -1,3 +1,4 @@ +import random import subprocess from datetime import datetime @@ -22,24 +23,29 @@ def run_demo(): store.materialize_incremental(end_date=datetime.now()) print("\n--- Online features ---") - fetch_online_features(store, use_feature_service=False) + fetch_online_features(store) print("\n--- Online features retrieved (instead) through a feature service---") - fetch_online_features(store, use_feature_service=True) + fetch_online_features(store, source="feature_service") + + print( + "\n--- Online features retrieved (using feature service v3, which uses a feature view with a push source---" + ) + fetch_online_features(store, source="push") print("\n--- Simulate a stream event ingestion of the hourly stats df ---") event_df = pd.DataFrame.from_dict( { "driver_id": [1001], "event_timestamp": [ - datetime(2021, 5, 13, 10, 59, 42), + datetime.now(), ], "created": [ - datetime(2021, 5, 13, 10, 59, 42), + datetime.now(), ], "conv_rate": [1.0], - "acc_rate": [1.0], - "avg_daily_trips": [1000], + "acc_rate": [1.0 + random.random()], + "avg_daily_trips": [int(1000 * random.random())], } ) print(event_df) @@ -48,7 +54,7 @@ def run_demo(): store.push("driver_stats_push_source", event_df, to=PushMode.ONLINE) print("\n--- Online features again with updated values from a stream push---") - fetch_online_features(store, use_feature_service=True) + fetch_online_features(store, source="push") print("\n--- Run feast teardown ---") subprocess.run(["feast", "teardown"]) @@ -91,7 +97,7 @@ def fetch_historical_features_entity_df(store: FeatureStore, for_batch_scoring: print(training_df.head()) -def fetch_online_features(store, use_feature_service: bool): +def fetch_online_features(store, source: str = ""): entity_rows = [ # {join_key: entity_value} { @@ -105,12 +111,13 @@ def fetch_online_features(store, use_feature_service: bool): "val_to_add_2": 2002, }, ] - if use_feature_service: + if source == "feature_service": features_to_fetch = store.get_feature_service("driver_activity_v1") + elif source == "push": + features_to_fetch = store.get_feature_service("driver_activity_v3") else: features_to_fetch = [ "driver_hourly_stats:acc_rate", - "driver_hourly_stats:avg_daily_trips", "transformed_conv_rate:conv_rate_plus_val1", "transformed_conv_rate:conv_rate_plus_val2", ] diff --git a/sdk/python/feast/templates/hbase/feature_repo/example_repo.py b/sdk/python/feast/templates/hbase/feature_repo/example_repo.py index b3c71154825..131f1bcaa61 100644 --- a/sdk/python/feast/templates/hbase/feature_repo/example_repo.py +++ b/sdk/python/feast/templates/hbase/feature_repo/example_repo.py @@ -54,12 +54,6 @@ tags={"team": "driver_performance"}, ) -# Defines a way to push data (to be available offline, online or both) into Feast. -driver_stats_push_source = PushSource( - name="driver_stats_push_source", - batch_source=driver_stats_source, -) - # Define a request data source which encodes features / information only # available at request time (e.g. part of the user initiated HTTP request) input_request = RequestSource( @@ -98,3 +92,48 @@ def transformed_conv_rate(inputs: pd.DataFrame) -> pd.DataFrame: driver_activity_v2 = FeatureService( name="driver_activity_v2", features=[driver_stats_fv, transformed_conv_rate] ) + +# Defines a way to push data (to be available offline, online or both) into Feast. +driver_stats_push_source = PushSource( + name="driver_stats_push_source", + batch_source=driver_stats_source, +) + +# Defines a slightly modified version of the feature view from above, where the source +# has been changed to the push source. This allows fresh features to be directly pushed +# to the online store for this feature view. +driver_stats_fresh_fv = FeatureView( + name="driver_hourly_stats_fresh", + entities=[driver], + ttl=timedelta(days=1), + schema=[ + Field(name="conv_rate", dtype=Float32), + Field(name="acc_rate", dtype=Float32), + Field(name="avg_daily_trips", dtype=Int64), + ], + online=True, + source=driver_stats_push_source, # Changed from above + tags={"team": "driver_performance"}, +) + + +# Define an on demand feature view which can generate new features based on +# existing feature views and RequestSource features +@on_demand_feature_view( + sources=[driver_stats_fresh_fv, input_request], # relies on fresh version of FV + schema=[ + Field(name="conv_rate_plus_val1", dtype=Float64), + Field(name="conv_rate_plus_val2", dtype=Float64), + ], +) +def transformed_conv_rate_fresh(inputs: pd.DataFrame) -> pd.DataFrame: + df = pd.DataFrame() + df["conv_rate_plus_val1"] = inputs["conv_rate"] + inputs["val_to_add"] + df["conv_rate_plus_val2"] = inputs["conv_rate"] + inputs["val_to_add_2"] + return df + + +driver_activity_v3 = FeatureService( + name="driver_activity_v3", + features=[driver_stats_fresh_fv, transformed_conv_rate_fresh], +) diff --git a/sdk/python/feast/templates/hbase/feature_repo/test_workflow.py b/sdk/python/feast/templates/hbase/feature_repo/test_workflow.py index 76b8d7836c2..eebeb113115 100644 --- a/sdk/python/feast/templates/hbase/feature_repo/test_workflow.py +++ b/sdk/python/feast/templates/hbase/feature_repo/test_workflow.py @@ -22,20 +22,25 @@ def run_demo(): store.materialize_incremental(end_date=datetime.now()) print("\n--- Online features ---") - fetch_online_features(store, use_feature_service=False) + fetch_online_features(store) print("\n--- Online features retrieved (instead) through a feature service---") - fetch_online_features(store, use_feature_service=True) + fetch_online_features(store, source="feature_service") + + print( + "\n--- Online features retrieved (using feature service v3, which uses a feature view with a push source---" + ) + fetch_online_features(store, source="push") print("\n--- Simulate a stream event ingestion of the hourly stats df ---") event_df = pd.DataFrame.from_dict( { "driver_id": [1001], "event_timestamp": [ - datetime(2021, 5, 13, 10, 59, 42), + datetime.now(), ], "created": [ - datetime(2021, 5, 13, 10, 59, 42), + datetime.now(), ], "conv_rate": [1.0], "acc_rate": [1.0], @@ -46,7 +51,7 @@ def run_demo(): store.push("driver_stats_push_source", event_df, to=PushMode.ONLINE_AND_OFFLINE) print("\n--- Online features again with updated values from a stream push---") - fetch_online_features(store, use_feature_service=True) + fetch_online_features(store, source="push") print("\n--- Run feast teardown ---") subprocess.run(["feast", "teardown"]) @@ -89,7 +94,7 @@ def fetch_historical_features_entity_df(store: FeatureStore, for_batch_scoring: print(training_df.head()) -def fetch_online_features(store, use_feature_service: bool): +def fetch_online_features(store, source: str = ""): entity_rows = [ # {join_key: entity_value} { @@ -103,12 +108,13 @@ def fetch_online_features(store, use_feature_service: bool): "val_to_add_2": 2002, }, ] - if use_feature_service: + if source == "feature_service": features_to_fetch = store.get_feature_service("driver_activity_v1") + elif source == "push": + features_to_fetch = store.get_feature_service("driver_activity_v3") else: features_to_fetch = [ "driver_hourly_stats:acc_rate", - "driver_hourly_stats:avg_daily_trips", "transformed_conv_rate:conv_rate_plus_val1", "transformed_conv_rate:conv_rate_plus_val2", ] diff --git a/sdk/python/feast/templates/local/README.md b/sdk/python/feast/templates/local/README.md index 8133b6e84e3..daf3a686fbc 100644 --- a/sdk/python/feast/templates/local/README.md +++ b/sdk/python/feast/templates/local/README.md @@ -3,13 +3,15 @@ If you haven't already, check out the quickstart guide on Feast's website (http: uses this repo. A quick view of what's in this repository's `feature_repo/` directory: * `data/` contains raw demo parquet data -* `example_repo.py` contains demo feature definitions -* `feature_store.yaml` contains a demo setup configuring where data sources are -* `test_workflow.py` showcases how to run all key Feast commands, including defining, retrieving, and pushing features. +* `feature_repo/example_repo.py` contains demo feature definitions +* `feature_repo/feature_store.yaml` contains a demo setup configuring where data sources are +* `feature_repo/test_workflow.py` showcases how to run all key Feast commands, including defining, retrieving, and pushing features. You can run the overall workflow with `python test_workflow.py`. ## To move from this into a more production ready workflow: +> See more details in [Running Feast in production](https://docs.feast.dev/how-to-guides/running-feast-in-production) + 1. First: you should start with a different Feast template, which delegates to a more scalable offline store. - For example, running `feast init -t gcp` or `feast init -t aws` or `feast init -t snowflake`. diff --git a/sdk/python/feast/templates/local/feature_repo/example_repo.py b/sdk/python/feast/templates/local/feature_repo/example_repo.py index b3c71154825..131f1bcaa61 100644 --- a/sdk/python/feast/templates/local/feature_repo/example_repo.py +++ b/sdk/python/feast/templates/local/feature_repo/example_repo.py @@ -54,12 +54,6 @@ tags={"team": "driver_performance"}, ) -# Defines a way to push data (to be available offline, online or both) into Feast. -driver_stats_push_source = PushSource( - name="driver_stats_push_source", - batch_source=driver_stats_source, -) - # Define a request data source which encodes features / information only # available at request time (e.g. part of the user initiated HTTP request) input_request = RequestSource( @@ -98,3 +92,48 @@ def transformed_conv_rate(inputs: pd.DataFrame) -> pd.DataFrame: driver_activity_v2 = FeatureService( name="driver_activity_v2", features=[driver_stats_fv, transformed_conv_rate] ) + +# Defines a way to push data (to be available offline, online or both) into Feast. +driver_stats_push_source = PushSource( + name="driver_stats_push_source", + batch_source=driver_stats_source, +) + +# Defines a slightly modified version of the feature view from above, where the source +# has been changed to the push source. This allows fresh features to be directly pushed +# to the online store for this feature view. +driver_stats_fresh_fv = FeatureView( + name="driver_hourly_stats_fresh", + entities=[driver], + ttl=timedelta(days=1), + schema=[ + Field(name="conv_rate", dtype=Float32), + Field(name="acc_rate", dtype=Float32), + Field(name="avg_daily_trips", dtype=Int64), + ], + online=True, + source=driver_stats_push_source, # Changed from above + tags={"team": "driver_performance"}, +) + + +# Define an on demand feature view which can generate new features based on +# existing feature views and RequestSource features +@on_demand_feature_view( + sources=[driver_stats_fresh_fv, input_request], # relies on fresh version of FV + schema=[ + Field(name="conv_rate_plus_val1", dtype=Float64), + Field(name="conv_rate_plus_val2", dtype=Float64), + ], +) +def transformed_conv_rate_fresh(inputs: pd.DataFrame) -> pd.DataFrame: + df = pd.DataFrame() + df["conv_rate_plus_val1"] = inputs["conv_rate"] + inputs["val_to_add"] + df["conv_rate_plus_val2"] = inputs["conv_rate"] + inputs["val_to_add_2"] + return df + + +driver_activity_v3 = FeatureService( + name="driver_activity_v3", + features=[driver_stats_fresh_fv, transformed_conv_rate_fresh], +) diff --git a/sdk/python/feast/templates/local/feature_repo/test_workflow.py b/sdk/python/feast/templates/local/feature_repo/test_workflow.py index 76b8d7836c2..eebeb113115 100644 --- a/sdk/python/feast/templates/local/feature_repo/test_workflow.py +++ b/sdk/python/feast/templates/local/feature_repo/test_workflow.py @@ -22,20 +22,25 @@ def run_demo(): store.materialize_incremental(end_date=datetime.now()) print("\n--- Online features ---") - fetch_online_features(store, use_feature_service=False) + fetch_online_features(store) print("\n--- Online features retrieved (instead) through a feature service---") - fetch_online_features(store, use_feature_service=True) + fetch_online_features(store, source="feature_service") + + print( + "\n--- Online features retrieved (using feature service v3, which uses a feature view with a push source---" + ) + fetch_online_features(store, source="push") print("\n--- Simulate a stream event ingestion of the hourly stats df ---") event_df = pd.DataFrame.from_dict( { "driver_id": [1001], "event_timestamp": [ - datetime(2021, 5, 13, 10, 59, 42), + datetime.now(), ], "created": [ - datetime(2021, 5, 13, 10, 59, 42), + datetime.now(), ], "conv_rate": [1.0], "acc_rate": [1.0], @@ -46,7 +51,7 @@ def run_demo(): store.push("driver_stats_push_source", event_df, to=PushMode.ONLINE_AND_OFFLINE) print("\n--- Online features again with updated values from a stream push---") - fetch_online_features(store, use_feature_service=True) + fetch_online_features(store, source="push") print("\n--- Run feast teardown ---") subprocess.run(["feast", "teardown"]) @@ -89,7 +94,7 @@ def fetch_historical_features_entity_df(store: FeatureStore, for_batch_scoring: print(training_df.head()) -def fetch_online_features(store, use_feature_service: bool): +def fetch_online_features(store, source: str = ""): entity_rows = [ # {join_key: entity_value} { @@ -103,12 +108,13 @@ def fetch_online_features(store, use_feature_service: bool): "val_to_add_2": 2002, }, ] - if use_feature_service: + if source == "feature_service": features_to_fetch = store.get_feature_service("driver_activity_v1") + elif source == "push": + features_to_fetch = store.get_feature_service("driver_activity_v3") else: features_to_fetch = [ "driver_hourly_stats:acc_rate", - "driver_hourly_stats:avg_daily_trips", "transformed_conv_rate:conv_rate_plus_val1", "transformed_conv_rate:conv_rate_plus_val2", ] diff --git a/sdk/python/feast/templates/postgres/feature_repo/example_repo.py b/sdk/python/feast/templates/postgres/feature_repo/example_repo.py index a7ba9d7eace..0d1783e1e5e 100644 --- a/sdk/python/feast/templates/postgres/feature_repo/example_repo.py +++ b/sdk/python/feast/templates/postgres/feature_repo/example_repo.py @@ -46,12 +46,6 @@ tags={"team": "driver_performance"}, ) -# Defines a way to push data (to be available offline, online or both) into Feast. -driver_stats_push_source = PushSource( - name="driver_stats_push_source", - batch_source=driver_stats_source, -) - # Define a request data source which encodes features / information only # available at request time (e.g. part of the user initiated HTTP request) input_request = RequestSource( @@ -90,3 +84,48 @@ def transformed_conv_rate(inputs: pd.DataFrame) -> pd.DataFrame: driver_activity_v2 = FeatureService( name="driver_activity_v2", features=[driver_stats_fv, transformed_conv_rate] ) + +# Defines a way to push data (to be available offline, online or both) into Feast. +driver_stats_push_source = PushSource( + name="driver_stats_push_source", + batch_source=driver_stats_source, +) + +# Defines a slightly modified version of the feature view from above, where the source +# has been changed to the push source. This allows fresh features to be directly pushed +# to the online store for this feature view. +driver_stats_fresh_fv = FeatureView( + name="driver_hourly_stats_fresh", + entities=[driver], + ttl=timedelta(days=1), + schema=[ + Field(name="conv_rate", dtype=Float32), + Field(name="acc_rate", dtype=Float32), + Field(name="avg_daily_trips", dtype=Int64), + ], + online=True, + source=driver_stats_push_source, # Changed from above + tags={"team": "driver_performance"}, +) + + +# Define an on demand feature view which can generate new features based on +# existing feature views and RequestSource features +@on_demand_feature_view( + sources=[driver_stats_fresh_fv, input_request], # relies on fresh version of FV + schema=[ + Field(name="conv_rate_plus_val1", dtype=Float64), + Field(name="conv_rate_plus_val2", dtype=Float64), + ], +) +def transformed_conv_rate_fresh(inputs: pd.DataFrame) -> pd.DataFrame: + df = pd.DataFrame() + df["conv_rate_plus_val1"] = inputs["conv_rate"] + inputs["val_to_add"] + df["conv_rate_plus_val2"] = inputs["conv_rate"] + inputs["val_to_add_2"] + return df + + +driver_activity_v3 = FeatureService( + name="driver_activity_v3", + features=[driver_stats_fresh_fv, transformed_conv_rate_fresh], +) diff --git a/sdk/python/feast/templates/postgres/feature_repo/test_workflow.py b/sdk/python/feast/templates/postgres/feature_repo/test_workflow.py index ca5c1ccf42d..f657aba15f7 100644 --- a/sdk/python/feast/templates/postgres/feature_repo/test_workflow.py +++ b/sdk/python/feast/templates/postgres/feature_repo/test_workflow.py @@ -22,20 +22,25 @@ def run_demo(): store.materialize_incremental(end_date=datetime.now()) print("\n--- Online features ---") - fetch_online_features(store, use_feature_service=False) + fetch_online_features(store) print("\n--- Online features retrieved (instead) through a feature service---") - fetch_online_features(store, use_feature_service=True) + fetch_online_features(store, source="feature_service") + + print( + "\n--- Online features retrieved (using feature service v3, which uses a feature view with a push source---" + ) + fetch_online_features(store, source="push") print("\n--- Simulate a stream event ingestion of the hourly stats df ---") event_df = pd.DataFrame.from_dict( { "driver_id": [1001], "event_timestamp": [ - datetime(2021, 5, 13, 10, 59, 42), + datetime.now(), ], "created": [ - datetime(2021, 5, 13, 10, 59, 42), + datetime.now(), ], "conv_rate": [1.0], "acc_rate": [1.0], @@ -46,7 +51,7 @@ def run_demo(): store.push("driver_stats_push_source", event_df, to=PushMode.ONLINE_AND_OFFLINE) print("\n--- Online features again with updated values from a stream push---") - fetch_online_features(store, use_feature_service=True) + fetch_online_features(store, source="push") print("\n--- Run feast teardown ---") subprocess.run(["feast", "teardown"]) @@ -89,7 +94,7 @@ def fetch_historical_features_entity_df(store: FeatureStore, for_batch_scoring: print(training_df.head()) -def fetch_online_features(store, use_feature_service: bool): +def fetch_online_features(store, source: str = ""): entity_rows = [ # {join_key: entity_value} { @@ -103,12 +108,13 @@ def fetch_online_features(store, use_feature_service: bool): "val_to_add_2": 2002, }, ] - if use_feature_service: + if source == "feature_service": features_to_fetch = store.get_feature_service("driver_activity_v1") + elif source == "push": + features_to_fetch = store.get_feature_service("driver_activity_v3") else: features_to_fetch = [ "driver_hourly_stats:acc_rate", - "driver_hourly_stats:avg_daily_trips", "transformed_conv_rate:conv_rate_plus_val1", "transformed_conv_rate:conv_rate_plus_val2", ] diff --git a/sdk/python/feast/templates/snowflake/README.md b/sdk/python/feast/templates/snowflake/README.md index 0c950de4358..d4f1ef6faf4 100644 --- a/sdk/python/feast/templates/snowflake/README.md +++ b/sdk/python/feast/templates/snowflake/README.md @@ -2,13 +2,15 @@ A quick view of what's in this repository: * `data/` contains raw demo parquet data -* `driver_repo.py` contains demo feature definitions -* `feature_store.yaml` contains a demo setup configuring where data sources are +* `feature_repo/driver_repo.py` contains demo feature definitions +* `feature_repo/feature_store.yaml` contains a demo setup configuring where data sources are * `test_workflow.py` showcases how to run all key Feast commands, including defining, retrieving, and pushing features. You can run the overall workflow with `python test_workflow.py`. ## To move from this into a more production ready workflow: +> See more details in [Running Feast in production](https://docs.feast.dev/how-to-guides/running-feast-in-production) + 1. `feature_store.yaml` points to a local file as a registry. You'll want to setup a remote file (e.g. in S3/GCS) or a SQL registry. See [registry docs](https://docs.feast.dev/getting-started/concepts/registry) for more details. 2. Setup CI/CD + dev vs staging vs prod environments to automatically update the registry as you change Feast feature definitions. See [docs](https://docs.feast.dev/how-to-guides/running-feast-in-production#1.-automatically-deploying-changes-to-your-feature-definitions). diff --git a/sdk/python/feast/templates/snowflake/feature_repo/driver_repo.py b/sdk/python/feast/templates/snowflake/feature_repo/driver_repo.py index 4befa693f99..dd05dac8455 100644 --- a/sdk/python/feast/templates/snowflake/feature_repo/driver_repo.py +++ b/sdk/python/feast/templates/snowflake/feature_repo/driver_repo.py @@ -66,12 +66,6 @@ tags={"team": "driver_performance"}, ) -# Defines a way to push data (to be available offline, online or both) into Feast. -driver_stats_push_source = PushSource( - name="driver_stats_push_source", - batch_source=driver_stats_source, -) - # Define a request data source which encodes features / information only # available at request time (e.g. part of the user initiated HTTP request) input_request = RequestSource( @@ -110,3 +104,48 @@ def transformed_conv_rate(inputs: pd.DataFrame) -> pd.DataFrame: driver_activity_v2 = FeatureService( name="driver_activity_v2", features=[driver_stats_fv, transformed_conv_rate] ) + +# Defines a way to push data (to be available offline, online or both) into Feast. +driver_stats_push_source = PushSource( + name="driver_stats_push_source", + batch_source=driver_stats_source, +) + +# Defines a slightly modified version of the feature view from above, where the source +# has been changed to the push source. This allows fresh features to be directly pushed +# to the online store for this feature view. +driver_stats_fresh_fv = FeatureView( + name="driver_hourly_stats_fresh", + entities=[driver], + ttl=timedelta(weeks=52 * 10), # Set to be very long for example purposes only + schema=[ + Field(name="conv_rate", dtype=Float32), + Field(name="acc_rate", dtype=Float32), + Field(name="avg_daily_trips", dtype=Int64), + ], + online=True, + source=driver_stats_push_source, # Changed from above + tags={"team": "driver_performance"}, +) + + +# Define an on demand feature view which can generate new features based on +# existing feature views and RequestSource features +@on_demand_feature_view( + sources=[driver_stats_fresh_fv, input_request], # relies on fresh version of FV + schema=[ + Field(name="conv_rate_plus_val1", dtype=Float64), + Field(name="conv_rate_plus_val2", dtype=Float64), + ], +) +def transformed_conv_rate_fresh(inputs: pd.DataFrame) -> pd.DataFrame: + df = pd.DataFrame() + df["conv_rate_plus_val1"] = inputs["conv_rate"] + inputs["val_to_add"] + df["conv_rate_plus_val2"] = inputs["conv_rate"] + inputs["val_to_add_2"] + return df + + +driver_activity_v3 = FeatureService( + name="driver_activity_v3", + features=[driver_stats_fresh_fv, transformed_conv_rate_fresh], +) diff --git a/sdk/python/feast/templates/snowflake/test_workflow.py b/sdk/python/feast/templates/snowflake/test_workflow.py index 6f5e33622a8..adeed279af5 100644 --- a/sdk/python/feast/templates/snowflake/test_workflow.py +++ b/sdk/python/feast/templates/snowflake/test_workflow.py @@ -23,20 +23,25 @@ def run_demo(): store.materialize_incremental(end_date=datetime.now()) print("\n--- Online features ---") - fetch_online_features(store, use_feature_service=False) + fetch_online_features(store) print("\n--- Online features retrieved (instead) through a feature service---") - fetch_online_features(store, use_feature_service=True) + fetch_online_features(store, source="feature_service") + + print( + "\n--- Online features retrieved (using feature service v3, which uses a feature view with a push source---" + ) + fetch_online_features(store, source="push") print("\n--- Simulate a stream event ingestion of the hourly stats df ---") event_df = pd.DataFrame.from_dict( { "driver_id": [1001], "event_timestamp": [ - datetime(2021, 5, 13, 10, 59, 42), + datetime.now(), ], "created": [ - datetime(2021, 5, 13, 10, 59, 42), + datetime.now(), ], "conv_rate": [1.0], "acc_rate": [1.0], @@ -46,8 +51,8 @@ def run_demo(): print(event_df) store.push("driver_stats_push_source", event_df, to=PushMode.ONLINE_AND_OFFLINE) - print("\n--- Online features again with updated values from a stream push---") - fetch_online_features(store, use_feature_service=True) + print("\n--- Online features again with updated values from a stream push ---") + fetch_online_features(store, source="push") print("\n--- Run feast teardown ---") command = "cd feature_repo; feast teardown" @@ -91,7 +96,7 @@ def fetch_historical_features_entity_df(store: FeatureStore, for_batch_scoring: print(training_df.head()) -def fetch_online_features(store, use_feature_service: bool): +def fetch_online_features(store, source: str = ""): entity_rows = [ # {join_key: entity_value} { @@ -105,12 +110,13 @@ def fetch_online_features(store, use_feature_service: bool): "val_to_add_2": 2002, }, ] - if use_feature_service: + if source == "feature_service": features_to_fetch = store.get_feature_service("driver_activity_v1") + elif source == "push": + features_to_fetch = store.get_feature_service("driver_activity_v3") else: features_to_fetch = [ "driver_hourly_stats:acc_rate", - "driver_hourly_stats:avg_daily_trips", "transformed_conv_rate:conv_rate_plus_val1", "transformed_conv_rate:conv_rate_plus_val2", ] diff --git a/sdk/python/feast/type_map.py b/sdk/python/feast/type_map.py index 2cb1c4fefb0..cc8d991da82 100644 --- a/sdk/python/feast/type_map.py +++ b/sdk/python/feast/type_map.py @@ -320,6 +320,8 @@ def _python_datetime_to_int_timestamp( int_timestamps.append(int(value.ToSeconds())) elif isinstance(value, np.datetime64): int_timestamps.append(value.astype("datetime64[s]").astype(np.int_)) + elif isinstance(value, type(np.nan)): + int_timestamps.append(NULL_TIMESTAMP_INT_VALUE) else: int_timestamps.append(int(value)) return int_timestamps diff --git a/sdk/python/tests/unit/infra/test_inference_unit_tests.py b/sdk/python/tests/unit/infra/test_inference_unit_tests.py index c5ed83c12f0..b4a0da7e47a 100644 --- a/sdk/python/tests/unit/infra/test_inference_unit_tests.py +++ b/sdk/python/tests/unit/infra/test_inference_unit_tests.py @@ -15,6 +15,7 @@ from feast.on_demand_feature_view import on_demand_feature_view from feast.repo_config import RepoConfig from feast.types import Float32, Float64, Int64, String, UnixTimestamp +from feast.value_type import ValueType from tests.utils.data_source_test_creator import prep_file_source @@ -216,6 +217,78 @@ def test_feature_view_inference_respects_basic_inference(): assert len(feature_view_2.entity_columns) == 2 +def test_feature_view_inference_on_entity_value_types(): + """ + Tests that feature view inference correctly uses the entity `value_type` attribute. + """ + entity1 = Entity( + name="test1", join_keys=["id_join_key"], value_type=ValueType.INT64 + ) + file_source = FileSource(path="some path") + feature_view_1 = FeatureView( + name="test1", + entities=[entity1], + schema=[Field(name="int64_col", dtype=Int64)], + source=file_source, + ) + + assert len(feature_view_1.schema) == 1 + assert len(feature_view_1.features) == 1 + assert len(feature_view_1.entity_columns) == 0 + + update_feature_views_with_inferred_features_and_entities( + [feature_view_1], + [entity1], + RepoConfig( + provider="local", project="test", entity_key_serialization_version=2 + ), + ) + + # The schema is only used as a parameter, as is therefore not updated during inference. + assert len(feature_view_1.schema) == 1 + + # Since there is already a feature specified, additional features are not inferred. + assert len(feature_view_1.features) == 1 + + # The single entity column is inferred correctly and has the expected type. + assert len(feature_view_1.entity_columns) == 1 + assert feature_view_1.entity_columns[0].dtype == Int64 + + +def test_conflicting_entity_value_types(): + """ + Tests that an error is thrown when the entity value types conflict. + """ + entity1 = Entity( + name="test1", join_keys=["id_join_key"], value_type=ValueType.INT64 + ) + file_source = FileSource(path="some path") + + with pytest.raises(ValueError): + _ = FeatureView( + name="test1", + entities=[entity1], + schema=[ + Field(name="int64_col", dtype=Int64), + Field( + name="id_join_key", dtype=Float64 + ), # Conflicts with the defined entity + ], + source=file_source, + ) + + # There should be no error here. + _ = FeatureView( + name="test1", + entities=[entity1], + schema=[ + Field(name="int64_col", dtype=Int64), + Field(name="id_join_key", dtype=Int64), # Conflicts with the defined entity + ], + source=file_source, + ) + + def test_feature_view_inference_on_entity_columns(simple_dataset_1): """ Tests that feature view inference correctly infers entity columns. diff --git a/sdk/python/tests/unit/test_feature_views.py b/sdk/python/tests/unit/test_feature_views.py index 0fe3f839e1f..379396e5c63 100644 --- a/sdk/python/tests/unit/test_feature_views.py +++ b/sdk/python/tests/unit/test_feature_views.py @@ -15,6 +15,20 @@ from feast.types import Float32 +def test_create_feature_view_with_conflicting_entities(): + user1 = Entity(name="user1", join_keys=["user_id"]) + user2 = Entity(name="user2", join_keys=["user_id"]) + batch_source = FileSource(path="some path") + + with pytest.raises(ValueError): + _ = FeatureView( + name="test", + entities=[user1, user2], + ttl=timedelta(days=30), + source=batch_source, + ) + + def test_create_batch_feature_view(): batch_source = FileSource(path="some path") BatchFeatureView( diff --git a/sdk/python/tests/utils/e2e_test_validation.py b/sdk/python/tests/utils/e2e_test_validation.py index e2b8b14eb47..43bdbefc004 100644 --- a/sdk/python/tests/utils/e2e_test_validation.py +++ b/sdk/python/tests/utils/e2e_test_validation.py @@ -112,8 +112,17 @@ def _check_offline_and_online_features( full_feature_names=full_feature_names, ).to_dict() - if full_feature_names: + # Wait for materialization to occur + if not response_dict[f"{fv.name}__value"][0]: + # Deal with flake with a retry + time.sleep(10) + response_dict = fs.get_online_features( + [f"{fv.name}:value"], + [{"driver_id": driver_id}], + full_feature_names=full_feature_names, + ).to_dict() + if full_feature_names: if expected_value: assert response_dict[f"{fv.name}__value"][0], f"Response: {response_dict}" assert ( diff --git a/ui/package.json b/ui/package.json index 7f0e7c3fbe1..6bd58663db1 100644 --- a/ui/package.json +++ b/ui/package.json @@ -1,6 +1,6 @@ { "name": "@feast-dev/feast-ui", - "version": "0.24.0", + "version": "0.24.1", "private": false, "files": [ "dist"