diff --git a/.github/workflows/complete.yml b/.github/workflows/complete.yml index b11a4862f29..06fd32d7811 100644 --- a/.github/workflows/complete.yml +++ b/.github/workflows/complete.yml @@ -132,26 +132,43 @@ jobs: - build-push-docker-images - publish-ingestion-jar runs-on: ubuntu-latest + env: + INGESTION_JAR_PATH: /shared/feast-ingestion-spark-develop.jar steps: - uses: actions/checkout@v2 + - name: Download ingestion jar + uses: actions/download-artifact@v2 + with: + name: ingestion-jar + path: ./infra/docker-compose/ - name: Test docker compose run: ./infra/scripts/test-docker-compose.sh ${GITHUB_SHA} publish-ingestion-jar: - runs-on: [self-hosted] + runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 - - uses: GoogleCloudPlatform/github-actions/setup-gcloud@master - with: - version: '290.0.1' - export_default_credentials: true - uses: actions/setup-java@v1 with: java-version: '11' - - uses: stCarolas/setup-maven@v3 + - name: Cache local Maven repository + uses: actions/cache@v2 with: - maven-version: 3.6.3 + path: ~/.m2/repository + key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} + restore-keys: | + ${{ runner.os }}-maven- - name: build-jar - run: make build-java-no-tests REVISION=${GITHUB_SHA} - - name: copy to gs - run: gsutil cp ./spark/ingestion/target/feast-ingestion-spark-${GITHUB_SHA}.jar gs://feast-jobs/spark/ingestion/ + env: + # Try to add retries to prevent connection resets + # https://github.community/t/getting-maven-could-not-transfer-artifact-with-500-error-when-using-github-actions/17570 + # https://github.com/actions/virtual-environments/issues/1499#issuecomment-718396233 + MAVEN_OPTS: -Dmaven.wagon.httpconnectionManager.ttlSeconds=25 -Dmaven.wagon.http.retryHandler.count=3 -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false + MAVEN_EXTRA_OPTS: -X + run: make build-java-no-tests REVISION=develop + - name: Upload ingestion jar + uses: actions/upload-artifact@v2 + with: + name: ingestion-jar + path: spark/ingestion/target/feast-ingestion-spark-develop.jar + retention-days: 1 diff --git a/.github/workflows/master_only.yml b/.github/workflows/master_only.yml index 20caebe0afc..aab08c73bbd 100644 --- a/.github/workflows/master_only.yml +++ b/.github/workflows/master_only.yml @@ -11,7 +11,7 @@ jobs: runs-on: [self-hosted] strategy: matrix: - component: [core, serving, jobcontroller, jupyter, ci] + component: [core, serving, jobservice, jupyter, ci] env: MAVEN_CACHE: gs://feast-templocation-kf-feast/.m2.2020-08-19.tar DOCKER_BUILDKIT: '1' @@ -27,6 +27,8 @@ jobs: infra/scripts/download-maven-cache.sh \ --archive-uri ${MAVEN_CACHE} \ --output-dir . + - name: Get version + run: echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV - name: Build image run: make build-${{ matrix.component }}-docker REGISTRY=gcr.io/kf-feast VERSION=${GITHUB_SHA} - name: Push image @@ -37,34 +39,9 @@ jobs: docker tag gcr.io/kf-feast/feast-${{ matrix.component }}:${GITHUB_SHA} gcr.io/kf-feast/feast-${{ matrix.component }}:develop docker push gcr.io/kf-feast/feast-${{ matrix.component }}:develop fi - - name: Get version - run: echo ::set-env name=RELEASE_VERSION::${GITHUB_REF#refs/*/} - - name: Push versioned Docker image - run: | - source infra/scripts/setup-common-functions.sh - # Build and push semver tagged commits - # Regular expression should match MAJOR.MINOR.PATCH[-PRERELEASE[.IDENTIFIER]] - # eg. v0.7.1 v0.7.2-alpha v0.7.2-rc.1 - SEMVER_REGEX='^v[0-9]+\.[0-9]+\.[0-9]+(-([0-9A-Za-z-]+(\.[0-9A-Za-z-]+)*))?$' - if echo "${RELEASE_VERSION}" | grep -P "$SEMVER_REGEX" &>/dev/null ; then - VERSION_WITHOUT_PREFIX=${RELEASE_VERSION:1} - - docker tag gcr.io/kf-feast/feast-${{ matrix.component }}:${GITHUB_SHA} gcr.io/kf-feast/feast-${{ matrix.component }}:${VERSION_WITHOUT_PREFIX} - docker push gcr.io/kf-feast/feast-${{ matrix.component }}:${VERSION_WITHOUT_PREFIX} - - # Also update "latest" image if tagged commit is pushed to stable branch - HIGHEST_SEMVER_TAG=$(get_tag_release -m) - echo "Only push to latest tag if tag is the highest semver version $HIGHEST_SEMVER_TAG" - - if [ "${VERSION_WITHOUT_PREFIX}" = "${HIGHEST_SEMVER_TAG:1}" ] - then - docker tag gcr.io/kf-feast/feast-${{ matrix.component }}:${GITHUB_SHA} gcr.io/kf-feast/feast-${{ matrix.component }}:latest - docker push gcr.io/kf-feast/feast-${{ matrix.component }}:latest - fi - fi publish-ingestion-jar: - runs-on: [ self-hosted ] + runs-on: ubuntu-latest env: PUBLISH_BUCKET: feast-jobs steps: @@ -73,12 +50,18 @@ jobs: with: version: '290.0.1' export_default_credentials: true + project_id: ${{ secrets.GCP_PROJECT_ID }} + service_account_key: ${{ secrets.GCP_SA_KEY }} - uses: actions/setup-java@v1 with: java-version: '11' - - uses: stCarolas/setup-maven@v3 + - name: Cache local Maven repository + uses: actions/cache@v2 with: - maven-version: 3.6.3 + path: ~/.m2/repository + key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} + restore-keys: | + ${{ runner.os }}-maven- - name: Publish develop version of ingestion job run: | if [ ${GITHUB_REF#refs/*/} == "master" ]; then @@ -86,7 +69,7 @@ jobs: gsutil cp ./spark/ingestion/target/feast-ingestion-spark-develop.jar gs://${PUBLISH_BUCKET}/spark/ingestion/ fi - name: Get version - run: echo ::set-env name=RELEASE_VERSION::${GITHUB_REF#refs/*/} + run: echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV - name: Publish tagged version of ingestion job run: | SEMVER_REGEX='^v[0-9]+\.[0-9]+\.[0-9]+(-([0-9A-Za-z-]+(\.[0-9A-Za-z-]+)*))?$' diff --git a/.github/workflows/mirror.yml b/.github/workflows/mirror.yml index d5c270d4c79..f9bfdd92a54 100644 --- a/.github/workflows/mirror.yml +++ b/.github/workflows/mirror.yml @@ -12,7 +12,7 @@ jobs: - uses: actions/checkout@v2 with: fetch-depth: 0 - - uses: webfactory/ssh-agent@v0.4.0 + - uses: webfactory/ssh-agent@v0.4.1 with: ssh-private-key: ${{ secrets.MIRROR_SSH_KEY }} - name: Mirror all origin branches and tags to internal repo diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 1d389859f42..2610f891349 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -13,6 +13,7 @@ jobs: version_without_prefix: ${{ steps.get_release_version_without_prefix.outputs.version_without_prefix }} highest_semver_tag: ${{ steps.get_highest_semver.outputs.highest_semver_tag }} steps: + - uses: actions/checkout@v2 - name: Get release version id: get_release_version run: echo ::set-output name=release_version::${GITHUB_REF#refs/*/} @@ -47,7 +48,7 @@ jobs: needs: get-version strategy: matrix: - component: [core, serving, jupyter] + component: [core, serving, jobservice, jupyter] env: MAVEN_CACHE: gs://feast-templocation-kf-feast/.m2.2020-08-19.tar DOCKER_BUILDKIT: '1' @@ -66,34 +67,34 @@ jobs: with: version: '290.0.1' export_default_credentials: true + project_id: ${{ secrets.GCP_PROJECT_ID }} + service_account_key: ${{ secrets.GCP_SA_KEY }} + - run: gcloud auth configure-docker --quiet - name: Get m2 cache run: | infra/scripts/download-maven-cache.sh \ --archive-uri ${MAVEN_CACHE} \ --output-dir . - - name: Build and push - uses: docker/build-push-action@v2 - env: - RELEASE_VERSION: ${{ needs.get-version.outputs.release_version }} - with: - push: true - file: infra/docker/${{ matrix.component }}/Dockerfile - tags: feastdev/feast-${{ matrix.component }}:${{ needs.get-version.outputs.release_version }} - build-args: | - REVISION=${RELEASE_VERSION} - - name: Build and push latest - uses: docker/build-push-action@v2 + - name: Build and push versioned images env: RELEASE_VERSION: ${{ needs.get-version.outputs.release_version }} VERSION_WITHOUT_PREFIX: ${{ needs.get-version.outputs.version_without_prefix }} HIGHEST_SEMVER_TAG: ${{ needs.get-version.outputs.highest_semver_tag }} - with: - if: ${VERSION_WITHOUT_PREFIX} == ${HIGHEST_SEMVER_TAG:1} - push: true - file: infra/docker/${{ matrix.component }}/Dockerfile - tags: feastdev/feast-${{ matrix.component }}:latest - build-args: | - REVISION=${RELEASE_VERSION} + run: | + docker build --build-arg VERSION=$RELEASE_VERSION \ + -t gcr.io/kf-feast/feast-${{ matrix.component }}:${GITHUB_SHA} \ + -t gcr.io/kf-feast/feast-${{ matrix.component }}:${VERSION_WITHOUT_PREFIX} \ + -f infra/docker/${{ matrix.component }}/Dockerfile . + docker push gcr.io/kf-feast/feast-${{ matrix.component }}:${VERSION_WITHOUT_PREFIX} + + echo "Only push to latest tag if tag is the highest semver version $HIGHEST_SEMVER_TAG" + if [ "${VERSION_WITHOUT_PREFIX}" = "${HIGHEST_SEMVER_TAG:1}" ] + then + docker tag feastdev/feast-${{ matrix.component }}:latest gcr.io/kf-feast/feast-${{ matrix.component }}:${VERSION_WITHOUT_PREFIX} + docker tag gcr.io/kf-feast/feast-${{ matrix.component }}:latest gcr.io/kf-feast/feast-${{ matrix.component }}:${VERSION_WITHOUT_PREFIX} + docker push feastdev/feast-${{ matrix.component }}:latest + docker push gcr.io/kf-feast/feast-${{ matrix.component }}:latest + fi publish-helm-charts: runs-on: ubuntu-latest diff --git a/CHANGELOG.md b/CHANGELOG.md index 6f866efce9d..e1c7abb2e46 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,31 @@ # Changelog + +## [v0.8.1](https://github.com/feast-dev/feast/tree/v0.8.1) (2020-11-24) + +[Full Changelog](https://github.com/feast-dev/feast/compare/v0.8.0...v0.8.1) + +**Implemented enhancements:** + +- Expires Redis Keys based on Feature Table Max Age [\#1161](https://github.com/feast-dev/feast/pull/1161) ([khorshuheng](https://github.com/khorshuheng)) +- Jobservice control loop \(based on \#1140\) [\#1156](https://github.com/feast-dev/feast/pull/1156) ([oavdeev](https://github.com/oavdeev)) + +**Fixed bugs:** + +- Lazy metrics initialization \(to correct pick up in executor\) [\#1195](https://github.com/feast-dev/feast/pull/1195) ([pyalex](https://github.com/pyalex)) +- Add missing third\_party folder [\#1185](https://github.com/feast-dev/feast/pull/1185) ([terryyylim](https://github.com/terryyylim)) +- Fix missing name variable instantiation [\#1166](https://github.com/feast-dev/feast/pull/1166) ([terryyylim](https://github.com/terryyylim)) + +**Merged pull requests:** + +- Bump ssh-agent version [\#1175](https://github.com/feast-dev/feast/pull/1175) ([terryyylim](https://github.com/terryyylim)) +- Refactor configurable options and add sphinx docs [\#1174](https://github.com/feast-dev/feast/pull/1174) ([terryyylim](https://github.com/terryyylim)) +- Stabilize flaky e2e tests [\#1173](https://github.com/feast-dev/feast/pull/1173) ([pyalex](https://github.com/pyalex)) +- Fix connection resets in CI for Maven [\#1164](https://github.com/feast-dev/feast/pull/1164) ([woop](https://github.com/woop)) +- Add dataproc executor resource config [\#1160](https://github.com/feast-dev/feast/pull/1160) ([terryyylim](https://github.com/terryyylim)) +- Fix github workflow deprecating env variable [\#1158](https://github.com/feast-dev/feast/pull/1158) ([terryyylim](https://github.com/terryyylim)) +- Ensure consistency of github workflow [\#1157](https://github.com/feast-dev/feast/pull/1157) ([terryyylim](https://github.com/terryyylim)) + + ## [v0.8.0](https://github.com/feast-dev/feast/tree/v0.8.0) (2020-11-10) [Full Changelog](https://github.com/feast-dev/feast/compare/v0.7.1...v0.8.0) diff --git a/Makefile b/Makefile index 985151b5728..479171aa649 100644 --- a/Makefile +++ b/Makefile @@ -17,6 +17,7 @@ ROOT_DIR := $(shell dirname $(realpath $(firstword $(MAKEFILE_LIST)))) PROTO_TYPE_SUBDIRS = core serving types storage PROTO_SERVICE_SUBDIRS = core serving +MVN := mvn ${MAVEN_EXTRA_OPTS} # General @@ -35,28 +36,28 @@ install-ci-dependencies: install-python-ci-dependencies install-go-ci-dependenci # Java install-java-ci-dependencies: - mvn verify clean --fail-never + ${MVN} verify clean --fail-never format-java: - mvn spotless:apply + ${MVN} spotless:apply lint-java: - mvn --no-transfer-progress spotless:check + ${MVN} --no-transfer-progress spotless:check test-java: - mvn --no-transfer-progress test + ${MVN} --no-transfer-progress test test-java-integration: - mvn --no-transfer-progress -Dmaven.javadoc.skip=true -Dgpg.skip -DskipUTs=true clean verify + ${MVN} --no-transfer-progress -Dmaven.javadoc.skip=true -Dgpg.skip -DskipUTs=true clean verify test-java-with-coverage: - mvn --no-transfer-progress test jacoco:report-aggregate + ${MVN} --no-transfer-progress test jacoco:report-aggregate build-java: - mvn clean verify + ${MVN} clean verify build-java-no-tests: - mvn --no-transfer-progress -Dmaven.javadoc.skip=true -Dgpg.skip -DskipUTs=true -Drevision=${REVISION} clean package + ${MVN} --no-transfer-progress -Dmaven.javadoc.skip=true -Dgpg.skip -DskipUTs=true -Drevision=${REVISION} clean package # Python SDK @@ -141,13 +142,13 @@ push-jupyter-docker: docker push $(REGISTRY)/feast-jupyter:$(VERSION) build-core-docker: - docker build -t $(REGISTRY)/feast-core:$(VERSION) -f infra/docker/core/Dockerfile . + docker build --build-arg VERSION=$(VERSION) -t $(REGISTRY)/feast-core:$(VERSION) -f infra/docker/core/Dockerfile . build-jobservice-docker: docker build -t $(REGISTRY)/feast-jobservice:$(VERSION) -f infra/docker/jobservice/Dockerfile . build-serving-docker: - docker build -t $(REGISTRY)/feast-serving:$(VERSION) -f infra/docker/serving/Dockerfile . + docker build --build-arg VERSION=$(VERSION) -t $(REGISTRY)/feast-serving:$(VERSION) -f infra/docker/serving/Dockerfile . build-ci-docker: docker build -t $(REGISTRY)/feast-ci:$(VERSION) -f infra/docker/ci/Dockerfile . diff --git a/datatypes/java/README.md b/datatypes/java/README.md index 0996af74b1c..22b37d848fa 100644 --- a/datatypes/java/README.md +++ b/datatypes/java/README.md @@ -16,7 +16,7 @@ Dependency Coordinates dev.feast datatypes-java - 0.8.0 + 0.8.1 ``` diff --git a/docs/.gitbook/assets/blank-diagram-4 (4).svg b/docs/.gitbook/assets/blank-diagram-4 (4).svg new file mode 100644 index 00000000000..fb5e0659e55 --- /dev/null +++ b/docs/.gitbook/assets/blank-diagram-4 (4).svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/docs/.gitbook/assets/concept_hierarchy (1).png b/docs/.gitbook/assets/concept_hierarchy (1).png new file mode 100644 index 00000000000..f5cf59ad673 Binary files /dev/null and b/docs/.gitbook/assets/concept_hierarchy (1).png differ diff --git a/docs/.gitbook/assets/feast-architecture-diagrams (1).svg b/docs/.gitbook/assets/feast-architecture-diagrams (1).svg new file mode 100644 index 00000000000..7335c131c44 --- /dev/null +++ b/docs/.gitbook/assets/feast-architecture-diagrams (1).svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/docs/.gitbook/assets/feast-on-aws-3- (1).png b/docs/.gitbook/assets/feast-on-aws-3- (1).png new file mode 100644 index 00000000000..e6de77dde9b Binary files /dev/null and b/docs/.gitbook/assets/feast-on-aws-3- (1).png differ diff --git a/docs/.gitbook/assets/image (4) (1).png b/docs/.gitbook/assets/image (4) (1).png new file mode 100644 index 00000000000..cd77f27cc45 Binary files /dev/null and b/docs/.gitbook/assets/image (4) (1).png differ diff --git a/docs/.gitbook/assets/image (4).png b/docs/.gitbook/assets/image (4).png new file mode 100644 index 00000000000..cd77f27cc45 Binary files /dev/null and b/docs/.gitbook/assets/image (4).png differ diff --git a/docs/.gitbook/assets/image (5).png b/docs/.gitbook/assets/image (5).png new file mode 100644 index 00000000000..49670e20054 Binary files /dev/null and b/docs/.gitbook/assets/image (5).png differ diff --git a/docs/.gitbook/assets/image (6).png b/docs/.gitbook/assets/image (6).png new file mode 100644 index 00000000000..49670e20054 Binary files /dev/null and b/docs/.gitbook/assets/image (6).png differ diff --git a/docs/.gitbook/assets/point_in_time_join (1) (1).png b/docs/.gitbook/assets/point_in_time_join (1) (1).png new file mode 100644 index 00000000000..331a090d719 Binary files /dev/null and b/docs/.gitbook/assets/point_in_time_join (1) (1).png differ diff --git a/docs/.gitbook/assets/rsz_untitled23 (2).jpg b/docs/.gitbook/assets/rsz_untitled23 (2).jpg new file mode 100644 index 00000000000..b92ec6fed72 Binary files /dev/null and b/docs/.gitbook/assets/rsz_untitled23 (2).jpg differ diff --git a/docs/.gitbook/assets/untitled-25-1- (2).jpg b/docs/.gitbook/assets/untitled-25-1- (2).jpg new file mode 100644 index 00000000000..93d010406bd Binary files /dev/null and b/docs/.gitbook/assets/untitled-25-1- (2).jpg differ diff --git a/docs/README.md b/docs/README.md index e84deb4ae77..995e30864ce 100644 --- a/docs/README.md +++ b/docs/README.md @@ -1,12 +1,12 @@ # Introduction -### What is Feast? +## What is Feast? Feast \(**Fea**ture **St**ore\) is an operational data system for managing and serving machine learning features to models in production. -![](.gitbook/assets/feast-architecture-diagrams.svg) +![](.gitbook/assets/feast-architecture-diagrams%20%281%29.svg) -### Problems Feast Solves +## Problems Feast Solves **Models need consistent access to data:** ML systems built on traditional data infrastructure are often coupled to databases, object stores, streams, and files. A result of this coupling, however, is that any change in data infrastructure may break dependent ML systems. Another challenge is that dual implementations of data retrieval for training and serving can lead to inconsistencies in data, which in turn can lead to training-serving skew. @@ -24,7 +24,7 @@ Feast solves the challenge of data leakage by providing point-in-time correct fe Feast addresses this problem by introducing feature reuse through a centralized system \(a registry\). This registry enables multiple teams working on different projects not only to contribute features, but also to reuse these same features. With Feast, data scientists can start new ML projects by selecting previously engineered features from a centralized registry, and are no longer required to develop new features for each project. -### Problems Feast does not yet solve +## Problems Feast does not yet solve **Feature engineering:** We aim for Feast to support light-weight feature engineering as part of our API. @@ -32,25 +32,25 @@ Feast addresses this problem by introducing feature reuse through a centralized **‌Feature validation:** We additionally aim for Feast to improve support for statistics generation of feature data and subsequent validation of these statistics. Current support is limited. -### What Feast is not +## What Feast is not -\*\*\*\*[**ETL**](https://en.wikipedia.org/wiki/Extract,_transform,_load) **or** [**ELT**](https://en.wikipedia.org/wiki/Extract,_load,_transform) **system:** Feast is not \(and does not plan to become\) a general purpose data transformation or pipelining system. Feast plans to include a light-weight feature engineering toolkit, but we encourage teams to integrate Feast with upstream ETL/ELT systems that are specialized in transformation. +[**ETL**](https://en.wikipedia.org/wiki/Extract,_transform,_load) **or** [**ELT**](https://en.wikipedia.org/wiki/Extract,_load,_transform) **system:** Feast is not \(and does not plan to become\) a general purpose data transformation or pipelining system. Feast plans to include a light-weight feature engineering toolkit, but we encourage teams to integrate Feast with upstream ETL/ELT systems that are specialized in transformation. **Data warehouse:** Feast is not a replacement for your data warehouse or the source of truth for all transformed data in your organization. Rather, Feast is a light-weight downstream layer that can serve data from an existing data warehouse \(or other data sources\) to models in production. **Data catalog:** Feast is not a general purpose data catalog for your organization. Feast is purely focused on cataloging features for use in ML pipelines or systems, and only to the extent of facilitating the reuse of features. -### How can I get started? +## How can I get started? {% hint style="info" %} The best way to learn Feast is to use it. Head over to our [Quickstart](quickstart.md) and try out our examples! {% endhint %} - Explore the following resources to get started with Feast: +Explore the following resources to get started with Feast: * [Getting Started](getting-started/) provides guides on [Installing Feast](getting-started/install-feast/) and [Connecting to Feast](getting-started/connect-to-feast/). * [Concepts](./) describes all important Feast API concepts. -* [User guide](user-guide/data-ingestion.md) provides guidance on completing Feast workflows. +* [User guide](user-guide/define-and-ingest-features.md) provides guidance on completing Feast workflows. * [Examples](https://github.com/feast-dev/feast/tree/master/examples) contains a Jupyter notebook that you can run on your Feast deployment. * [Advanced](advanced/troubleshooting.md) contains information about both advanced and operational aspects of Feast. * [Reference](reference/api/) contains detailed API and design documents for advanced users. diff --git a/docs/SUMMARY.md b/docs/SUMMARY.md index 81c35f4e9eb..b2495aff982 100644 --- a/docs/SUMMARY.md +++ b/docs/SUMMARY.md @@ -8,11 +8,11 @@ * [Amazon EKS \(with Terraform\)](getting-started/install-feast/kubernetes-amazon-eks-with-terraform.md) * [Connect to Feast](getting-started/connect-to-feast/README.md) * [Python SDK](getting-started/connect-to-feast/python-sdk.md) - * [Feast CLI](getting-started/connect-to-feast/connecting-to-feast.md) + * [Feast CLI](getting-started/connect-to-feast/feast-cli.md) * [Learn Feast](getting-started/learn-feast.md) * [Roadmap](roadmap.md) * [Changelog](https://github.com/feast-dev/feast/blob/master/CHANGELOG.md) -* [Community](getting-help.md) +* [Community](community.md) ## Concepts @@ -21,12 +21,14 @@ * [Entities](concepts/entities.md) * [Sources](concepts/sources.md) * [Feature Tables](concepts/feature-tables.md) -* [Feature References](concepts/feature-references.md) +* [Stores](concepts/stores.md) +* [Glossary](concepts/glossary.md) ## User Guide -* [Getting data into Feast](user-guide/data-ingestion.md) -* [Getting training features](user-guide/feature-retrieval.md) +* [Overview](user-guide/overview.md) +* [Define and ingest features](user-guide/define-and-ingest-features.md) +* [Getting training features](user-guide/getting-training-features.md) * [Getting online features](user-guide/getting-online-features.md) ## Tutorials @@ -44,14 +46,14 @@ ## Reference * [API Reference](reference/api/README.md) - * [Core gRPC API](https://api.docs.feast.dev/grpc/feast.core.pb.html) - * [Serving gRPC API](https://api.docs.feast.dev/grpc/feast.serving.pb.html) - * [gRPC Types](https://api.docs.feast.dev/grpc/feast.types.pb.html) -* [Configuration Reference](reference/configuration-reference/README.md) * [Go SDK](https://godoc.org/github.com/feast-dev/feast/sdk/go) -* [Metrics Reference](reference/metrics-reference/README.md) * [Java SDK](https://javadoc.io/doc/dev.feast/feast-sdk) + * [Core gRPC API](https://api.docs.feast.dev/grpc/feast.core.pb.html) * [Python SDK](https://api.docs.feast.dev/python/) + * [Serving gRPC API](https://api.docs.feast.dev/grpc/feast.serving.pb.html) + * [gRPC Types](https://api.docs.feast.dev/grpc/feast.types.pb.html) +* [Configuration Reference](reference/configuration-reference.md) +* [Metrics Reference](reference/metrics-reference.md) * [Limitations](reference/limitations.md) ## Contributing diff --git a/docs/advanced/audit-logging.md b/docs/advanced/audit-logging.md index 269f063c597..8fc8e8fc9b2 100644 --- a/docs/advanced/audit-logging.md +++ b/docs/advanced/audit-logging.md @@ -1,5 +1,9 @@ # Audit Logging +{% hint style="warning" %} +This page applies to Feast 0.7. The content may be out of date for Feast 0.8+ +{% endhint %} + ## Introduction Feast provides audit logging functionality in order to debug problems and to trace the lineage of events. diff --git a/docs/advanced/metrics.md b/docs/advanced/metrics.md index 57b9ebfc059..23e941d78be 100644 --- a/docs/advanced/metrics.md +++ b/docs/advanced/metrics.md @@ -1,23 +1,27 @@ # Metrics -### Overview +{% hint style="warning" %} +This page applies to Feast 0.7. The content may be out of date for Feast 0.8+ +{% endhint %} + +## Overview Feast Components export metrics that can provide insight into Feast behavior: * [Feast Ingestion Jobs can be configured to push metrics into StatsD](metrics.md#2-exporting-feast-metrics-to-prometheus) * [Prometheus can be configured to scrape metrics from Feast Core and Serving.](metrics.md#2-exporting-feast-metrics-to-prometheus) -See the [Metrics Reference ](../reference/metrics-reference/)for documentation on metrics are exported by Feast. +See the [Metrics Reference ](../reference/metrics-reference.md)for documentation on metrics are exported by Feast. {% hint style="info" %} Feast Job Controller currently does not export any metrics on its own. However its `application.yml` is used to configure metrics export for ingestion jobs. {% endhint %} -### Pushing Ingestion Metrics to StatsD +## Pushing Ingestion Metrics to StatsD -#### **Feast Ingestion Job** +### **Feast Ingestion Job** -Feast Ingestion Job can be configured to push Ingestion metrics to a StatsD instance. Metrics export to StatsD for Ingestion Job is configured in Job Controller's `application.yml` under `feast.jobs.metrics` +Feast Ingestion Job can be configured to push Ingestion metrics to a StatsD instance. Metrics export to StatsD for Ingestion Job is configured in Job Controller's `application.yml` under `feast.jobs.metrics` ```yaml feast: @@ -32,12 +36,12 @@ Feast Ingestion Job can be configured to push Ingestion metrics to a StatsD inst ``` {% hint style="info" %} -If you need Ingestion Metrics in Prometheus or some other metrics backend, use a metrics forwarder to forward Ingestion Metrics from StatsD to the metrics backend of choice. \(ie Use [`prometheus-statsd-exporter`](https://github.com/prometheus/statsd_exporter) to forward metrics to Prometheus\). +If you need Ingestion Metrics in Prometheus or some other metrics backend, use a metrics forwarder to forward Ingestion Metrics from StatsD to the metrics backend of choice. \(ie Use [`prometheus-statsd-exporter`](https://github.com/prometheus/statsd_exporter) to forward metrics to Prometheus\). {% endhint %} -### Exporting Feast Metrics to Prometheus +## Exporting Feast Metrics to Prometheus -#### **Feast Core and Serving** +### **Feast Core and Serving** Feast Core and Serving exports metrics to a Prometheus instance via Prometheus scraping its `/metrics` endpoint. Metrics export to Prometheus for Core and Serving can be configured via their corresponding `application.yml` @@ -49,9 +53,7 @@ server: [Direct Prometheus](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#scrape_config) to scrape directly from Core and Serving's `/metrics` endpoint. -### Further Reading - -See the [Metrics Reference ](../reference/metrics-reference/)for documentation on metrics are exported by Feast. +## Further Reading -## +See the [Metrics Reference ](../reference/metrics-reference.md)for documentation on metrics are exported by Feast. diff --git a/docs/advanced/security.md b/docs/advanced/security.md index 937dadb3bdc..d3febf993e0 100644 --- a/docs/advanced/security.md +++ b/docs/advanced/security.md @@ -5,14 +5,12 @@ description: 'Secure Feast with SSL/TLS, Authentication and Authorization.' # Security {% hint style="warning" %} -Currently, Security functionality applies only to Feast Core and Feast Online Serving. - -Security for Historical Serving will become available once offline storage is introduced in Feast 0.9. +This page applies to Feast 0.7. The content may be out of date for Feast 0.8+ {% endhint %} -### Overview +## Overview -![Overview of Feast's Security Methods.](../.gitbook/assets/untitled-25-1-.jpg) +![Overview of Feast's Security Methods.](../.gitbook/assets/untitled-25-1-%20%282%29.jpg) Feast supports the following security methods: @@ -22,7 +20,7 @@ Feast supports the following security methods: [Important considerations when integrating Authentication/Authorization](security.md#5-authentication-and-authorization). -## **1. SSL/TLS** +## **SSL/TLS** Feast supports SSL/TLS encrypted inter-service communication among Feast Core, Feast Online Serving, and Feast SDKs. @@ -34,13 +32,13 @@ The following properties configure SSL/TLS. These properties are located in thei | :--- | :--- | | `grpc.server.security.enabled` | Enables SSL/TLS functionality if `true` | | `grpc.server.security.certificateChain` | Provide the path to certificate chain. | -| `grpc.server.security.privateKey` | Provide the to private key. | +| `grpc.server.security.privateKey` | Provide the to private key. | > Read more on enabling SSL/TLS in the[ gRPC starter docs.](https://yidongnan.github.io/grpc-spring-boot-starter/en/server/security.html#enable-transport-layer-security) ### Configuring SSL/TLS on Python SDK/CLI -To enable SSL/TLS in the [Feast Python SDK](https://api.docs.feast.dev/python/#feast.client.Client) or [Feast CLI](../getting-started/connect-to-feast/connecting-to-feast.md), set the config options via `feast config`: +To enable SSL/TLS in the [Feast Python SDK](https://api.docs.feast.dev/python/#feast.client.Client) or [Feast CLI](../getting-started/connect-to-feast/feast-cli.md), set the config options via `feast config`: | Configuration Option | Description | | :--- | :--- | @@ -60,7 +58,7 @@ Configure SSL/TLS on the [Go SDK](https://godoc.org/github.com/feast-dev/feast/s ```go cli, err := feast.NewSecureGrpcClient("localhost", 6566, feast.SecurityConfig{ EnableTLS: true, - TLSCertPath: "/path/to/cert.pem", + TLSCertPath: "/path/to/cert.pem", })Option ``` @@ -86,15 +84,15 @@ FeastClient client = FeastClient.createSecure("localhost", 6566, | `setTLSEnabled()` | Enables SSL/TLS functionality when connecting to Feast if `true` | | `setCertificatesPath()` | Optional. Set the path of the root certificate used to verify Feast Service's identity. If omitted, uses system certificates. | -## **2. Authentication** +## **Authentication** {% hint style="warning" %} To prevent man in the middle attacks, we recommend that SSL/TLS be implemented prior to authentication. {% endhint %} -Authentication can be implemented to identify and validate client requests to Feast Core and Feast Online Serving. Currently, Feast uses[ ](https://auth0.com/docs/protocols/openid-connect-protocol)[Open ID Connect \(OIDC\)](https://auth0.com/docs/protocols/openid-connect-protocol) ID tokens \(i.e. [Google Open ID Connect](https://developers.google.com/identity/protocols/oauth2/openid-connect)\) to authenticate client requests. +Authentication can be implemented to identify and validate client requests to Feast Core and Feast Online Serving. Currently, Feast uses[ ](https://auth0.com/docs/protocols/openid-connect-protocol)[Open ID Connect \(OIDC\)](https://auth0.com/docs/protocols/openid-connect-protocol) ID tokens \(i.e. [Google Open ID Connect](https://developers.google.com/identity/protocols/oauth2/openid-connect)\) to authenticate client requests. -#### Configuring Authentication in Feast Core and Feast Online Serving +### Configuring Authentication in Feast Core and Feast Online Serving Authentication can be configured for Feast Core and Feast Online Serving via properties in their corresponding `application.yml` files: @@ -120,14 +118,14 @@ Behind the scenes, Feast Core and Feast Online Serving authenticate by: ### **Authenticating Serving with Feast Core** -Feast Online Serving communicates with Feast Core during normal operation. When both authentication and authorization are enabled on Feast Core, Feast Online Serving is forced to authenticate its requests to Feast Core. Otherwise, Feast Online Serving produces an Authentication failure error when connecting to Feast Core. +Feast Online Serving communicates with Feast Core during normal operation. When both authentication and authorization are enabled on Feast Core, Feast Online Serving is forced to authenticate its requests to Feast Core. Otherwise, Feast Online Serving produces an Authentication failure error when connecting to Feast Core. - Properties used to configure Serving authentication via `application.yml`: +Properties used to configure Serving authentication via `application.yml`: | Configuration Property | Description | | :--- | :--- | | `feast.core-authentication.enabled` | Requires Feast Online Serving to authenticate when communicating with Feast Core. | -| `feast.core-authentication.provider` | Selects provider Feast Online Serving uses to retrieve credentials then used to authenticate requests to Feast Core. Valid providers are `google` and `oauth`. | +| `feast.core-authentication.provider` | Selects provider Feast Online Serving uses to retrieve credentials then used to authenticate requests to Feast Core. Valid providers are `google` and `oauth`. | {% tabs %} {% tab title="Google Provider" %} @@ -139,56 +137,48 @@ Google Provider automatically extracts the credential from the credential JSON f {% tab title="OAuth Provider" %} OAuth Provider makes an OAuth [client credentials](https://auth0.com/docs/flows/call-your-api-using-the-client-credentials-flow) request to obtain the credential. OAuth requires the following options to be set at `feast.security.core-authentication.options.`: +| Configuration Property | Description | +| :--- | :--- | + + +| `oauth_url` | Target URL receiving the client-credentials request. | +| :--- | :--- | + + +| `grant_type` | OAuth grant type. Set as `client_credentials` | +| :--- | :--- | + + +| `client_id` | Client Id used in the client-credentials request. | +| :--- | :--- | + + +| `client_secret` | Client secret used in the client-credentials request. | +| :--- | :--- | + + - - - - - - - - - - - - - - - - - - - - - - - - + - - - + - + +
Configuration PropertyDescription
oauth_url - Target URL receiving the client-credentials request.
grant_type - OAuth grant type. Set as client_credentials -
client_id - Client Id used in the client-credentials request.
client_secret - Client secret used in the client-credentials request.
audience - + audience +

Target audience of the credential. Set to host URL of Feast Core.

(i.e. https://localhost if Feast Core listens on localhost).

- -
jwkEndpointURI - HTTPS URL used to retrieve a JWK that can be used to decode the credential.
+ +| `jwkEndpointURI` | HTTPS URL used to retrieve a JWK that can be used to decode the credential. | +| :--- | :--- | {% endtab %} {% endtabs %} ### **Enabling Authentication in Python SDK/CLI** -Configure the [Feast Python SDK](https://api.docs.feast.dev/python/) and [Feast CLI](../getting-started/connect-to-feast/connecting-to-feast.md) to use authentication via `feast config`: +Configure the [Feast Python SDK](https://api.docs.feast.dev/python/) and [Feast CLI](../getting-started/connect-to-feast/feast-cli.md) to use authentication via `feast config`: ```python $ feast config set enable_auth true @@ -197,7 +187,7 @@ $ feast config set enable_auth true | Configuration Option | Description | | :--- | :--- | | `enable_auth` | Enables authentication functionality if set to `true`. | -| `auth_provider` | Use an authentication provider to obtain a credential for authentication. Currently supports `google` and `oauth`. | +| `auth_provider` | Use an authentication provider to obtain a credential for authentication. Currently supports `google` and `oauth`. | | `auth_token` | Manually specify a static token for use in authentication. Overrules `auth_provider` if both are set. | {% tabs %} @@ -220,44 +210,38 @@ $ export GOOGLE_APPLICATION_CREDENTIALS="path/to/key.json" {% tab title="OAuth Provider" %} OAuth Provider makes an OAuth [client credentials](https://auth0.com/docs/flows/call-your-api-using-the-client-credentials-flow) request to obtain the credential/token used to authenticate Feast requests. The OAuth provider requires the following config options to be set via `feast config`: +| Configuration Property | Description | +| :--- | :--- | + + +| `oauth_token_request_url` | Target URL receiving the client-credentials request. | +| :--- | :--- | + + +| `oauth_grant_type` | OAuth grant type. Set as `client_credentials` | +| :--- | :--- | + + +| `oauth_client_id` | Client Id used in the client-credentials request. | +| :--- | :--- | + + +| `oauth_client_secret` | Client secret used in the client-credentials request. | +| :--- | :--- | + + - - - - - - - - - - - - - - - - - - - - - - - - + - + +
Configuration PropertyDescription
oauth_token_request_url - Target URL receiving the client-credentials request.
oauth_grant_type - OAuth grant type. Set as client_credentials -
oauth_client_id - Client Id used in the client-credentials request.
oauth_client_secret - Client secret used in the client-credentials request.
oauth_audience - + oauth_audience +

Target audience of the credential. Set to host URL of target Service.

(https://localhost if Service listens on localhost).

- +
{% endtab %} {% endtabs %} @@ -272,7 +256,7 @@ Configure the [Feast Java SDK](https://javadoc.io/doc/dev.feast/feast-sdk/latest cred, _ := feast.NewGoogleCredential("localhost:6566") cli, _ := feast.NewSecureGrpcClient("localhost", 6566, feast.SecurityConfig{ // Specify the credential to provide tokens for Feast Authentication. - Credential: cred, + Credential: cred, }) ``` @@ -304,39 +288,34 @@ OAuth Credential makes an OAuth [client credentials](https://auth0.com/docs/flow cred := feast.NewOAuthCredential("localhost:6566", "client_id", "secret", "https://oauth.endpoint/auth") ``` +| Parameter | Description | +| :--- | :--- | + + - - - - - - - - + - - - - - - - - - - - + - + +
ParameterDescription
audience - + audience +

Target audience of the credential. Set to host URL of target Service.

( https://localhost if Service listens on localhost).

- -
clientId - Client Id used in the client-credentials request.
clientSecret - Client secret used in the client-credentials request.
endpointURL - Target URL to make the client-credentials request to.
+ +| `clientId` | Client Id used in the client-credentials request. | +| :--- | :--- | + + +| `clientSecret` | Client secret used in the client-credentials request. | +| :--- | :--- | + + +| `endpointURL` | Target URL to make the client-credentials request to. | +| :--- | :--- | {% endtab %} {% endtabs %} @@ -391,54 +370,46 @@ CallCredentials credentials = new OAuthCredentials(Map.of( "jwkEndpointURI", "https://jwk.endpoint/jwk")); ``` +| Parameter | Description | +| :--- | :--- | + + - - - - - - - - + - - - - - - - - - - - + - - - - - - - - - + +
ParameterDescription
audience - + audience +

Target audience of the credential. Set to host URL of target Service.

( https://localhost if Service listens on localhost).

- -
grant_type - OAuth grant type. Set as client_credentials -
client_id - Client Id used in the client-credentials request.
client_secret - Client secret used in the client-credentials request.
oauth_url - Target URL to make the client-credentials request to obtain credential.
jwkEndpointURI - HTTPS URL used to retrieve a JWK that can be used to decode the credential.
+ +| `grant_type` | OAuth grant type. Set as `client_credentials` | +| :--- | :--- | + + +| `client_id` | Client Id used in the client-credentials request. | +| :--- | :--- | + + +| `client_secret` | Client secret used in the client-credentials request. | +| :--- | :--- | + + +| `oauth_url` | Target URL to make the client-credentials request to obtain credential. | +| :--- | :--- | + + +| `jwkEndpointURI` | HTTPS URL used to retrieve a JWK that can be used to decode the credential. | +| :--- | :--- | {% endtab %} {% endtabs %} -## 3. Authorization +## Authorization {% hint style="info" %} Authorization requires that authentication be configured to obtain a user identity for use in authorizing requests. @@ -451,7 +422,7 @@ Authorization provides access control to FeatureTables and/or Features based on ### **Authorization API/Server** -![Feast Authorization Flow](../.gitbook/assets/rsz_untitled23.jpg) +![Feast Authorization Flow](../.gitbook/assets/rsz_untitled23%20%282%29.jpg) Feast delegates Authorization grants to an external Authorization Server that implements the [Authorization Open API specification](https://github.com/feast-dev/feast/blob/master/common/src/main/resources/api.yaml). @@ -471,12 +442,10 @@ Authorization can be configured for Feast Core and Feast Online Serving via prop This example of the [Authorization Server with Keto](https://github.com/feast-dev/feast-keto-auth-server) can be used as a reference implementation for implementing an Authorization Server that Feast supports. {% endhint %} -## **4. Authentication & Authorization** +## **Authentication & Authorization** When using Authentication & Authorization, consider: * Enabling Authentication without Authorization makes authentication **optional**. You can still send unauthenticated requests. * Enabling Authorization forces all requests to be authenticated. Requests that are not authenticated are **dropped.** - - diff --git a/docs/advanced/troubleshooting.md b/docs/advanced/troubleshooting.md index 8936a6b77c7..1b1098eff4d 100644 --- a/docs/advanced/troubleshooting.md +++ b/docs/advanced/troubleshooting.md @@ -1,6 +1,10 @@ # Troubleshooting -If at any point in time you cannot resolve a problem, please see the [Community](../getting-help.md) section for reaching out to the Feast community. +{% hint style="warning" %} +This page applies to Feast 0.7. The content may be out of date for Feast 0.8+ +{% endhint %} + +If at any point in time you cannot resolve a problem, please see the [Community](../community.md) section for reaching out to the Feast community. ## How can I verify that all services are operational? @@ -24,7 +28,7 @@ kubectl get pods First locate the the host and port of the Feast Services. -#### **Docker Compose \(from inside the docker network\)** +### **Docker Compose \(from inside the docker network\)** You will probably need to connect using the hostnames of services and standard Feast ports: @@ -35,7 +39,7 @@ export FEAST_HISTORICAL_SERVING_URL=historical_serving:6567 export FEAST_JOBCONTROLLER_URL=jobcontroller:6570 ``` -#### **Docker Compose \(from outside the docker network\)** +### **Docker Compose \(from outside the docker network\)** You will probably need to connect using `localhost` and standard ports: @@ -46,7 +50,7 @@ export FEAST_HISTORICAL_SERVING_URL=localhost:6567 export FEAST_JOBCONTROLLER_URL=localhost:6570 ``` -#### **Google Kubernetes Engine \(GKE\)** +### **Google Kubernetes Engine \(GKE\)** You will need to find the external IP of one of the nodes as well as the NodePorts. Please make sure that your firewall is open for these ports: diff --git a/docs/advanced/upgrading.md b/docs/advanced/upgrading.md index 8e1c426584c..6e1744ade17 100644 --- a/docs/advanced/upgrading.md +++ b/docs/advanced/upgrading.md @@ -1,6 +1,6 @@ # Upgrading Feast -## Migration v0.6 to v0.7 +## Migration from v0.6 to v0.7 ### Feast Core Validation changes @@ -22,7 +22,7 @@ Feast now prevents feature sets from being applied if no store is subscribed to ### Feast Core's Job Coordinator is now Feast Job Controller -In v0.7, Feast Core's Job Coordinator has been decoupled from Feast Core and runs as a separate Feast Job Controller application. See its [Configuration reference](../reference/configuration-reference/#2-feast-core-serving-and-job-controller) for how to configure Feast Job Controller. +In v0.7, Feast Core's Job Coordinator has been decoupled from Feast Core and runs as a separate Feast Job Controller application. See its [Configuration reference](../reference/configuration-reference.md#2-feast-core-serving-and-job-controller) for how to configure Feast Job Controller. **Ingestion Job API** @@ -48,7 +48,7 @@ Users of Ingestion Job via Python SDK \(ie `feast ingest-jobs list` or `client.s * Rename`feast.security.authorization.options.subjectClaim` to `feast.security.authentication.options.subjectClaim` * Rename `feast.logging.audit.messageLoggingEnabled` to `feast.audit.messageLogging.enabled` -## Migration v0.5 to v0.6 +## Migration from v0.5 to v0.6 ### Database schema @@ -94,7 +94,7 @@ Minor changes: has now `version` and `delivery_status`. -## Migrate v0.4 to v0.6 +## Migration from v0.4 to v0.6 ### Database diff --git a/docs/getting-help.md b/docs/community.md similarity index 68% rename from docs/getting-help.md rename to docs/community.md index e9ead181caa..c98421aeb8b 100644 --- a/docs/getting-help.md +++ b/docs/community.md @@ -1,6 +1,6 @@ # Community -### Links & Resources +## Links & Resources * [Slack](https://kubeflow.slack.com/messages/CE0L8T267): We use the channel [\#Feast](https://kubeflow.slack.com/messages/CE0L8T267) in [kubeflow.slack.com](https://join.slack.com/t/kubeflow/shared_invite/zt-cpr020z4-PfcAue_2nw67~iIDy7maAQ). Feel free to ask questions or say hello! * [Mailing list](https://groups.google.com/d/forum/feast-dev): We have both a user and developer mailing list. @@ -11,19 +11,26 @@ * User surveys and meeting minutes. * Slide decks of conferences our contributors have spoken at. * [Feast GitHub Repository](https://github.com/feast-dev/feast/): Find the complete Feast codebase on GitHub. +* [Feast Linux Foundation Wiki](https://wiki.lfaidata.foundation/display/FEAST/Feast+Home): Our LFAI wiki page contains links to resources for contributors and maintainers. -### How can I get help? +## How can I get help? -* **Slack:** Need to speak to a human? Come ask a question in our Slack channel \(link above\) +* **Slack:** Need to speak to a human? Come ask a question in our Slack channel \(link above\). * **GitHub Issues:** Found a bug or need a feature? [Create an issue on GitHub](https://github.com/feast-dev/feast/issues/new). * **StackOverflow:** Need to ask a question on how to use Feast? We also monitor and respond to [StackOverflow](https://stackoverflow.com/questions/tagged/feast). -### Community Call +## Community Call -We have a community call every 2 weeks. Alternating between two times: +We have a user and contributor community call every two weeks \(Asia & US friendly\). -* 11 am \(UTC + 8\) -* 5 pm \(UTC + 8\) +### Frequency \(every 2 weeks\) -Join the [feast-dev](getting-help.md#feast-development) mailing list to receive a Google calendar invitation. +* **Asia \(UTC+08:00\):** Wednesday 10:00 am to 10:30 am. +* **US West Coast \(PT\):** Tuesday 18:00 pm to 18:30 pm. + +### Links + +* Calendar: [Feast Community Calendar \(Linux Foundation\)](https://wiki.lfaidata.foundation/pages/viewpage.action?pageId=30408973) +* Zoom: [https://zoom.us/j/6325193230](https://zoom.us/j/6325193230) +* Meeting notes: [https://bit.ly/feast-notes](https://bit.ly/feast-notes%20) diff --git a/docs/concepts/architecture.md b/docs/concepts/architecture.md index eab6941d8e7..84ffa01739d 100644 --- a/docs/concepts/architecture.md +++ b/docs/concepts/architecture.md @@ -1,36 +1,51 @@ # Architecture -![Feast high-level flow](../.gitbook/assets/blank-diagram-4.svg) - -### **Feast Core** - -Feast Core is the central management service of a Feast deployment. It's role is to: - -* Allow users to create [entities](entities.md). -* Allow users to create features through the creation of [feature tables](feature-tables.md). -* Act as a source of truth and central registry of feature tables. - -### **Feast Ingestion** - -Before you ingest data into Feast, first register one or more entity, then register feature tables. These [feature tables](feature-tables.md) tell Feast where to find their data and how to ingest it. The feature tables also describe the characteristics of the data for validation purposes. After a feature table is registered, you can start a Spark job to populate a store with data from the defined source in the feature table specification. - -To ensure stores are populated with data, you must publish the data to a [source](sources.md). Currently, Feast supports a few batch and stream sources. Feast users \(or pipelines\) ingest batch data through the [Feast Python SDK](../getting-started/connect-to-feast/python-sdk.md) using its `ingest()` method. The SDK publishes the data into the batch source specified for the feature table's batch source. - -Streaming systems can also ingest data into Feast. This is done by publishing to the correct stream source from the feature table specification in the expected format. The topic and brokers can be found on the feature table's stream source if specified during registration. - -### **Stores** - -Stores are nothing more than databases used to store feature data. Feast loads data into stores through an ingestion process, after which the data can be served through the [Feast Online Serving API](https://api.docs.feast.dev/grpc/feast.serving.pb.html). Stores are documented in the following section. - -### **Feast Online Serving** - -`Feast Online Serving` is the data-access layer through which end users and production systems retrieve feature data. Each `Serving` instance is backed by a [store](). - -Because Feast supports multiple store types \(online, historical\), multiple instances of a deployed `Feast Online Serving` is common: those for online serving and those for historical. This means Feast allows for any number of `Feast Online Serving` deployments, presenting the possibility to use a `Feast Online Serving` deployment per production system, with its own stores and population jobs. - -`Feast Online Serving` deployments subscribe to all feature data, consuming all features known to a `Feast Core` deployment. - -Feature retrieval \(and feature references\) are documented in more detail in subsequent sections. - -{% page-ref page="../user-guide/feature-retrieval.md" %} +![](../.gitbook/assets/image%20%285%29.png) + +## Sequence description + +1. **Log Raw Events:** Production backend applications are configured to emit internal state changes as events to a stream. +2. **Create Stream Features:** Stream processing systems like Flink, Spark, and Beam are used to transform and refine events and to produce features that are logged back to the stream. +3. **Log Streaming Features:** Both raw and refined events are logged into a data lake or batch storage location. +4. **Create Batch Features:** ELT/ETL systems like Spark and SQL are used to transform data in the batch store. +5. **Define and Ingest Features:** The Feast user defines [feature tables](feature-tables.md) based on the features available in batch and streaming sources and publish these definitions to Feast Core. +6. **Poll Feature Definitions:** The Feast Job Service polls for new or changed feature definitions. +7. **Start Ingestion Jobs:** Every new feature table definition results in a new ingestion job being provisioned \(see limitations\). +8. **Batch Ingestion:** Batch ingestion jobs are short-lived jobs that load data from batch sources into either an offline or online store \(see limitations\). +9. **Stream Ingestion:** Streaming ingestion jobs are long-lived jobs that load data from stream sources into online stores. A stream source and batch source on a feature table must have the same features/fields. +10. **Model Training:** A model training pipeline is launched. It uses the Feast Python SDK to retrieve a training dataset and trains a model. +11. **Get Historical Features:** Feast exports a point-in-time correct training dataset based on the list of features and entity DataFrame provided by the model training pipeline. +12. **Deploy Model:** The trained model binary \(and list of features\) are deployed into a model serving system. +13. **Get Prediction:** A backend system makes a request for a prediction from the model serving service. +14. **Retrieve Online Features:** The model serving service makes a request to the Feast Online Serving service for online features using a Feast SDK. +15. **Return Prediction:** The model serving service makes a prediction using the returned features and returns the outcome. + +{% hint style="warning" %} +Limitations + +* Feast 0.8 has no offline store. Batch retrieval is direct from source. We plan to implement an optional offline store in Feast 0.9 +* Only Redis is supported for online storage. +* Batch ingestion jobs must be triggered from your own scheduler like Airflow. Streaming ingestion jobs are automatically launched by the Feast Job Service. +{% endhint %} + +## Components: + +A complete Feast deployment contains the following components: + +* **Feast Core:** Acts as the central registry for feature and entity definitions in Feast. +* **Feast Job Service:** Manages data processing jobs that load data from sources into stores, and jobs that export training datasets. +* **Feast Online Serving:** Provides low-latency access to feature values in an online store. +* **Feast Python SDK:** The primary user facing SDK. Used to: + * Manage feature definitions with Feast Core. + * Launch jobs through the Feast Job Service. + * Retrieve training datasets. + * Retrieve online features. +* **Online Store:** The online store is a database that stores only the latest feature values for each entity entity. The online store can be populated by either batch ingestion jobs \(in the case the user has no streaming source\), or can be populated by a streaming ingestion job from a streaming source. Feast Online Serving looks up feature values from the online store. +* **Offline Store:** The offline store persists batch data that has been ingested into Feast. This data is used for producing training datasets. + +Please see the [configuration reference](../reference/configuration-reference.md#overview) for more details on configuring these components. + +{% hint style="info" %} +Java and Go SDKs are also available for online feature retrieval. See [API Reference](https://github.com/feast-dev/feast/tree/dc71813b7abbdb01139c0ba80da539350c9eef83/docs/reference/api/README.md). +{% endhint %} diff --git a/docs/concepts/concepts.md b/docs/concepts/concepts.md deleted file mode 100644 index e54bd488d6a..00000000000 --- a/docs/concepts/concepts.md +++ /dev/null @@ -1,124 +0,0 @@ -# Concepts - -## Architecture - -![Logical diagram of a typical Feast deployment](../.gitbook/assets/basic-architecture-diagram%20%282%29.svg) - -The core components of a Feast deployment are - -* **Feast Core:** Feast Core is a centralized service that acts as the authority on features within an organization. Typically there is only one "Core" deployment per organization, with all feature management happening through it. -* **Feast Ingestion Jobs:** Feast ingestion jobs retrieve feature data from user defined data sources and populate serving stores with this feature data. These jobs are managed by Feast Core. Data can either be sources from existing sources \(like [Kafka](https://kafka.apache.org/)\), or it can be loaded into Feast through its API. -* **Feast Serving:** Feast Serving is the data access layer through which end users and production systems retrieve feature data. Each Serving store is backed by one or more databases. These databases are updated by the Feast ingestion jobs. There are two types of stores: batch and online. Batch stores hold large volumes historical data, while online stores only hold the latest feature values. - -## Data Model - -### Feature Set - -User data is typically in the form of dataframes, tables in data warehouses, or events on a stream. These data sources are loaded into Feast in order to serve features for model training or serving. - -Feature sets allow for groups of fields in these data sources to be ingested and stored together. This allows for efficient storage and logical namespacing of data. - -When data is loaded from these sources, each field in the feature set must be found in every record of the data source. Fields from these data sources must be either a timestamp, an entity, or a feature. - -{% hint style="info" %} -Feature sets are a grouping of feature sets based on how they are loaded into Feast. They ensure that data is efficiently stored during ingestion. Feature sets are not a grouping of features for retrieval of features. During retrieval it is possible to retrieve feature values from any number of feature sets. -{% endhint %} - -#### Customer Transactions Example - -Below is an example of a basic `customer transactions` feature set that has been exported to YAML: - -{% tabs %} -{% tab title="customer\_transactions\_feature\_set.yaml" %} -```yaml -name: customer_transactions -kind: feature_set -entities: -- name: customer_id - valueType: INT64 -features: -- name: daily_transactions - valueType: FLOAT -- name: total_transactions - valueType: FLOAT - maxAge: 3600s -``` -{% endtab %} -{% endtabs %} - -The dataframe below \(`customer_data.csv`\) contains the features and entities of the above feature set - -| datetime | customer\_id | daily\_transactions | total\_tra**nsactions** | -| :--- | :--- | :--- | :--- | -| 2019-01-01 01:00:00 | 20001 | 5.0 | 14.0 | -| 2019-01-01 01:00:00 | 20002 | 2.6 | 43.0 | -| 2019-01-01 01:00:00 | 20003 | 4.1 | 154.0 | -| 2019-01-01 01:00:00 | 20004 | 3.4 | 74.0 | - -In order to ingest feature data into Feast for this specific feature set: - -```python -# Load dataframe -customer_df = pd.read_csv("customer_data.csv") - -# Create feature set from YAML (using YAML is optional) -cust_trans_fs = FeatureSet.from_yaml("customer_transactions_feature_set.yaml") - -# Load feature data into Feast for this specific feature set -client.ingest(cust_trans_fs, customer_data) -``` - -### Feature - -A feature is an individual measurable property or characteristic of a phenomenon being observed. Features are the most important concepts within a feature store. Feature data is used both as input to models during training and when models are served in production. - -In the context of Feast, features are values that are associated with either one or more entities over time. In Feast, these values are either primitives or lists of primitives. Each feature can also have additional information attached to it. For example whether it is a categorical feature or numerical. - -{% hint style="info" %} -Features in Feast are defined within Feature Sets and are not treated as standalone concepts. -{% endhint %} - -### Entity - -An entity type is any object in an organization that needs to be modeled and on which information should be stored. Entity types are usually recognizable concepts, either concrete or abstract, such as persons, places, things, or events which have relevance to the modeled system. - -An entity is an instance of an entity type. - -* Examples of entity types in the context of ride-hailing and food delivery: `customer`, `order`, `driver`, `restaurant`, `dish`, `area`. -* A specific driver, for example a driver with ID `D011234` would be an entity of the entity type `driver` - -An entity is the object on which features are observed. For example we could have a feature `total_trips_24h` on the driver `D01123` with a feature value of `11`. - -In the context of Feast, entities are important because they are used as keys when looking up feature values. Entities are also used when joining feature values between different feature sets in order to build one large data set to train a model, or to serve a model. - -{% hint style="info" %} -Entities in Feast are defined within Feature Sets and are not treated as standalone concepts. -{% endhint %} - -### Types - -Feast supports the following types for feature values - -* BYTES -* STRING -* INT32 -* INT64 -* DOUBLE -* FLOAT -* BOOL -* BYTES\_LIST -* STRING\_LIST -* INT32\_LIST -* INT64\_LIST -* DOUBLE\_LIST -* FLOAT\_LIST -* BOOL\_LIST - -## Glossary - -| Term | Description | -| :--- | :--- | -| Feast deployment | A complete Feast system as it is deployed. Consists out of a single Feast Core deployment and one or more Feast Serving deployments. | -| Feast Core | The centralized service which acts as a registry and authority of features. Organizations should only deploy a single Feast Core instance. Feast Core also manages the ingestion of feature data and population of Feast Serving data stores. | -| Feast Serving | Feast Serving is a service used to access both online and batch feature data. Feast Serving deployments are backed by one or more databases. | - diff --git a/docs/concepts/entities.md b/docs/concepts/entities.md index 3691f711431..d0d88b923b3 100644 --- a/docs/concepts/entities.md +++ b/docs/concepts/entities.md @@ -1,41 +1,41 @@ # Entities -### Overview +## Overview -An entity is any domain object that can be modelled and about which information can be stored. Entities are usually recognisable concepts, either concrete or abstract, such as persons, places, things, or events which have relevance to the modelled system. +An entity is any domain object that can be modeled and about which information can be stored. Entities are usually recognizable concepts, either concrete or abstract, such as persons, places, things, or events. -* Examples of entity types in the context of ride-hailing and food delivery: `customer`, `order`, `driver`, `restaurant`, `dish`, `area`. -* A specific driver, for example a driver with ID `D011234` would be an entity of the entity type `driver` +Examples of entities in the context of ride-hailing and food delivery: `customer`, `order`, `driver`, `restaurant`, `dish`, `area`. -An entity is the domain object on which features are observed. For example, we could have a feature `total_trips_24h` for driver `D011234` with a feature value of `11`. +Entities are important in the context of feature stores since features are always properties of a specific entity. For example, we could have a feature `total_trips_24h` for driver `D011234` with a feature value of `11`. -Entities are important for Feast because they are used as keys when searching for feature values. Entities are also used when joining feature values from different feature tables to build a large data set that is used to train or serve models. +Feast uses entities in the following way: -### Structure of an Entity +* Entities serve as the keys used to look up features for producing training datasets and online feature values. +* Entities serve as a natural grouping of features in a feature table. A feature table must belong to an entity \(which could be a composite entity\) + +## Structure of an Entity When creating an entity specification, consider the following fields: -* **name**: Name of the entity -* **description**: Description of the entity -* **value\_type**: Value type of the entity -* **labels**: User-defined metadata +* **Name**: Name of the entity +* **Description**: Description of the entity +* **Value Type**: Value type of the entity. Feast will attempt to coerce entity columns in your data sources into this type. +* **Labels**: Labels are maps that allow users to attach their own metadata to entities A valid entity specification is shown below: ```python -from feast import Entity, ValueType - -# Create a customer entity customer = Entity( - "customer_id", - "Customer id for ride customer", - ValueType.INT64 + name="customer_id", + description="Customer id for ride customer", + value_type=ValueType.INT64, + labels={} ) ``` -### Working with an Entity +## Working with an Entity -Creating an Entity: +### Creating an Entity: ```python # Create a customer entity @@ -43,7 +43,7 @@ customer_entity = Entity(name="customer_id", description="ID of car customer") client.apply_entity(customer_entity) ``` -Updating an Entity: +### Updating an Entity: ```python # Update a customer entity @@ -56,12 +56,9 @@ Permitted changes include: * The entity's description and labels -{% hint style="warning" %} -You **cannot** change the following: - -* Project or name of an entity -* Types of entity -{% endhint %} +The following changes are note permitted: -Visit [EntitySpec](https://api.docs.feast.dev/grpc/feast.core.pb.html#EntitySpecV2) for the entity-specification API. +* Project +* Name of an entity +* Type diff --git a/docs/concepts/feature-references.md b/docs/concepts/feature-references.md deleted file mode 100644 index 4f712a24c45..00000000000 --- a/docs/concepts/feature-references.md +++ /dev/null @@ -1,36 +0,0 @@ -# Feature References - -## Overview - -In Feast, each feature can be uniquely addressed through a feature reference. A feature reference is composed of the following components: - -* Feature Table name -* Feature name - -## Structure of a Feature Reference - -A string based feature reference takes on the following format: - -`:` - -```python -# Feature references -feature_refs = [ - "driver_trips:average_daily_rides", - "driver_trips:maximum_daily_rides", - "driver_trips:rating", -] -``` - -Feature references only apply to a single `project`. Features cannot be retrieved across projects in a single request. - -## Working with a Feature Reference - -#### Feature Retrieval - -Feature retrieval \(or serving\) is the process of retrieving either historical features or online features from Feast, for the purposes of training or serving a model. - -Feast attempts to unify the process of retrieving features in both the historical and online case. It does this through the creation of feature references. One of the major advantages of using Feast is that you have a single semantic reference to a feature. These feature references can then be stored alongside your model and loaded into a serving layer where it can be used for online feature retrieval. - -More information about how to perform feature retrieval for historical and online features can be found in the sections under **User Guide**. - diff --git a/docs/concepts/feature-tables.md b/docs/concepts/feature-tables.md index 470be7a755e..92eb9963f62 100644 --- a/docs/concepts/feature-tables.md +++ b/docs/concepts/feature-tables.md @@ -6,33 +6,25 @@ Feature tables are both a schema and a logical means of grouping features, data Feature tables serve the following purposes: -* They are a means for defining the location and properties of data [sources](sources.md). -* They are used to create within Feast a database-level structure for the storage of feature values. -* The data sources described within feature tables enable Feast to ingest and store features within Feast. -* They ensure data is efficiently stored during [ingestion](../user-guide/data-ingestion.md). +* Feature tables are a means for defining the location and properties of data [sources](sources.md). +* Feature tables are used to create within Feast a database-level structure for the storage of feature values. +* The data sources described within feature tables allow Feast to find and ingest feature data into stores within Feast. +* Feature tables ensure data is efficiently stored during [ingestion](../user-guide/define-and-ingest-features.md) by providing a grouping mechanism of features values that occur on the same event timestamp. {% hint style="info" %} -Feast does not yet apply feature transformations. Transformations are currently expected to happen before data is ingested into Feast. The data sources described within feature tables should reference feature values in their already computed form. +Feast does not yet apply feature transformations. Transformations are currently expected to happen before data is ingested into Feast. The data sources described within feature tables should reference feature values in their already transformed form. {% endhint %} ### Features -A feature is an individual measurable property or characteristic of an observable phenomenon. For example, in a bank, a feature could be `total_foreign_transactions_24h` for a specific class of credit cards the bank issues. Feature data is the input both for training models, and for models served in production. +A feature is an individual measurable property observed on an entity. For example the amount of transactions \(feature\) a customer \(entity\) has completed. Features are used for both model training and scoring \(batch, online\). -{% hint style="info" %} -Features are the most important concepts within a feature store. -{% endhint %} - -In Feast, features are values that are associated with one or more [entities](entities.md). These values are either primitives or lists of primitives. Each feature can also have additional information attached to it. - -You define a feature by providing a name and value type. In our example, we use a name and value type that might be used in a ride-hailing company: +Features are defined as part of feature tables. Since Feast does not apply transformations, a feature is basically a schema that only contains a name and a type: ```python avg_daily_ride = Feature("average_daily_rides", ValueType.FLOAT) ``` -Features act purely as a schema within feature tables. Feature tables and features act as normal database tables and columns. - Visit [FeatureSpec](https://api.docs.feast.dev/grpc/feast.core.pb.html#FeatureSpecV2) for the complete feature specification API. ## Structure of a Feature Table @@ -41,20 +33,20 @@ Feature tables contain the following fields: * **Name:** Name of feature table. This name must be unique within a project. * **Entities:** List of [entities](entities.md) to associate with the features defined in this feature table. Entities are used as lookup keys when retrieving features from a feature table. -* **Features:** List of features within this feature table. +* **Features:** List of features within a feature table. * **Labels:** Labels are arbitrary key-value properties that can be defined by users. -* **Max age:** Max age affect the retrieval of features from a feature table. Age is measured as the duration of time between the event timestamp of a feature and the lookup time on an entity key used to retrieve the feature. Feature values outside max age will be returned as unset values. Max age allows for eviction of keys from online stores and limits the amount of scanning for historical feature values during retrieval. -* **Batch Source:** The batch data source from which you can ingest feature values into Feast. Visit [Sources](sources.md) to learn more about them. -* **Stream Source:** The streaming data source from which you can ingest streaming feature values into Feast. Visit [Sources](sources.md) to learn more about them. +* **Max age:** Max age affect the retrieval of features from a feature table. Age is measured as the duration of time between the event timestamp of a feature and the lookup time on an [entity key](glossary.md#entity-key) used to retrieve the feature. Feature values outside max age will be returned as unset values. Max age allows for eviction of keys from online stores and limits the amount of historical scanning required for historical feature values during retrieval. +* **Batch Source:** The batch data source from which Feast will ingest feature values into stores. This can either be used to back-fill stores before switching over to a streaming source, or it can be used as the primary source of data for a feature table. Visit [Sources](sources.md) to learn more about batch sources. +* **Stream Source:** The streaming data source from which you can ingest streaming feature values into Feast. Streaming sources must be paired with a batch source containing the same feature values. A streaming source is only used to populate online stores. The batch equivalent source that is paired with a streaming source is used during the generation of historical feature datasets. Visit [Sources](sources.md) to learn more about stream sources. -Here is a ride-hailing example of a valid feature-table specification: +Here is a ride-hailing example of a valid feature table specification: {% tabs %} {% tab title="driver\_trips\_feature\_table.py" %} ```python from feast import BigQuerySource, FeatureTable, Feature, ValueType +from google.protobuf.duration_pb2 import Duration -# Create an empty feature table driver_ft = FeatureTable( name="driver_trips", entities=["driver_id"], @@ -62,7 +54,7 @@ driver_ft = FeatureTable( Feature("average_daily_rides", ValueType.FLOAT), Feature("rating", ValueType.FLOAT) ], - max_age=14400, + max_age=Duration(seconds=3600), labels={ "team": "driver_matching" }, @@ -79,13 +71,11 @@ driver_ft = FeatureTable( {% endtab %} {% endtabs %} -When you register a feature table, at a minimum specify a batch source to populate the feature table. Stream sources are optional. They are used to stream feature values into online stores. - By default, Feast assumes that features specified in the feature-table specification corresponds one-to-one to the fields found in the sources. All features defined in a feature table should be available in the defined sources. -However, if the names of the fields in the batch source are different from the names of features, you can use `field_mappings` to ensure the names correspond. +Field mappings can be used to map features defined in Feast to fields as they occur in data sources. -In the example feature-specification table above, we use `field_mappings` to ensure the field named `rating` in the batch source is mapped to the feature named `driver_rating`. +In the example feature-specification table above, we use field mappings to ensure the feature named `rating` in the batch source is mapped to the field named `driver_rating`. ## Working with a Feature Table @@ -98,8 +88,6 @@ client.apply_feature_table(driver_ft) #### Updating a Feature Table -Feature table definitions may need to change over time to reflect more accurately your use case. In our ride-hailing example below, we update the max age: - ```python driver_ft = FeatureTable() @@ -110,17 +98,17 @@ driver_ft.labels = {"team": "marketplace"} client.apply_feature_table(driver_ft) ``` -Feast currently supports the following changes to feature tables: +#### Feast currently supports the following changes to feature tables: * Adding new features. -* Deleting existing features -* Changing the feature table's source, max age, and labels. +* Removing features. +* Updating source, max age, and labels. {% hint style="warning" %} Deleted features are archived, rather than removed completely. Importantly, new features cannot use the names of these deleted features. {% endhint %} -Feast currently does not support the following changes to feature tables: +#### Feast currently does not support the following changes to feature tables: * Changes to the project or name of a feature table. * Changes to entities related to a feature table. diff --git a/docs/concepts/glossary.md b/docs/concepts/glossary.md new file mode 100644 index 00000000000..04d9dbe7844 --- /dev/null +++ b/docs/concepts/glossary.md @@ -0,0 +1,36 @@ +# Glossary + +## **Entity key** + +The combination of entities that uniquely identify a row. For example a feature table with the composite entity of \(customer, country\) might have an entity key of \(1001, 5\). They key is used during lookups of feature values and for deduplicating historical rows. + +## Entity timestamp + +The timestamp on which an event occurred. The entity timestamp could describe the event time at which features were calculated, or it could describe the event timestamps at which outcomes were observed. + +Entity timestamps are commonly found on the entity dataframe and associated with the target variable \(outcome\) that needs to be predicted. These timestamps are the target on which point-in-time joins should be made. + +## Entity rows + +A combination of a single [entity key ](glossary.md#entity-key)and a single [entity timestamp](glossary.md#entity-timestamp). + +## Entity dataframe + +A collection of [entity rows](glossary.md#entity-rows). This dataframe is enriched with feature values before being used for model training. + +## Feature References + +Feature references uniquely identify feature values throughout Feast. Feature references can either be defined as objects or as strings. + +The structure of a feature reference in string form is as follows: + +`feature_table:feature` + +Example: + +`drivers_stream:unique_drivers` + +Feature references are unique within a project. It is not possible to reference \(or retrieve\) features from multiple projects at the same time. + +\*\*\*\* + diff --git a/docs/concepts/overview.md b/docs/concepts/overview.md index 0becb135531..e56dcd9da64 100644 --- a/docs/concepts/overview.md +++ b/docs/concepts/overview.md @@ -1,64 +1,21 @@ # Overview -## Using Feast +## Concepts -Feast is the bridge between your ML models and data. Feast enables your team to: +[Entities](entities.md) are objects in an organization like customers, transactions, and drivers, products, etc. -1. Create feature specifications to manage features, and load data that you want managed -2. Retrieve historical features for training models -3. Retrieve online features for serving models +[Sources](sources.md) are external sources of data where feature data can be found. -{% hint style="info" %} -Feast currently does not apply feature transformations to data. -{% endhint %} - -### Creating and managing features - -Feature creators model the data within their organization into Feast through the creation of [feature tables](feature-tables.md). - -Feature tables are both a schema and a means of identifying data sources for features. They allow Feast to know how to interpret your data, and optionally where to find it. Feature tables allow you to define domain [entities](entities.md) along with the features that are available on these entities. Feature tables also allow you to define schemas that describe properties of the respective data, which in turn can be used for validation purposes. - -After you register a feature table, Feast creates the relevant schemas to store feature data within its feature [stores](). These stores are then populated by [ingestion jobs](../user-guide/data-ingestion.md) that ingest data from data [sources](sources.md). The now data-rich stores enable Feast to provide access to features for training and serving. Alternatively, you can [ingest](../user-guide/data-ingestion.md) data into Feast instead of using an external source. - -Visit [feature tables](feature-tables.md) to learn more about them. - -### Retrieving historical features during training - -Historical retrieval uses [feature references](../user-guide/feature-retrieval.md) through the[ Feast SDK](https://api.docs.feast.dev/python/) to retrieve historical features. For historical serving, Feast requires that you provide the entities and timestamps for the corresponding feature data. Feast produces a point-in-time correct dataset using the requested features. These features can be requested from an unlimited number of feature sets. - -{% hint style="info" %} -For historical serving, Feast stores all historical values. -{% endhint %} - -Stores supported: [BigQuery](https://cloud.google.com/bigquery) - -### Retrieving online features during serving - -Online retrieval uses feature references through the [Feast Online Serving API](https://api.docs.feast.dev/grpc/feast.serving.pb.html) to retrieve online features. Online serving allows for very low latency requests to feature data at very high throughput. - -{% hint style="info" %} -During online serving, Feast stores **only** the latest values for each feature. -{% endhint %} - -Stores supported: [Redis](https://redis.io/), [Redis Cluster](https://redis.io/topics/cluster-tutorial) +[Feature Tables](feature-tables.md) are objects that define logical groupings of features, data sources, and other related metadata. ## Concept Hierarchy -![](../.gitbook/assets/concept_hierarchy.png) - -Feast resources are arranged in the above hierarchy, with projects grouping one or more [entities](entities.md), which in turn groups [feature tables](feature-tables.md). These feature tables consist of [data sources](sources.md) and multiple features. - -The logical grouping of these resources is important for namespacing and retrieval. Retrieval requires referencing individual features through feature references. These references uniquely identify a feature within a Feast deployment. - -### Concepts - -[Entities](entities.md) are objects in an organization that model a specific construct. Examples of these include customers, transactions, and drivers. - -[Sources](sources.md) are either internal or external data sources where feature data can be found. - -[Feature Tables](feature-tables.md) are schemas that define logical groupings of features, data sources, and other related metadata. +![](../.gitbook/assets/image%20%284%29%20%281%29.png) -[Stores]() are databases that maintain feature data that gets served to models during training or inference. +Feast contains the following core concepts: -[Ingestion](../user-guide/data-ingestion.md) is the process of loading data into Feast. +* **Projects:** Serve as a top level namespace for all Feast resources. Each project is a completely independent environment in Feast. Users can only work in a single project at a time. +* **Entities:** Entities are the objects in an organization on which features occur. They map to your business domain \(users, products, transactions, locations\). +* **Feature Tables:** Defines a group of features that occur on a specific entity. +* **Features:** Individual feature within a feature table. diff --git a/docs/concepts/sources.md b/docs/concepts/sources.md index 1069e8dfaa7..67f16f4e9a3 100644 --- a/docs/concepts/sources.md +++ b/docs/concepts/sources.md @@ -1,26 +1,34 @@ # Sources -### Overview +## Overview -A `source` is a data source that can be used to find feature data. Users define sources as part of [feature tables](feature-tables.md). Currently, Feast supports the following source types: +Sources are descriptions of external feature data and are registered to Feast as part of [feature tables](feature-tables.md). Once registered, Feast can ingest feature data from these sources into stores. -* Batch Source - * File - * [BigQuery](https://cloud.google.com/bigquery) -* Stream Source - * [Kafka](https://kafka.apache.org/) - * [Kinesis](https://aws.amazon.com/kinesis/) +Currently, Feast supports the following source types: -### Structure of a Source +### Batch Source -For both batch and stream sources, the following configurations are **necessary**: +* File \(as in Spark\): Parquet and CSV files supported. +* BigQuery -* **created\_timestamp\_column**: Name of column containing timestamp when data is created. -* **event\_timestamp\_column**: Name of column containing timestamp when event data occurred. +### Stream Source -When configuring data source options, see the [Feast Python API documentation](https://api.docs.feast.dev/python/) for more details. +* Kafka +* Kinesis -Some valid source specifications are shown below: +The following encodings are supported on streams + +* Avro +* Protobuf + +## Structure of a Source + +For both batch and stream sources, the following configurations are necessary: + +* **Event timestamp column**: Name of column containing timestamp when event data occurred. Used during point-in-time join of feature values to [entity timestamps](glossary.md#entity-timestamp). +* **Created timestamp column**: Name of column containing timestamp when data is created. Used to deduplicate data when multiple copies of the same [entity key](glossary.md#entity-key) is ingested. + +Example data source specifications: {% tabs %} {% tab title="batch\_sources.py" %} @@ -30,7 +38,7 @@ from feast.data_format import ParquetFormat batch_file_source = FileSource( file_format=ParquetFormat(), - file_url="file://feast/*", + file_url="file:///feast/customer.parquet", event_timestamp_column="event_timestamp", created_timestamp_column="created_timestamp", ) @@ -55,15 +63,11 @@ stream_kafka_source = KafkaSource( The [Feast Python API documentation](https://api.docs.feast.dev/python/) provides more information about options to specify for the above sources. -{% hint style="info" %} -When creating a Feature Table for use in training datasets, specify a batch source already containing materialized data. -{% endhint %} - -### Working with a Source +## Working with a Source -#### Creating a Source +### Creating a Source -Sources are required when specifying a [feature table](feature-tables.md): +Sources are defined as part of [feature tables](feature-tables.md): ```python batch_bigquery_source = BigQuerySource( diff --git a/docs/concepts/stores.md b/docs/concepts/stores.md new file mode 100644 index 00000000000..f50746a84b6 --- /dev/null +++ b/docs/concepts/stores.md @@ -0,0 +1,24 @@ +# Stores + +In Feast, a store is a database that is populated with feature data that will ultimately be served to models. + +## Offline \(Historical\) Store + +The offline store maintains historical copies of feature values. These features are grouped and stored in feature tables. During retrieval of historical data, features are queries from these feature tables in order to produce training datasets. + +{% hint style="warning" %} +Feast 0.8 does not support offline storage. Support will be added in Feast 0.9. +{% endhint %} + +## Online Store + +The online store maintains only the latest values for a specific feature. + +* Feature values are stored based on their [entity keys](glossary.md#entity-key) +* Feast currently supports Redis as an online store. +* Online stores are meant for very high throughput writes from ingestion jobs and very low latency access to features during online serving. + +{% hint style="info" %} +Feast only supports a single online store in production +{% endhint %} + diff --git a/docs/contributing/adding-a-new-store-1.md b/docs/contributing/adding-a-new-store-1.md deleted file mode 100644 index 56c06b9b4b6..00000000000 --- a/docs/contributing/adding-a-new-store-1.md +++ /dev/null @@ -1,87 +0,0 @@ -# Adding a New Store - -The following guide will explain the process of adding a new store through the introduction of a storage connector. - -## 1. Storage API - -Feast has an external module where storage interfaces are defined: [Storage API](https://github.com/gojek/feast/tree/master/storage/api/src/main/java/feast/storage/api) - -Feast interacts with a store at three points . - -1. **During initialization:** Store configuration is loaded into memory by Feast Serving and synchronized with Feast Core -2. **During ingestion of feature data.** [writer interfaces](https://github.com/gojek/feast/tree/master/storage/api/src/main/java/feast/storage/api/writer) are used by the Apache Beam ingestion jobs in order to populate stores \(historical or online\). -3. **During retrieval of feature data:** [Retrieval interfaces](https://github.com/gojek/feast/tree/master/storage/api/src/main/java/feast/storage/api/retriever) are used by Feast Serving in order to read data from stores in order to create training datasets or to serve online data. - -All three of these components should be implemented in order to have a complete storage connector. - -## 2. Adding a Storage Connector - -### 2.1 Initialization and configuration - -Stores are configured in Feast Serving. Feast Serving publishes its store configuration to Feast Core, after which Feast Core can start ingestion/population jobs to populate it. - -Store configuration is always in the form of a map<String, String>. The keys and configuration for stores are defined in [protos](https://github.com/gojek/feast/blob/master/protos/feast/core/Store.proto). This must be added in order to define a new store - -Then the store must be configured to be loaded through Feast Serving. The above configuration is loaded through [FeastProperties.java](https://github.com/gojek/feast/blob/a1937c374a4e39b7a75d828e7b7c3b87a64d9d6e/serving/src/main/java/feast/serving/config/FeastProperties.java#L175). - -Once configuration is loaded, the store will then be instantiated. - -* Feast Core: The [StoreUtil.java](https://github.com/gojek/feast/blob/master/ingestion/src/main/java/feast/ingestion/utils/StoreUtil.java#L85) instantiates new stores for the purposes of feature ingestion. -* Feast Serving: The [ServingServiceConfig](https://github.com/gojek/feast/blob/a1937c374a4e39b7a75d828e7b7c3b87a64d9d6e/serving/src/main/java/feast/serving/config/ServingServiceConfig.java#L56) instantiates new stores for the purposes of retrieval - -{% hint style="info" %} -In the future we plan to provide a plugin interface for adding stores. -{% endhint %} - -### 2.2 Feature Ingestion \(Writer\) - -Feast creates and manages ingestion/population jobs that stream in data from upstream data sources. Currently Feast only supports Kafka as a data source, meaning these jobs are all long running. Batch ingestion \(from users\) results in data being pushed to Kafka topics after which they are picked up by these "population" jobs and written to stores. - -In order for ingestion to succeed, the destination store must be writable. This means that Feast must be able to create the appropriate tables/schemas in the store and also write data from the population job into the store. - -Currently Feast Core starts and manages these population jobs that ingest data into stores \(although we are planning to move this responsibility to the serving layer\). Feast Core starts an [Apache Beam](https://beam.apache.org/) job which synchronously runs migrations on the destination store and subsequently starts consuming [FeatureRows](https://github.com/gojek/feast/blob/master/protos/feast/types/FeatureRow.proto) from Kafka and writing it into stores using a [writer](https://github.com/gojek/feast/tree/master/storage/api/src/main/java/feast/storage/api/writer). - -Below is a "happy path" of a batch ingestion process which includes a blocking step at the Python SDK. - -![](https://user-images.githubusercontent.com/6728866/74807906-91e73c00-5324-11ea-8ba5-2b43c7c5282b.png) - - - -The complete ingestion flow is executed by a [FeatureSink](https://github.com/gojek/feast/blob/master/storage/api/src/main/java/feast/storage/api/writer/FeatureSink.java). Two methods should be implemented - -* [prepareWrite\(\)](https://github.com/gojek/feast/blob/a1937c374a4e39b7a75d828e7b7c3b87a64d9d6e/storage/api/src/main/java/feast/storage/api/writer/FeatureSink.java#L45): Sets up storage backend for writing/ingestion. This method will be called once during pipeline initialisation. Typically this is used to apply schemas. -* [writer\(\)](https://github.com/gojek/feast/blob/a1937c374a4e39b7a75d828e7b7c3b87a64d9d6e/storage/api/src/main/java/feast/storage/api/writer/FeatureSink.java#L53): Retrieves an Apache Beam PTransform that is used to write data to this store. - -### 2.2 Feature Serving \(Retriever\) - -Feast Serving can serve both historical/batch features and online features. Depending on the store that is being added, you should implement either a historical/batch store or an online storage. - -#### 2.2.1 Historical Serving - -The historical serving interface is defined through the [HistoricalRetriever](https://github.com/gojek/feast/blob/master/storage/api/src/main/java/feast/storage/api/retriever/HistoricalRetriever.java) interface. Historical retrieval is an asynchronous process. The client submits a request for a dataset to be produced, and polls until it is ready. - -![High-level flow for batch retrieval](https://user-images.githubusercontent.com/6728866/74797157-702a8c80-5305-11ea-8901-bf6f4eb075f9.png) - -The current implementation of batch retrieval starts and ends with a file \(dataset\) in a Google Cloud Storage bucket. The user ingests an entity dataset. This dataset is loaded into a store \(BigQuery0, joined to features in a point-in-time correct way, then exported again to the bucket. - -Additionally, we have also implemented a [batch retrieval method ](https://github.com/gojek/feast/blob/a1937c374a4e39b7a75d828e7b7c3b87a64d9d6e/sdk/python/feast/client.py#L509)in the Python SDK. Depending on the means through which this new store will export data, this client may have to change. At the very least it would change if Google Cloud Storage isn't used as the staging bucket. - -The means through which you implement the export/import of data into the store will depend on your store. - -#### 2.2.2 Online Serving - -In the case of online serving it is necessary to implement an [OnlineRetriever](https://github.com/gojek/feast/blob/master/storage/api/src/main/java/feast/storage/api/retriever/OnlineRetriever.java). This online retriever will read rows directly and synchronously from an online database. The exact encoding strategy you use to store your data in the store would be defined in the FeatureSink. The OnlineRetriever is expected to read and decode those rows. - -## 3. Storage Connectors Examples - -Feast currently provides support for the following storage types - -Historical storage - -* [BigQuery](https://github.com/gojek/feast/tree/master/storage/connectors/bigquery) - -Online storage - -* [Redis](https://github.com/gojek/feast/tree/master/storage/connectors/redis) -* [Redis Cluster](https://github.com/gojek/feast/tree/master/storage/connectors/rediscluster) - diff --git a/docs/contributing/contributing.md b/docs/contributing/contributing.md index 2be097d6ffd..d9378ca6847 100644 --- a/docs/contributing/contributing.md +++ b/docs/contributing/contributing.md @@ -1,10 +1,10 @@ # Contribution Process -We use [RFCs](https://en.wikipedia.org/wiki/Request_for_Comments) and [GitHub issues](https://github.com/gojek/feast/issues) to communicate development ideas. The simplest way to contribute to Feast is to leave comments in our [RFCs](https://drive.google.com/drive/u/0/folders/1Lj1nIeRB868oZvKTPLYqAvKQ4O0BksjY) in the [Feast Google Drive](https://drive.google.com/drive/u/0/folders/0AAe8j7ZK3sxSUk9PVA) or our GitHub issues. You will need to join our [Google Group](../getting-help.md) in order to get access. +We use [RFCs](https://en.wikipedia.org/wiki/Request_for_Comments) and [GitHub issues](https://github.com/feast-dev/feast/issues) to communicate development ideas. The simplest way to contribute to Feast is to leave comments in our [RFCs](https://drive.google.com/drive/u/0/folders/1Lj1nIeRB868oZvKTPLYqAvKQ4O0BksjY) in the [Feast Google Drive](https://drive.google.com/drive/u/0/folders/0AAe8j7ZK3sxSUk9PVA) or our GitHub issues. You will need to join our [Google Group](../community.md) in order to get access. -We follow a process of [lazy consensus](http://community.apache.org/committers/lazyConsensus.html). If you believe you know what the project needs then just start development. If you are unsure about which direction to take with development then please communicate your ideas through a GitHub issue or through our [Slack Channel](../getting-help.md) before starting development. +We follow a process of [lazy consensus](http://community.apache.org/committers/lazyConsensus.html). If you believe you know what the project needs then just start development. If you are unsure about which direction to take with development then please communicate your ideas through a GitHub issue or through our [Slack Channel](../community.md) before starting development. -Please [submit a PR ](https://github.com/gojek/feast/pulls)to the master branch of the Feast repository once you are ready to submit your contribution. Code submission to Feast \(including submission from project maintainers\) require review and approval from maintainers or code owners. +Please [submit a PR ](https://github.com/feast-dev/feast/pulls)to the master branch of the Feast repository once you are ready to submit your contribution. Code submission to Feast \(including submission from project maintainers\) require review and approval from maintainers or code owners. PRs that are submitted by the general public need to be identified as `ok-to-test`. Once enabled, [Prow](https://github.com/kubernetes/test-infra/tree/master/prow) will run a range of tests to verify the submission, after which community members will help to review the pull request. diff --git a/docs/contributing/development-guide.md b/docs/contributing/development-guide.md index 9c9ec2bdf51..4c1d38ff97d 100644 --- a/docs/contributing/development-guide.md +++ b/docs/contributing/development-guide.md @@ -18,6 +18,7 @@ The following software is required for Feast development * Java SE Development Kit 11 * Python version 3.6 \(or above\) and pip * [Maven](https://maven.apache.org/install.html) version 3.6.x +* PySpark 2.4.2 ### **Services** @@ -108,10 +109,10 @@ Feast Serving has a dependency on Feast Core, thus always start Feast Core first ```bash # Start Feast Core locally -java -jar core/target/feast-core-0.8.0-exec.jar +java -jar core/target/feast-core-0.8.1-exec.jar # Start Feast Serving locally -java -jar serving/target/feast-serving-0.8.0-exec.jar +java -jar serving/target/feast-serving-0.8.1-exec.jar ``` Test whether Feast Core, Feast Serving are started and running correctly: @@ -122,8 +123,8 @@ feast version --core-url="localhost:6565" --serving-url="localhost:6566" ```javascript { - 'serving': {'url': 'localhost:6566', 'version': '0.8.0'}, - 'core': {'url': 'localhost:6565', 'version': '0.8.0'} + 'serving': {'url': 'localhost:6566', 'version': '0.8.1'}, + 'core': {'url': 'localhost:6565', 'version': '0.8.1'} } ``` diff --git a/docs/contributing/release-process.md b/docs/contributing/release-process.md index 277e549e561..2106e3a9154 100644 --- a/docs/contributing/release-process.md +++ b/docs/contributing/release-process.md @@ -38,13 +38,13 @@ For Feast maintainers, these are the concrete steps for making a new release. 3. Check that versions are updated with `env TARGET_MERGE_BRANCH=master make lint-versions` 7. Create a [GitHub release](https://github.com/feast-dev/feast/releases) which includes a summary of im~~p~~ortant changes as well as any artifacts associated with the release. Make sure to include the same change log as added in [CHANGELOG.md](https://github.com/feast-dev/feast/blob/master/CHANGELOG.md). Use `Feast vX.Y.Z` as the title. 8. Update the[ Upgrade Guide](../advanced/upgrading.md) to include the action required instructions for users to upgrade to this new release. Instructions should include a migration for each breaking change made to this release. -9. Update[ Feast Supported Versions]() to include the supported versions of each component. +9. Update[ Feast Supported Versions](release-process.md) to include the supported versions of each component. When a tag that matches a Semantic Version string is pushed, CI will automatically build and push the relevant artifacts to their repositories or package managers \(docker images, Python wheels, etc\). JVM artifacts are promoted from Sonatype OSSRH to Maven Central, but it sometimes takes some time for them to be available. The `sdk/go/v tag` is required to version the Go SDK go module so that users can go get a specific tagged release of the Go SDK. ### Creating a change log -We use an [open source change log generator](https://hub.docker.com/r/ferrarimarco/github-changelog-generator/) to generate change logs. The process still requires a little bit of manual effort. +We use an [open source change log generator](https://hub.docker.com/r/ferrarimarco/github-changelog-generator/) to generate change logs. The process still requires a little bit of manual effort. 1. Create a GitHub token as [per these instructions](https://github.com/github-changelog-generator/github-changelog-generator#github-token). The token is used as an input argument \(`-t`\) to the change log generator. 2. The change log generator configuration below will look for unreleased changes on a specific branch. The branch will be `master` for a major/minor release, or a release branch \(`v0.4-branch`\) for a patch release. You will need to set the branch using the `--release-branch` argument. @@ -77,5 +77,5 @@ docker run -it --rm ferrarimarco/github-changelog-generator \ It's important to flag breaking changes and deprecation to the API for each release so that we can maintain API compatibility. -Developers should have flagged PRs with breaking changes with the `compat/breaking` label. However, it's important to double check each PR's release notes and contents for changes that will break API compatibility and manually label `compat/breaking` to PRs with undeclared breaking changes. The change log will have to be regenerated if any new labels have to be added. +Developers should have flagged PRs with breaking changes with the `compat/breaking` label. However, it's important to double check each PR's release notes and contents for changes that will break API compatibility and manually label `compat/breaking` to PRs with undeclared breaking changes. The change log will have to be regenerated if any new labels have to be added. diff --git a/docs/getting-started/README.md b/docs/getting-started/README.md index 4a5296eba3c..716dd63ff63 100644 --- a/docs/getting-started/README.md +++ b/docs/getting-started/README.md @@ -1,18 +1,18 @@ # Getting Started -### Install Feast +## Install Feast If you would like to deploy a new installation of Feast, click on [Install Feast](install-feast/) {% page-ref page="install-feast/" %} -### Connect to Feast +## Connect to Feast If you would like to connect to an existing Feast deployment, click on [Connect to Feast](connect-to-feast/) {% page-ref page="connect-to-feast/" %} -### Learn Feast +## Learn Feast If you would like to learn more about Feast, click on [Learn Feast](learn-feast.md) diff --git a/docs/getting-started/connect-to-feast/README.md b/docs/getting-started/connect-to-feast/README.md index 214cf89fa3b..c2350d78850 100644 --- a/docs/getting-started/connect-to-feast/README.md +++ b/docs/getting-started/connect-to-feast/README.md @@ -1,6 +1,6 @@ # Connect to Feast -### Feast Python SDK +## Feast Python SDK The Feast Python SDK is used as a library to interact with a Feast deployment. @@ -11,7 +11,7 @@ The Feast Python SDK is used as a library to interact with a Feast deployment. {% page-ref page="python-sdk.md" %} -### Feast CLI +## Feast CLI The Feast CLI is a command line implementation of the Feast Python SDK. @@ -19,9 +19,9 @@ The Feast CLI is a command line implementation of the Feast Python SDK. * Ingest data into Feast * Manage ingestion jobs -{% page-ref page="connecting-to-feast.md" %} +{% page-ref page="feast-cli.md" %} -### Online Serving Clients +## Online Serving Clients The following clients can be used to retrieve online feature values: diff --git a/docs/getting-started/connect-to-feast/connecting-to-feast.md b/docs/getting-started/connect-to-feast/feast-cli.md similarity index 100% rename from docs/getting-started/connect-to-feast/connecting-to-feast.md rename to docs/getting-started/connect-to-feast/feast-cli.md diff --git a/docs/getting-started/install-feast/README.md b/docs/getting-started/install-feast/README.md index c358c2e12ae..dbd96215988 100644 --- a/docs/getting-started/install-feast/README.md +++ b/docs/getting-started/install-feast/README.md @@ -1,12 +1,12 @@ # Install Feast -### Kubernetes \(with Helm\) +## Kubernetes \(with Helm\) This guide installs Feast into an existing Kubernetes cluster using Helm. The installation is not specific to any cloud platform or environment, but requires Kubernetes and Helm. {% page-ref page="kubernetes-with-helm.md" %} -### Amazon EKS \(with Terraform\) +## Amazon EKS \(with Terraform\) This guide installs Feast into an AWS environment using Terraform. The Terraform script is opinionated and intended to allow you to start quickly. diff --git a/docs/getting-started/install-feast/kubernetes-amazon-eks-with-terraform.md b/docs/getting-started/install-feast/kubernetes-amazon-eks-with-terraform.md index d3058913d1f..6bb138bc8f1 100644 --- a/docs/getting-started/install-feast/kubernetes-amazon-eks-with-terraform.md +++ b/docs/getting-started/install-feast/kubernetes-amazon-eks-with-terraform.md @@ -1,6 +1,6 @@ # Amazon EKS \(with Terraform\) -### Overview +## Overview This guide installs Feast on AWS using our [reference Terraform configuration](https://github.com/feast-dev/feast/tree/master/infra/terraform/aws). @@ -17,15 +17,15 @@ This Terraform configuration creates the following resources: * Amazon EMR cluster to run Spark \(3x spot m4.xlarge\) * Staging S3 bucket to store temporary data -![](../../.gitbook/assets/feast-on-aws-3-.png) +![](../../.gitbook/assets/feast-on-aws-3-%20%281%29.png) -### 1. Requirements +## 1. Requirements * Create an AWS account and [configure credentials locally](https://docs.aws.amazon.com/cli/latest/userguide/cli-chap-configure.html) * Install [Terraform](https://www.terraform.io/) > = 0.12 \(tested with 0.13.3\) * Install [Helm](https://helm.sh/docs/intro/install/) \(tested with v3.3.4\) -### 2. Configure Terraform +## 2. Configure Terraform Create a `.tfvars` file under`feast/infra/terraform/aws`. Name the file. In our example, we use `my_feast.tfvars`. You can see the full list of configuration variables in `variables.tf`. At a minimum, you need to set `name_prefix` and an AWS region: @@ -36,7 +36,7 @@ region = "us-east-1" ``` {% endcode %} -### 3. Apply +## 3. Apply After completing the configuration, initialize Terraform and apply: @@ -46,13 +46,13 @@ $ terraform init $ terraform apply -var-file=my_feast.tfvars ``` -Starting may take a minute. A kubectl configuration file is also created in this directory, and the file's name will start with `kubeconfig_` and end with a random suffix. +Starting may take a minute. A kubectl configuration file is also created in this directory, and the file's name will start with `kubeconfig_` and end with a random suffix. -### 4. Connect to Feast using Jupyter +## 4. Connect to Feast using Jupyter After all pods are running, connect to the Jupyter Notebook Server running in the cluster. -To connect to the remote Feast server you just created, forward a port from the remote k8s cluster to your local machine. Replace `kubeconfig_XXXXXXX` below with the kubeconfig file name Terraform generates for you. +To connect to the remote Feast server you just created, forward a port from the remote k8s cluster to your local machine. Replace `kubeconfig_XXXXXXX` below with the kubeconfig file name Terraform generates for you. ```bash KUBECONFIG=kubeconfig_XXXXXXX kubectl port-forward \ diff --git a/docs/getting-started/install-feast/kubernetes-with-helm.md b/docs/getting-started/install-feast/kubernetes-with-helm.md index 1f38a8ba50c..82473ffe016 100644 --- a/docs/getting-started/install-feast/kubernetes-with-helm.md +++ b/docs/getting-started/install-feast/kubernetes-with-helm.md @@ -1,6 +1,6 @@ # Kubernetes \(with Helm\) -### Overview +## Overview This guide installs Feast on an existing Kubernetes cluster, and ensures the following services are running: @@ -11,12 +11,12 @@ This guide installs Feast on an existing Kubernetes cluster, and ensures the fol * Feast Jupyter \(Optional\) * Prometheus \(Optional\) -### 1. Requirements +## 1. Requirements 1. Install and configure [Kubectl](https://kubernetes.io/docs/tasks/tools/install-kubectl/) 2. Install [Helm 3](https://helm.sh/) -### 2. Preparation +## 2. Preparation Add the Feast Helm repository and download the latest charts: @@ -27,13 +27,13 @@ helm repo update Feast includes a Helm chart that installs all necessary components to run Feast Core, Feast Online Serving, and an example Jupyter notebook. -Feast Core requires Postgres to run, which requires a secret to be set on Kubernetes: +Feast Core requires Postgres to run, which requires a secret to be set on Kubernetes: ```bash kubectl create secret generic feast-postgresql --from-literal=postgresql-password=password ``` -### 3. Installation +## 3. Installation Install Feast using Helm. The pods may take a few minutes to initialize. @@ -41,7 +41,7 @@ Install Feast using Helm. The pods may take a few minutes to initialize. helm install feast-release feast-charts/feast ``` -### 4. Use Jupyter to connect to Feast +## 4. Use Jupyter to connect to Feast After all the pods are in a `RUNNING` state, port-forward to the Jupyter Notebook Server in the cluster: @@ -59,10 +59,10 @@ You can now connect to the bundled Jupyter Notebook Server at `localhost:8888` a {% embed url="http://localhost:8888/tree?" caption="" %} -### 5. Further Reading +## 5. Further Reading * [Feast Concepts](../../concepts/overview.md) * [Feast Examples/Tutorials](https://github.com/feast-dev/feast/tree/master/examples) * [Feast Helm Chart Documentation](https://github.com/feast-dev/feast/blob/master/infra/charts/feast/README.md) -* [Configuring Feast components](../../reference/configuration-reference/) +* [Configuring Feast components](../../reference/configuration-reference.md) diff --git a/docs/getting-started/learn-feast.md b/docs/getting-started/learn-feast.md index 004b1075aaa..5f15877d507 100644 --- a/docs/getting-started/learn-feast.md +++ b/docs/getting-started/learn-feast.md @@ -3,10 +3,10 @@ Explore the following resources to learn more about Feast: * [Concepts](../) describes all important Feast API concepts. -* [User guide](../user-guide/data-ingestion.md) provides guidance on completing Feast workflows. +* [User guide](../user-guide/define-and-ingest-features.md) provides guidance on completing Feast workflows. * [Examples](https://github.com/feast-dev/feast/tree/master/examples) contains Jupyter notebooks that you can run on your Feast deployment. * [Advanced](../advanced/troubleshooting.md) contains information about both advanced and operational aspects of Feast. -* [Reference](../reference/api/) contains detailed API and design documents for advanced users. +* [Reference](https://github.com/feast-dev/feast/tree/dc71813b7abbdb01139c0ba80da539350c9eef83/docs/reference/api/README.md) contains detailed API and design documents for advanced users. * [Contributing](../contributing/contributing.md) contains resources for anyone who wants to contribute to Feast. {% hint style="info" %} diff --git a/docs/getting-started/quickstart.md b/docs/getting-started/quickstart.md deleted file mode 100644 index ce315dbf27f..00000000000 --- a/docs/getting-started/quickstart.md +++ /dev/null @@ -1,56 +0,0 @@ -# Quickstart - -## Overview - -This guide will give a walkthrough on deploying Feast using Docker Compose, which allows the user to quickly explore the functionalities in Feast with minimal infrastructure setup. It includes a built in Jupyter Notebook Server that is preloaded with PySpark and Feast SDK, as well as Feast example notebooks to get you started. - -## 0. Requirements - -* [Docker Compose](https://docs.docker.com/compose/install/) should be installed. -* Optional dependancies: - * a [GCP service account](https://cloud.google.com/iam/docs/creating-managing-service-account-keys) that has access to [Google Cloud Storage](https://cloud.google.com/storage). - -## 1. Set up environment - -Clone the latest stable version of the [Feast repository](https://github.com/gojek/feast/) and setup before we deploy: - -```text -git clone https://github.com/feast-dev/feast.git -cd feast/infra/docker-compose -cp .env.sample .env -``` - -## 2. Start Feast Services - -Start the Feast services. Make sure that the following ports are free on the host machines: 6565, 6566, 8888, 9094, 5432. Alternatively, change the port mapping to use a different port on the host. - -```javascript -docker-compose up -d -``` - -{% hint style="info" %} -The Docker Compose deployment will take some time fully startup: - -* During this time Feast Serving container may restart, which should be automatically corrected after Feast Core is up and ready. -* If container restarts do not stop after 10 minutes, check the docker compose log to see if there is any error that prevents Feast Core from starting successfully. -{% endhint %} - -Once deployed, you should be able to connect at `localhost:8888` to the bundled Jupyter Notebook Server and follow the example notebooks: - -{% embed url="http://localhost:8888/tree?" caption="" %} - -## 3. Optional dependancies - -### 3.1 Set up Google Cloud Platform - -The example Jupyter notebook does not require any GCP dependancies by default. If you would like to modify the example such that a GCP service is required \(eg. Google Cloud Storage\), you would need to set up a [service account](https://cloud.google.com/iam/docs/creating-managing-service-accounts) that is associated with the notebook. Make sure that the service account has sufficient privileges to access the required GCP services. - -Once the service account is created, download the associated JSON key file and copy the file to the path configured in `.env` , under `GCP_SERVICE_ACCOUNT` . - -## 4. Further Reading - -* [Feast Concepts](../concepts/overview.md) -* [Feast Examples/Tutorials](https://github.com/feast-dev/feast/tree/master/examples) -* [Configuring Feast Components](../reference/configuration-reference.md) -* [Configuration Reference](https://app.gitbook.com/@feast/s/docs/v/master/reference/configuration-reference) - diff --git a/docs/installation/docker-compose.md b/docs/installation/docker-compose.md deleted file mode 100644 index 3c8c50862e3..00000000000 --- a/docs/installation/docker-compose.md +++ /dev/null @@ -1,112 +0,0 @@ -# Docker Compose - -### Overview - -This guide will bring Feast up using Docker Compose. This will allow you to: - -* Create, register, and manage feature sets -* Ingest feature data into Feast -* Retrieve features for online serving -* Retrieve features for batch serving \(only if using Google Cloud Platform\) - -This guide is split into three parts: - -1. Setting up your environment -2. Starting Feast with **online serving support only** \(does not require GCP\). -3. Starting Feast with support for **both online and batch** serving \(requires GCP\) - -{% hint style="info" %} -The docker compose setup uses Direct Runner for the Apache Beam jobs that populate data stores. Running Beam with the Direct Runner means it does not need a dedicated runner like Flink or Dataflow, but this comes at the cost of performance. We recommend the use of a dedicated runner when running Feast with very large workloads. -{% endhint %} - -### 0. Requirements - -* [Docker compose](https://docs.docker.com/compose/install/) must be installed. -* The following list of TCP ports must be free: - * 6565, 6566, 8888, and 9094. - * Alternatively it is possible to modify port mappings in `/docker-compose/docker-compose.yml`. -* \(for batch serving only\) For batch serving you will also need a [GCP service account key](https://cloud.google.com/iam/docs/creating-managing-service-account-keys) that has access to [Google Cloud Storage](https://cloud.google.com/storage) and [BigQuery](https://cloud.google.com/bigquery). -* \(for batch serving only\) [Google Cloud SDK ](https://cloud.google.com/sdk/install)installed, authenticated, and configured to the project you will use. - -## 1. Set up environment - -Clone the [Feast repository](https://github.com/feast-dev/feast/) and navigate to the `docker-compose` sub-directory: - -```bash -git clone https://github.com/feast-dev/feast.git && \ -cd feast && export FEAST_HOME_DIR=$(pwd) && \ -cd infra/docker-compose -``` - -Make a copy of the `.env.sample` file: - -```bash -cp .env.sample .env -``` - -## 2. Docker Compose for Online Serving Only - -### 2.1 Start Feast \(without batch retrieval support\) - -If you do not require batch serving, then its possible to simply bring up Feast: - -```javascript -docker-compose up -d -``` - -A Jupyter Notebook environment is now available to use Feast: - -[http://localhost:8888/tree/feast/examples](http://localhost:8888/tree/feast/examples) - -## 3. Docker Compose for Online and Batch Serving - -{% hint style="info" %} -Batch serving requires Google Cloud Storage to function, specifically Google Cloud Storage \(GCP\) and BigQuery. -{% endhint %} - -### 3.1 Set up Google Cloud Platform - -Create a [service account ](https://cloud.google.com/iam/docs/creating-managing-service-accounts)from the GCP console and copy it to the `infra/docker-compose/gcp-service-accounts` folder: - -```javascript -cp my-service-account.json ${FEAST_HOME_DIR}/infra/docker-compose/gcp-service-accounts -``` - -Create a Google Cloud Storage bucket. Make sure that your service account above has read/write permissions to this bucket: - -```bash -gsutil mb gs://my-feast-staging-bucket -``` - -### 3.2 Configure .env - -Configure the `.env` file based on your environment. At the very least you have to modify: - -| Parameter | Description | -| :--- | :--- | -| FEAST\_CORE\_GCP\_SERVICE\_ACCOUNT\_KEY | This should be your service account file name, for example `key.json`. | -| FEAST\_HISTORICAL\_SERVING\_GCP\_SERVICE\_ACCOUNT\_KEY | This should be your service account file name, for example `key.json` | -| FEAST\_JUPYTER\_GCP\_SERVICE\_ACCOUNT\_KEY | This should be your service account file name, for example `key.json` | -| FEAST\_JOB\_STAGING\_LOCATION | Google Cloud Storage bucket that Feast will use to stage data exports and batch retrieval requests, for example `gs://your-gcs-bucket/staging` | - -### 3.3 Configure .bq-store.yml - -We will also need to configure the `bq-store.yml` file inside `infra/docker-compose/serving/` to configure the BigQuery storage configuration as well as the feature sets that the store subscribes to. At a minimum you will need to set: - -| Parameter | Description | -| :--- | :--- | -| bigquery\_config.project\_id | This is you [GCP project Id](https://cloud.google.com/resource-manager/docs/creating-managing-projects). | -| bigquery\_config.dataset\_id | This is the name of the BigQuery dataset that tables will be created in. Each feature set will have one table in BigQuery. | - -### 3.4 Start Feast \(with batch retrieval support\) - -Start Feast: - -```javascript -docker-compose up -d -``` - -A Jupyter Notebook environment is now available to use Feast: - -[http://localhost:8888/tree/feast/examples](http://localhost:8888/tree/feast/examples) - diff --git a/docs/installation/gke.md b/docs/installation/gke.md deleted file mode 100644 index 66041887786..00000000000 --- a/docs/installation/gke.md +++ /dev/null @@ -1,211 +0,0 @@ -# Google Kubernetes Engine \(GKE\) - -### Overview - -This guide will install Feast into a Kubernetes cluster on GCP. It assumes that all of your services will run within a single Kubernetes cluster. Once Feast is installed you will be able to: - -* Define and register features. -* Load feature data from both batch and streaming sources. -* Retrieve features for model training. -* Retrieve features for online serving. - -{% hint style="info" %} -This guide requires [Google Cloud Platform](https://cloud.google.com/) for installation. - -* [BigQuery](https://cloud.google.com/bigquery/) is used for storing historical features. -* [Google Cloud Storage](https://cloud.google.com/storage/) is used for intermediate data storage. -{% endhint %} - -## 0. Requirements - -1. [Google Cloud SDK ](https://cloud.google.com/sdk/install)installed, authenticated, and configured to the project you will use. -2. [Kubectl](https://kubernetes.io/docs/tasks/tools/install-kubectl/) installed. -3. [Helm](https://helm.sh/3) \(2.16.0 or greater\) installed on your local machine with Tiller installed in your cluster. Helm 3 has not been tested yet. - -## 1. Set up GCP - -First define the environmental variables that we will use throughout this installation. Please customize these to reflect your environment. - -```bash -export FEAST_GCP_PROJECT_ID=my-gcp-project -export FEAST_GCP_REGION=us-central1 -export FEAST_GCP_ZONE=us-central1-a -export FEAST_BIGQUERY_DATASET_ID=feast -export FEAST_GCS_BUCKET=${FEAST_GCP_PROJECT_ID}_feast_bucket -export FEAST_GKE_CLUSTER_NAME=feast -export FEAST_SERVICE_ACCOUNT_NAME=feast-sa -``` - -Create a Google Cloud Storage bucket for Feast to stage batch data exports: - -```bash -gsutil mb gs://${FEAST_GCS_BUCKET} -``` - -Create the service account that Feast will run as: - -```bash -gcloud iam service-accounts create ${FEAST_SERVICE_ACCOUNT_NAME} - -gcloud projects add-iam-policy-binding ${FEAST_GCP_PROJECT_ID} \ - --member serviceAccount:${FEAST_SERVICE_ACCOUNT_NAME}@${FEAST_GCP_PROJECT_ID}.iam.gserviceaccount.com \ - --role roles/editor - -gcloud iam service-accounts keys create key.json --iam-account \ -${FEAST_SERVICE_ACCOUNT_NAME}@${FEAST_GCP_PROJECT_ID}.iam.gserviceaccount.com -``` - -## 2. Set up a Kubernetes \(GKE\) cluster - -{% hint style="warning" %} -Provisioning a GKE cluster can expose your services publicly. This guide does not cover securing access to the cluster. -{% endhint %} - -Create a GKE cluster: - -```bash -gcloud container clusters create ${FEAST_GKE_CLUSTER_NAME} \ - --machine-type n1-standard-4 -``` - -Create a secret in the GKE cluster based on your local key `key.json`: - -```bash -kubectl create secret generic feast-gcp-service-account --from-file=key.json -``` - -For this guide we will use `NodePort` for exposing Feast services. In order to do so, we must find an External IP of at least one GKE node. This should be a public IP. - -```bash -export FEAST_IP=$(kubectl describe nodes | grep ExternalIP | awk '{print $2}' | head -n 1) -export FEAST_CORE_URL=${FEAST_IP}:32090 -export FEAST_ONLINE_SERVING_URL=${FEAST_IP}:32091 -export FEAST_HISTORICAL_SERVING_URL=${FEAST_IP}:32092 -``` - -Add firewall rules to open up ports on your Google Cloud Platform project: - -```bash -gcloud compute firewall-rules create feast-core-port --allow tcp:32090 -gcloud compute firewall-rules create feast-online-port --allow tcp:32091 -gcloud compute firewall-rules create feast-batch-port --allow tcp:32092 -gcloud compute firewall-rules create feast-redis-port --allow tcp:32101 -gcloud compute firewall-rules create feast-kafka-ports --allow tcp:31090-31095 -``` - -## 3. Set up Helm - -Run the following command to provide Tiller with authorization to install Feast: - -```bash -kubectl apply -f - < @@ -38,10 +38,15 @@ -### Ingestion +## Ingestion | Limitation | Motivation | | :--- | :--- | | Once data has been ingested into Feast, there is currently no way to delete the data without manually going to the database and deleting it. However, during retrieval only the latest rows will be returned for a specific key \(`event_timestamp`, `entity`\) based on its `created_timestamp`. | This functionality simply doesn't exist yet as a Feast API | -| During the ingestion of data into BigQuery, `event_timestamp` is rounded down to seconds. E.g., `2020-08-21T08:40:19.906 -> 2020-08-21T08:40:19.000` | This ensures that floating point rounding errors do not occur during the retrieval of feature data, since this step requires time based joins | + +## Storage + +| Limitation | Motivation | +| :--- | :--- | +| Feast does not support offline storage in Feast 0.8 | As part of our re-architecture of Feast, we moved from GCP to cloud-agnostic deployments. Developing offline storage support that is available in all cloud environments is a pending action. | diff --git a/docs/reference/metrics-reference/README.md b/docs/reference/metrics-reference.md similarity index 88% rename from docs/reference/metrics-reference/README.md rename to docs/reference/metrics-reference.md index 4ccd6cb9ce0..5e4ca857374 100644 --- a/docs/reference/metrics-reference/README.md +++ b/docs/reference/metrics-reference.md @@ -1,12 +1,16 @@ # Metrics Reference +{% hint style="warning" %} +This page applies to Feast 0.7. The content may be out of date for Feast 0.8+ +{% endhint %} + Reference of the metrics that each Feast component exports: -* [Feast Core](./#feast-core) -* [Feast Serving](./#feast-serving) -* [Feast Ingestion Job](./#feast-ingestion-job) +* [Feast Core](metrics-reference.md#feast-core) +* [Feast Serving](metrics-reference.md#feast-serving) +* [Feast Ingestion Job](metrics-reference.md#feast-ingestion-job) -For how to configure Feast to export Metrics, see the [Metrics user guide.](../../advanced/metrics.md) +For how to configure Feast to export Metrics, see the [Metrics user guide.](../advanced/metrics.md) ## Feast Core @@ -44,8 +48,8 @@ Feast Serving exports the following metrics: | :--- | :--- | :--- | | `feast_serving_request_latency_seconds` | Feast Serving's latency in serving Requests in Seconds. | `method` | | `feast_serving_request_feature_count` | No. of requests retrieving a Feature from Feast Serving. | `project`, `feature_name` | -| `feast_serving_not_found_feature_count` | No. of requests retrieving a Feature has resulted in a [`NOT_FOUND` field status.](../../user-guide/feature-retrieval.md#online-field-statuses) | `project`, `feature_name` | -| `feast_serving_stale_feature_count` | No. of requests retrieving a Feature resulted in a [`OUTSIDE_MAX_AGE` field status.](../../user-guide/feature-retrieval.md#online-field-statuses) | `project`, `feature_name` | +| `feast_serving_not_found_feature_count` | No. of requests retrieving a Feature has resulted in a [`NOT_FOUND` field status.](../user-guide/getting-training-features.md#online-field-statuses) | `project`, `feature_name` | +| `feast_serving_stale_feature_count` | No. of requests retrieving a Feature resulted in a [`OUTSIDE_MAX_AGE` field status.](../user-guide/getting-training-features.md#online-field-statuses) | `project`, `feature_name` | | `feast_serving_grpc_request_count` | Total gRPC requests served. | `method` | **Metric Tags** @@ -61,20 +65,16 @@ Exported Feast Serving metrics may be filtered by the following tags/keys ## Feast Ingestion Job -Feast Ingestion computes both metrics an statistics on [data ingestion.](../../user-guide/data-ingestion.md) Make sure you familar with data ingestion concepts before proceeding. - -{% hint style="info" %} -For documentation on Feature value statistics computed by the Ingestion Job see [Statistics]() -{% endhint %} +Feast Ingestion computes both metrics an statistics on [data ingestion.](../user-guide/define-and-ingest-features.md) Make sure you familar with data ingestion concepts before proceeding. **Metrics Namespace** -Metrics are computed at two stages of the Feature Row's/Feature Value's life cycle when being processed by the Ingestion Job: +Metrics are computed at two stages of the Feature Row's/Feature Value's life cycle when being processed by the Ingestion Job: * `Inflight`- Prior to writing data to stores, but after successful validation of data. * `WriteToStoreSucess`- After a successful store write. -Metrics processed by each staged will be tagged with `metrics_namespace` to the stage where the metric was computed. +Metrics processed by each staged will be tagged with `metrics_namespace` to the stage where the metric was computed. **Metrics Bucketing** @@ -165,7 +165,7 @@ Metrics with a `{BUCKET}` are computed on a 60 second window/bucket. Suffix with **Metric Tags** -Exported Feast Ingestion Job metrics may be filtered by the following tags/keys +Exported Feast Ingestion Job metrics may be filtered by the following tags/keys | Tag | Description | | :--- | :--- | diff --git a/docs/roadmap.md b/docs/roadmap.md index 5ec26f85a1c..4f1a59c751c 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -1,32 +1,44 @@ # Roadmap -### Feast 0.8 +## Feast 0.9 + +[Discussion](https://github.com/feast-dev/feast/issues/1131) + +### New Functionality + +* Feast Job Service +* Delta offline store support. Optional for users +* Push based ingestion into offline store +* On-prem support \(Open source storage and launcher\) +* Azure support + +## Feast 0.8 [Discussion](https://github.com/feast-dev/feast/issues/1018) [Feast 0.8 RFC](https://docs.google.com/document/d/1snRxVb8ipWZjCiLlfkR4Oc28p7Fkv_UXjvxBFWjRBj4/edit#heading=h.yvkhw2cuvx5) -#### **New Functionality** +### **New Functionality** 1. Add support for AWS \(data sources and deployment\) 2. Add support for local deployment 3. Add support for Spark based ingestion 4. Add support for Spark based historical retrieval -#### **Technical debt, refactoring, or housekeeping** +### **Technical debt, refactoring, or housekeeping** 1. Move job management functionality to SDK 2. Remove Apache Beam based ingestion 3. Allow direct ingestion from batch sources that does not pass through stream 4. Remove Feast Historical Serving abstraction to allow direct access from Feast SDK to data sources for retrieval -### Feast 0.7 +## Feast 0.7 [Discussion](https://github.com/feast-dev/feast/issues/834) [GitHub Milestone](https://github.com/feast-dev/feast/milestone/4) -#### **New Functionality** +### **New Functionality** 1. Label based Ingestion Job selector for Job Controller [\#903](https://github.com/feast-dev/feast/pull/903) 2. Authentication Support for Java & Go SDKs [\#971](https://github.com/feast-dev/feast/pull/971) @@ -35,19 +47,19 @@ 5. Request Response Logging support via Fluentd [\#961](https://github.com/feast-dev/feast/pull/961) 6. Feast Core Rest Endpoints [\#878](https://github.com/feast-dev/feast/pull/878) -#### **Technical debt, refactoring, or housekeeping** +### **Technical debt, refactoring, or housekeeping** 1. Improved integration testing framework [\#886](https://github.com/feast-dev/feast/pull/886) 2. Rectify all flaky batch tests [\#953](https://github.com/feast-dev/feast/pull/953), [\#982](https://github.com/feast-dev/feast/pull/982) 3. Decouple job management from Feast Core [\#951](https://github.com/feast-dev/feast/pull/951) -### Feast 0.6 +## Feast 0.6 [Discussion](https://github.com/feast-dev/feast/issues/767) [GitHub Milestone](https://github.com/feast-dev/feast/milestone/3) -#### New functionality +### New functionality 1. Batch statistics and validation [\#612](https://github.com/feast-dev/feast/pull/612) 2. Authentication and authorization [\#554](https://github.com/feast-dev/feast/pull/554) @@ -55,29 +67,29 @@ 4. Improved searching and filtering of features and entities 5. Python support for labels [\#663](https://github.com/feast-dev/feast/issues/663) -#### Technical debt, refactoring, or housekeeping +### Technical debt, refactoring, or housekeeping 1. Improved job life cycle management [\#761](https://github.com/feast-dev/feast/issues/761) 2. Compute and write metrics for rows prior to store writes [\#763](https://github.com/feast-dev/feast/pull/763) -### Feast 0.5 +## Feast 0.5 -[Discussion](https://github.com/gojek/feast/issues/527) +[Discussion](https://github.com/feast-dev/feast/issues/527) -#### New functionality +### New functionality 1. Streaming statistics and validation \(M1 from [Feature Validation RFC](https://docs.google.com/document/d/1TPmd7r4mniL9Y-V_glZaWNo5LMXLshEAUpYsohojZ-8/edit)\) -2. Support for Redis Clusters \([\#478](https://github.com/gojek/feast/issues/478), [\#502](https://github.com/gojek/feast/issues/502)\) -3. Add feature and feature set labels, i.e. key/value registry metadata \([\#463](https://github.com/gojek/feast/issues/463)\) -4. Job management API \([\#302](https://github.com/gojek/feast/issues/302)\) - -#### Technical debt, refactoring, or housekeeping - -1. Clean up and document all configuration options \([\#525](https://github.com/gojek/feast/issues/525)\) -2. Externalize storage interfaces \([\#402](https://github.com/gojek/feast/issues/402)\) -3. Reduce memory usage in Redis \([\#515](https://github.com/gojek/feast/issues/515)\) -4. Support for handling out of order ingestion \([\#273](https://github.com/gojek/feast/issues/273)\) -5. Remove feature versions and enable automatic data migration \([\#386](https://github.com/gojek/feast/issues/386)\) \([\#462](https://github.com/gojek/feast/issues/462)\) -6. Tracking of batch ingestion by with dataset\_id/job\_id \([\#461](https://github.com/gojek/feast/issues/461)\) -7. Write Beam metrics after ingestion to store \(not prior\) \([\#489](https://github.com/gojek/feast/issues/489)\) +2. Support for Redis Clusters \([\#478](https://github.com/feast-dev/feast/issues/478), [\#502](https://github.com/feast-dev/feast/issues/502)\) +3. Add feature and feature set labels, i.e. key/value registry metadata \([\#463](https://github.com/feast-dev/feast/issues/463)\) +4. Job management API \([\#302](https://github.com/feast-dev/feast/issues/302)\) + +### Technical debt, refactoring, or housekeeping + +1. Clean up and document all configuration options \([\#525](https://github.com/feast-dev/feast/issues/525)\) +2. Externalize storage interfaces \([\#402](https://github.com/feast-dev/feast/issues/402)\) +3. Reduce memory usage in Redis \([\#515](https://github.com/feast-dev/feast/issues/515)\) +4. Support for handling out of order ingestion \([\#273](https://github.com/feast-dev/feast/issues/273)\) +5. Remove feature versions and enable automatic data migration \([\#386](https://github.com/feast-dev/feast/issues/386)\) \([\#462](https://github.com/feast-dev/feast/issues/462)\) +6. Tracking of batch ingestion by with dataset\_id/job\_id \([\#461](https://github.com/feast-dev/feast/issues/461)\) +7. Write Beam metrics after ingestion to store \(not prior\) \([\#489](https://github.com/feast-dev/feast/issues/489)\) diff --git a/docs/user-guide/data-ingestion.md b/docs/user-guide/data-ingestion.md deleted file mode 100644 index e29953d173e..00000000000 --- a/docs/user-guide/data-ingestion.md +++ /dev/null @@ -1,42 +0,0 @@ -# Getting data into Feast - -In order to retrieve features for both training and serving, Feast requires data being ingested into the offline and online stores. - -{% hint style="warning" %} -Offline storage support will not be available until v0.9. Only Online storage support exists currently. -{% endhint %} - -Users are expected to already have either a batch or stream source with data materialized in it, ready to be ingested into Feast. Upon providing their external data sources in feature table specifications and registering them, users can now ingest data into Feast using Spark jobs. - -The following depicts an example ingestion flow from the specified data source to online store. - -### Batch Source to Online Store - -```python -from feast import Client -from datetime import datetime, timedelta - -client = Client(core_url="localhost:6565") -driver_ft = client.get_feature_table("driver_trips") - -# Initialize date ranges -today = datetime.now() -yesterday = today - timedelta(1) - -client.start_offline_to_online_ingestion( - driver_ft, yesterday, today -) -``` - -### Stream Source to Online Store - -```python -from feast import Client -from datetime import datetime, timedelta - -client = Client(core_url="localhost:6565") -driver_ft = client.get_feature_table("driver_trips") - -client.start_stream_to_online_ingestion(driver_ft) -``` - diff --git a/docs/user-guide/define-and-ingest-features.md b/docs/user-guide/define-and-ingest-features.md new file mode 100644 index 00000000000..6300a7ef610 --- /dev/null +++ b/docs/user-guide/define-and-ingest-features.md @@ -0,0 +1,56 @@ +# Define and ingest features + +In order to retrieve features for both training and serving, Feast requires data being ingested into its offline and online stores. + +{% hint style="warning" %} +Feast 0.8 does not have an offline store. Only Online storage support exists currently. Feast 0.9 will have offline storage support. In Feast 0.8, historical data is retrieved directly from batch sources. +{% endhint %} + +Users are expected to already have either a batch or stream source with data stored in it, ready to be ingested into Feast. Once a feature table \(with the corresponding sources\) has been registered with Feast, it is possible to load data from this source into stores. + +The following depicts an example ingestion flow from a data source to the online store. + +## Batch Source to Online Store + +```python +from feast import Client +from datetime import datetime, timedelta + +client = Client(core_url="localhost:6565") +driver_ft = client.get_feature_table("driver_trips") + +# Initialize date ranges +today = datetime.now() +yesterday = today - timedelta(1) + +# Launches a short-lived job that ingests data over the provided date range. +client.start_offline_to_online_ingestion( + driver_ft, yesterday, today +) +``` + +## Stream Source to Online Store + +```python +from feast import Client +from datetime import datetime, timedelta + +client = Client(core_url="localhost:6565") +driver_ft = client.get_feature_table("driver_trips") + +# Launches a long running streaming ingestion job +client.start_stream_to_online_ingestion(driver_ft) +``` + +## Batch Source to Offline Store + +{% hint style="danger" %} +Not supported in Feast 0.8 +{% endhint %} + +## Stream Source to Offline Store + +{% hint style="danger" %} +Not supported in Feast 0.8 +{% endhint %} + diff --git a/docs/user-guide/feature-retrieval.md b/docs/user-guide/feature-retrieval.md deleted file mode 100644 index 3a6d072a410..00000000000 --- a/docs/user-guide/feature-retrieval.md +++ /dev/null @@ -1,61 +0,0 @@ -# Getting training features - -Feast provides a historical retrieval interface for exporting feature data to train machine learning models. Essentially, users are able to retrieve features from any feature tables and join them together in a single response dataset. The only requirement is that the user provides the correct entities and timestamps in order to look up the features. - -Historical feature retrieval can be done through the [Feast SDK](https://api.docs.feast.dev/python). - -{% hint style="warning" %} -Historical Retrieval currently pulls from batch sources for Feast v0.8, and offline storage support will not be available until v0.9. -{% endhint %} - -{% hint style="info" %} -By default, Feast infers that the features specified belong to the `default` project. To retrieve from another project, specify the `project` parameter when retrieving features. -{% endhint %} - -## **Point-in-time-correct Join** - -Feast does a point in time correct query from a single feature table. For each entity key and event timestamp combination that is provided by `entity_source`, Feast determines the values of all the features in the `feature_refs` list at that respective point in time and then joins features values to that specific entity value and event timestamp, and repeats this process for all timestamps. - -This is called a point in time correct join. - -Below is an example of how a point-in-time-correct join works. We have two DataFrames. The first is the `entity dataframe` that contains timestamps, entities, and labels. The user would like to have driver features joined onto this `entity dataframe` from the `driver dataframe` to produce a `joined dataframe` upon materializing the view that contains both labels and features. They would then like to train their model on this output - -![](../.gitbook/assets/point_in_time_join%20%281%29.png) - -Typically the `input 1` DataFrame would be provided by the user through `entity_source`, and the `input 2` DataFrame would already be ingested into Feast. To join these two, the user would call Feast as follows: - -```python -# Feature references with target feature -feature_refs = [ - "driver_trips:average_daily_rides", - "driver_trips:maximum_daily_rides", - "driver_trips:rating", - "trip_completed", -] - -# Define entity source -entity_source = FileSource( - "event_timestamp", - ParquetFormat(), - "gs://some-bucket/customer" -) - -# Retrieve historical dataset from Feast. -historical_feature_retrieval_job = client.get_historical_features( - feature_refs=feature_refs, - entity_rows=entity_source -) - -# Retrieve the output uri to materialize the dataset object into a Pandas DataFrame etc. -# Eg. gs://some-bucket/output/, s3://*, file://* -output_file_uri = historical_feature_retrieval_job.get_output_file_uri() -``` - -Feast is able to intelligently join feature data with different timestamps to a single basis table in a point-in-time-correct way. This allows users to join daily batch data with high-frequency event data transparently. They simply need to provide the feature references. - -{% hint style="info" %} -Feast can retrieve features from any amount of feature tables, as long as they occur on the same entities. -{% endhint %} - -Point-in-time-correct joins also prevents the occurrence of feature leakage by trying to accurate the state of the world at a single point in time, instead of just joining features based on the nearest timestamps. - diff --git a/docs/user-guide/getting-online-features.md b/docs/user-guide/getting-online-features.md index 722bb2ade4c..83523046cb4 100644 --- a/docs/user-guide/getting-online-features.md +++ b/docs/user-guide/getting-online-features.md @@ -1,10 +1,10 @@ # Getting online features -Feast provides an online retrieval interface for serving. Data ingested into the online store comes from both batch and stream sources. +Feast provides an API through which online feature values can be retrieved. This allows teams to look up feature values at low latency in production during model serving, in order to make online predictions. -When data is ingested from a batch source, users can retrieve the same features used in training models from the low latency online store to be used in production. When data is ingested from a stream source, features that are retrieved by users are of the latest values, which are not yet used in training models. - -Online feature retrieval works in much the same way as batch retrieval, with one important distinction: Online stores only maintain the current state of features, i.e latest feature values. No historical data is served. +{% hint style="info" %} +Online stores only maintain the current state of features, i.e latest feature values. No historical data is stored or served. +{% endhint %} ```python from feast import Client @@ -36,35 +36,19 @@ response_dict = response.to_dict() print(response_dict) ``` -{% hint style="info" %} -When no project is specified when retrieving features with get\_online\_feature\(\), Feast infers that the features specified belong to the default project. To retrieve from another project, specify the project parameter when retrieving features. -{% endhint %} - -Feast Serving provides a [gRPC API](https://api.docs.feast.dev/grpc/feast.serving.pb.html) that is backed by [Redis](https://redis.io/). We also provide support for [Python](https://api.docs.feast.dev/python/), [Go](https://godoc.org/github.com/gojek/feast/sdk/go), and [Java](https://javadoc.io/doc/dev.feast) clients. +The online store must be populated through [ingestion jobs](define-and-ingest-features.md#batch-source-to-online-store) prior to being used for online serving. -### Online Field Statuses +Feast Serving provides a [gRPC API](https://api.docs.feast.dev/grpc/feast.serving.pb.html) that is backed by [Redis](https://redis.io/). We have native clients in [Python](https://api.docs.feast.dev/python/), [Go](https://godoc.org/github.com/gojek/feast/sdk/go), and [Java](https://javadoc.io/doc/dev.feast). -Online Serving also returns Online Field Statuses when retrieving features. These status values gives useful insight into situations where Online Serving returns unset values. It also allows better of handling of the different possible cases represented by each status:for feature in features: +## Online Field Statuses -```python -response_dict = response.to_dict() - -for feature_ref in feature_refs: - # field status can be obtained from the response's field values - status = response_dict["field_values"]["statuses"][feature_ref] - - if status == GetOnlineFeaturesResponse.FieldStatus.NOT_FOUND: - # handle case where feature value has not been ingested - elif status == GetOnlineFeaturesResponse.FieldStatus.PRESENT: - # feature value is present and can be used - value = response_dict["field_values"]["statuses"][feature_ref] -``` +Feast also returns status codes when retrieving features from the Feast Serving API. These status code give useful insight into the quality of data being served. | Status | Meaning | | :--- | :--- | -| NOT\_FOUND | Unset values returned as the feature value was not found in the online store. This might mean that no feature value was ingested for this feature. | -| NULL\_VALUE | Unset values returned as the ingested feature value was also unset. | -| OUTSIDE\_MAX\_AGE | Unset values returned as the age of the feature value \(time since the value was ingested\) has exceeded the Feature Set's max age, which the feature was defined in. | -| PRESENT | Set values are returned for the requested feature. | -| UNKNOWN | Status signifies the field status is unset for the requested feature. Might mean that the Feast version does not support Field Statuses. | +| NOT\_FOUND | The feature value was not found in the online store. This might mean that no feature value was ingested for this feature. | +| NULL\_VALUE | A entity key was successfully found but no feature values had been set. This status code should not occur during normal operation. | +| OUTSIDE\_MAX\_AGE | The age of the feature row in the online store \(in terms of its event timestamp\) has exceeded the maximum age defined within the feature table. | +| PRESENT | The feature values have been found and are within the maximum age. | +| UNKNOWN | Indicates a system failure. | diff --git a/docs/user-guide/getting-training-features.md b/docs/user-guide/getting-training-features.md new file mode 100644 index 00000000000..d9be256fd83 --- /dev/null +++ b/docs/user-guide/getting-training-features.md @@ -0,0 +1,72 @@ +# Getting training features + +Feast provides a historical retrieval interface for exporting feature data in order to train machine learning models. Essentially, users are able to enrich their data with features from any feature tables. + +## Retrieving historical features + +Below is an example of the process required to produce a training dataset: + +```python +# Feature references with target feature +feature_refs = [ + "driver_trips:average_daily_rides", + "driver_trips:maximum_daily_rides", + "driver_trips:rating", + "driver_trips:rating:trip_completed", +] + +# Define entity source +entity_source = FileSource( + "event_timestamp", + ParquetFormat(), + "gs://some-bucket/customer" +) + +# Retrieve historical dataset from Feast. +historical_feature_retrieval_job = client.get_historical_features( + feature_refs=feature_refs, + entity_rows=entity_source +) + +output_file_uri = historical_feature_retrieval_job.get_output_file_uri() +``` + +### 1. Define feature references + +[Feature references](../concepts/glossary.md#feature-references) define the specific features that will be retrieved from Feast. These features can come from multiple feature tables. The only requirement is that the feature tables that make up the feature references have the same entity \(or composite entity\). + +**2. Define an entity dataframe** + +Feast needs to join feature values onto specific entities at specific points in time. Thus, it is necessary to provide an [entity dataframe](../concepts/glossary.md#entity-dataframe) as part of the `get_historical_features` method. In the example above we are defining an entity source. This source is an external file that provides Feast with the entity dataframe. + +**3. Launch historical retrieval job** + +Once the feature references and an entity source are defined, it is possible to call `get_historical_features()`. This method launches a job that extracts features from the sources defined in the provided feature tables, joins them onto the provided entity source, and returns a reference to the training dataset that is produced. + +Please see the [Feast SDK](https://api.docs.feast.dev/python) for more details. + +## Point-in-time Joins + +Feast always joins features onto entity data in a point-in-time correct way. The process can be described through an example. + +In the example below there are two tables \(or dataframes\): + +* The dataframe on the left is the [entity dataframe](../concepts/glossary.md#entity-dataframe) that contains timestamps, entities, and the target variable \(trip\_completed\). This dataframe is provided to Feast through an entity source. +* The dataframe on the right contains driver features. This dataframe is represented in Feast through a feature table and its accompanying data source\(s\). + +The user would like to have the driver features joined onto the entity dataframe to produce a training dataset that contains both the target \(trip\_completed\) and features \(average\_daily\_rides, maximum\_daily\_rides, rating\). This dataset will then be used to train their model. + +![](../.gitbook/assets/point_in_time_join%20%281%29%20%281%29.png) + +Feast is able to intelligently join feature data with different timestamps to a single entity dataframe. It does this through a point-in-time join as follows: + +1. Feast loads the entity dataframe and all feature tables \(driver dataframe\) into the same location. This can either be a database or in memory. +2. For each [entity row](../concepts/glossary.md#entity-rows) in the [entity dataframe](getting-online-features.md), Feast tries to find feature values in each feature table to join to it. Feast extracts the timestamp and entity key of each row in the entity dataframe and scans backward through the feature table until it finds a matching entity key. +3. If the event timestamp of the matching entity key within the driver feature table is within the maximum age configured for the feature table, then the features at that entity key are joined onto the entity dataframe. If the event timestamp is outside of the maximum age, then only null values are returned. +4. If multiple entity keys are found with the same event timestamp, then they are deduplicated by the created timestamp, with newer values taking precedence. +5. Feast repeats this joining process for all feature tables and returns the resulting dataset. + +{% hint style="info" %} +Point-in-time correct joins attempts to prevent the occurrence of feature leakage by trying to recreate the state of the world at a single point in time, instead of joining features based on exact timestamps only. +{% endhint %} + diff --git a/docs/user-guide/overview.md b/docs/user-guide/overview.md new file mode 100644 index 00000000000..c47f6c71dd1 --- /dev/null +++ b/docs/user-guide/overview.md @@ -0,0 +1,32 @@ +# Overview + +## Using Feast + +Feast development happens through three key workflows: + +1. [Define and load feature data into Feast](define-and-ingest-features.md) +2. [Retrieve historical features for training models](getting-training-features.md) +3. [Retrieve online features for serving models](getting-online-features.md) + +## Defining feature tables and ingesting data into Feast + +Feature creators model the data within their organization into Feast through the definition of [feature tables](../concepts/feature-tables.md) that contain [data sources](../concepts/sources.md). Feature tables are both a schema and a means of identifying data sources for features, and allow Feast to know how to interpret your data, and where to find it. + +After registering a feature table with Feast, users can trigger an ingestion from their data source into Feast. This loads feature values from an upstream data source into Feast stores through ingestion jobs. + +Visit [feature tables](../concepts/feature-tables.md#overview) to learn more about them. + +{% page-ref page="define-and-ingest-features.md" %} + +## Retrieving historical features for training + +In order to generate a training dataset it is necessary to provide both an [entity dataframe ](../concepts/glossary.md#entity-dataframe)and feature references through the[ Feast SDK](https://api.docs.feast.dev/python/) to retrieve historical features. For historical serving, Feast requires that you provide the entities and timestamps for the corresponding feature data. Feast produces a point-in-time correct dataset using the requested features. These features can be requested from an unlimited number of feature sets. + +{% page-ref page="getting-training-features.md" %} + +## Retrieving online features for online serving + +Online retrieval uses feature references through the [Feast Online Serving API](https://api.docs.feast.dev/grpc/feast.serving.pb.html) to retrieve online features. Online serving allows for very low latency requests to feature data at very high throughput. + +{% page-ref page="getting-online-features.md" %} + diff --git a/infra/charts/feast/Chart.yaml b/infra/charts/feast/Chart.yaml index 323f4f3389d..c1d7ed849cf 100644 --- a/infra/charts/feast/Chart.yaml +++ b/infra/charts/feast/Chart.yaml @@ -1,4 +1,4 @@ apiVersion: v1 description: Feature store for machine learning. name: feast -version: 0.8.0 +version: 0.8.1 diff --git a/infra/charts/feast/README.md b/infra/charts/feast/README.md index 5260efc4823..7ac1837eda6 100644 --- a/infra/charts/feast/README.md +++ b/infra/charts/feast/README.md @@ -1,7 +1,7 @@ feast ===== -Feature store for machine learning. Current chart version is `0.8.0` +Feature store for machine learning. Current chart version is `0.8.1` ## Installation @@ -11,9 +11,9 @@ https://docs.feast.dev/v/master/getting-started/deploying-feast/kubernetes | Repository | Name | Version | |------------|------|---------| -| | feast-core | 0.8.0 | -| | feast-jupyter | 0.8.0 | -| | feast-serving | 0.8.0 | +| | feast-core | 0.8.1 | +| | feast-jupyter | 0.8.1 | +| | feast-serving | 0.8.1 | | | prometheus-statsd-exporter | 0.1.2 | | https://charts.bitnami.com/bitnami/ | kafka | 11.8.8 | | https://kubernetes-charts.storage.googleapis.com/ | grafana | 5.0.5 | diff --git a/infra/charts/feast/charts/feast-core/Chart.yaml b/infra/charts/feast/charts/feast-core/Chart.yaml index 0857cfe4a5d..221c3a5d819 100644 --- a/infra/charts/feast/charts/feast-core/Chart.yaml +++ b/infra/charts/feast/charts/feast-core/Chart.yaml @@ -1,4 +1,4 @@ apiVersion: v1 description: Feast Core registers feature specifications. name: feast-core -version: 0.8.0 +version: 0.8.1 diff --git a/infra/charts/feast/charts/feast-core/README.md b/infra/charts/feast/charts/feast-core/README.md index 45607047a3c..90985e27e97 100644 --- a/infra/charts/feast/charts/feast-core/README.md +++ b/infra/charts/feast/charts/feast-core/README.md @@ -2,7 +2,7 @@ feast-core ========== Feast Core registers feature specifications. -Current chart version is `0.8.0` +Current chart version is `0.8.1` diff --git a/infra/charts/feast/charts/feast-jobservice/Chart.yaml b/infra/charts/feast/charts/feast-jobservice/Chart.yaml index d4183b197ab..d0b1c53bf51 100644 --- a/infra/charts/feast/charts/feast-jobservice/Chart.yaml +++ b/infra/charts/feast/charts/feast-jobservice/Chart.yaml @@ -1,4 +1,4 @@ apiVersion: v1 description: Feast Job Service manage ingestion jobs. name: feast-jobservice -version: 0.8.0 +version: 0.8.1 diff --git a/infra/charts/feast/charts/feast-jobservice/README.md b/infra/charts/feast/charts/feast-jobservice/README.md index 859889978ac..3f01dce016e 100644 --- a/infra/charts/feast/charts/feast-jobservice/README.md +++ b/infra/charts/feast/charts/feast-jobservice/README.md @@ -2,7 +2,7 @@ feast-jobservice ================ Feast Job Service manage ingestion jobs. -Current chart version is `0.8.0` +Current chart version is `0.8.1` diff --git a/infra/charts/feast/charts/feast-jupyter/Chart.yaml b/infra/charts/feast/charts/feast-jupyter/Chart.yaml index 3f8131fa427..7e89f5981ff 100644 --- a/infra/charts/feast/charts/feast-jupyter/Chart.yaml +++ b/infra/charts/feast/charts/feast-jupyter/Chart.yaml @@ -1,4 +1,4 @@ apiVersion: v1 description: Feast Jupyter provides a Jupyter server with pre-installed Feast SDK name: feast-jupyter -version: 0.8.0 +version: 0.8.1 diff --git a/infra/charts/feast/charts/feast-jupyter/README.md b/infra/charts/feast/charts/feast-jupyter/README.md index 5bb26b4357a..f541547a4a8 100644 --- a/infra/charts/feast/charts/feast-jupyter/README.md +++ b/infra/charts/feast/charts/feast-jupyter/README.md @@ -2,7 +2,7 @@ feast-jupyter ============= Feast Jupyter provides a Jupyter server with pre-installed Feast SDK -Current chart version is `0.8.0` +Current chart version is `0.8.1` diff --git a/infra/charts/feast/charts/feast-serving/Chart.yaml b/infra/charts/feast/charts/feast-serving/Chart.yaml index fb007dd04dc..13b409081f6 100644 --- a/infra/charts/feast/charts/feast-serving/Chart.yaml +++ b/infra/charts/feast/charts/feast-serving/Chart.yaml @@ -1,4 +1,4 @@ apiVersion: v1 description: Feast Serving serves low-latency latest features and historical batch features. name: feast-serving -version: 0.8.0 +version: 0.8.1 diff --git a/infra/charts/feast/charts/feast-serving/README.md b/infra/charts/feast/charts/feast-serving/README.md index acfdc0040f9..908f22348d2 100644 --- a/infra/charts/feast/charts/feast-serving/README.md +++ b/infra/charts/feast/charts/feast-serving/README.md @@ -2,7 +2,7 @@ feast-serving ============= Feast Serving serves low-latency latest features and historical batch features. -Current chart version is `0.8.0` +Current chart version is `0.8.1` diff --git a/infra/charts/feast/requirements.lock b/infra/charts/feast/requirements.lock index 2936f5007d4..f44e039379d 100644 --- a/infra/charts/feast/requirements.lock +++ b/infra/charts/feast/requirements.lock @@ -1,13 +1,13 @@ dependencies: - name: feast-core repository: "" - version: 0.8.0 + version: 0.8.1 - name: feast-serving repository: "" - version: 0.8.0 + version: 0.8.1 - name: feast-jupyter repository: "" - version: 0.8.0 + version: 0.8.1 - name: postgresql repository: https://kubernetes-charts.storage.googleapis.com/ version: 8.6.1 diff --git a/infra/charts/feast/requirements.yaml b/infra/charts/feast/requirements.yaml index dd47411da23..db501d872ba 100644 --- a/infra/charts/feast/requirements.yaml +++ b/infra/charts/feast/requirements.yaml @@ -1,13 +1,13 @@ dependencies: - name: feast-core - version: 0.8.0 + version: 0.8.1 condition: feast-core.enabled - name: feast-serving alias: feast-online-serving - version: 0.8.0 + version: 0.8.1 condition: feast-online-serving.enabled - name: feast-jupyter - version: 0.8.0 + version: 0.8.1 condition: feast-jupyter.enabled - name: postgresql version: 8.6.1 diff --git a/infra/docker-compose/.env.sample b/infra/docker-compose/.env.sample index ea98b441d6e..8c48fbc976b 100644 --- a/infra/docker-compose/.env.sample +++ b/infra/docker-compose/.env.sample @@ -2,4 +2,5 @@ COMPOSE_PROJECT_NAME=feast FEAST_VERSION=develop FEAST_CORE_CONFIG=./core/core.yml FEAST_ONLINE_SERVING_CONFIG=./serving/online-serving.yml -GCP_SERVICE_ACCOUNT=./gcp-service-accounts/placeholder.json \ No newline at end of file +GCP_SERVICE_ACCOUNT=./gcp-service-accounts/placeholder.json +INGESTION_JAR_PATH=https://storage.googleapis.com/feast-jobs/spark/ingestion/feast-ingestion-spark-develop.jar \ No newline at end of file diff --git a/infra/docker-compose/docker-compose.yml b/infra/docker-compose/docker-compose.yml index 90c94e0055c..4fea101478a 100644 --- a/infra/docker-compose/docker-compose.yml +++ b/infra/docker-compose/docker-compose.yml @@ -36,6 +36,7 @@ services: FEAST_HISTORICAL_FEATURE_OUTPUT_LOCATION: file:///shared/historical_feature_output FEAST_HISTORICAL_FEATURE_OUTPUT_FORMAT: parquet FEAST_REDIS_HOST: redis + FEAST_SPARK_INGESTION_JAR: ${INGESTION_JAR_PATH} jupyter: image: gcr.io/kf-feast/feast-jupyter:${FEAST_VERSION} diff --git a/infra/docker/ci/Dockerfile b/infra/docker/ci/Dockerfile index c6ef17aabd6..29d077b8d3c 100644 --- a/infra/docker/ci/Dockerfile +++ b/infra/docker/ci/Dockerfile @@ -1,5 +1,7 @@ FROM maven:3.6-jdk-11 +ARG REVISION + # Install Google Cloud SDK RUN echo "deb [signed-by=/usr/share/keyrings/cloud.google.gpg] http://packages.cloud.google.com/apt cloud-sdk main" \ | tee -a /etc/apt/sources.list.d/google-cloud-sdk.list && \ diff --git a/infra/docker/core/Dockerfile b/infra/docker/core/Dockerfile index bf2e17cf076..997d8484dcd 100644 --- a/infra/docker/core/Dockerfile +++ b/infra/docker/core/Dockerfile @@ -29,8 +29,8 @@ RUN mvn dependency:go-offline -DexcludeGroupIds:dev.feast 2>/dev/null || true COPY . . -ARG REVISION=dev -RUN mvn --also-make --projects core -Drevision=$REVISION \ +ARG VERSION=dev +RUN mvn --also-make --projects core -Drevision=$VERSION \ -DskipUTs=true --batch-mode clean package # @@ -46,9 +46,9 @@ RUN wget -q https://github.com/grpc-ecosystem/grpc-health-probe/releases/downloa # ============================================================ FROM openjdk:11-jre as production -ARG REVISION=dev +ARG VERSION=dev -COPY --from=builder /build/core/target/feast-core-$REVISION-exec.jar /opt/feast/feast-core.jar +COPY --from=builder /build/core/target/feast-core-$VERSION-exec.jar /opt/feast/feast-core.jar COPY --from=builder /usr/bin/grpc-health-probe /usr/bin/grpc-health-probe CMD ["java",\ diff --git a/infra/docker/jobservice/Dockerfile b/infra/docker/jobservice/Dockerfile index f3f8127ec75..9fd991f76c1 100644 --- a/infra/docker/jobservice/Dockerfile +++ b/infra/docker/jobservice/Dockerfile @@ -14,7 +14,7 @@ RUN apt-get update && apt-get -y install make git wget RUN make compile-protos-python # Install Feast SDK -COPY .git .git +RUN git init . COPY README.md README.md RUN pip install -U -e sdk/python RUN pip install "s3fs" "boto3" "urllib3>=1.25.4" diff --git a/infra/docker/jupyter/Dockerfile b/infra/docker/jupyter/Dockerfile index 69aa3622caf..567182a0028 100644 --- a/infra/docker/jupyter/Dockerfile +++ b/infra/docker/jupyter/Dockerfile @@ -15,7 +15,7 @@ RUN make compile-protos-python RUN pip install -r sdk/python/requirements-ci.txt # Install Feast SDK -COPY .git .git +RUN git init . COPY README.md README.md RUN pip install -e sdk/python -U RUN pip install "s3fs" "boto3" "urllib3>=1.25.4" diff --git a/infra/docker/serving/Dockerfile b/infra/docker/serving/Dockerfile index 960e2848906..1b63a7b2389 100644 --- a/infra/docker/serving/Dockerfile +++ b/infra/docker/serving/Dockerfile @@ -29,8 +29,8 @@ RUN mvn dependency:go-offline -DexcludeGroupIds:dev.feast 2>/dev/null || true COPY . . -ARG REVISION=dev -RUN mvn --also-make --projects serving -Drevision=$REVISION \ +ARG VERSION=dev +RUN mvn --also-make --projects serving -Drevision=$VERSION \ -DskipUTs=true --batch-mode clean package # # Download grpc_health_probe to run health check for Feast Serving @@ -45,8 +45,8 @@ RUN wget -q https://github.com/grpc-ecosystem/grpc-health-probe/releases/downloa # ============================================================ FROM openjdk:11-jre-slim as production -ARG REVISION=dev -COPY --from=builder /build/serving/target/feast-serving-$REVISION-exec.jar /opt/feast/feast-serving.jar +ARG VERSION=dev +COPY --from=builder /build/serving/target/feast-serving-$VERSION-exec.jar /opt/feast/feast-serving.jar COPY --from=builder /usr/bin/grpc-health-probe /usr/bin/grpc-health-probe CMD ["java",\ "-Xms1024m",\ diff --git a/infra/scripts/setup-e2e-env-aws.sh b/infra/scripts/setup-e2e-env-aws.sh index 6521f941e25..d74492b82c8 100755 --- a/infra/scripts/setup-e2e-env-aws.sh +++ b/infra/scripts/setup-e2e-env-aws.sh @@ -8,7 +8,8 @@ python -m pip install -qr sdk/python/requirements-dev.txt python -m pip install -qr tests/requirements.txt # Using mvn -q to make it less verbose. This step happens after docker containers were -# succesfully built so it should be unlikely to fail. +# succesfully built so it should be unlikely to fail, therefore we likely won't need detailed logs. echo "########## Building ingestion jar" TIMEFORMAT='########## took %R seconds' -time mvn -q --no-transfer-progress -Dmaven.javadoc.skip=true -Dgpg.skip -DskipUTs=true clean package + +time make build-java-no-tests REVISION=develop MAVEN_EXTRA_OPTS="-q --no-transfer-progress" diff --git a/infra/scripts/sync-helm-charts.sh b/infra/scripts/sync-helm-charts.sh index 8c242aeae69..acb4effe66e 100755 --- a/infra/scripts/sync-helm-charts.sh +++ b/infra/scripts/sync-helm-charts.sh @@ -30,6 +30,8 @@ fi exit_code=0 +helm repo add bitnami https://charts.bitnami.com/bitnami + for dir in "$repo_dir"/*; do if helm dep update "$dir" && helm dep build "$dir"; then helm package --destination "$sync_dir" "$dir" diff --git a/infra/scripts/test-docker-compose.sh b/infra/scripts/test-docker-compose.sh index 6ac950aec04..348444956ff 100755 --- a/infra/scripts/test-docker-compose.sh +++ b/infra/scripts/test-docker-compose.sh @@ -69,4 +69,4 @@ docker exec \ -e DISABLE_FEAST_SERVICE_FIXTURES=true \ --user root \ feast_jupyter_1 bash \ - -c 'cd /feast/tests && python -m pip install -r requirements.txt && pytest e2e/ --ingestion-jar https://storage.googleapis.com/feast-jobs/spark/ingestion/feast-ingestion-spark-${FEAST_VERSION}.jar --redis-url redis:6379 --core-url core:6565 --serving-url online_serving:6566 --job-service-url jobservice:6568 --staging-path file:///shared/staging/ --kafka-brokers kafka:9092' + -c 'cd /feast/tests && python -m pip install -r requirements.txt && pytest e2e/ --ingestion-jar https://storage.googleapis.com/feast-jobs/spark/ingestion/feast-ingestion-spark-${FEAST_VERSION}.jar --redis-url redis:6379 --core-url core:6565 --serving-url online_serving:6566 --job-service-url jobservice:6568 --staging-path file:///shared/staging/ --kafka-brokers kafka:9092 --feast-version develop' diff --git a/infra/scripts/test-end-to-end-aws.sh b/infra/scripts/test-end-to-end-aws.sh index 981118fd2e5..807f439f6d6 100755 --- a/infra/scripts/test-end-to-end-aws.sh +++ b/infra/scripts/test-end-to-end-aws.sh @@ -8,6 +8,7 @@ export DISABLE_FEAST_SERVICE_FIXTURES=1 export DISABLE_SERVICE_FIXTURES=1 PYTHONPATH=sdk/python pytest tests/e2e/ \ + --feast-version develop \ --core-url cicd-feast-core:6565 \ --serving-url cicd-feast-online-serving:6566 \ --env aws \ diff --git a/infra/scripts/test-end-to-end-gcp.sh b/infra/scripts/test-end-to-end-gcp.sh index 3dab0513f6c..4679cbb9b1f 100755 --- a/infra/scripts/test-end-to-end-gcp.sh +++ b/infra/scripts/test-end-to-end-gcp.sh @@ -1,7 +1,10 @@ #!/usr/bin/env bash export DISABLE_SERVICE_FIXTURES=1 +export MAVEN_OPTS="-Dmaven.repo.local=/tmp/.m2/repository -DdependencyLocationsEnabled=false" +export MAVEN_CACHE="gs://feast-templocation-kf-feast/.m2.2020-11-17.tar" +infra/scripts/download-maven-cache.sh --archive-uri ${MAVEN_CACHE} --output-dir /tmp apt-get update && apt-get install -y redis-server postgresql libpq-dev make build-java-no-tests REVISION=develop diff --git a/infra/scripts/test-end-to-end.sh b/infra/scripts/test-end-to-end.sh index 60f9c33a140..03beb22ab4d 100755 --- a/infra/scripts/test-end-to-end.sh +++ b/infra/scripts/test-end-to-end.sh @@ -1,5 +1,9 @@ #!/usr/bin/env bash +export MAVEN_OPTS="-Dmaven.repo.local=/tmp/.m2/repository -DdependencyLocationsEnabled=false" +export MAVEN_CACHE="gs://feast-templocation-kf-feast/.m2.2020-11-17.tar" + +infra/scripts/download-maven-cache.sh --archive-uri ${MAVEN_CACHE} --output-dir /tmp apt-get update && apt-get install -y redis-server postgresql libpq-dev make build-java-no-tests REVISION=develop diff --git a/pom.xml b/pom.xml index 409e60e6f4e..f80770a2828 100644 --- a/pom.xml +++ b/pom.xml @@ -41,7 +41,7 @@ - 0.8.0 + 0.8.1 https://github.com/feast-dev/feast UTF-8 diff --git a/protos/feast/core/JobService.proto b/protos/feast/core/JobService.proto index d3924ecc71f..d0ae6ac05f2 100644 --- a/protos/feast/core/JobService.proto +++ b/protos/feast/core/JobService.proto @@ -72,6 +72,8 @@ message Job { JobType type = 2; // Current job status JobStatus status = 3; + // Deterministic hash of the Job + string hash = 8; message RetrievalJobMeta { string output_location = 4; diff --git a/sdk/python/docs/index.rst b/sdk/python/docs/index.rst index 57d3b5976ed..782ec9c83e2 100644 --- a/sdk/python/docs/index.rst +++ b/sdk/python/docs/index.rst @@ -7,19 +7,10 @@ Client .. automodule:: feast.client :members: - -Feature Set -================== - -.. automodule:: feast.feature_set - :members: - - -Feature +Data Source ================== -.. automodule:: feast.feature - :inherited-members: +.. automodule:: feast.data_source :members: @@ -31,24 +22,22 @@ Entity :members: -Value +Feature Table ================== -.. automodule:: feast.value_type +.. automodule:: feast.feature_table :members: - -Source +Feature ================== -.. automodule:: feast.source +.. automodule:: feast.feature + :inherited-members: :members: - -Job +Constants ================== -.. automodule:: feast.job +.. automodule:: feast.constants :members: - - + :exclude-members: AuthProvider, ConfigMeta diff --git a/sdk/python/feast/cli.py b/sdk/python/feast/cli.py index 9d79c405186..43f9c581a6e 100644 --- a/sdk/python/feast/cli.py +++ b/sdk/python/feast/cli.py @@ -23,7 +23,7 @@ from feast.client import Client from feast.config import Config -from feast.constants import CONFIG_SPARK_LAUNCHER +from feast.constants import ConfigOptions as opt from feast.entity import Entity from feast.feature_table import FeatureTable from feast.job_service import start_job_service @@ -422,7 +422,7 @@ def stop_stream_to_online(feature_table: str): Stop stream to online sync job """ - spark_launcher = Config().get(CONFIG_SPARK_LAUNCHER) + spark_launcher = Config().get(opt.SPARK_LAUNCHER) if spark_launcher == "emr": import feast.pyspark.aws.jobs @@ -441,7 +441,7 @@ def list_jobs(): """ from tabulate import tabulate - spark_launcher = Config().get(CONFIG_SPARK_LAUNCHER) + spark_launcher = Config().get(opt.SPARK_LAUNCHER) if spark_launcher == "emr": import feast.pyspark.aws.jobs diff --git a/sdk/python/feast/client.py b/sdk/python/feast/client.py index 5e4b78b5ff1..231230a155e 100644 --- a/sdk/python/feast/client.py +++ b/sdk/python/feast/client.py @@ -24,24 +24,7 @@ import pandas as pd from feast.config import Config -from feast.constants import ( - CONFIG_CORE_ENABLE_SSL_KEY, - CONFIG_CORE_SERVER_SSL_CERT_KEY, - CONFIG_CORE_URL_KEY, - CONFIG_ENABLE_AUTH_KEY, - CONFIG_GRPC_CONNECTION_TIMEOUT_DEFAULT_KEY, - CONFIG_JOB_SERVICE_ENABLE_SSL_KEY, - CONFIG_JOB_SERVICE_SERVER_SSL_CERT_KEY, - CONFIG_JOB_SERVICE_URL_KEY, - CONFIG_PROJECT_KEY, - CONFIG_SERVING_ENABLE_SSL_KEY, - CONFIG_SERVING_SERVER_SSL_CERT_KEY, - CONFIG_SERVING_URL_KEY, - CONFIG_SPARK_HISTORICAL_FEATURE_OUTPUT_FORMAT, - CONFIG_SPARK_HISTORICAL_FEATURE_OUTPUT_LOCATION, - CONFIG_SPARK_STAGING_LOCATION, - FEAST_DEFAULT_OPTIONS, -) +from feast.constants import ConfigOptions as opt from feast.core.CoreService_pb2 import ( ApplyEntityRequest, ApplyEntityResponse, @@ -81,7 +64,6 @@ from feast.grpc import auth as feast_auth from feast.grpc.grpc import create_grpc_channel from feast.loaders.ingest import ( - BATCH_INGESTION_PRODUCTION_TIMEOUT, _check_field_mappings, _read_table_from_source, _upload_to_bq_source, @@ -158,7 +140,7 @@ def __init__(self, options: Optional[Dict[str, str]] = None, **kwargs): self._auth_metadata: Optional[grpc.AuthMetadataPlugin] = None # Configure Auth Metadata Plugin if auth is enabled - if self._config.getboolean(CONFIG_ENABLE_AUTH_KEY): + if self._config.getboolean(opt.ENABLE_AUTH): self._auth_metadata = feast_auth.get_auth_metadata_plugin(self._config) @property @@ -170,12 +152,12 @@ def _core_service(self): """ if not self._core_service_stub: channel = create_grpc_channel( - url=self._config.get(CONFIG_CORE_URL_KEY), - enable_ssl=self._config.getboolean(CONFIG_CORE_ENABLE_SSL_KEY), - enable_auth=self._config.getboolean(CONFIG_ENABLE_AUTH_KEY), - ssl_server_cert_path=self._config.get(CONFIG_CORE_SERVER_SSL_CERT_KEY), + url=self._config.get(opt.CORE_URL), + enable_ssl=self._config.getboolean(opt.CORE_ENABLE_SSL), + enable_auth=self._config.getboolean(opt.ENABLE_AUTH), + ssl_server_cert_path=self._config.get(opt.CORE_SERVER_SSL_CERT), auth_metadata_plugin=self._auth_metadata, - timeout=self._config.getint(CONFIG_GRPC_CONNECTION_TIMEOUT_DEFAULT_KEY), + timeout=self._config.getint(opt.GRPC_CONNECTION_TIMEOUT), ) self._core_service_stub = CoreServiceStub(channel) return self._core_service_stub @@ -189,21 +171,19 @@ def _serving_service(self): """ if not self._serving_service_stub: channel = create_grpc_channel( - url=self._config.get(CONFIG_SERVING_URL_KEY), - enable_ssl=self._config.getboolean(CONFIG_SERVING_ENABLE_SSL_KEY), - enable_auth=self._config.getboolean(CONFIG_ENABLE_AUTH_KEY), - ssl_server_cert_path=self._config.get( - CONFIG_SERVING_SERVER_SSL_CERT_KEY - ), + url=self._config.get(opt.SERVING_URL), + enable_ssl=self._config.getboolean(opt.SERVING_ENABLE_SSL), + enable_auth=self._config.getboolean(opt.ENABLE_AUTH), + ssl_server_cert_path=self._config.get(opt.SERVING_SERVER_SSL_CERT), auth_metadata_plugin=self._auth_metadata, - timeout=self._config.getint(CONFIG_GRPC_CONNECTION_TIMEOUT_DEFAULT_KEY), + timeout=self._config.getint(opt.GRPC_CONNECTION_TIMEOUT), ) self._serving_service_stub = ServingServiceStub(channel) return self._serving_service_stub @property def _use_job_service(self) -> bool: - return self._config.exists(CONFIG_JOB_SERVICE_URL_KEY) + return self._config.exists(opt.JOB_SERVICE_URL) @property def _job_service(self): @@ -218,21 +198,19 @@ def _job_service(self): if not self._job_service_stub: channel = create_grpc_channel( - url=self._config.get(CONFIG_JOB_SERVICE_URL_KEY), - enable_ssl=self._config.getboolean(CONFIG_JOB_SERVICE_ENABLE_SSL_KEY), - enable_auth=self._config.getboolean(CONFIG_ENABLE_AUTH_KEY), - ssl_server_cert_path=self._config.get( - CONFIG_JOB_SERVICE_SERVER_SSL_CERT_KEY - ), + url=self._config.get(opt.JOB_SERVICE_URL), + enable_ssl=self._config.getboolean(opt.JOB_SERVICE_ENABLE_SSL), + enable_auth=self._config.getboolean(opt.ENABLE_AUTH), + ssl_server_cert_path=self._config.get(opt.JOB_SERVICE_SERVER_SSL_CERT), auth_metadata_plugin=self._auth_metadata, - timeout=self._config.getint(CONFIG_GRPC_CONNECTION_TIMEOUT_DEFAULT_KEY), + timeout=self._config.getint(opt.GRPC_CONNECTION_TIMEOUT), ) self._job_service_service_stub = JobServiceStub(channel) return self._job_service_service_stub def _extra_grpc_params(self) -> Dict[str, Any]: return dict( - timeout=self._config.getint(CONFIG_GRPC_CONNECTION_TIMEOUT_DEFAULT_KEY), + timeout=self._config.getint(opt.GRPC_CONNECTION_TIMEOUT), metadata=self._get_grpc_metadata(), ) @@ -244,7 +222,7 @@ def core_url(self) -> str: Returns: Feast Core URL string """ - return self._config.get(CONFIG_CORE_URL_KEY) + return self._config.get(opt.CORE_URL) @core_url.setter def core_url(self, value: str): @@ -254,7 +232,7 @@ def core_url(self, value: str): Args: value: Feast Core URL """ - self._config.set(CONFIG_CORE_URL_KEY, value) + self._config.set(opt.CORE_URL, value) @property def serving_url(self) -> str: @@ -264,7 +242,7 @@ def serving_url(self) -> str: Returns: Feast Serving URL string """ - return self._config.get(CONFIG_SERVING_URL_KEY) + return self._config.get(opt.SERVING_URL) @serving_url.setter def serving_url(self, value: str): @@ -274,7 +252,7 @@ def serving_url(self, value: str): Args: value: Feast Serving URL """ - self._config.set(CONFIG_SERVING_URL_KEY, value) + self._config.set(opt.SERVING_URL, value) @property def job_service_url(self) -> str: @@ -284,7 +262,7 @@ def job_service_url(self) -> str: Returns: Feast Job Service URL string """ - return self._config.get(CONFIG_JOB_SERVICE_URL_KEY) + return self._config.get(opt.JOB_SERVICE_URL) @job_service_url.setter def job_service_url(self, value: str): @@ -294,7 +272,7 @@ def job_service_url(self, value: str): Args: value: Feast Job Service URL """ - self._config.set(CONFIG_JOB_SERVICE_URL_KEY, value) + self._config.set(opt.JOB_SERVICE_URL, value) @property def core_secure(self) -> bool: @@ -304,7 +282,7 @@ def core_secure(self) -> bool: Returns: Whether client-side SSL/TLS is enabled """ - return self._config.getboolean(CONFIG_CORE_ENABLE_SSL_KEY) + return self._config.getboolean(opt.CORE_ENABLE_SSL) @core_secure.setter def core_secure(self, value: bool): @@ -314,7 +292,7 @@ def core_secure(self, value: bool): Args: value: True to enable client-side SSL/TLS """ - self._config.set(CONFIG_CORE_ENABLE_SSL_KEY, value) + self._config.set(opt.CORE_ENABLE_SSL, value) @property def serving_secure(self) -> bool: @@ -324,7 +302,7 @@ def serving_secure(self) -> bool: Returns: Whether client-side SSL/TLS is enabled """ - return self._config.getboolean(CONFIG_SERVING_ENABLE_SSL_KEY) + return self._config.getboolean(opt.SERVING_ENABLE_SSL) @serving_secure.setter def serving_secure(self, value: bool): @@ -334,7 +312,7 @@ def serving_secure(self, value: bool): Args: value: True to enable client-side SSL/TLS """ - self._config.set(CONFIG_SERVING_ENABLE_SSL_KEY, value) + self._config.set(opt.SERVING_ENABLE_SSL, value) @property def job_service_secure(self) -> bool: @@ -344,7 +322,7 @@ def job_service_secure(self) -> bool: Returns: Whether client-side SSL/TLS is enabled """ - return self._config.getboolean(CONFIG_JOB_SERVICE_ENABLE_SSL_KEY) + return self._config.getboolean(opt.JOB_SERVICE_ENABLE_SSL) @job_service_secure.setter def job_service_secure(self, value: bool): @@ -354,7 +332,7 @@ def job_service_secure(self, value: bool): Args: value: True to enable client-side SSL/TLS """ - self._config.set(CONFIG_JOB_SERVICE_ENABLE_SSL_KEY, value) + self._config.set(opt.JOB_SERVICE_ENABLE_SSL, value) def version(self): """ @@ -371,7 +349,7 @@ def version(self): if self.serving_url: serving_version = self._serving_service.GetFeastServingInfo( GetFeastServingInfoRequest(), - timeout=self._config.getint(CONFIG_GRPC_CONNECTION_TIMEOUT_DEFAULT_KEY), + timeout=self._config.getint(opt.GRPC_CONNECTION_TIMEOUT), metadata=self._get_grpc_metadata(), ).version result["serving"] = {"url": self.serving_url, "version": serving_version} @@ -379,7 +357,7 @@ def version(self): if self.core_url: core_version = self._core_service.GetFeastCoreVersion( GetFeastCoreVersionRequest(), - timeout=self._config.getint(CONFIG_GRPC_CONNECTION_TIMEOUT_DEFAULT_KEY), + timeout=self._config.getint(opt.GRPC_CONNECTION_TIMEOUT), metadata=self._get_grpc_metadata(), ).version result["core"] = {"url": self.core_url, "version": core_version} @@ -394,9 +372,9 @@ def project(self) -> str: Returns: Project name """ - if not self._config.get(CONFIG_PROJECT_KEY): + if not self._config.get(opt.PROJECT): raise ValueError("No project has been configured.") - return self._config.get(CONFIG_PROJECT_KEY) + return self._config.get(opt.PROJECT) def set_project(self, project: Optional[str] = None): """ @@ -406,8 +384,8 @@ def set_project(self, project: Optional[str] = None): project: Project to set as active. If unset, will reset to the default project. """ if project is None: - project = FEAST_DEFAULT_OPTIONS[CONFIG_PROJECT_KEY] - self._config.set(CONFIG_PROJECT_KEY, project) + project = opt().PROJECT + self._config.set(opt.PROJECT, project) def list_projects(self) -> List[str]: """ @@ -420,7 +398,7 @@ def list_projects(self) -> List[str]: response = self._core_service.ListProjects( ListProjectsRequest(), - timeout=self._config.getint(CONFIG_GRPC_CONNECTION_TIMEOUT_DEFAULT_KEY), + timeout=self._config.getint(opt.GRPC_CONNECTION_TIMEOUT), metadata=self._get_grpc_metadata(), ) # type: ListProjectsResponse return list(response.projects) @@ -435,7 +413,7 @@ def create_project(self, project: str): self._core_service.CreateProject( CreateProjectRequest(name=project), - timeout=self._config.getint(CONFIG_GRPC_CONNECTION_TIMEOUT_DEFAULT_KEY), + timeout=self._config.getint(opt.GRPC_CONNECTION_TIMEOUT), metadata=self._get_grpc_metadata(), ) # type: CreateProjectResponse @@ -452,7 +430,7 @@ def archive_project(self, project): try: self._core_service_stub.ArchiveProject( ArchiveProjectRequest(name=project), - timeout=self._config.getint(CONFIG_GRPC_CONNECTION_TIMEOUT_DEFAULT_KEY), + timeout=self._config.getint(opt.GRPC_CONNECTION_TIMEOUT), metadata=self._get_grpc_metadata(), ) # type: ArchiveProjectResponse except grpc.RpcError as e: @@ -460,7 +438,7 @@ def archive_project(self, project): # revert to the default project if self._project == project: - self._project = FEAST_DEFAULT_OPTIONS[CONFIG_PROJECT_KEY] + self._project = opt().PROJECT def apply_entity(self, entities: Union[List[Entity], Entity], project: str = None): """ @@ -513,7 +491,7 @@ def _apply_entity(self, project: str, entity: Entity): try: apply_entity_response = self._core_service.ApplyEntity( ApplyEntityRequest(project=project, spec=entity_proto), # type: ignore - timeout=self._config.getint(CONFIG_GRPC_CONNECTION_TIMEOUT_DEFAULT_KEY), + timeout=self._config.getint(opt.GRPC_CONNECTION_TIMEOUT), metadata=self._get_grpc_metadata(), ) # type: ApplyEntityResponse except grpc.RpcError as e: @@ -625,7 +603,7 @@ def _apply_feature_table(self, project: str, feature_table: FeatureTable): try: apply_feature_table_response = self._core_service.ApplyFeatureTable( ApplyFeatureTableRequest(project=project, table_spec=feature_table_proto), # type: ignore - timeout=self._config.getint(CONFIG_GRPC_CONNECTION_TIMEOUT_DEFAULT_KEY), + timeout=self._config.getint(opt.GRPC_CONNECTION_TIMEOUT), metadata=self._get_grpc_metadata(), ) # type: ApplyFeatureTableResponse except grpc.RpcError as e: @@ -722,7 +700,7 @@ def ingest( project: str = None, chunk_size: int = 10000, max_workers: int = max(CPU_COUNT - 1, 1), - timeout: int = BATCH_INGESTION_PRODUCTION_TIMEOUT, + timeout: int = int(opt().BATCH_INGESTION_PRODUCTION_TIMEOUT), ) -> None: """ Batch load feature data into a FeatureTable. @@ -768,6 +746,8 @@ def ingest( if project is None: project = self.project + if isinstance(feature_table, str): + name = feature_table if isinstance(feature_table, FeatureTable): name = feature_table.name @@ -847,7 +827,7 @@ def _get_grpc_metadata(self): Returns: Tuple of metadata to attach to each gRPC call """ - if self._config.getboolean(CONFIG_ENABLE_AUTH_KEY) and self._auth_metadata: + if self._config.getboolean(opt.ENABLE_AUTH) and self._auth_metadata: return self._auth_metadata.get_signed_meta() return () @@ -893,7 +873,7 @@ def get_online_features( entity_rows=_infer_online_entity_rows(entity_rows), project=project if project is not None else self.project, ), - timeout=self._config.getint(CONFIG_GRPC_CONNECTION_TIMEOUT_DEFAULT_KEY), + timeout=self._config.getint(opt.GRPC_CONNECTION_TIMEOUT), metadata=self._get_grpc_metadata(), ) except grpc.RpcError as e: @@ -952,10 +932,10 @@ def get_historical_features( if output_location is None: output_location = os.path.join( - self._config.get(CONFIG_SPARK_HISTORICAL_FEATURE_OUTPUT_LOCATION), + self._config.get(opt.HISTORICAL_FEATURE_OUTPUT_LOCATION), str(uuid.uuid4()), ) - output_format = self._config.get(CONFIG_SPARK_HISTORICAL_FEATURE_OUTPUT_FORMAT) + output_format = self._config.get(opt.HISTORICAL_FEATURE_OUTPUT_FORMAT) feature_sources = [ feature_table.batch_source for feature_table in feature_tables ] @@ -976,7 +956,7 @@ def get_historical_features( else: entity_source = stage_entities_to_fs( entity_source, - staging_location=self._config.get(CONFIG_SPARK_STAGING_LOCATION), + staging_location=self._config.get(opt.SPARK_STAGING_LOCATION), ) if self._use_job_service: @@ -1098,12 +1078,15 @@ def start_offline_to_online_ingestion( ) def start_stream_to_online_ingestion( - self, feature_table: FeatureTable, extra_jars: Optional[List[str]] = None, + self, + feature_table: FeatureTable, + extra_jars: Optional[List[str]] = None, + project: str = None, ) -> SparkJob: if not self._use_job_service: return start_stream_to_online_ingestion( client=self, - project=self.project, + project=project or self.project, feature_table=feature_table, extra_jars=extra_jars or [], ) @@ -1113,7 +1096,7 @@ def start_stream_to_online_ingestion( ) response = self._job_service.StartStreamToOnlineIngestionJob(request) return RemoteStreamIngestionJob( - self._job_service, self._extra_grpc_params, response.id, + self._job_service, self._extra_grpc_params, response.id ) def list_jobs(self, include_terminated: bool) -> List[SparkJob]: diff --git a/sdk/python/feast/config.py b/sdk/python/feast/config.py index 1bbab4edbcf..a3e3f1bd0bf 100644 --- a/sdk/python/feast/config.py +++ b/sdk/python/feast/config.py @@ -24,9 +24,9 @@ CONFIG_FILE_DEFAULT_DIRECTORY, CONFIG_FILE_NAME, CONFIG_FILE_SECTION, - FEAST_CONFIG_FILE_ENV_KEY, + FEAST_CONFIG_FILE_ENV, ) -from feast.constants import FEAST_DEFAULT_OPTIONS as DEFAULTS +from feast.constants import ConfigOptions as opt _logger = logging.getLogger(__name__) @@ -50,7 +50,7 @@ def _init_config(path: str): os.makedirs(os.path.dirname(config_dir), exist_ok=True) # Create the configuration file itself - config = ConfigParser(defaults=DEFAULTS) + config = ConfigParser(defaults=opt().defaults()) if os.path.exists(path): config.read(path) @@ -104,9 +104,7 @@ def __init__( if not path: path = join( expanduser("~"), - os.environ.get( - FEAST_CONFIG_FILE_ENV_KEY, CONFIG_FILE_DEFAULT_DIRECTORY, - ), + os.environ.get(FEAST_CONFIG_FILE_ENV, CONFIG_FILE_DEFAULT_DIRECTORY,), CONFIG_FILE_NAME, ) diff --git a/sdk/python/feast/constants.py b/sdk/python/feast/constants.py index 0a4b4f52e85..4d677bc38d8 100644 --- a/sdk/python/feast/constants.py +++ b/sdk/python/feast/constants.py @@ -14,6 +14,7 @@ # limitations under the License. # from enum import Enum +from typing import Optional class AuthProvider(Enum): @@ -21,132 +22,216 @@ class AuthProvider(Enum): OAUTH = "oauth" -DATETIME_COLUMN = "datetime" - -# Environmental variable to specify Feast configuration file location -FEAST_CONFIG_FILE_ENV_KEY = "FEAST_CONFIG" - -# Default prefix to Feast environmental variables -CONFIG_FEAST_ENV_VAR_PREFIX = "FEAST_" - -# Default directory to Feast configuration file -CONFIG_FILE_DEFAULT_DIRECTORY = ".feast" - -# Default Feast configuration file name -CONFIG_FILE_NAME = "config" - -# Default section in Feast configuration file to specify options -CONFIG_FILE_SECTION = "general" - -# Feast Configuration Options -CONFIG_PROJECT_KEY = "project" -CONFIG_CORE_URL_KEY = "core_url" -CONFIG_CORE_ENABLE_SSL_KEY = "core_enable_ssl" -CONFIG_ENABLE_AUTH_KEY = "enable_auth" -CONFIG_ENABLE_AUTH_TOKEN_KEY = "auth_token" -CONFIG_CORE_SERVER_SSL_CERT_KEY = "core_server_ssl_cert" -CONFIG_JOB_CONTROLLER_SERVER_KEY = "jobcontroller_url" -CONFIG_SERVING_URL_KEY = "serving_url" -CONFIG_SERVING_ENABLE_SSL_KEY = "serving_enable_ssl" -CONFIG_SERVING_SERVER_SSL_CERT_KEY = "serving_server_ssl_cert" -CONFIG_JOB_SERVICE_URL_KEY = "job_service_url" -CONFIG_JOB_SERVICE_ENABLE_SSL_KEY = "job_service_enable_ssl" -CONFIG_JOB_SERVICE_SERVER_SSL_CERT_KEY = "job_service_server_ssl_cert" -CONFIG_GRPC_CONNECTION_TIMEOUT_DEFAULT_KEY = "grpc_connection_timeout_default" -CONFIG_GRPC_CONNECTION_TIMEOUT_APPLY_KEY = "grpc_connection_timeout_apply" -CONFIG_BATCH_FEATURE_REQUEST_WAIT_TIME_SECONDS_KEY = ( - "batch_feature_request_wait_time_seconds" -) -CONFIG_OAUTH_GRANT_TYPE_KEY = "oauth_grant_type" -CONFIG_OAUTH_CLIENT_ID_KEY = "oauth_client_id" -CONFIG_OAUTH_CLIENT_SECRET_KEY = "oauth_client_secret" -CONFIG_OAUTH_AUDIENCE_KEY = "oauth_audience" -CONFIG_OAUTH_TOKEN_REQUEST_URL_KEY = "oauth_token_request_url" -CONFIG_AUTH_PROVIDER = "auth_provider" - -CONFIG_TIMEOUT_KEY = "timeout" -CONFIG_MAX_WAIT_INTERVAL_KEY = "max_wait_interval" - -# Spark Job Config -CONFIG_SPARK_LAUNCHER = "spark_launcher" # standalone, dataproc, emr - -CONFIG_SPARK_STAGING_LOCATION = "spark_staging_location" - -CONFIG_SPARK_INGESTION_JOB_JAR = "spark_ingestion_jar" - -CONFIG_SPARK_STANDALONE_MASTER = "spark_standalone_master" -CONFIG_SPARK_HOME = "spark_home" - -CONFIG_SPARK_DATAPROC_CLUSTER_NAME = "dataproc_cluster_name" -CONFIG_SPARK_DATAPROC_PROJECT = "dataproc_project" -CONFIG_SPARK_DATAPROC_REGION = "dataproc_region" - -CONFIG_SPARK_HISTORICAL_FEATURE_OUTPUT_FORMAT = "historical_feature_output_format" -CONFIG_SPARK_HISTORICAL_FEATURE_OUTPUT_LOCATION = "historical_feature_output_location" - -CONFIG_REDIS_HOST = "redis_host" -CONFIG_REDIS_PORT = "redis_port" -CONFIG_REDIS_SSL = "redis_ssl" - -CONFIG_STATSD_ENABLED = "statsd_enabled" -CONFIG_STATSD_HOST = "statsd_host" -CONFIG_STATSD_PORT = "statsd_port" - -CONFIG_DEADLETTER_PATH = "deadletter_path" -CONFIG_STENCIL_URL = "stencil_url" - -CONFIG_SPARK_EMR_REGION = "emr_region" -CONFIG_SPARK_EMR_CLUSTER_ID = "emr_cluster_id" -CONFIG_SPARK_EMR_CLUSTER_TEMPLATE_PATH = "emr_cluster_template_path" -CONFIG_SPARK_EMR_LOG_LOCATION = "emr_log_location" - -# Configuration option default values -FEAST_DEFAULT_OPTIONS = { - # Default Feast project to use - CONFIG_PROJECT_KEY: "default", - # Default Feast Core URL - CONFIG_CORE_URL_KEY: "localhost:6565", - # Enable or disable TLS/SSL to Feast Core - CONFIG_CORE_ENABLE_SSL_KEY: "False", - # Enable user authentication to Feast Core - CONFIG_ENABLE_AUTH_KEY: "False", - # Path to certificate(s) to secure connection to Feast Core - CONFIG_CORE_SERVER_SSL_CERT_KEY: "", - # Default Feast Job Controller URL - CONFIG_JOB_CONTROLLER_SERVER_KEY: "localhost:6570", - # Default Feast Serving URL - CONFIG_SERVING_URL_KEY: "localhost:6565", - # Enable or disable TLS/SSL to Feast Serving - CONFIG_SERVING_ENABLE_SSL_KEY: "False", - # Path to certificate(s) to secure connection to Feast Serving - CONFIG_SERVING_SERVER_SSL_CERT_KEY: "", - # Default connection timeout to Feast Serving, Feast Core, and Feast Job Service (in seconds) - CONFIG_GRPC_CONNECTION_TIMEOUT_DEFAULT_KEY: "10", - # Default gRPC connection timeout when sending an ApplyFeatureSet command to - # Feast Core (in seconds) - CONFIG_GRPC_CONNECTION_TIMEOUT_APPLY_KEY: "600", - # Time to wait for batch feature requests before timing out. - CONFIG_BATCH_FEATURE_REQUEST_WAIT_TIME_SECONDS_KEY: "600", - CONFIG_TIMEOUT_KEY: "21600", - CONFIG_MAX_WAIT_INTERVAL_KEY: "60", - # Authentication Provider - Google OpenID/OAuth - CONFIG_AUTH_PROVIDER: "google", - CONFIG_SPARK_LAUNCHER: "dataproc", - CONFIG_SPARK_INGESTION_JOB_JAR: "https://storage.googleapis.com/feast-jobs/spark/" - "ingestion/feast-ingestion-spark-develop.jar", - CONFIG_SPARK_STANDALONE_MASTER: "local[*]", - CONFIG_REDIS_HOST: "localhost", - CONFIG_REDIS_PORT: "6379", - CONFIG_REDIS_SSL: "False", - CONFIG_SPARK_HISTORICAL_FEATURE_OUTPUT_FORMAT: "parquet", - # Enable or disable TLS/SSL to Feast Service - CONFIG_JOB_SERVICE_ENABLE_SSL_KEY: "False", - # Path to certificate(s) to secure connection to Feast Job Service - CONFIG_JOB_SERVICE_SERVER_SSL_CERT_KEY: "", - CONFIG_STATSD_ENABLED: "False", - # IngestionJob DeadLetter Destination - CONFIG_DEADLETTER_PATH: "", - # ProtoRegistry Address (currently only Stencil Server is supported as registry) - # https://github.com/gojekfarm/stencil - CONFIG_STENCIL_URL: "", -} +class Option: + def __init__(self, name, default): + self._name = name + self._default = default + + def __get__(self, instance, owner): + if instance is None: + return self._name.lower() + + return self._default + + +class ConfigMeta(type): + """ + Class factory which customizes ConfigOptions class instantiation. + Specifically, setting configuration option's name to lowercase of capitalized variable. + """ + + def __new__(cls, name, bases, attrs): + keys = [ + k for k, v in attrs.items() if not k.startswith("_") and not callable(v) + ] + attrs["__config_keys__"] = keys + attrs.update({k: Option(k, attrs[k]) for k in keys}) + return super().__new__(cls, name, bases, attrs) + + +#: Default datetime column name for point-in-time join +DATETIME_COLUMN: str = "datetime" + +#: Environmental variable to specify Feast configuration file location +FEAST_CONFIG_FILE_ENV: str = "FEAST_CONFIG" + +#: Default prefix to Feast environmental variables +CONFIG_FEAST_ENV_VAR_PREFIX: str = "FEAST_" + +#: Default directory to Feast configuration file +CONFIG_FILE_DEFAULT_DIRECTORY: str = ".feast" + +#: Default Feast configuration file name +CONFIG_FILE_NAME: str = "config" + +#: Default section in Feast configuration file to specify options +CONFIG_FILE_SECTION: str = "general" + +# Maximum interval(secs) to wait between retries for retry function +MAX_WAIT_INTERVAL: str = "60" + + +class ConfigOptions(metaclass=ConfigMeta): + """ Feast Configuration Options """ + + #: Feast project namespace to use + PROJECT: str = "default" + + #: Default Feast Core URL + CORE_URL: str = "localhost:6565" + + #: Enable or disable TLS/SSL to Feast Core + CORE_ENABLE_SSL: str = "False" + + #: Enable user authentication to Feast Core + ENABLE_AUTH: str = "False" + + #: JWT Auth token for user authentication to Feast + AUTH_TOKEN: Optional[str] = None + + #: Path to certificate(s) to secure connection to Feast Core + CORE_SERVER_SSL_CERT: str = "" + + #: Default Feast Serving URL + SERVING_URL: str = "localhost:6566" + + #: Enable or disable TLS/SSL to Feast Serving + SERVING_ENABLE_SSL: str = "False" + + #: Path to certificate(s) to secure connection to Feast Serving + SERVING_SERVER_SSL_CERT: str = "" + + #: Default Feast Job Service URL + JOB_SERVICE_URL: Optional[str] = None + + #: Enable or disable TLS/SSL to Feast Job Service + JOB_SERVICE_ENABLE_SSL: str = "False" + + #: Path to certificate(s) to secure connection to Feast Job Service + JOB_SERVICE_SERVER_SSL_CERT: str = "" + + #: Enable or disable control loop for Feast Job Service + JOB_SERVICE_ENABLE_CONTROL_LOOP: str = "False" + + #: Default connection timeout to Feast Serving, Feast Core, and Feast Job Service (in seconds) + GRPC_CONNECTION_TIMEOUT: str = "10" + + #: Default gRPC connection timeout when sending an ApplyFeatureTable command to Feast Core (in seconds) + GRPC_CONNECTION_TIMEOUT_APPLY: str = "600" + + #: Default timeout when running batch ingestion + BATCH_INGESTION_PRODUCTION_TIMEOUT: str = "120" + + #: Time to wait for historical feature requests before timing out. + BATCH_FEATURE_REQUEST_WAIT_TIME_SECONDS: str = "600" + + #: Authentication Provider - Google OpenID/OAuth + #: + #: Options: "google" / "oauth" + AUTH_PROVIDER: str = "google" + + #: Spark Job launcher. The choice of storage is connected to the choice of SPARK_LAUNCHER. + #: + #: Options: "standalone", "dataproc", "emr" + SPARK_LAUNCHER: Optional[str] = None + + #: Feast Spark Job ingestion jobs staging location. The choice of storage is connected to the choice of SPARK_LAUNCHER. + #: + #: Eg. gs://some-bucket/output/, s3://some-bucket/output/, file://data/subfolder/ + SPARK_STAGING_LOCATION: Optional[str] = None + + #: Feast Spark Job ingestion jar file. The choice of storage is connected to the choice of SPARK_LAUNCHER. + #: + #: Eg. "dataproc" (http and gs), "emr" (http and s3), "standalone" (http and file) + SPARK_INGESTION_JAR: str = "https://storage.googleapis.com/feast-jobs/spark/ingestion/feast-ingestion-spark-develop.jar" + + #: Spark resource manager master url + SPARK_STANDALONE_MASTER: str = "local[*]" + + #: Directory where Spark is installed + SPARK_HOME: Optional[str] = None + + #: Dataproc cluster to run Feast Spark Jobs in + DATAPROC_CLUSTER_NAME: Optional[str] = None + + #: Project of Dataproc cluster + DATAPROC_PROJECT: Optional[str] = None + + #: Region of Dataproc cluster + DATAPROC_REGION: Optional[str] = None + + #: No. of executor instances for Dataproc cluster + DATAPROC_EXECUTOR_INSTANCES = "2" + + #: No. of executor cores for Dataproc cluster + DATAPROC_EXECUTOR_CORES = "2" + + #: No. of executor memory for Dataproc cluster + DATAPROC_EXECUTOR_MEMORY = "2g" + + #: File format of historical retrieval features + HISTORICAL_FEATURE_OUTPUT_FORMAT: str = "parquet" + + #: File location of historical retrieval features + HISTORICAL_FEATURE_OUTPUT_LOCATION: Optional[str] = None + + #: Default Redis host + REDIS_HOST: str = "localhost" + + #: Default Redis port + REDIS_PORT: str = "6379" + + #: Enable or disable TLS/SSL to Redis + REDIS_SSL: str = "False" + + #: Enable or disable StatsD + STATSD_ENABLED: str = "False" + + #: Default StatsD port + STATSD_HOST: Optional[str] = None + + #: Default StatsD port + STATSD_PORT: Optional[str] = None + + #: Ingestion Job DeadLetter Destination. The choice of storage is connected to the choice of SPARK_LAUNCHER. + #: + #: Eg. gs://some-bucket/output/, s3://some-bucket/output/, file://data/subfolder/ + DEADLETTER_PATH: str = "" + + #: ProtoRegistry Address (currently only Stencil Server is supported as registry) + #: https://github.com/gojekfarm/stencil + STENCIL_URL: str = "" + + #: EMR cluster to run Feast Spark Jobs in + EMR_CLUSTER_ID: Optional[str] = None + + #: Region of EMR cluster + EMR_REGION: Optional[str] = None + + #: Template path of EMR cluster + EMR_CLUSTER_TEMPLATE_PATH: Optional[str] = None + + #: Log path of EMR cluster + EMR_LOG_LOCATION: Optional[str] = None + + #: Oauth grant type + OAUTH_GRANT_TYPE: Optional[str] = None + + #: Oauth client ID + OAUTH_CLIENT_ID: Optional[str] = None + + #: Oauth client secret + OAUTH_CLIENT_SECRET: Optional[str] = None + + #: Oauth intended recipients + OAUTH_AUDIENCE: Optional[str] = None + + #: Oauth token request url + OAUTH_TOKEN_REQUEST_URL: Optional[str] = None + + def defaults(self): + return { + k: getattr(self, k) + for k in self.__config_keys__ + if getattr(self, k) is not None + } diff --git a/sdk/python/feast/grpc/auth.py b/sdk/python/feast/grpc/auth.py index 9680607b8e3..8614015f456 100644 --- a/sdk/python/feast/grpc/auth.py +++ b/sdk/python/feast/grpc/auth.py @@ -18,16 +18,8 @@ from google.auth.exceptions import DefaultCredentialsError from feast.config import Config -from feast.constants import ( - CONFIG_AUTH_PROVIDER, - CONFIG_ENABLE_AUTH_TOKEN_KEY, - CONFIG_OAUTH_AUDIENCE_KEY, - CONFIG_OAUTH_CLIENT_ID_KEY, - CONFIG_OAUTH_CLIENT_SECRET_KEY, - CONFIG_OAUTH_GRANT_TYPE_KEY, - CONFIG_OAUTH_TOKEN_REQUEST_URL_KEY, - AuthProvider, -) +from feast.constants import AuthProvider +from feast.constants import ConfigOptions as opt def get_auth_metadata_plugin(config: Config) -> grpc.AuthMetadataPlugin: @@ -44,9 +36,9 @@ def get_auth_metadata_plugin(config: Config) -> grpc.AuthMetadataPlugin: Args: config: Feast Configuration object """ - if AuthProvider(config.get(CONFIG_AUTH_PROVIDER)) == AuthProvider.GOOGLE: + if AuthProvider(config.get(opt.AUTH_PROVIDER)) == AuthProvider.GOOGLE: return GoogleOpenIDAuthMetadataPlugin(config) - elif AuthProvider(config.get(CONFIG_AUTH_PROVIDER)) == AuthProvider.OAUTH: + elif AuthProvider(config.get(opt.AUTH_PROVIDER)) == AuthProvider.OAUTH: return OAuthMetadataPlugin(config) else: raise RuntimeError( @@ -75,15 +67,15 @@ def __init__(self, config: Config): self._token = None # If provided, set a static token - if config.exists(CONFIG_ENABLE_AUTH_TOKEN_KEY): - self._static_token = config.get(CONFIG_ENABLE_AUTH_TOKEN_KEY) + if config.exists(opt.AUTH_TOKEN): + self._static_token = config.get(opt.AUTH_TOKEN) self._refresh_token(config) elif ( - config.exists(CONFIG_OAUTH_GRANT_TYPE_KEY) - and config.exists(CONFIG_OAUTH_CLIENT_ID_KEY) - and config.exists(CONFIG_OAUTH_CLIENT_SECRET_KEY) - and config.exists(CONFIG_OAUTH_AUDIENCE_KEY) - and config.exists(CONFIG_OAUTH_TOKEN_REQUEST_URL_KEY) + config.exists(opt.OAUTH_GRANT_TYPE) + and config.exists(opt.OAUTH_CLIENT_ID) + and config.exists(opt.OAUTH_CLIENT_SECRET) + and config.exists(opt.OAUTH_AUDIENCE) + and config.exists(opt.OAUTH_TOKEN_REQUEST_URL) ): self._refresh_token(config) else: @@ -112,14 +104,14 @@ def _refresh_token(self, config: Config): headers_token = {"content-type": "application/json"} data_token = { - "grant_type": config.get(CONFIG_OAUTH_GRANT_TYPE_KEY), - "client_id": config.get(CONFIG_OAUTH_CLIENT_ID_KEY), - "client_secret": config.get(CONFIG_OAUTH_CLIENT_SECRET_KEY), - "audience": config.get(CONFIG_OAUTH_AUDIENCE_KEY), + "grant_type": config.get(opt.OAUTH_GRANT_TYPE), + "client_id": config.get(opt.OAUTH_CLIENT_ID), + "client_secret": config.get(opt.OAUTH_CLIENT_SECRET), + "audience": config.get(opt.OAUTH_AUDIENCE), } data_token = json.dumps(data_token) response_token = requests.post( - config.get(CONFIG_OAUTH_TOKEN_REQUEST_URL_KEY), + config.get(opt.OAUTH_TOKEN_REQUEST_URL), headers=headers_token, data=data_token, ) @@ -171,8 +163,8 @@ def __init__(self, config: Config): self._token = None # If provided, set a static token - if config.exists(CONFIG_ENABLE_AUTH_TOKEN_KEY): - self._static_token = config.get(CONFIG_ENABLE_AUTH_TOKEN_KEY) + if config.exists(opt.AUTH_TOKEN): + self._static_token = config.get(opt.AUTH_TOKEN) self._request = requests.Request() self._refresh_token() diff --git a/sdk/python/feast/job_service.py b/sdk/python/feast/job_service.py index 62f8f3bed99..6742ff14931 100644 --- a/sdk/python/feast/job_service.py +++ b/sdk/python/feast/job_service.py @@ -1,9 +1,16 @@ import logging +import os +import signal +import threading +import time +import traceback from concurrent.futures import ThreadPoolExecutor +from typing import Dict, List, Tuple import grpc import feast +from feast.constants import ConfigOptions as opt from feast.core import JobService_pb2_grpc from feast.core.JobService_pb2 import ( CancelJobResponse, @@ -31,6 +38,7 @@ ) from feast.pyspark.launcher import ( get_job_by_id, + get_stream_to_online_ingestion_params, list_jobs, start_historical_feature_retrieval_job, start_offline_to_online_ingestion, @@ -43,36 +51,37 @@ ) +def _job_to_proto(spark_job: SparkJob) -> JobProto: + job = JobProto() + job.id = spark_job.get_id() + status = spark_job.get_status() + if status == SparkJobStatus.COMPLETED: + job.status = JobStatus.JOB_STATUS_DONE + elif status == SparkJobStatus.IN_PROGRESS: + job.status = JobStatus.JOB_STATUS_RUNNING + elif status == SparkJobStatus.FAILED: + job.status = JobStatus.JOB_STATUS_ERROR + elif status == SparkJobStatus.STARTING: + job.status = JobStatus.JOB_STATUS_PENDING + else: + raise ValueError(f"Invalid job status {status}") + + if isinstance(spark_job, RetrievalJob): + job.type = JobType.RETRIEVAL_JOB + job.retrieval.output_location = spark_job.get_output_file_uri(block=False) + elif isinstance(spark_job, BatchIngestionJob): + job.type = JobType.BATCH_INGESTION_JOB + elif isinstance(spark_job, StreamIngestionJob): + job.type = JobType.STREAM_INGESTION_JOB + else: + raise ValueError(f"Invalid job type {job}") + + return job + + class JobServiceServicer(JobService_pb2_grpc.JobServiceServicer): - def __init__(self): - self.client = feast.Client() - - def _job_to_proto(self, spark_job: SparkJob) -> JobProto: - job = JobProto() - job.id = spark_job.get_id() - status = spark_job.get_status() - if status == SparkJobStatus.COMPLETED: - job.status = JobStatus.JOB_STATUS_DONE - elif status == SparkJobStatus.IN_PROGRESS: - job.status = JobStatus.JOB_STATUS_RUNNING - elif status == SparkJobStatus.FAILED: - job.status = JobStatus.JOB_STATUS_ERROR - elif status == SparkJobStatus.STARTING: - job.status = JobStatus.JOB_STATUS_PENDING - else: - raise ValueError(f"Invalid job status {status}") - - if isinstance(spark_job, RetrievalJob): - job.type = JobType.RETRIEVAL_JOB - job.retrieval.output_location = spark_job.get_output_file_uri(block=False) - elif isinstance(spark_job, BatchIngestionJob): - job.type = JobType.BATCH_INGESTION_JOB - elif isinstance(spark_job, StreamIngestionJob): - job.type = JobType.STREAM_INGESTION_JOB - else: - raise ValueError(f"Invalid job type {job}") - - return job + def __init__(self, client): + self.client = client def StartOfflineToOnlineIngestionJob( self, request: StartOfflineToOnlineIngestionJobRequest, context @@ -117,6 +126,20 @@ def StartStreamToOnlineIngestionJob( feature_table = self.client.get_feature_table( request.table_name, request.project ) + + if self.client._config.getboolean(opt.JOB_SERVICE_ENABLE_CONTROL_LOOP): + # If the control loop is enabled, return existing stream ingestion job id instead of starting a new one + params = get_stream_to_online_ingestion_params( + self.client, request.project, feature_table, [] + ) + job_hash = params.get_job_hash() + for job in list_jobs(include_terminated=True, client=self.client): + if isinstance(job, StreamIngestionJob) and job.get_hash() == job_hash: + return StartStreamToOnlineIngestionJobResponse(id=job.get_id()) + raise RuntimeError( + "Feast Job Service has control loop enabled, but couldn't find the existing stream ingestion job for the given FeatureTable" + ) + # TODO: add extra_jars to request job = start_stream_to_online_ingestion( client=self.client, @@ -131,7 +154,7 @@ def ListJobs(self, request, context): jobs = list_jobs( include_terminated=request.include_terminated, client=self.client ) - return ListJobsResponse(jobs=[self._job_to_proto(job) for job in jobs]) + return ListJobsResponse(jobs=[_job_to_proto(job) for job in jobs]) def CancelJob(self, request, context): """Stop a single job""" @@ -142,7 +165,30 @@ def CancelJob(self, request, context): def GetJob(self, request, context): """Get details of a single job""" job = get_job_by_id(request.job_id, client=self.client) - return GetJobResponse(job=self._job_to_proto(job)) + return GetJobResponse(job=_job_to_proto(job)) + + +def start_control_loop() -> None: + """Starts control loop that continuously ensures that correct jobs are being run. + + Currently this affects only the stream ingestion jobs. Please refer to + ensure_stream_ingestion_jobs for full documentation on how the check works. + + """ + logging.info( + "Feast Job Service is starting a control loop in a background thread, " + "which will ensure that stream ingestion jobs are successfully running." + ) + try: + client = feast.Client() + while True: + ensure_stream_ingestion_jobs(client, all_projects=True) + time.sleep(1) + except Exception: + traceback.print_exc() + finally: + # Send interrupt signal to the main thread to kill the server if control loop fails + os.kill(os.getpid(), signal.SIGINT) class HealthServicer(HealthService_pb2_grpc.HealthServicer): @@ -156,7 +202,7 @@ def intercept_service(self, continuation, handler_call_details): return continuation(handler_call_details) -def start_job_service(): +def start_job_service() -> None: """ Start Feast Job Service """ @@ -164,10 +210,102 @@ def start_job_service(): log_fmt = "%(asctime)s %(levelname)s %(message)s" logging.basicConfig(level=logging.INFO, format=log_fmt) + client = feast.Client() + + if client._config.getboolean(opt.JOB_SERVICE_ENABLE_CONTROL_LOOP): + # Start the control loop thread only if it's enabled from configs + thread = threading.Thread(target=start_control_loop, daemon=True) + thread.start() + server = grpc.server(ThreadPoolExecutor(), interceptors=(LoggingInterceptor(),)) - JobService_pb2_grpc.add_JobServiceServicer_to_server(JobServiceServicer(), server) + JobService_pb2_grpc.add_JobServiceServicer_to_server( + JobServiceServicer(client), server + ) HealthService_pb2_grpc.add_HealthServicer_to_server(HealthServicer(), server) server.add_insecure_port("[::]:6568") server.start() - print("Feast job server listening on port :6568") + logging.info("Feast Job Service is listening on port :6568") server.wait_for_termination() + + +def _get_expected_job_hash_to_table_refs( + client: feast.Client, projects: List[str] +) -> Dict[str, Tuple[str, str]]: + """ + Checks all feature tables for the requires project(s) and determines all required stream + ingestion jobs from them. Outputs a map of the expected job_hash to a tuple of (project, table_name). + + Args: + all_projects (bool): If true, runs the check for all project. + Otherwise only checks the current project. + + Returns: + Dict[str, Tuple[str, str]]: Map of job_hash -> (project, table_name) for expected stream ingestion jobs + """ + job_hash_to_table_refs = {} + + for project in projects: + feature_tables = client.list_feature_tables(project) + for feature_table in feature_tables: + if feature_table.stream_source is not None: + params = get_stream_to_online_ingestion_params( + client, project, feature_table, [] + ) + job_hash = params.get_job_hash() + job_hash_to_table_refs[job_hash] = (project, feature_table.name) + + return job_hash_to_table_refs + + +def ensure_stream_ingestion_jobs(client: feast.Client, all_projects: bool): + """Ensures all required stream ingestion jobs are running and cleans up the unnecessary jobs. + + More concretely, it will determine + - which stream ingestion jobs are running + - which stream ingestion jobs should be running + And it'll do 2 kinds of operations + - Cancel all running jobs that should not be running + - Start all non-existent jobs that should be running + + Args: + all_projects (bool): If true, runs the check for all project. + Otherwise only checks the client's current project. + """ + + projects = client.list_projects() if all_projects else [client.project] + + expected_job_hash_to_table_refs = _get_expected_job_hash_to_table_refs( + client, projects + ) + + expected_job_hashes = set(expected_job_hash_to_table_refs.keys()) + + jobs_by_hash: Dict[str, StreamIngestionJob] = {} + for job in client.list_jobs(include_terminated=False): + if isinstance(job, StreamIngestionJob): + jobs_by_hash[job.get_hash()] = job + + existing_job_hashes = set(jobs_by_hash.keys()) + + job_hashes_to_cancel = existing_job_hashes - expected_job_hashes + job_hashes_to_start = expected_job_hashes - existing_job_hashes + + logging.debug( + f"existing_job_hashes = {sorted(list(existing_job_hashes))} expected_job_hashes = {sorted(list(expected_job_hashes))}" + ) + + for job_hash in job_hashes_to_cancel: + job = jobs_by_hash[job_hash] + logging.info( + f"Cancelling a stream ingestion job with job_hash={job_hash} job_id={job.get_id()} status={job.get_status()}" + ) + job.cancel() + + for job_hash in job_hashes_to_start: + # Any job that we wish to start should be among expected table refs map + project, table_name = expected_job_hash_to_table_refs[job_hash] + logging.info( + f"Starting a stream ingestion job for project={project}, table_name={table_name} with job_hash={job_hash}" + ) + feature_table = client.get_feature_table(name=table_name, project=project) + client.start_stream_to_online_ingestion(feature_table, [], project=project) diff --git a/sdk/python/feast/loaders/ingest.py b/sdk/python/feast/loaders/ingest.py index dc87d5b32e5..b4dc1e4239c 100644 --- a/sdk/python/feast/loaders/ingest.py +++ b/sdk/python/feast/loaders/ingest.py @@ -11,13 +11,6 @@ from feast.staging.storage_client import get_staging_client -GRPC_CONNECTION_TIMEOUT_DEFAULT = 3 # type: int -GRPC_CONNECTION_TIMEOUT_APPLY = 300 # type: int -FEAST_SERVING_URL_ENV_KEY = "FEAST_SERVING_URL" # type: str -FEAST_CORE_URL_ENV_KEY = "FEAST_CORE_URL" # type: str -BATCH_FEATURE_REQUEST_WAIT_TIME_SECONDS = 300 -BATCH_INGESTION_PRODUCTION_TIMEOUT = 120 # type: int - def _check_field_mappings( column_names: List[str], diff --git a/sdk/python/feast/pyspark/abc.py b/sdk/python/feast/pyspark/abc.py index d3935ff65b8..19ba9ab174b 100644 --- a/sdk/python/feast/pyspark/abc.py +++ b/sdk/python/feast/pyspark/abc.py @@ -1,4 +1,5 @@ import abc +import hashlib import json import os from datetime import datetime @@ -456,6 +457,13 @@ def get_arguments(self) -> List[str]: "online", ] + def get_job_hash(self) -> str: + job_json = json.dumps( + {"source": self._source, "feature_table": self._feature_table}, + sort_keys=True, + ) + return hashlib.md5(job_json.encode()).hexdigest() + class BatchIngestionJob(SparkJob): """ @@ -468,6 +476,17 @@ class StreamIngestionJob(SparkJob): Container for the streaming ingestion job result """ + def get_hash(self) -> str: + """Gets the consistent hash of this stream ingestion job. + + The hash needs to be persisted at the data processing layer, so that we can get the same + hash when retrieving the job from Spark. + + Returns: + str: The hash for this streaming ingestion job + """ + raise NotImplementedError + class JobLauncher(abc.ABC): """ diff --git a/sdk/python/feast/pyspark/launcher.py b/sdk/python/feast/pyspark/launcher.py index e3e7c03e51d..d928a4f129e 100644 --- a/sdk/python/feast/pyspark/launcher.py +++ b/sdk/python/feast/pyspark/launcher.py @@ -2,28 +2,7 @@ from typing import TYPE_CHECKING, List, Union from feast.config import Config -from feast.constants import ( - CONFIG_DEADLETTER_PATH, - CONFIG_REDIS_HOST, - CONFIG_REDIS_PORT, - CONFIG_REDIS_SSL, - CONFIG_SPARK_DATAPROC_CLUSTER_NAME, - CONFIG_SPARK_DATAPROC_PROJECT, - CONFIG_SPARK_DATAPROC_REGION, - CONFIG_SPARK_EMR_CLUSTER_ID, - CONFIG_SPARK_EMR_CLUSTER_TEMPLATE_PATH, - CONFIG_SPARK_EMR_LOG_LOCATION, - CONFIG_SPARK_EMR_REGION, - CONFIG_SPARK_HOME, - CONFIG_SPARK_INGESTION_JOB_JAR, - CONFIG_SPARK_LAUNCHER, - CONFIG_SPARK_STAGING_LOCATION, - CONFIG_SPARK_STANDALONE_MASTER, - CONFIG_STATSD_ENABLED, - CONFIG_STATSD_HOST, - CONFIG_STATSD_PORT, - CONFIG_STENCIL_URL, -) +from feast.constants import ConfigOptions as opt from feast.data_source import BigQuerySource, DataSource, FileSource, KafkaSource from feast.feature_table import FeatureTable from feast.pyspark.abc import ( @@ -47,7 +26,7 @@ def _standalone_launcher(config: Config) -> JobLauncher: from feast.pyspark.launchers import standalone return standalone.StandaloneClusterLauncher( - config.get(CONFIG_SPARK_STANDALONE_MASTER), config.get(CONFIG_SPARK_HOME) + config.get(opt.SPARK_STANDALONE_MASTER), config.get(opt.SPARK_HOME), ) @@ -55,10 +34,13 @@ def _dataproc_launcher(config: Config) -> JobLauncher: from feast.pyspark.launchers import gcloud return gcloud.DataprocClusterLauncher( - config.get(CONFIG_SPARK_DATAPROC_CLUSTER_NAME), - config.get(CONFIG_SPARK_STAGING_LOCATION), - config.get(CONFIG_SPARK_DATAPROC_REGION), - config.get(CONFIG_SPARK_DATAPROC_PROJECT), + cluster_name=config.get(opt.DATAPROC_CLUSTER_NAME), + staging_location=config.get(opt.SPARK_STAGING_LOCATION), + region=config.get(opt.DATAPROC_REGION), + project_id=config.get(opt.DATAPROC_PROJECT), + executor_instances=config.get(opt.DATAPROC_EXECUTOR_INSTANCES), + executor_cores=config.get(opt.DATAPROC_EXECUTOR_CORES), + executor_memory=config.get(opt.DATAPROC_EXECUTOR_MEMORY), ) @@ -70,11 +52,11 @@ def _get_optional(option): return config.get(option) return aws.EmrClusterLauncher( - region=config.get(CONFIG_SPARK_EMR_REGION), - existing_cluster_id=_get_optional(CONFIG_SPARK_EMR_CLUSTER_ID), - new_cluster_template_path=_get_optional(CONFIG_SPARK_EMR_CLUSTER_TEMPLATE_PATH), - staging_location=config.get(CONFIG_SPARK_STAGING_LOCATION), - emr_log_location=config.get(CONFIG_SPARK_EMR_LOG_LOCATION), + region=config.get(opt.EMR_REGION), + existing_cluster_id=_get_optional(opt.EMR_CLUSTER_ID), + new_cluster_template_path=_get_optional(opt.EMR_CLUSTER_TEMPLATE_PATH), + staging_location=config.get(opt.SPARK_STAGING_LOCATION), + emr_log_location=config.get(opt.EMR_LOG_LOCATION), ) @@ -86,7 +68,7 @@ def _get_optional(option): def resolve_launcher(config: Config) -> JobLauncher: - return _launchers[config.get(CONFIG_SPARK_LAUNCHER)](config) + return _launchers[config.get(opt.SPARK_LAUNCHER)](config) def _source_to_argument(source: DataSource): @@ -241,28 +223,48 @@ def start_offline_to_online_ingestion( return launcher.offline_to_online_ingestion( BatchIngestionJobParameters( - jar=client._config.get(CONFIG_SPARK_INGESTION_JOB_JAR), + jar=client._config.get(opt.SPARK_INGESTION_JAR), source=_source_to_argument(feature_table.batch_source), feature_table=_feature_table_to_argument(client, project, feature_table), start=start, end=end, - redis_host=client._config.get(CONFIG_REDIS_HOST), - redis_port=client._config.getint(CONFIG_REDIS_PORT), - redis_ssl=client._config.getboolean(CONFIG_REDIS_SSL), + redis_host=client._config.get(opt.REDIS_HOST), + redis_port=client._config.getint(opt.REDIS_PORT), + redis_ssl=client._config.getboolean(opt.REDIS_SSL), statsd_host=( - client._config.getboolean(CONFIG_STATSD_ENABLED) - and client._config.get(CONFIG_STATSD_HOST) + client._config.getboolean(opt.STATSD_ENABLED) + and client._config.get(opt.STATSD_HOST) ), statsd_port=( - client._config.getboolean(CONFIG_STATSD_ENABLED) - and client._config.getint(CONFIG_STATSD_PORT) + client._config.getboolean(opt.STATSD_ENABLED) + and client._config.getint(opt.STATSD_PORT) ), - deadletter_path=client._config.get(CONFIG_DEADLETTER_PATH), - stencil_url=client._config.get(CONFIG_STENCIL_URL), + deadletter_path=client._config.get(opt.DEADLETTER_PATH), + stencil_url=client._config.get(opt.STENCIL_URL), ) ) +def get_stream_to_online_ingestion_params( + client: "Client", project: str, feature_table: FeatureTable, extra_jars: List[str] +) -> StreamIngestionJobParameters: + return StreamIngestionJobParameters( + jar=client._config.get(opt.SPARK_INGESTION_JAR), + extra_jars=extra_jars, + source=_source_to_argument(feature_table.stream_source), + feature_table=_feature_table_to_argument(client, project, feature_table), + redis_host=client._config.get(opt.REDIS_HOST), + redis_port=client._config.getint(opt.REDIS_PORT), + redis_ssl=client._config.getboolean(opt.REDIS_SSL), + statsd_host=client._config.getboolean(opt.STATSD_ENABLED) + and client._config.get(opt.STATSD_HOST), + statsd_port=client._config.getboolean(opt.STATSD_ENABLED) + and client._config.getint(opt.STATSD_PORT), + deadletter_path=client._config.get(opt.DEADLETTER_PATH), + stencil_url=client._config.get(opt.STENCIL_URL), + ) + + def start_stream_to_online_ingestion( client: "Client", project: str, feature_table: FeatureTable, extra_jars: List[str] ) -> StreamIngestionJob: @@ -270,20 +272,8 @@ def start_stream_to_online_ingestion( launcher = resolve_launcher(client._config) return launcher.start_stream_to_online_ingestion( - StreamIngestionJobParameters( - jar=client._config.get(CONFIG_SPARK_INGESTION_JOB_JAR), - extra_jars=extra_jars, - source=_source_to_argument(feature_table.stream_source), - feature_table=_feature_table_to_argument(client, project, feature_table), - redis_host=client._config.get(CONFIG_REDIS_HOST), - redis_port=client._config.getint(CONFIG_REDIS_PORT), - redis_ssl=client._config.getboolean(CONFIG_REDIS_SSL), - statsd_host=client._config.getboolean(CONFIG_STATSD_ENABLED) - and client._config.get(CONFIG_STATSD_HOST), - statsd_port=client._config.getboolean(CONFIG_STATSD_ENABLED) - and client._config.getint(CONFIG_STATSD_PORT), - deadletter_path=client._config.get(CONFIG_DEADLETTER_PATH), - stencil_url=client._config.get(CONFIG_STENCIL_URL), + get_stream_to_online_ingestion_params( + client, project, feature_table, extra_jars ) ) diff --git a/sdk/python/feast/pyspark/launchers/aws/emr.py b/sdk/python/feast/pyspark/launchers/aws/emr.py index 02cb59c12c6..42b5348c298 100644 --- a/sdk/python/feast/pyspark/launchers/aws/emr.py +++ b/sdk/python/feast/pyspark/launchers/aws/emr.py @@ -118,8 +118,12 @@ class EmrStreamIngestionJob(EmrJobMixin, StreamIngestionJob): Ingestion streaming job for a EMR cluster """ - def __init__(self, emr_client, job_ref: EmrJobRef): + def __init__(self, emr_client, job_ref: EmrJobRef, job_hash: str): super().__init__(emr_client, job_ref) + self._job_hash = job_hash + + def get_hash(self) -> str: + return self._job_hash class EmrClusterLauncher(JobLauncher): @@ -283,16 +287,19 @@ def start_stream_to_online_ingestion( else: extra_jar_paths.append(_upload_jar(self._staging_location, extra_jar)) + job_hash = ingestion_job_params.get_job_hash() + step = _stream_ingestion_step( jar_s3_path, extra_jar_paths, ingestion_job_params.get_feature_table_name(), args=ingestion_job_params.get_arguments(), + job_hash=job_hash, ) job_ref = self._submit_emr_job(step) - return EmrStreamIngestionJob(self._emr_client(), job_ref) + return EmrStreamIngestionJob(self._emr_client(), job_ref, job_hash) def stage_dataframe(self, df: pandas.DataFrame, event_timestamp: str) -> FileSource: with tempfile.NamedTemporaryFile() as f: @@ -322,8 +329,12 @@ def _job_from_job_info(self, job_info: JobInfo) -> SparkJob: emr_client=self._emr_client(), job_ref=job_info.job_ref, ) elif job_info.job_type == STREAM_TO_ONLINE_JOB_TYPE: + # job_hash must not be None for stream ingestion jobs + assert job_info.job_hash is not None return EmrStreamIngestionJob( - emr_client=self._emr_client(), job_ref=job_info.job_ref, + emr_client=self._emr_client(), + job_ref=job_info.job_ref, + job_hash=job_info.job_hash, ) else: # We should never get here diff --git a/sdk/python/feast/pyspark/launchers/aws/emr_utils.py b/sdk/python/feast/pyspark/launchers/aws/emr_utils.py index 634d82ce787..72df3f92d8c 100644 --- a/sdk/python/feast/pyspark/launchers/aws/emr_utils.py +++ b/sdk/python/feast/pyspark/launchers/aws/emr_utils.py @@ -183,6 +183,7 @@ class JobInfo(NamedTuple): state: str table_name: Optional[str] output_file_uri: Optional[str] + job_hash: Optional[str] def _list_jobs( @@ -228,6 +229,8 @@ def _list_jobs( "feast.step_metadata.historical_retrieval.output_file_uri" ) + job_hash = props.get("feast.step_metadata.job_hash") + if table_name and step_table_name != table_name: continue @@ -241,6 +244,7 @@ def _list_jobs( state=step["Status"]["State"], table_name=step_table_name, output_file_uri=output_file_uri, + job_hash=job_hash, ) ) return res @@ -364,7 +368,11 @@ def _historical_retrieval_step( def _stream_ingestion_step( - jar_path: str, extra_jar_paths: List[str], feature_table_name: str, args: List[str], + jar_path: str, + extra_jar_paths: List[str], + feature_table_name: str, + args: List[str], + job_hash: str, ) -> Dict[str, Any]: if extra_jar_paths: @@ -384,6 +392,7 @@ def _stream_ingestion_step( "Key": "feast.step_metadata.stream_to_online.table_name", "Value": feature_table_name, }, + {"Key": "feast.step_metadata.job_hash", "Value": job_hash}, ], "Args": ["spark-submit", "--class", "feast.ingestion.IngestionJob"] + jars_args diff --git a/sdk/python/feast/pyspark/launchers/gcloud/dataproc.py b/sdk/python/feast/pyspark/launchers/gcloud/dataproc.py index 1dfce4ce444..cfbfb828108 100644 --- a/sdk/python/feast/pyspark/launchers/gcloud/dataproc.py +++ b/sdk/python/feast/pyspark/launchers/gcloud/dataproc.py @@ -174,6 +174,19 @@ class DataprocStreamingIngestionJob(DataprocJobMixin, StreamIngestionJob): Streaming Ingestion job result for a Dataproc cluster """ + def __init__( + self, + job: Job, + refresh_fn: Callable[[], Job], + cancel_fn: Callable[[], None], + job_hash: str, + ) -> None: + super().__init__(job, refresh_fn, cancel_fn) + self._job_hash = job_hash + + def get_hash(self) -> str: + return self._job_hash + class DataprocClusterLauncher(JobLauncher): """ @@ -184,9 +197,17 @@ class DataprocClusterLauncher(JobLauncher): EXTERNAL_JARS = ["gs://spark-lib/bigquery/spark-bigquery-latest_2.12.jar"] JOB_TYPE_LABEL_KEY = "feast_job_type" + JOB_HASH_LABEL_KEY = "feast_job_hash" def __init__( - self, cluster_name: str, staging_location: str, region: str, project_id: str, + self, + cluster_name: str, + staging_location: str, + region: str, + project_id: str, + executor_instances: str, + executor_cores: str, + executor_memory: str, ): """ Initialize a dataproc job controller client, used internally for job submission and result @@ -199,8 +220,14 @@ def __init__( GCS directory for the storage of files generated by the launcher, such as the pyspark scripts. region (str): Dataproc cluster region. - project_id (str: + project_id (str): GCP project id for the dataproc cluster. + executor_instances (str): + Number of executor instances for dataproc job. + executor_cores (str): + Number of cores for dataproc job. + executor_memory (str): + Amount of memory for dataproc job. """ self.cluster_name = cluster_name @@ -217,6 +244,9 @@ def __init__( self.job_client = JobControllerClient( client_options={"api_endpoint": f"{region}-dataproc.googleapis.com:443"} ) + self.executor_instances = executor_instances + self.executor_cores = executor_cores + self.executor_memory = executor_memory def _stage_file(self, file_path: str, job_id: str) -> str: if not os.path.isfile(file_path): @@ -238,6 +268,11 @@ def dataproc_submit( "placement": {"cluster_name": self.cluster_name}, "labels": {self.JOB_TYPE_LABEL_KEY: job_params.get_job_type().name.lower()}, } + + # Add job hash to labels only for the stream ingestion job + if isinstance(job_params, StreamIngestionJobParameters): + job_config["labels"][self.JOB_HASH_LABEL_KEY] = job_params.get_job_hash() + if job_params.get_class_name(): job_config.update( { @@ -245,7 +280,12 @@ def dataproc_submit( "jar_file_uris": [main_file_uri] + self.EXTERNAL_JARS, "main_class": job_params.get_class_name(), "args": job_params.get_arguments(), - "properties": {"spark.yarn.user.classpath.first": "true"}, + "properties": { + "spark.yarn.user.classpath.first": "true", + "spark.executor.instances": self.executor_instances, + "spark.executor.cores": self.executor_cores, + "spark.executor.memory": self.executor_memory, + }, } } ) @@ -301,7 +341,8 @@ def start_stream_to_online_ingestion( self, ingestion_job_params: StreamIngestionJobParameters ) -> StreamIngestionJob: job, refresh_fn, cancel_fn = self.dataproc_submit(ingestion_job_params) - return DataprocStreamingIngestionJob(job, refresh_fn, cancel_fn) + job_hash = ingestion_job_params.get_job_hash() + return DataprocStreamingIngestionJob(job, refresh_fn, cancel_fn, job_hash) def stage_dataframe(self, df, event_timestamp_column: str): raise NotImplementedError @@ -331,7 +372,8 @@ def _dataproc_job_to_spark_job(self, job: Job) -> SparkJob: return DataprocBatchIngestionJob(job, refresh_fn, cancel_fn) if job_type == SparkJobType.STREAM_INGESTION.name.lower(): - return DataprocStreamingIngestionJob(job, refresh_fn, cancel_fn) + job_hash = job.labels[self.JOB_HASH_LABEL_KEY] + return DataprocStreamingIngestionJob(job, refresh_fn, cancel_fn, job_hash) raise ValueError(f"Unrecognized job type: {job_type}") diff --git a/sdk/python/feast/pyspark/launchers/standalone/__init__.py b/sdk/python/feast/pyspark/launchers/standalone/__init__.py index 1c44e5497fb..433d9ed1246 100644 --- a/sdk/python/feast/pyspark/launchers/standalone/__init__.py +++ b/sdk/python/feast/pyspark/launchers/standalone/__init__.py @@ -1,3 +1,11 @@ -from .local import StandaloneClusterLauncher, StandaloneClusterRetrievalJob +from .local import ( + StandaloneClusterLauncher, + StandaloneClusterRetrievalJob, + reset_job_cache, +) -__all__ = ["StandaloneClusterRetrievalJob", "StandaloneClusterLauncher"] +__all__ = [ + "StandaloneClusterRetrievalJob", + "StandaloneClusterLauncher", + "reset_job_cache", +] diff --git a/sdk/python/feast/pyspark/launchers/standalone/local.py b/sdk/python/feast/pyspark/launchers/standalone/local.py index 821a962ed3e..9783cbe52e7 100644 --- a/sdk/python/feast/pyspark/launchers/standalone/local.py +++ b/sdk/python/feast/pyspark/launchers/standalone/local.py @@ -1,9 +1,10 @@ import os import socket import subprocess +import threading import uuid from contextlib import closing -from typing import Dict, List +from typing import Dict, List, Optional import requests from requests.exceptions import RequestException @@ -22,9 +23,66 @@ StreamIngestionJobParameters, ) -# In-memory cache of Spark jobs -# This is necessary since we can't query Spark jobs in local mode -JOB_CACHE: Dict[str, SparkJob] = {} + +class JobCache: + """ + A *global* in-memory cache of Spark jobs. + + This is necessary since we can't easily keep track of running Spark jobs in local mode, since + there is no external state (unlike EMR and Dataproc which keep track of the running jobs for + us). + """ + + # Map of job_id -> spark job + job_by_id: Dict[str, SparkJob] + + # Map of job_id -> job_hash. The value can be None, indicating this job was + # manually created and Job Service isn't maintaining the state of this job + hash_by_id: Dict[str, Optional[str]] + + # This reentrant lock is necessary for multi-threading access + lock: threading.RLock + + def __init__(self): + self.job_by_id = {} + self.hash_by_id = {} + self.lock = threading.RLock() + + def add_job(self, job: SparkJob) -> None: + """Add a Spark job to the cache. + + Args: + job (SparkJob): The new Spark job to add. + """ + with self.lock: + self.job_by_id[job.get_id()] = job + if isinstance(job, StreamIngestionJob): + self.hash_by_id[job.get_id()] = job.get_hash() + + def list_jobs(self) -> List[SparkJob]: + """List all Spark jobs in the cache.""" + with self.lock: + return list(self.job_by_id.values()) + + def get_job_by_id(self, job_id: str) -> SparkJob: + """Get a Spark job with the given ID. Throws an exception if such job doesn't exist. + + Args: + job_id (str): External ID of the Spark job to get. + + Returns: + SparkJob: The Spark job with the given ID. + """ + with self.lock: + return self.job_by_id[job_id] + + +global_job_cache = JobCache() + + +def reset_job_cache(): + global global_job_cache + global_job_cache = JobCache() def _find_free_port(): @@ -100,7 +158,19 @@ class StandaloneClusterStreamingIngestionJob( Streaming Ingestion job result for a standalone spark cluster """ - pass + def __init__( + self, + job_id: str, + job_name: str, + process: subprocess.Popen, + ui_port: int, + job_hash: str, + ) -> None: + super().__init__(job_id, job_name, process, ui_port) + self._job_hash = job_hash + + def get_hash(self) -> str: + return self._job_hash class StandaloneClusterRetrievalJob(StandaloneClusterJobMixin, RetrievalJob): @@ -230,7 +300,7 @@ def historical_feature_retrieval( self.spark_submit(job_params), job_params.get_destination_path(), ) - JOB_CACHE[job_id] = job + global_job_cache.add_job(job) return job def offline_to_online_ingestion( @@ -244,7 +314,7 @@ def offline_to_online_ingestion( self.spark_submit(ingestion_job_params, ui_port), ui_port, ) - JOB_CACHE[job_id] = job + global_job_cache.add_job(job) return job def start_stream_to_online_ingestion( @@ -257,23 +327,24 @@ def start_stream_to_online_ingestion( ingestion_job_params.get_name(), self.spark_submit(ingestion_job_params, ui_port), ui_port, + ingestion_job_params.get_job_hash(), ) - JOB_CACHE[job_id] = job + global_job_cache.add_job(job) return job def stage_dataframe(self, df, event_timestamp_column: str): raise NotImplementedError def get_job_by_id(self, job_id: str) -> SparkJob: - return JOB_CACHE[job_id] + return global_job_cache.get_job_by_id(job_id) def list_jobs(self, include_terminated: bool) -> List[SparkJob]: if include_terminated is True: - return list(JOB_CACHE.values()) + return global_job_cache.list_jobs() else: return [ job - for job in JOB_CACHE.values() + for job in global_job_cache.list_jobs() if job.get_status() in (SparkJobStatus.STARTING, SparkJobStatus.IN_PROGRESS) ] diff --git a/sdk/python/feast/remote_job.py b/sdk/python/feast/remote_job.py index 0a766cf7961..6af6081749f 100644 --- a/sdk/python/feast/remote_job.py +++ b/sdk/python/feast/remote_job.py @@ -124,13 +124,12 @@ class RemoteStreamIngestionJob(RemoteJobMixin, StreamIngestionJob): Stream ingestion job result. """ - def __init__( - self, - service: JobServiceStub, - grpc_extra_param_provider: GrpcExtraParamProvider, - job_id: str, - ): - super().__init__(service, grpc_extra_param_provider, job_id) + def get_hash(self) -> str: + response = self._service.GetJob( + GetJobRequest(job_id=self._job_id), **self._grpc_extra_param_provider() + ) + + return response.job.hash def get_remote_job_from_proto( diff --git a/sdk/python/feast/third_party/__init__.py b/sdk/python/feast/third_party/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/sdk/python/feast/wait.py b/sdk/python/feast/wait.py index c32897606ec..daa0b0b7dc5 100644 --- a/sdk/python/feast/wait.py +++ b/sdk/python/feast/wait.py @@ -15,15 +15,14 @@ import time from typing import Any, Callable, Optional, Tuple -from feast.constants import CONFIG_MAX_WAIT_INTERVAL_KEY -from feast.constants import FEAST_DEFAULT_OPTIONS as defaults +from feast.constants import MAX_WAIT_INTERVAL def wait_retry_backoff( retry_fn: Callable[[], Tuple[Any, bool]], timeout_secs: int = 0, timeout_msg: Optional[str] = "Timeout while waiting for retry_fn() to return True", - max_interval_secs: int = int(defaults[CONFIG_MAX_WAIT_INTERVAL_KEY]), + max_interval_secs: int = int(MAX_WAIT_INTERVAL), ) -> Any: """ Repeatedly try calling given retry_fn until it returns a True boolean success flag. diff --git a/sdk/python/tests/feast_core_server.py b/sdk/python/tests/feast_core_server.py index 85f09175bf3..0c7191f35b4 100644 --- a/sdk/python/tests/feast_core_server.py +++ b/sdk/python/tests/feast_core_server.py @@ -11,6 +11,8 @@ ApplyEntityResponse, ApplyFeatureTableRequest, ApplyFeatureTableResponse, + DeleteFeatureTableRequest, + DeleteFeatureTableResponse, GetEntityRequest, GetEntityResponse, GetFeastCoreVersionResponse, @@ -20,6 +22,7 @@ ListEntitiesResponse, ListFeatureTablesRequest, ListFeatureTablesResponse, + ListProjectsResponse, ) from feast.core.Entity_pb2 import Entity as EntityProto from feast.core.Entity_pb2 import EntityMeta @@ -66,6 +69,7 @@ class CoreServicer(Core.CoreServiceServicer): def __init__(self): self._feature_tables = dict() self._entities = dict() + self._projects = ["default"] def GetFeastCoreVersion(self, request, context): return GetFeastCoreVersionResponse(version="0.10.0") @@ -105,6 +109,10 @@ def ApplyFeatureTable(self, request: ApplyFeatureTableRequest, context): return ApplyFeatureTableResponse(table=applied_feature_table,) + def DeleteFeatureTable(self, request: DeleteFeatureTableRequest, context): + del self._feature_tables[request.name] + return DeleteFeatureTableResponse() + def GetEntity(self, request: GetEntityRequest, context): filtered_entities = [ entity @@ -119,6 +127,9 @@ def ListEntities(self, request: ListEntitiesRequest, context): return ListEntitiesResponse(entities=filtered_entities_response) + def ListProjects(self, request, context): + return ListProjectsResponse(projects=self._projects) + def ApplyEntity(self, request: ApplyEntityRequest, context): entity_spec = request.spec diff --git a/sdk/python/tests/grpc/test_auth.py b/sdk/python/tests/grpc/test_auth.py index 7f023aabcfd..29f781cbedb 100644 --- a/sdk/python/tests/grpc/test_auth.py +++ b/sdk/python/tests/grpc/test_auth.py @@ -14,6 +14,7 @@ # limitations under the License. import json +from configparser import NoOptionError from http import HTTPStatus from unittest.mock import call, patch @@ -141,7 +142,7 @@ def test_get_auth_metadata_plugin_oauth_should_raise_when_response_is_not_200( def test_get_auth_metadata_plugin_oauth_should_raise_when_config_is_incorrect( config_with_missing_variable, ): - with raises(RuntimeError): + with raises((RuntimeError, NoOptionError)): get_auth_metadata_plugin(config_with_missing_variable) diff --git a/sdk/python/tests/test_streaming_control_loop.py b/sdk/python/tests/test_streaming_control_loop.py new file mode 100644 index 00000000000..1c85efc2b05 --- /dev/null +++ b/sdk/python/tests/test_streaming_control_loop.py @@ -0,0 +1,188 @@ +import os +import subprocess +from concurrent import futures +from contextlib import contextmanager +from typing import List +from unittest.mock import patch + +import grpc +import pyspark + +from feast.client import Client +from feast.core.CoreService_pb2_grpc import add_CoreServiceServicer_to_server +from feast.data_format import ParquetFormat, ProtoFormat +from feast.data_source import FileSource, KafkaSource +from feast.entity import Entity +from feast.feature import Feature +from feast.feature_table import FeatureTable +from feast.job_service import ensure_stream_ingestion_jobs +from feast.pyspark.launchers.standalone import ( + StandaloneClusterLauncher, + reset_job_cache, +) +from feast.value_type import ValueType +from tests.feast_core_server import CoreServicer as MockCoreServicer + + +@contextmanager +def mock_server(servicer, add_fn): + """Instantiate a server and return its address for use in tests""" + server = grpc.server(futures.ThreadPoolExecutor(max_workers=10)) + add_fn(servicer, server) + port = server.add_insecure_port("[::]:0") + server.start() + + try: + address = "localhost:%d" % port + with grpc.insecure_channel(address): + yield address + finally: + server.stop(None) + + +SERVING_URL = "serving.example.com" + + +class TestStreamingControlLoop: + table_name = "my-feature-table-1" + + features_1 = [ + Feature(name="fs1-my-feature-1", dtype=ValueType.INT64), + Feature(name="fs1-my-feature-2", dtype=ValueType.STRING), + Feature(name="fs1-my-feature-3", dtype=ValueType.STRING_LIST), + Feature(name="fs1-my-feature-4", dtype=ValueType.BYTES_LIST), + ] + + features_2 = features_1 + [ + Feature(name="fs1-my-feature-5", dtype=ValueType.BYTES_LIST), + ] + + def _create_ft(self, client: Client, features) -> None: + entity = Entity( + name="driver_car_id", + description="Car driver id", + value_type=ValueType.STRING, + labels={"team": "matchmaking"}, + ) + + # Register Entity with Core + client.apply_entity(entity) + + # Create Feature Tables + batch_source = FileSource( + file_format=ParquetFormat(), + file_url="file://feast/*", + event_timestamp_column="ts_col", + created_timestamp_column="timestamp", + date_partition_column="date_partition_col", + ) + + stream_source = KafkaSource( + bootstrap_servers="localhost:9094", + message_format=ProtoFormat("class.path"), + topic="test_topic", + event_timestamp_column="ts_col", + created_timestamp_column="timestamp", + ) + + ft1 = FeatureTable( + name=self.table_name, + features=features, + entities=["driver_car_id"], + labels={"team": "matchmaking"}, + batch_source=batch_source, + stream_source=stream_source, + ) + + # Register Feature Table with Core + client.apply_feature_table(ft1) + + def _delete_ft(self, client: Client): + client.delete_feature_table(self.table_name) + + def test_streaming_job_control_loop(self) -> None: + """ Test streaming job control loop logic. """ + + reset_job_cache() + + core_servicer = MockCoreServicer() + + processes: List[subprocess.Popen] = [] + + def _mock_spark_submit(self, *args, **kwargs) -> subprocess.Popen: + # We mock StandaloneClusterLauncher.spark_submit to run a dummy process and pretend + # that this is a spark structured streaming process. In addition, this implementation + # will keep track of launched processes in an array. + result = subprocess.Popen(args=["/bin/bash", "-c", "sleep 600"]) + processes.append(result) + return result + + with patch.object( + StandaloneClusterLauncher, "spark_submit", new=_mock_spark_submit + ), mock_server( + core_servicer, add_CoreServiceServicer_to_server + ) as core_service_url: + client = Client( + core_url=core_service_url, + serving_url=SERVING_URL, + spark_launcher="standalone", + spark_home=os.path.dirname(pyspark.__file__), + ) + + # Run one iteration of the control loop. It should do nothing since we have no + # feature tables. + ensure_stream_ingestion_jobs(client=client, all_projects=True) + + # No jobs should be running at this point. + assert len(client.list_jobs(include_terminated=True)) == 0 + + # Now, create a new feature table. + self._create_ft(client, self.features_1) + + # Run another iteration of the control loop. + ensure_stream_ingestion_jobs(client=client, all_projects=True) + + # We expect a streaming job to be created for the new Feature Table. + assert len(client.list_jobs(include_terminated=False)) == 1 + assert len(processes) == 1 + + first_job_id = client.list_jobs(include_terminated=False)[0].get_id() + + # Pretend that the streaming job has terminated for no reason. + processes[0].kill() + + # The control loop is expected to notice the killed job and start it again. + ensure_stream_ingestion_jobs(client=client, all_projects=True) + + # We expect to find one terminated job and one restarted job. + assert len(client.list_jobs(include_terminated=False)) == 1 + assert len(client.list_jobs(include_terminated=True)) == 2 + + id_after_restart = client.list_jobs(include_terminated=False)[0].get_id() + + # Indeed it is a new job with a new id. + assert id_after_restart != first_job_id + + # Update the feature table. + self._create_ft(client, self.features_2) + + # Run another iteration of the job control loop. We expect to restart the streaming + # job since the feature table has changed. + ensure_stream_ingestion_jobs(client=client, all_projects=True) + + # We expect to find two terminated job and one live job. + assert len(client.list_jobs(include_terminated=False)) == 1 + assert len(client.list_jobs(include_terminated=True)) == 3 + + id_after_change = client.list_jobs(include_terminated=False)[0].get_id() + assert id_after_restart != id_after_change + + # Delete the feature table. + self._delete_ft(client) + + # Run another iteration of the job control loop. We expect it to terminate the streaming + # job. + ensure_stream_ingestion_jobs(client=client, all_projects=True) + + assert len(client.list_jobs(include_terminated=False)) == 0 + assert len(client.list_jobs(include_terminated=True)) == 3 diff --git a/spark/ingestion/pom.xml b/spark/ingestion/pom.xml index d78984837c9..e055006964c 100644 --- a/spark/ingestion/pom.xml +++ b/spark/ingestion/pom.xml @@ -182,14 +182,14 @@ com.dimafeng testcontainers-scala-scalatest_${scala.version} - 0.38.3 + 0.38.6 test com.dimafeng testcontainers-scala-kafka_${scala.version} - 0.38.3 + 0.38.6 test diff --git a/spark/ingestion/src/main/scala/feast/ingestion/BasePipeline.scala b/spark/ingestion/src/main/scala/feast/ingestion/BasePipeline.scala index cc1f451ae74..679785bc2a5 100644 --- a/spark/ingestion/src/main/scala/feast/ingestion/BasePipeline.scala +++ b/spark/ingestion/src/main/scala/feast/ingestion/BasePipeline.scala @@ -41,11 +41,7 @@ trait BasePipeline { case Some(c: StatsDConfig) => conf .set( - "spark.metrics.conf.*.source.redis.class", - "org.apache.spark.metrics.source.RedisSinkMetricSource" - ) - .set( - "spark.metrics.conf.*.source.redis.labels", + "spark.metrics.labels", s"feature_table=${jobConfig.featureTable.name}" ) .set( @@ -56,7 +52,7 @@ trait BasePipeline { .set("spark.metrics.conf.*.sink.statsd.port", c.port.toString) .set("spark.metrics.conf.*.sink.statsd.period", "30") .set("spark.metrics.conf.*.sink.statsd.unit", "seconds") - .set("spark.metrics.namespace", jobConfig.mode.toString) + .set("spark.metrics.namespace", jobConfig.mode.toString.toLowerCase) case None => () } diff --git a/spark/ingestion/src/main/scala/feast/ingestion/BatchPipeline.scala b/spark/ingestion/src/main/scala/feast/ingestion/BatchPipeline.scala index a54c83140f1..1348914b86d 100644 --- a/spark/ingestion/src/main/scala/feast/ingestion/BatchPipeline.scala +++ b/spark/ingestion/src/main/scala/feast/ingestion/BatchPipeline.scala @@ -72,6 +72,7 @@ object BatchPipeline extends BasePipeline { .option("namespace", featureTable.name) .option("project_name", featureTable.project) .option("timestamp_column", config.source.eventTimestampColumn) + .option("max_age", config.featureTable.maxAge.getOrElse(0)) .save() config.deadLetterPath match { diff --git a/spark/ingestion/src/main/scala/feast/ingestion/IngestionJob.scala b/spark/ingestion/src/main/scala/feast/ingestion/IngestionJob.scala index f4d0dcc1159..e393ac3015b 100644 --- a/spark/ingestion/src/main/scala/feast/ingestion/IngestionJob.scala +++ b/spark/ingestion/src/main/scala/feast/ingestion/IngestionJob.scala @@ -17,7 +17,6 @@ package feast.ingestion import org.joda.time.DateTime - import org.json4s._ import org.json4s.jackson.JsonMethods.{parse => parseJSON} import org.json4s.ext.JavaEnumNameSerializer @@ -29,9 +28,9 @@ object IngestionJob { new JavaEnumNameSerializer[feast.proto.types.ValueProto.ValueType.Enum]() + ShortTypeHints(List(classOf[ProtoFormat], classOf[AvroFormat])) - val parser = new scopt.OptionParser[IngestionJobConfig]("IngestionJon") { + val parser = new scopt.OptionParser[IngestionJobConfig]("IngestionJob") { // ToDo: read version from Manifest - head("feast.ingestion.IngestionJob", "0.8.0") + head("feast.ingestion.IngestionJob", "0.8.1") opt[Modes]("mode") .action((x, c) => c.copy(mode = x)) diff --git a/spark/ingestion/src/main/scala/feast/ingestion/IngestionJobConfig.scala b/spark/ingestion/src/main/scala/feast/ingestion/IngestionJobConfig.scala index c922a1c096e..8b0ae25d7a0 100644 --- a/spark/ingestion/src/main/scala/feast/ingestion/IngestionJobConfig.scala +++ b/spark/ingestion/src/main/scala/feast/ingestion/IngestionJobConfig.scala @@ -91,7 +91,8 @@ case class FeatureTable( name: String, project: String, entities: Seq[Field], - features: Seq[Field] + features: Seq[Field], + maxAge: Option[Int] = None ) case class IngestionJobConfig( diff --git a/spark/ingestion/src/main/scala/feast/ingestion/StreamingPipeline.scala b/spark/ingestion/src/main/scala/feast/ingestion/StreamingPipeline.scala index 1945d4aa0f2..99d5e66c882 100644 --- a/spark/ingestion/src/main/scala/feast/ingestion/StreamingPipeline.scala +++ b/spark/ingestion/src/main/scala/feast/ingestion/StreamingPipeline.scala @@ -87,6 +87,7 @@ object StreamingPipeline extends BasePipeline with Serializable { .option("namespace", featureTable.name) .option("project_name", featureTable.project) .option("timestamp_column", config.source.eventTimestampColumn) + .option("max_age", config.featureTable.maxAge.getOrElse(0)) .save() config.deadLetterPath match { diff --git a/spark/ingestion/src/main/scala/feast/ingestion/metrics/StatsdReporterWithTags.scala b/spark/ingestion/src/main/scala/feast/ingestion/metrics/StatsdReporterWithTags.scala index 66b48dd444e..894014b6fdf 100644 --- a/spark/ingestion/src/main/scala/feast/ingestion/metrics/StatsdReporterWithTags.scala +++ b/spark/ingestion/src/main/scala/feast/ingestion/metrics/StatsdReporterWithTags.scala @@ -169,19 +169,28 @@ class StatsdReporterWithTags( reportMetered(name, timer) } + private val nameWithTag = """(\S+)#(\S+)""".r + private def send(name: String, value: String, metricType: String)(implicit socket: DatagramSocket ): Unit = { - val bytes = sanitize(s"$name:$value|$metricType").getBytes(UTF_8) + val bytes = name match { + case nameWithTag(name, tags) => + val tagsWithSemicolon = tags.replace('=', ':') + sanitize(s"$name:$value|$metricType|#$tagsWithSemicolon").getBytes(UTF_8) + case _ => + sanitize(s"$name:$value|$metricType").getBytes(UTF_8) + } val packet = new DatagramPacket(bytes, bytes.length, address) socket.send(packet) } - private val nameWithTag = """(\S+)#(\S+)""".r - private def fullName(name: String, suffixes: String*): String = name match { case nameWithTag(name, tags) => - MetricRegistry.name(prefix, name +: suffixes: _*) ++ "#" ++ tags + // filter out parts that consists only from numbers + // that could be executor-id for example + val stableName = name.split('.').filterNot(_ forall Character.isDigit).mkString(".") + MetricRegistry.name(prefix, stableName +: suffixes: _*) ++ "#" ++ tags case _ => MetricRegistry.name(prefix, name +: suffixes: _*) } diff --git a/spark/ingestion/src/main/scala/feast/ingestion/stores/redis/HashTypePersistence.scala b/spark/ingestion/src/main/scala/feast/ingestion/stores/redis/HashTypePersistence.scala index b34f0667c0e..00ab8736302 100644 --- a/spark/ingestion/src/main/scala/feast/ingestion/stores/redis/HashTypePersistence.scala +++ b/spark/ingestion/src/main/scala/feast/ingestion/stores/redis/HashTypePersistence.scala @@ -16,16 +16,17 @@ */ package feast.ingestion.stores.redis -import org.apache.spark.sql.Row -import org.apache.spark.sql.types._ -import redis.clients.jedis.{Pipeline, Response} import java.nio.charset.StandardCharsets +import java.util import com.google.common.hash.Hashing - -import scala.jdk.CollectionConverters._ import com.google.protobuf.Timestamp import feast.ingestion.utils.TypeConversion +import org.apache.spark.sql.Row +import org.apache.spark.sql.types._ +import redis.clients.jedis.{Pipeline, Response} + +import scala.jdk.CollectionConverters._ /** * Use Redis hash type as storage layout. Every feature is stored as separate entry in Hash. @@ -35,10 +36,10 @@ import feast.ingestion.utils.TypeConversion * Values are serialized with protobuf (`ValueProto`). */ class HashTypePersistence(config: SparkRedisConfig) extends Persistence with Serializable { - def encodeRow( - keyColumns: Array[String], - timestampField: String, - value: Row + + private def encodeRow( + value: Row, + maxExpiryTimestamp: java.sql.Timestamp ): Map[Array[Byte], Array[Byte]] = { val fields = value.schema.fields.map(_.name) val types = value.schema.fields.map(f => (f.name, f.dataType)).toMap @@ -51,49 +52,87 @@ class HashTypePersistence(config: SparkRedisConfig) extends Persistence with Ser } .filter { case (k, _) => // don't store entities & timestamp - !keyColumns.contains(k) && k != config.timestampColumn + !config.entityColumns.contains(k) && k != config.timestampColumn } .map { case (k, v) => encodeKey(k) -> encodeValue(v, types(k)) } - val timestamp = Seq( + val timestampHash = Seq( ( - timestampField.getBytes, + timestampHashKey(config.namespace).getBytes, encodeValue(value.getAs[Timestamp](config.timestampColumn), TimestampType) ) ) - values ++ timestamp + val expiryUnixTimestamp = { + if (config.maxAge > 0) + value.getAs[java.sql.Timestamp](config.timestampColumn).getTime + config.maxAge * 1000 + else maxExpiryTimestamp.getTime + } + val expiryTimestamp = new java.sql.Timestamp(expiryUnixTimestamp) + val expiryTimestampHash = Seq( + ( + expiryTimestampHashKey(config.namespace).getBytes, + encodeValue(expiryTimestamp, TimestampType) + ) + ) + + values ++ timestampHash ++ expiryTimestampHash } - def encodeValue(value: Any, `type`: DataType): Array[Byte] = { + private def encodeValue(value: Any, `type`: DataType): Array[Byte] = { TypeConversion.sqlTypeToProtoValue(value, `type`).toByteArray } - def encodeKey(key: String): Array[Byte] = { + private def encodeKey(key: String): Array[Byte] = { val fullFeatureReference = s"${config.namespace}:$key" Hashing.murmur3_32.hashString(fullFeatureReference, StandardCharsets.UTF_8).asBytes() } - def save( + private def timestampHashKey(namespace: String): String = { + s"${config.timestampPrefix}:${namespace}" + } + + private def expiryTimestampHashKey(namespace: String): String = { + s"${config.expiryPrefix}:${namespace}" + } + + private def decodeTimestamp(encodedTimestamp: Array[Byte]): java.sql.Timestamp = { + new java.sql.Timestamp(Timestamp.parseFrom(encodedTimestamp).getSeconds * 1000) + } + + override def save( pipeline: Pipeline, key: Array[Byte], - value: Map[Array[Byte], Array[Byte]], - ttl: Int + row: Row, + expiryTimestamp: java.sql.Timestamp, + maxExpiryTimestamp: java.sql.Timestamp ): Unit = { - pipeline.hset(key, value.asJava) - if (ttl > 0) { - pipeline.expire(key, ttl) + val value = encodeRow(row, maxExpiryTimestamp).asJava + pipeline.hset(key, value) + if (expiryTimestamp.equals(maxExpiryTimestamp)) { + pipeline.persist(key) + } else { + pipeline.expireAt(key, expiryTimestamp.getTime / 1000) } } - def getTimestamp( + override def get( pipeline: Pipeline, - key: Array[Byte], - timestampField: String - ): Response[Array[Byte]] = { - pipeline.hget(key, timestampField.getBytes) + key: Array[Byte] + ): Response[util.Map[Array[Byte], Array[Byte]]] = { + pipeline.hgetAll(key) } + override def storedTimestamp( + value: util.Map[Array[Byte], Array[Byte]] + ): Option[java.sql.Timestamp] = { + value.asScala.toMap + .map { case (key, value) => + (key.map(_.toChar).mkString, value) + } + .get(timestampHashKey(config.namespace)) + .map(value => decodeTimestamp(value)) + } } diff --git a/spark/ingestion/src/main/scala/feast/ingestion/stores/redis/Persistence.scala b/spark/ingestion/src/main/scala/feast/ingestion/stores/redis/Persistence.scala index 47161358c2e..4c4b1690c0b 100644 --- a/spark/ingestion/src/main/scala/feast/ingestion/stores/redis/Persistence.scala +++ b/spark/ingestion/src/main/scala/feast/ingestion/stores/redis/Persistence.scala @@ -16,26 +16,58 @@ */ package feast.ingestion.stores.redis +import java.sql.Timestamp +import java.util + import org.apache.spark.sql.Row import redis.clients.jedis.{Pipeline, Response} +/** + * Determine how a Spark row should be serialized and stored on Redis. + */ trait Persistence { - def encodeRow( - keyColumns: Array[String], - timestampField: String, - value: Row - ): Map[Array[Byte], Array[Byte]] + /** + * Persist a Spark row to Redis + * + * @param pipeline Redis pipeline + * @param key Redis key in serialized bytes format + * @param row Row representing the value to be persist + * @param expiryTimestamp Expiry timestamp for the row + * @param maxExpiryTimestamp No ttl should be set if the expiry timestamp + * is equal to the maxExpiryTimestamp + */ def save( pipeline: Pipeline, key: Array[Byte], - value: Map[Array[Byte], Array[Byte]], - ttl: Int + row: Row, + expiryTimestamp: Timestamp, + maxExpiryTimestamp: Timestamp ): Unit - def getTimestamp( + /** + * Returns a Redis response, which can be used by `storedTimestamp` and `newExpiryTimestamp` to + * derive the currently stored event timestamp, and the updated expiry timestamp. This method will + * be called prior to persisting the row to Redis, so that `RedisSinkRelation` can decide whether + * the currently stored value should be updated. + * + * @param pipeline Redis pipeline + * @param key Redis key in serialized bytes format + * @return Redis response representing the row value + */ + def get( pipeline: Pipeline, - key: Array[Byte], - timestampField: String - ): Response[Array[Byte]] + key: Array[Byte] + ): Response[util.Map[Array[Byte], Array[Byte]]] + + /** + * Returns the currently stored event timestamp for the key and the feature table associated with the ingestion job. + * + * @param value Response returned from `get` + * @return Stored event timestamp associated with the key. Returns `None` if + * the key is not present in Redis, or if timestamp information is + * unavailable on the stored value. + */ + def storedTimestamp(value: util.Map[Array[Byte], Array[Byte]]): Option[Timestamp] + } diff --git a/spark/ingestion/src/main/scala/feast/ingestion/stores/redis/RedisSinkRelation.scala b/spark/ingestion/src/main/scala/feast/ingestion/stores/redis/RedisSinkRelation.scala index d880a6461c8..f70af5d7e53 100644 --- a/spark/ingestion/src/main/scala/feast/ingestion/stores/redis/RedisSinkRelation.scala +++ b/spark/ingestion/src/main/scala/feast/ingestion/stores/redis/RedisSinkRelation.scala @@ -16,21 +16,24 @@ */ package feast.ingestion.stores.redis +import java.util + import com.google.protobuf.Timestamp -import com.redislabs.provider.redis.{ReadWriteConfig, RedisConfig, RedisEndpoint, RedisNode} -import redis.clients.jedis.util.JedisClusterCRC16 +import com.google.protobuf.util.Timestamps import com.redislabs.provider.redis.util.PipelineUtils.{foreachWithPipeline, mapWithPipeline} +import com.redislabs.provider.redis.{ReadWriteConfig, RedisConfig, RedisEndpoint, RedisNode} import feast.ingestion.utils.TypeConversion +import feast.proto.storage.RedisProto.RedisKeyV2 +import feast.proto.types.ValueProto import org.apache.spark.SparkEnv import org.apache.spark.metrics.source.RedisSinkMetricSource +import org.apache.spark.sql.functions.col import org.apache.spark.sql.sources.{BaseRelation, InsertableRelation} import org.apache.spark.sql.types.StructType -import org.apache.spark.sql.functions.col import org.apache.spark.sql.{DataFrame, Row, SQLContext} +import redis.clients.jedis.util.JedisClusterCRC16 -import collection.JavaConverters._ -import feast.proto.storage.RedisProto.RedisKeyV2 -import feast.proto.types.ValueProto +import scala.collection.JavaConverters._ /** * High-level writer to Redis. Relies on `Persistence` implementation for actual storage layout. @@ -45,6 +48,9 @@ class RedisSinkRelation(override val sqlContext: SQLContext, config: SparkRedisC extends BaseRelation with InsertableRelation with Serializable { + + import RedisSinkRelation._ + private implicit val redisConfig: RedisConfig = { new RedisConfig( new RedisEndpoint(sqlContext.sparkContext.getConf) @@ -57,6 +63,8 @@ class RedisSinkRelation(override val sqlContext: SQLContext, config: SparkRedisC override def schema: StructType = ??? + val MAX_EXPIRED_TIMESTAMP = new java.sql.Timestamp(Timestamps.MAX_VALUE.getSeconds * 1000) + val persistence: Persistence = new HashTypePersistence(config) override def insert(data: DataFrame, overwrite: Boolean): Unit = { @@ -75,27 +83,27 @@ class RedisSinkRelation(override val sqlContext: SQLContext, config: SparkRedisC groupKeysByNode(redisConfig.hosts, rowsWithKey.keysIterator).foreach { case (node, keys) => val conn = node.connect() - // retrieve latest stored timestamp per key - val timestamps = mapWithPipeline(conn, keys) { (pipeline, key) => - persistence.getTimestamp(pipeline, key.toByteArray, timestampField) - } - - val timestampByKey = timestamps - .map(_.asInstanceOf[Array[Byte]]) - .map( - Option(_) - .map(Timestamp.parseFrom) - .map(t => new java.sql.Timestamp(t.getSeconds * 1000)) - ) - .zip(keys) - .map(_.swap) + // retrieve latest stored values + val storedValues = mapWithPipeline(conn, keys) { (pipeline, key) => + persistence.get(pipeline, key.toByteArray) + }.map(_.asInstanceOf[util.Map[Array[Byte], Array[Byte]]]) + + val timestamps = storedValues.map(persistence.storedTimestamp) + val timestampByKey = keys.zip(timestamps).toMap + + val expiryTimestampByKey = keys + .zip(storedValues) + .map { case (key, storedValue) => + (key, newExpiryTimestamp(rowsWithKey(key), storedValue)) + } .toMap foreachWithPipeline(conn, keys) { (pipeline, key) => val row = rowsWithKey(key) timestampByKey(key) match { - case Some(t) if !t.before(row.getAs[java.sql.Timestamp](config.timestampColumn)) => () + case Some(t) if (t.after(row.getAs[java.sql.Timestamp](config.timestampColumn))) => + () case _ => if (metricSource.nonEmpty) { val lag = System.currentTimeMillis() - row @@ -105,9 +113,13 @@ class RedisSinkRelation(override val sqlContext: SQLContext, config: SparkRedisC metricSource.get.METRIC_TOTAL_ROWS_INSERTED.inc() metricSource.get.METRIC_ROWS_LAG.update(lag) } - - val encodedRow = persistence.encodeRow(config.entityColumns, timestampField, row) - persistence.save(pipeline, key.toByteArray, encodedRow, ttl = 0) + persistence.save( + pipeline, + key.toByteArray, + row, + expiryTimestampByKey(key), + MAX_EXPIRED_TIMESTAMP + ) } } conn.close() @@ -142,15 +154,21 @@ class RedisSinkRelation(override val sqlContext: SQLContext, config: SparkRedisC .build } - private def timestampField: String = { - s"${config.timestampPrefix}:${config.namespace}" - } + private lazy val metricSource: Option[RedisSinkMetricSource] = { + MetricInitializationLock.synchronized { + // RedisSinkMetricSource needs to be registered on executor and SparkEnv must already exist. + // Which is problematic, since metrics system is initialized before SparkEnv set. + // That's why I moved source registering here + if (SparkEnv.get.metricsSystem.getSourcesByName(RedisSinkMetricSource.sourceName).isEmpty) { + SparkEnv.get.metricsSystem.registerSource(new RedisSinkMetricSource) + } + } - private lazy val metricSource: Option[RedisSinkMetricSource] = SparkEnv.get.metricsSystem.getSourcesByName(RedisSinkMetricSource.sourceName) match { case Seq(head) => Some(head.asInstanceOf[RedisSinkMetricSource]) case _ => None } + } private def groupKeysByNode( nodes: Array[RedisNode], @@ -169,4 +187,35 @@ class RedisSinkRelation(override val sqlContext: SQLContext, config: SparkRedisC nodes.filter { node => node.startSlot <= slot && node.endSlot >= slot }.filter(_.idx == 0)(0) } + + private def newExpiryTimestamp( + row: Row, + value: util.Map[Array[Byte], Array[Byte]] + ): java.sql.Timestamp = { + val maxExpiryOtherFeatureTables: Long = value.asScala.toMap + .map { case (key, value) => + (key.map(_.toChar).mkString, value) + } + .filterKeys(_.startsWith(config.expiryPrefix)) + .filterKeys(_.split(":").last != config.namespace) + .values + .map(value => Timestamp.parseFrom(value).getSeconds * 1000) + .reduceOption(_ max _) + .getOrElse(0) + + val rowExpiry: Long = + if (config.maxAge > 0) + (row + .getAs[java.sql.Timestamp](config.timestampColumn) + .getTime + config.maxAge * 1000) + else MAX_EXPIRED_TIMESTAMP.getTime + + val maxExpiry = maxExpiryOtherFeatureTables max rowExpiry + new java.sql.Timestamp(maxExpiry) + + } +} + +object RedisSinkRelation { + object MetricInitializationLock } diff --git a/spark/ingestion/src/main/scala/feast/ingestion/stores/redis/SparkRedisConfig.scala b/spark/ingestion/src/main/scala/feast/ingestion/stores/redis/SparkRedisConfig.scala index 389607ce99e..cac12a6c279 100644 --- a/spark/ingestion/src/main/scala/feast/ingestion/stores/redis/SparkRedisConfig.scala +++ b/spark/ingestion/src/main/scala/feast/ingestion/stores/redis/SparkRedisConfig.scala @@ -23,7 +23,9 @@ case class SparkRedisConfig( timestampColumn: String, iteratorGroupingSize: Int = 1000, timestampPrefix: String = "_ts", - repartitionByEntity: Boolean = true + repartitionByEntity: Boolean = true, + maxAge: Int = 0, + expiryPrefix: String = "_ex" ) object SparkRedisConfig { @@ -32,6 +34,7 @@ object SparkRedisConfig { val TS_COLUMN = "timestamp_column" val ENTITY_REPARTITION = "entity_repartition" val PROJECT_NAME = "project_name" + val MAX_AGE = "max_age" def parse(parameters: Map[String, String]): SparkRedisConfig = SparkRedisConfig( @@ -39,6 +42,7 @@ object SparkRedisConfig { projectName = parameters.getOrElse(PROJECT_NAME, "default"), entityColumns = parameters.getOrElse(ENTITY_COLUMNS, "").split(","), timestampColumn = parameters.getOrElse(TS_COLUMN, "event_timestamp"), - repartitionByEntity = parameters.getOrElse(ENTITY_REPARTITION, "true") == "true" + repartitionByEntity = parameters.getOrElse(ENTITY_REPARTITION, "true") == "true", + maxAge = parameters.get(MAX_AGE).map(_.toInt).getOrElse(0) ) } diff --git a/spark/ingestion/src/main/scala/org/apache/spark/metrics/source/RedisSinkMetricSource.scala b/spark/ingestion/src/main/scala/org/apache/spark/metrics/source/RedisSinkMetricSource.scala index 77c9218a7ec..bc4747828ee 100644 --- a/spark/ingestion/src/main/scala/org/apache/spark/metrics/source/RedisSinkMetricSource.scala +++ b/spark/ingestion/src/main/scala/org/apache/spark/metrics/source/RedisSinkMetricSource.scala @@ -17,23 +17,28 @@ package org.apache.spark.metrics.source import com.codahale.metrics.MetricRegistry -import org.apache.spark.{SparkConf, SparkEnv} +import org.apache.spark.SparkEnv class RedisSinkMetricSource extends Source { override val sourceName: String = RedisSinkMetricSource.sourceName override val metricRegistry: MetricRegistry = new MetricRegistry - private val sparkConfig = Option(SparkEnv.get).map(_.conf).getOrElse(new SparkConf(true)) + private val sparkConfig = SparkEnv.get.conf - private val metricLabels = sparkConfig.get("spark.metrics.conf.*.source.redis.labels") + private val metricLabels = sparkConfig.get("spark.metrics.labels", "") - private def nameWithLabels(name: String) = + private val appId = sparkConfig.get("spark.app.id", "") + + private val executorId = sparkConfig.get("spark.executor.id", "") + + private def nameWithLabels(name: String) = { if (metricLabels.isEmpty) { name } else { - s"$name#$metricLabels" + s"$name#$metricLabels,job_id=$appId-$executorId" } + } val METRIC_TOTAL_ROWS_INSERTED = metricRegistry.counter(nameWithLabels("feast_ingestion_feature_row_ingested_count")) diff --git a/spark/ingestion/src/test/scala/feast/ingestion/BatchPipelineIT.scala b/spark/ingestion/src/test/scala/feast/ingestion/BatchPipelineIT.scala index 6ccfe9ee345..70f8a1f7187 100644 --- a/spark/ingestion/src/test/scala/feast/ingestion/BatchPipelineIT.scala +++ b/spark/ingestion/src/test/scala/feast/ingestion/BatchPipelineIT.scala @@ -17,9 +17,11 @@ package feast.ingestion import java.nio.file.Paths +import java.sql.Timestamp import collection.JavaConverters._ import com.dimafeng.testcontainers.{ForAllTestContainer, GenericContainer} +import com.google.protobuf.util.Timestamps import feast.proto.types.ValueProto.ValueType import org.apache.spark.SparkConf import org.joda.time.{DateTime, Seconds} @@ -106,14 +108,253 @@ class BatchPipelineIT extends SparkSpec with ForAllTestContainer { val featureKeyEncoder: String => String = encodeFeatureKey(config.featureTable) rows.foreach(r => { - val storedValues = jedis.hgetAll(encodeEntityKey(r, config.featureTable)).asScala.toMap + val encodedEntityKey = encodeEntityKey(r, config.featureTable) + val storedValues = jedis.hgetAll(encodedEntityKey).asScala.toMap storedValues should beStoredRow( Map( featureKeyEncoder("feature1") -> r.feature1, featureKeyEncoder("feature2") -> r.feature2, - "_ts:test-fs" -> r.eventTimestamp + "_ts:test-fs" -> r.eventTimestamp, + "_ex:test-fs" -> new Timestamp(Timestamps.MAX_VALUE.getSeconds * 1000) + ) + ) + val keyTTL = jedis.ttl(encodedEntityKey).toInt + keyTTL shouldEqual -1 + + }) + } + + "Parquet source file" should "be ingested in redis with expiry time equal to the largest of (event_timestamp + max_age) for" + + "all feature tables associated with the entity" in new Scope { + val startDate = new DateTime().minusDays(1).withTimeAtStartOfDay() + val endDate = new DateTime().withTimeAtStartOfDay() + val gen = rowGenerator(startDate, endDate) + val rows = generateDistinctRows(gen, 1000, groupByEntity) + val tempPath = storeAsParquet(sparkSession, rows) + val maxAge = 86400 * 2 + val configWithMaxAge = config.copy( + source = FileSource(tempPath, Map.empty, "eventTimestamp"), + featureTable = config.featureTable.copy(maxAge = Some(maxAge)), + startTime = startDate, + endTime = endDate + ) + + val ingestionTimeUnix = System.currentTimeMillis() + BatchPipeline.createPipeline(sparkSession, configWithMaxAge) + + val featureKeyEncoder: String => String = encodeFeatureKey(config.featureTable) + + rows.foreach(r => { + val encodedEntityKey = encodeEntityKey(r, config.featureTable) + val storedValues = jedis.hgetAll(encodedEntityKey).asScala.toMap + val expectedExpiryTimestamp = + new java.sql.Timestamp(r.eventTimestamp.getTime + 1000 * maxAge) + storedValues should beStoredRow( + Map( + featureKeyEncoder("feature1") -> r.feature1, + featureKeyEncoder("feature2") -> r.feature2, + "_ts:test-fs" -> r.eventTimestamp, + "_ex:test-fs" -> expectedExpiryTimestamp + ) ) + val keyTTL = jedis.ttl(encodedEntityKey).toLong + keyTTL should (be <= (expectedExpiryTimestamp.getTime - ingestionTimeUnix) / 1000 and be > 0L) + + }) + + val increasedMaxAge = 86400 * 3 + val configWithSecondFeatureTable = config.copy( + source = FileSource(tempPath, Map.empty, "eventTimestamp"), + featureTable = config.featureTable.copy( + name = "test-fs-2", + maxAge = Some(increasedMaxAge) + ), + startTime = startDate, + endTime = endDate ) + + val secondIngestionTimeUnix = System.currentTimeMillis() + BatchPipeline.createPipeline(sparkSession, configWithSecondFeatureTable) + + val featureKeyEncoderSecondTable: String => String = + encodeFeatureKey(configWithSecondFeatureTable.featureTable) + + rows.foreach(r => { + val encodedEntityKey = encodeEntityKey(r, config.featureTable) + val storedValues = jedis.hgetAll(encodedEntityKey).asScala.toMap + val expectedExpiryTimestamp1 = + new java.sql.Timestamp(r.eventTimestamp.getTime + 1000 * maxAge) + val expectedExpiryTimestamp2 = + new java.sql.Timestamp(r.eventTimestamp.getTime + 1000 * increasedMaxAge) + storedValues should beStoredRow( + Map( + featureKeyEncoder("feature1") -> r.feature1, + featureKeyEncoder("feature2") -> r.feature2, + featureKeyEncoderSecondTable("feature1") -> r.feature1, + featureKeyEncoderSecondTable("feature2") -> r.feature2, + "_ts:test-fs" -> r.eventTimestamp, + "_ts:test-fs-2" -> r.eventTimestamp, + "_ex:test-fs" -> expectedExpiryTimestamp1, + "_ex:test-fs-2" -> expectedExpiryTimestamp2 + ) + ) + val keyTTL = jedis.ttl(encodedEntityKey).toLong + keyTTL should (be <= (expectedExpiryTimestamp2.getTime - secondIngestionTimeUnix) / 1000 and be > (expectedExpiryTimestamp1.getTime - secondIngestionTimeUnix) / 1000) + + }) + } + + "Redis key TTL" should "not be updated, when a second feature table associated with the same entity is registered and ingested, if (event_timestamp + max_age) of the second " + + "Feature Table is not later than the expiry timestamp of the first feature table" in new Scope { + val startDate = new DateTime().minusDays(1).withTimeAtStartOfDay() + val endDate = new DateTime().withTimeAtStartOfDay() + val gen = rowGenerator(startDate, endDate) + val rows = generateDistinctRows(gen, 1000, groupByEntity) + val tempPath = storeAsParquet(sparkSession, rows) + val maxAge = 86400 * 3 + val configWithMaxAge = config.copy( + source = FileSource(tempPath, Map.empty, "eventTimestamp"), + featureTable = config.featureTable.copy(maxAge = Some(maxAge)), + startTime = startDate, + endTime = endDate + ) + + val ingestionTimeUnix = System.currentTimeMillis() + BatchPipeline.createPipeline(sparkSession, configWithMaxAge) + + val reducedMaxAge = 86400 * 2 + val configWithSecondFeatureTable = config.copy( + source = FileSource(tempPath, Map.empty, "eventTimestamp"), + featureTable = config.featureTable.copy( + name = "test-fs-2", + maxAge = Some(reducedMaxAge) + ), + startTime = startDate, + endTime = endDate + ) + + BatchPipeline.createPipeline(sparkSession, configWithSecondFeatureTable) + + val featureKeyEncoder: String => String = encodeFeatureKey(config.featureTable) + val featureKeyEncoderSecondTable: String => String = + encodeFeatureKey(configWithSecondFeatureTable.featureTable) + + rows.foreach(r => { + val encodedEntityKey = encodeEntityKey(r, config.featureTable) + val storedValues = jedis.hgetAll(encodedEntityKey).asScala.toMap + val expectedExpiryTimestamp1 = + new java.sql.Timestamp(r.eventTimestamp.getTime + 1000 * maxAge) + val expectedExpiryTimestamp2 = + new java.sql.Timestamp(r.eventTimestamp.getTime + 1000 * reducedMaxAge) + storedValues should beStoredRow( + Map( + featureKeyEncoder("feature1") -> r.feature1, + featureKeyEncoder("feature2") -> r.feature2, + featureKeyEncoderSecondTable("feature1") -> r.feature1, + featureKeyEncoderSecondTable("feature2") -> r.feature2, + "_ts:test-fs" -> r.eventTimestamp, + "_ts:test-fs-2" -> r.eventTimestamp, + "_ex:test-fs" -> expectedExpiryTimestamp1, + "_ex:test-fs-2" -> expectedExpiryTimestamp2 + ) + ) + val keyTTL = jedis.ttl(encodedEntityKey).toLong + keyTTL should (be <= (expectedExpiryTimestamp1.getTime - ingestionTimeUnix) / 1000 and + be > (expectedExpiryTimestamp2.getTime - ingestionTimeUnix) / 1000) + + }) + } + + "Redis key TTL" should "be updated, when the same feature table is re-ingested, with a smaller max age" in new Scope { + val startDate = new DateTime().minusDays(1).withTimeAtStartOfDay() + val endDate = new DateTime().withTimeAtStartOfDay() + val gen = rowGenerator(startDate, endDate) + val rows = generateDistinctRows(gen, 1000, groupByEntity) + val tempPath = storeAsParquet(sparkSession, rows) + val maxAge = 86400 * 3 + val configWithMaxAge = config.copy( + source = FileSource(tempPath, Map.empty, "eventTimestamp"), + featureTable = config.featureTable.copy(maxAge = Some(maxAge)), + startTime = startDate, + endTime = endDate + ) + + val ingestionTimeUnix = System.currentTimeMillis() + BatchPipeline.createPipeline(sparkSession, configWithMaxAge) + + val reducedMaxAge = 86400 * 2 + val configWithUpdatedFeatureTable = config.copy( + source = FileSource(tempPath, Map.empty, "eventTimestamp"), + featureTable = config.featureTable.copy( + maxAge = Some(reducedMaxAge) + ), + startTime = startDate, + endTime = endDate + ) + + BatchPipeline.createPipeline(sparkSession, configWithUpdatedFeatureTable) + + val featureKeyEncoder: String => String = encodeFeatureKey(config.featureTable) + + rows.foreach(r => { + val encodedEntityKey = encodeEntityKey(r, config.featureTable) + val storedValues = jedis.hgetAll(encodedEntityKey).asScala.toMap + val expiryTimestampAfterUpdate = + new java.sql.Timestamp(r.eventTimestamp.getTime + 1000 * reducedMaxAge) + storedValues should beStoredRow( + Map( + featureKeyEncoder("feature1") -> r.feature1, + featureKeyEncoder("feature2") -> r.feature2, + "_ts:test-fs" -> r.eventTimestamp, + "_ex:test-fs" -> expiryTimestampAfterUpdate + ) + ) + val keyTTL = jedis.ttl(encodedEntityKey).toLong + keyTTL should (be <= (expiryTimestampAfterUpdate.getTime - ingestionTimeUnix) / 1000 and be > 0L) + + }) + } + + "Redis key TTL" should "be removed, when the same feature table is re-ingested without max age" in new Scope { + val startDate = new DateTime().minusDays(1).withTimeAtStartOfDay() + val endDate = new DateTime().withTimeAtStartOfDay() + val gen = rowGenerator(startDate, endDate) + val rows = generateDistinctRows(gen, 1000, groupByEntity) + val tempPath = storeAsParquet(sparkSession, rows) + val maxAge = 86400 * 3 + val configWithMaxAge = config.copy( + source = FileSource(tempPath, Map.empty, "eventTimestamp"), + featureTable = config.featureTable.copy(maxAge = Some(maxAge)), + startTime = startDate, + endTime = endDate + ) + + BatchPipeline.createPipeline(sparkSession, configWithMaxAge) + + val configWithoutMaxAge = config.copy( + source = FileSource(tempPath, Map.empty, "eventTimestamp"), + startTime = startDate, + endTime = endDate + ) + + BatchPipeline.createPipeline(sparkSession, configWithoutMaxAge) + + val featureKeyEncoder: String => String = encodeFeatureKey(config.featureTable) + + rows.foreach(r => { + val encodedEntityKey = encodeEntityKey(r, config.featureTable) + val storedValues = jedis.hgetAll(encodedEntityKey).asScala.toMap + storedValues should beStoredRow( + Map( + featureKeyEncoder("feature1") -> r.feature1, + featureKeyEncoder("feature2") -> r.feature2, + "_ts:test-fs" -> r.eventTimestamp, + "_ex:test-fs" -> new Timestamp(Timestamps.MAX_VALUE.getSeconds * 1000) + ) + ) + val keyTTL = jedis.ttl(encodedEntityKey).toInt + keyTTL shouldEqual -1 + }) } diff --git a/spark/ingestion/src/test/scala/feast/ingestion/StreamingPipelineIT.scala b/spark/ingestion/src/test/scala/feast/ingestion/StreamingPipelineIT.scala index 39c44ad55a8..20e41dc17b9 100644 --- a/spark/ingestion/src/test/scala/feast/ingestion/StreamingPipelineIT.scala +++ b/spark/ingestion/src/test/scala/feast/ingestion/StreamingPipelineIT.scala @@ -17,6 +17,7 @@ package feast.ingestion import java.nio.file.Paths +import java.sql import java.util.Properties import com.dimafeng.testcontainers.{ @@ -30,6 +31,7 @@ import org.apache.spark.SparkConf import org.joda.time.DateTime import org.apache.kafka.clients.producer._ import com.example.protos.{AllTypesMessage, InnerMessage, TestMessage, VehicleType} +import com.google.protobuf.util.Timestamps import com.google.protobuf.{AbstractMessage, ByteString, Timestamp} import org.scalacheck.Gen import redis.clients.jedis.Jedis @@ -39,10 +41,8 @@ import feast.ingestion.helpers.RedisStorageHelper._ import feast.ingestion.helpers.DataHelper._ import feast.proto.storage.RedisProto.RedisKeyV2 import feast.proto.types.ValueProto -import org.apache.spark.sql.Row import org.apache.spark.sql.avro.to_avro import org.apache.spark.sql.functions.{col, struct} -import org.apache.spark.sql.types.StructType class StreamingPipelineIT extends SparkSpec with ForAllTestContainer { val redisContainer = GenericContainer("redis:6.0.8", exposedPorts = Seq(6379)) @@ -136,16 +136,86 @@ class StreamingPipelineIT extends SparkSpec with ForAllTestContainer { query.processAllAvailable() rows.foreach { r => - val storedValues = jedis.hgetAll(encodeEntityKey(r, config.featureTable)).asScala.toMap + val encodedEntityKey = encodeEntityKey(r, config.featureTable) + val storedValues = jedis.hgetAll(encodedEntityKey).asScala.toMap storedValues should beStoredRow( Map( featureKeyEncoder("unique_drivers") -> r.getUniqueDrivers, - "_ts:driver-fs" -> new java.sql.Timestamp(r.getEventTimestamp.getSeconds * 1000) + "_ts:driver-fs" -> new java.sql.Timestamp(r.getEventTimestamp.getSeconds * 1000), + "_ex:driver-fs" -> new java.sql.Timestamp(Timestamps.MAX_VALUE.getSeconds * 1000) ) ) + val keyTTL = jedis.ttl(encodedEntityKey).toInt + keyTTL shouldEqual -1 } } + "Streaming pipeline" should "store messages from kafka to redis with expiry time equal to the largest of (event_timestamp + max_age) for all feature " + + "tables associated with the entity" in new Scope { + val maxAge = 86400 + val configWithMaxAge = config.copy( + source = kafkaSource, + featureTable = config.featureTable.copy(maxAge = Some(maxAge)) + ) + val query = StreamingPipeline.createPipeline(sparkSession, configWithMaxAge).get + query.processAllAvailable() // to init kafka consumer + + val rows = generateDistinctRows(rowGenerator, 100, groupByEntity) + + val ingestionTimeUnix = System.currentTimeMillis() + rows.foreach(sendToKafka(kafkaSource.topic, _)) + + query.processAllAvailable() + + rows.foreach { r => + val encodedEntityKey = encodeEntityKey(r, config.featureTable) + val storedValues = jedis.hgetAll(encodedEntityKey).asScala.toMap + storedValues should beStoredRow( + Map( + featureKeyEncoder("unique_drivers") -> r.getUniqueDrivers, + "_ts:driver-fs" -> new java.sql.Timestamp(r.getEventTimestamp.getSeconds * 1000), + "_ex:driver-fs" -> new java.sql.Timestamp( + (r.getEventTimestamp.getSeconds + maxAge) * 1000 + ) + ) + ) + val keyTTL = jedis.ttl(encodedEntityKey).toLong + keyTTL should (be <= (r.getEventTimestamp.getSeconds + maxAge - ingestionTimeUnix / 1000) and be > 0L) + } + + val kafkaSourceSecondFeatureTable = kafkaSource.copy(topic = "topic-2") + val configWithSecondFeatureTable = config.copy( + source = kafkaSourceSecondFeatureTable, + featureTable = config.featureTable.copy(name = "driver-fs-2") + ) + val querySecondFeatureTable = + StreamingPipeline.createPipeline(sparkSession, configWithSecondFeatureTable).get + querySecondFeatureTable.processAllAvailable() // to init kafka consumer + rows.foreach(sendToKafka(kafkaSourceSecondFeatureTable.topic, _)) + querySecondFeatureTable.processAllAvailable() + + val featureKeyEncoderSecondFeatureTable: String => String = + encodeFeatureKey(configWithSecondFeatureTable.featureTable) + rows.foreach { r => + val encodedEntityKey = encodeEntityKey(r, config.featureTable) + val storedValues = jedis.hgetAll(encodedEntityKey).asScala.toMap + storedValues should beStoredRow( + Map( + featureKeyEncoder("unique_drivers") -> r.getUniqueDrivers, + featureKeyEncoderSecondFeatureTable("unique_drivers") -> r.getUniqueDrivers, + "_ts:driver-fs" -> new java.sql.Timestamp(r.getEventTimestamp.getSeconds * 1000), + "_ex:driver-fs" -> new java.sql.Timestamp( + (r.getEventTimestamp.getSeconds + maxAge) * 1000 + ), + "_ex:driver-fs-2" -> new java.sql.Timestamp(Timestamps.MAX_VALUE.getSeconds * 1000) + ) + ) + val keyTTL = jedis.ttl(encodedEntityKey).toInt + keyTTL shouldEqual -1 + } + + } + "Streaming pipeline" should "store invalid proto messages to deadletter path" in new Scope { val configWithDeadletter = config.copy( source = kafkaSource, diff --git a/spark/ingestion/src/test/scala/feast/ingestion/helpers/RedisStorageHelper.scala b/spark/ingestion/src/test/scala/feast/ingestion/helpers/RedisStorageHelper.scala index 921d65d4778..d15126d9d37 100644 --- a/spark/ingestion/src/test/scala/feast/ingestion/helpers/RedisStorageHelper.scala +++ b/spark/ingestion/src/test/scala/feast/ingestion/helpers/RedisStorageHelper.scala @@ -38,14 +38,15 @@ object RedisStorageHelper { m compose { (_: Map[Array[Byte], Array[Byte]]) - .map { case (k, v) => - if (k.length == 4) + .map { + case (k, v) if k.length == 4 => ( ByteBuffer.wrap(k).order(ByteOrder.LITTLE_ENDIAN).getInt.toHexString, ValueProto.Value.parseFrom(v).asScala ) - else + case (k, v) if k.startsWith("_ts".getBytes) || k.startsWith("_ex".getBytes) => (new String(k), Timestamp.parseFrom(v).asScala) + case (k, v) => (new String(k), ValueProto.Value.parseFrom(v).asScala) } } } diff --git a/spark/ingestion/src/test/scala/feast/ingestion/metrics/StatsReporterSpec.scala b/spark/ingestion/src/test/scala/feast/ingestion/metrics/StatsReporterSpec.scala index 1ae61724ed1..3b674de8b7a 100644 --- a/spark/ingestion/src/test/scala/feast/ingestion/metrics/StatsReporterSpec.scala +++ b/spark/ingestion/src/test/scala/feast/ingestion/metrics/StatsReporterSpec.scala @@ -89,19 +89,19 @@ class StatsReporterSpec extends UnitSpec { server.receive should contain("test:0|g") } - "Statsd reporter" should "keep tags part in the name's end" in new Scope { + "Statsd reporter" should "keep tags part in the message's end" in new Scope { reporter.report( gauges = Collections.emptySortedMap(), counters = Collections.emptySortedMap(), histograms = new util.TreeMap( Map( - "test#fs=name" -> histogram((1 to 100)) + "prefix.1111.test#fs=name,job=aaa" -> histogram((1 to 100)) ).asJava ), meters = Collections.emptySortedMap(), timers = Collections.emptySortedMap() ) - server.receive should contain("test.p95#fs=name:95.95|ms") + server.receive should contain("prefix.test.p95:95.95|ms|#fs:name,job:aaa") } } diff --git a/tests/e2e/conftest.py b/tests/e2e/conftest.py index 17524573dd3..6e340b49ef2 100644 --- a/tests/e2e/conftest.py +++ b/tests/e2e/conftest.py @@ -17,6 +17,9 @@ def pytest_addoption(parser): parser.addoption("--emr-cluster-id", action="store") parser.addoption("--emr-region", action="store") parser.addoption("--dataproc-project", action="store") + parser.addoption("--dataproc-executor-instances", action="store", default="2") + parser.addoption("--dataproc-executor-cores", action="store", default="2") + parser.addoption("--dataproc-executor-memory", action="store", default="2g") parser.addoption("--ingestion-jar", action="store") parser.addoption("--redis-url", action="store", default="localhost:6379") parser.addoption("--redis-cluster", action="store_true") diff --git a/tests/e2e/fixtures/base.py b/tests/e2e/fixtures/base.py index 68b9be5a4da..abdf9c76c01 100644 --- a/tests/e2e/fixtures/base.py +++ b/tests/e2e/fixtures/base.py @@ -12,5 +12,5 @@ def project_root(): def project_version(pytestconfig): if pytestconfig.getoption("feast_version"): return pytestconfig.getoption("feast_version") - - return "0.8-SNAPSHOT" + else: + raise Exception("feast_version not set") diff --git a/tests/e2e/fixtures/feast_services.py b/tests/e2e/fixtures/feast_services.py index 441864ba503..f2afd2bf6d5 100644 --- a/tests/e2e/fixtures/feast_services.py +++ b/tests/e2e/fixtures/feast_services.py @@ -47,7 +47,9 @@ def _wait_port_open(port, max_wait=60): return -@pytest.fixture(scope="session", params=[True, False]) +@pytest.fixture( + scope="session", params=[False], +) def enable_auth(request): return request.param @@ -185,6 +187,15 @@ def feast_jobservice( ) env["FEAST_DATAPROC_PROJECT"] = pytestconfig.getoption("dataproc_project") env["FEAST_DATAPROC_REGION"] = pytestconfig.getoption("dataproc_region") + env["FEAST_DATAPROC_EXECUTOR_INSTANCES"] = pytestconfig.getoption( + "dataproc_executor_instances" + ) + env["FEAST_DATAPROC_EXECUTOR_CORES"] = pytestconfig.getoption( + "dataproc_executor_cores" + ) + env["FEAST_DATAPROC_EXECUTOR_MEMORY"] = pytestconfig.getoption( + "dataproc_executor_memory" + ) env["FEAST_SPARK_STAGING_LOCATION"] = os.path.join( global_staging_path, "dataproc" ) diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 88ab8743a50..a80e6948295 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -3,5 +3,8 @@ def pytest_addoption(parser): parser.addoption("--dataproc-region", action="store") parser.addoption("--dataproc-project", action="store") parser.addoption("--dataproc-staging-location", action="store") + parser.addoption("--dataproc-executor-instances", action="store", default="2") + parser.addoption("--dataproc-executor-cores", action="store", default="2") + parser.addoption("--dataproc-executor-memory", action="store", default="2g") parser.addoption("--redis-url", action="store") parser.addoption("--redis-cluster", action="store_true") diff --git a/tests/integration/fixtures/launchers.py b/tests/integration/fixtures/launchers.py index ebe93172d1e..d289d974ac4 100644 --- a/tests/integration/fixtures/launchers.py +++ b/tests/integration/fixtures/launchers.py @@ -9,9 +9,15 @@ def dataproc_launcher(pytestconfig) -> DataprocClusterLauncher: region = pytestconfig.getoption("--dataproc-region") project_id = pytestconfig.getoption("--dataproc-project") staging_location = pytestconfig.getoption("--dataproc-staging-location") + executor_instances = pytestconfig.getoption("dataproc_executor_instances") + executor_cores = pytestconfig.getoption("dataproc_executor_cores") + executor_memory = pytestconfig.getoption("dataproc_executor_memory") return DataprocClusterLauncher( cluster_name=cluster_name, staging_location=staging_location, region=region, project_id=project_id, + executor_instances=executor_instances, + executor_cores=executor_cores, + executor_memory=executor_memory, )