diff --git a/.dockerignore b/.dockerignore index bef7cf70696..e28f081aff3 100644 --- a/.dockerignore +++ b/.dockerignore @@ -2,3 +2,4 @@ docs !docs/coverage charts env +**/.terraform diff --git a/.github/workflows/complete.yml b/.github/workflows/complete.yml index b11a4862f29..cb910ba7de0 100644 --- a/.github/workflows/complete.yml +++ b/.github/workflows/complete.yml @@ -7,7 +7,7 @@ jobs: runs-on: [self-hosted] strategy: matrix: - component: [core, serving, jobservice, jupyter] + component: [core, serving, jobservice, jupyter, ci] env: GITHUB_PR_SHA: ${{ github.event.pull_request.head.sha }} REGISTRY: gcr.io/kf-feast @@ -65,7 +65,6 @@ jobs: run: make lint-go lint-versions: - container: gcr.io/kf-feast/feast-ci:latest runs-on: [ubuntu-latest] steps: - uses: actions/checkout@v2 @@ -75,15 +74,20 @@ jobs: unit-test-java: runs-on: ubuntu-latest needs: lint-java - container: gcr.io/kf-feast/feast-ci:latest steps: - uses: actions/checkout@v2 - - uses: actions/cache@v1 + - name: Set up JDK 11 + uses: actions/setup-java@v1 + with: + java-version: '11' + java-package: jdk + architecture: x64 + - uses: actions/cache@v2 with: path: ~/.m2/repository - key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} + key: ${{ runner.os }}-ut-maven-${{ hashFiles('**/pom.xml') }} restore-keys: | - ${{ runner.os }}-maven- + ${{ runner.os }}-ut-maven- - name: Test java run: make test-java-with-coverage - uses: actions/upload-artifact@v2 @@ -118,40 +122,74 @@ jobs: needs: unit-test-java steps: - uses: actions/checkout@v2 - - name: Set up JDK 1.8 + - name: Set up JDK 11 uses: actions/setup-java@v1 with: java-version: '11' java-package: jdk architecture: x64 + - uses: actions/setup-python@v2 + with: + python-version: '3.6' + architecture: 'x64' + - uses: actions/cache@v2 + with: + path: ~/.m2/repository + key: ${{ runner.os }}-it-maven-${{ hashFiles('**/pom.xml') }} + restore-keys: | + ${{ runner.os }}-it-maven- - name: Run integration tests run: make test-java-integration + - name: Save report + uses: actions/upload-artifact@v2 + if: failure() + with: + name: it-report + path: spark/ingestion/target/test-reports/TestSuite.txt + retention-days: 5 tests-docker-compose: needs: - build-push-docker-images - publish-ingestion-jar runs-on: ubuntu-latest + env: + INGESTION_JAR_PATH: /shared/feast-ingestion-spark-develop.jar steps: - uses: actions/checkout@v2 + - name: Download ingestion jar + uses: actions/download-artifact@v2 + with: + name: ingestion-jar + path: ./infra/docker-compose/ - name: Test docker compose run: ./infra/scripts/test-docker-compose.sh ${GITHUB_SHA} publish-ingestion-jar: - runs-on: [self-hosted] + runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 - - uses: GoogleCloudPlatform/github-actions/setup-gcloud@master - with: - version: '290.0.1' - export_default_credentials: true - uses: actions/setup-java@v1 with: java-version: '11' - - uses: stCarolas/setup-maven@v3 + - name: Cache local Maven repository + uses: actions/cache@v2 with: - maven-version: 3.6.3 + path: ~/.m2/repository + key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} + restore-keys: | + ${{ runner.os }}-maven- - name: build-jar - run: make build-java-no-tests REVISION=${GITHUB_SHA} - - name: copy to gs - run: gsutil cp ./spark/ingestion/target/feast-ingestion-spark-${GITHUB_SHA}.jar gs://feast-jobs/spark/ingestion/ + env: + # Try to add retries to prevent connection resets + # https://github.community/t/getting-maven-could-not-transfer-artifact-with-500-error-when-using-github-actions/17570 + # https://github.com/actions/virtual-environments/issues/1499#issuecomment-718396233 + MAVEN_OPTS: -Dmaven.wagon.httpconnectionManager.ttlSeconds=25 -Dmaven.wagon.http.retryHandler.count=3 -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false + MAVEN_EXTRA_OPTS: -X + run: make build-java-no-tests REVISION=develop + - name: Upload ingestion jar + uses: actions/upload-artifact@v2 + with: + name: ingestion-jar + path: spark/ingestion/target/feast-ingestion-spark-develop.jar + retention-days: 1 diff --git a/.github/workflows/master_only.yml b/.github/workflows/master_only.yml index 20caebe0afc..495d95163f7 100644 --- a/.github/workflows/master_only.yml +++ b/.github/workflows/master_only.yml @@ -11,10 +11,9 @@ jobs: runs-on: [self-hosted] strategy: matrix: - component: [core, serving, jobcontroller, jupyter, ci] + component: [core, serving, jobservice, jupyter, ci] env: MAVEN_CACHE: gs://feast-templocation-kf-feast/.m2.2020-08-19.tar - DOCKER_BUILDKIT: '1' steps: - uses: actions/checkout@v2 - uses: GoogleCloudPlatform/github-actions/setup-gcloud@master @@ -27,6 +26,8 @@ jobs: infra/scripts/download-maven-cache.sh \ --archive-uri ${MAVEN_CACHE} \ --output-dir . + - name: Get version + run: echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV - name: Build image run: make build-${{ matrix.component }}-docker REGISTRY=gcr.io/kf-feast VERSION=${GITHUB_SHA} - name: Push image @@ -37,34 +38,9 @@ jobs: docker tag gcr.io/kf-feast/feast-${{ matrix.component }}:${GITHUB_SHA} gcr.io/kf-feast/feast-${{ matrix.component }}:develop docker push gcr.io/kf-feast/feast-${{ matrix.component }}:develop fi - - name: Get version - run: echo ::set-env name=RELEASE_VERSION::${GITHUB_REF#refs/*/} - - name: Push versioned Docker image - run: | - source infra/scripts/setup-common-functions.sh - # Build and push semver tagged commits - # Regular expression should match MAJOR.MINOR.PATCH[-PRERELEASE[.IDENTIFIER]] - # eg. v0.7.1 v0.7.2-alpha v0.7.2-rc.1 - SEMVER_REGEX='^v[0-9]+\.[0-9]+\.[0-9]+(-([0-9A-Za-z-]+(\.[0-9A-Za-z-]+)*))?$' - if echo "${RELEASE_VERSION}" | grep -P "$SEMVER_REGEX" &>/dev/null ; then - VERSION_WITHOUT_PREFIX=${RELEASE_VERSION:1} - - docker tag gcr.io/kf-feast/feast-${{ matrix.component }}:${GITHUB_SHA} gcr.io/kf-feast/feast-${{ matrix.component }}:${VERSION_WITHOUT_PREFIX} - docker push gcr.io/kf-feast/feast-${{ matrix.component }}:${VERSION_WITHOUT_PREFIX} - - # Also update "latest" image if tagged commit is pushed to stable branch - HIGHEST_SEMVER_TAG=$(get_tag_release -m) - echo "Only push to latest tag if tag is the highest semver version $HIGHEST_SEMVER_TAG" - - if [ "${VERSION_WITHOUT_PREFIX}" = "${HIGHEST_SEMVER_TAG:1}" ] - then - docker tag gcr.io/kf-feast/feast-${{ matrix.component }}:${GITHUB_SHA} gcr.io/kf-feast/feast-${{ matrix.component }}:latest - docker push gcr.io/kf-feast/feast-${{ matrix.component }}:latest - fi - fi publish-ingestion-jar: - runs-on: [ self-hosted ] + runs-on: ubuntu-latest env: PUBLISH_BUCKET: feast-jobs steps: @@ -73,12 +49,18 @@ jobs: with: version: '290.0.1' export_default_credentials: true + project_id: ${{ secrets.GCP_PROJECT_ID }} + service_account_key: ${{ secrets.GCP_SA_KEY }} - uses: actions/setup-java@v1 with: java-version: '11' - - uses: stCarolas/setup-maven@v3 + - name: Cache local Maven repository + uses: actions/cache@v2 with: - maven-version: 3.6.3 + path: ~/.m2/repository + key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} + restore-keys: | + ${{ runner.os }}-maven- - name: Publish develop version of ingestion job run: | if [ ${GITHUB_REF#refs/*/} == "master" ]; then @@ -86,7 +68,7 @@ jobs: gsutil cp ./spark/ingestion/target/feast-ingestion-spark-develop.jar gs://${PUBLISH_BUCKET}/spark/ingestion/ fi - name: Get version - run: echo ::set-env name=RELEASE_VERSION::${GITHUB_REF#refs/*/} + run: echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV - name: Publish tagged version of ingestion job run: | SEMVER_REGEX='^v[0-9]+\.[0-9]+\.[0-9]+(-([0-9A-Za-z-]+(\.[0-9A-Za-z-]+)*))?$' @@ -94,4 +76,31 @@ jobs: VERSION=${RELEASE_VERSION:1} make build-java-no-tests REVISION=${VERSION} gsutil cp ./spark/ingestion/target/feast-ingestion-spark-${VERSION}.jar gs://${PUBLISH_BUCKET}/spark/ingestion/ - fi \ No newline at end of file + fi + + publish-ingestion-pylibs: + strategy: + matrix: + os: [ ubuntu-latest, macos-latest ] + python-version: [ 3.6, 3.7, 3.8 ] + env: + PUBLISH_BUCKET: feast-jobs + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v2 + - uses: GoogleCloudPlatform/github-actions/setup-gcloud@master + with: + version: '290.0.1' + export_default_credentials: true + project_id: ${{ secrets.GCP_PROJECT_ID }} + service_account_key: ${{ secrets.GCP_SA_KEY }} + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - name: Create libs archive + env: + PY_VERSION: ${{ matrix.python-version }} + run: | + export PLATFORM=$(python -c 'import platform; print(platform.system().lower())') + ./infra/scripts/build-ingestion-py-dependencies.sh "py${PY_VERSION}-$PLATFORM" gs://${PUBLISH_BUCKET}/spark/validation/ diff --git a/.github/workflows/mirror.yml b/.github/workflows/mirror.yml index d5c270d4c79..f9bfdd92a54 100644 --- a/.github/workflows/mirror.yml +++ b/.github/workflows/mirror.yml @@ -12,7 +12,7 @@ jobs: - uses: actions/checkout@v2 with: fetch-depth: 0 - - uses: webfactory/ssh-agent@v0.4.0 + - uses: webfactory/ssh-agent@v0.4.1 with: ssh-private-key: ${{ secrets.MIRROR_SSH_KEY }} - name: Mirror all origin branches and tags to internal repo diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 1d389859f42..157a593d35f 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -13,6 +13,7 @@ jobs: version_without_prefix: ${{ steps.get_release_version_without_prefix.outputs.version_without_prefix }} highest_semver_tag: ${{ steps.get_highest_semver.outputs.highest_semver_tag }} steps: + - uses: actions/checkout@v2 - name: Get release version id: get_release_version run: echo ::set-output name=release_version::${GITHUB_REF#refs/*/} @@ -47,10 +48,9 @@ jobs: needs: get-version strategy: matrix: - component: [core, serving, jupyter] + component: [core, serving, jobservice, jupyter] env: MAVEN_CACHE: gs://feast-templocation-kf-feast/.m2.2020-08-19.tar - DOCKER_BUILDKIT: '1' steps: - uses: actions/checkout@v2 - name: Set up QEMU @@ -66,40 +66,42 @@ jobs: with: version: '290.0.1' export_default_credentials: true + project_id: ${{ secrets.GCP_PROJECT_ID }} + service_account_key: ${{ secrets.GCP_SA_KEY }} + - run: gcloud auth configure-docker --quiet - name: Get m2 cache run: | infra/scripts/download-maven-cache.sh \ --archive-uri ${MAVEN_CACHE} \ --output-dir . - - name: Build and push - uses: docker/build-push-action@v2 - env: - RELEASE_VERSION: ${{ needs.get-version.outputs.release_version }} - with: - push: true - file: infra/docker/${{ matrix.component }}/Dockerfile - tags: feastdev/feast-${{ matrix.component }}:${{ needs.get-version.outputs.release_version }} - build-args: | - REVISION=${RELEASE_VERSION} - - name: Build and push latest - uses: docker/build-push-action@v2 + - name: Build and push versioned images env: RELEASE_VERSION: ${{ needs.get-version.outputs.release_version }} VERSION_WITHOUT_PREFIX: ${{ needs.get-version.outputs.version_without_prefix }} HIGHEST_SEMVER_TAG: ${{ needs.get-version.outputs.highest_semver_tag }} - with: - if: ${VERSION_WITHOUT_PREFIX} == ${HIGHEST_SEMVER_TAG:1} - push: true - file: infra/docker/${{ matrix.component }}/Dockerfile - tags: feastdev/feast-${{ matrix.component }}:latest - build-args: | - REVISION=${RELEASE_VERSION} + run: | + docker build --build-arg VERSION=$RELEASE_VERSION \ + -t gcr.io/kf-feast/feast-${{ matrix.component }}:${GITHUB_SHA} \ + -t gcr.io/kf-feast/feast-${{ matrix.component }}:${VERSION_WITHOUT_PREFIX} \ + -t feastdev/feast-${{ matrix.component }}:${VERSION_WITHOUT_PREFIX} \ + -f infra/docker/${{ matrix.component }}/Dockerfile . + docker push gcr.io/kf-feast/feast-${{ matrix.component }}:${VERSION_WITHOUT_PREFIX} + docker push feastdev/feast-${{ matrix.component }}:${VERSION_WITHOUT_PREFIX} + + echo "Only push to latest tag if tag is the highest semver version $HIGHEST_SEMVER_TAG" + if [ "${VERSION_WITHOUT_PREFIX}" = "${HIGHEST_SEMVER_TAG:1}" ] + then + docker tag feastdev/feast-${{ matrix.component }}:${VERSION_WITHOUT_PREFIX} feastdev/feast-${{ matrix.component }}:latest + docker tag gcr.io/kf-feast/feast-${{ matrix.component }}:${VERSION_WITHOUT_PREFIX} gcr.io/kf-feast/feast-${{ matrix.component }}:latest + docker push feastdev/feast-${{ matrix.component }}:latest + docker push gcr.io/kf-feast/feast-${{ matrix.component }}:latest + fi publish-helm-charts: runs-on: ubuntu-latest needs: get-version env: - HELM_VERSION: v2.16.9 + HELM_VERSION: v2.17.0 steps: - uses: actions/checkout@v2 - uses: GoogleCloudPlatform/github-actions/setup-gcloud@master @@ -109,8 +111,10 @@ jobs: project_id: ${{ secrets.GCP_PROJECT_ID }} service_account_key: ${{ secrets.GCP_SA_KEY }} - run: gcloud auth configure-docker --quiet - - name: Validate chart versions + - name: Validate repository versions run: make lint-versions + - name: Validate chart release versions + run: ./infra/scripts/validate-helm-chart-docker-image.sh - name: Remove previous Helm run: sudo rm -rf $(which helm) - name: Install Helm diff --git a/.gitignore b/.gitignore index b2c3f77f8c3..546aff03830 100644 --- a/.gitignore +++ b/.gitignore @@ -184,3 +184,8 @@ sdk/python/docs/html *_pb2.py *_pb2.pyi *_pb2_grpc.py + +# VSCode +.bloop +.metals +*.code-workspace diff --git a/.prow/config.yaml b/.prow/config.yaml index 781e311a8c8..944d1baf2b3 100644 --- a/.prow/config.yaml +++ b/.prow/config.yaml @@ -1,24 +1,32 @@ -prowjob_namespace: default +prowjob_namespace: prow pod_namespace: test-pods +in_repo_config: + enabled: + "*": true + plank: job_url_prefix_config: - "*": http://prow.feast.ai/view/gcs - report_template: '[Full PR test history](https://prow.feast.ai/pr-history?org={{.Spec.Refs.Org}}&repo={{.Spec.Refs.Repo}}&pr={{with index .Spec.Refs.Pulls 0}}{{.Number}}{{end}})' + "*": https://prow.feast.dev/view/gcs pod_pending_timeout: 60m + report_templates: + '*': >- + [Full PR test history](https://prow.feast.dev/pr-history?org={{.Spec.Refs.Org}}&repo={{.Spec.Refs.Repo}}&pr={{with index .Spec.Refs.Pulls 0}}{{.Number}}{{end}}). + [Your PR dashboard](https://prow.feast.dev/pr?query=is:pr+state:open+author:{{with + index .Spec.Refs.Pulls 0}}{{.Author}}{{end}}). default_decoration_configs: "*": timeout: 1h grace_period: 15s - utility_images: - clonerefs: gcr.io/k8s-prow/clonerefs:v20190221-d14461a - initupload: gcr.io/k8s-prow/initupload:v20190221-d14461a - entrypoint: gcr.io/k8s-prow/entrypoint:v20190221-d14461a - sidecar: gcr.io/k8s-prow/sidecar:v20190221-d14461a gcs_configuration: - bucket: feast-templocation-kf-feast + bucket: gs://feast-prow-artifacts path_strategy: explicit - gcs_credentials_secret: prow-service-account + gcs_credentials_secret: gcs-credentials + utility_images: + clonerefs: gcr.io/k8s-prow/clonerefs:v20201112-00537d1bb4 + entrypoint: gcr.io/k8s-prow/entrypoint:v20201112-00537d1bb4 + initupload: gcr.io/k8s-prow/initupload:v20201112-00537d1bb4 + sidecar: gcr.io/k8s-prow/sidecar:v20201112-00537d1bb4 deck: tide_update_period: 1s @@ -42,6 +50,7 @@ tide: queries: - repos: - feast-dev/feast + - feast-dev/feast-spark labels: - lgtm - approved @@ -54,6 +63,7 @@ tide: - needs-kind merge_method: feast-dev/feast: squash + feast-dev/feast-spark: squash blocker_label: merge-blocker squash_label: tide/squash @@ -222,7 +232,7 @@ presubmits: spec: containers: - image: gcr.io/kf-feast/feast-ci:latest - command: [ "infra/scripts/codebuild_runner.py", "--location-from-prow" ] + command: [ "infra/scripts/codebuild_runner.py", "--location-from-prow", "--project-name", "feast-ci-project" ] resources: requests: cpu: "2" @@ -240,6 +250,89 @@ presubmits: key: AWS_SECRET_ACCESS_KEY - name: AWS_DEFAULT_REGION value: us-west-2 + - name: test-end-to-end-sparkop + decorate: true + always_run: true + max_concurrency: 1 + spec: + containers: + - image: gcr.io/kf-feast/feast-ci:latest + command: [ "infra/scripts/codebuild_runner.py", "--location-from-prow", "--project-name", "feast-ci-sparkop-project" ] + resources: + requests: + cpu: "2" + memory: "2048Mi" + env: + - name: AWS_ACCESS_KEY_ID + valueFrom: + secretKeyRef: + name: feast-aws-creds + key: AWS_ACCESS_KEY_ID + - name: AWS_SECRET_ACCESS_KEY + valueFrom: + secretKeyRef: + name: feast-aws-creds + key: AWS_SECRET_ACCESS_KEY + - name: AWS_DEFAULT_REGION + value: us-west-2 + + - name: test-end-to-end-azure + decorate: true + always_run: true + max_concurrency: 1 + spec: + containers: + - image: gcr.io/kf-feast/feast-ci:develop + command: [ "infra/scripts/azure-runner.sh" ] + resources: + requests: + cpu: "2" + memory: "2048Mi" + env: + - name: GOOGLE_APPLICATION_CREDENTIALS + value: /etc/gcloud/service-account.json + - name: RESOURCE_GROUP + value: Feast + - name: AKS_CLUSTER_NAME + value: main + - name: DOCKER_REPOSITORY + value: gcr.io/kf-feast + - name: STAGING_PATH + value: wasbs://staging@feastcicd.blob.core.windows.net/cicd-staging + - name: AZ_SERVICE_PRINCIPAL_ID + valueFrom: + secretKeyRef: + name: feast-az-creds + key: AZ_SERVICE_PRINCIPAL_ID + - name: AZ_SERVICE_PRINCIPAL_PASS + valueFrom: + secretKeyRef: + name: feast-az-creds + key: AZ_SERVICE_PRINCIPAL_PASS + - name: AZ_SERVICE_PRINCIPAL_TENANT_ID + valueFrom: + secretKeyRef: + name: feast-az-creds + key: AZ_SERVICE_PRINCIPAL_TENANT_ID + - name: AZURE_BLOB_ACCOUNT_NAME + valueFrom: + secretKeyRef: + name: feast-az-creds + key: AZURE_BLOB_ACCOUNT_NAME + - name: AZURE_BLOB_ACCOUNT_ACCESS_KEY + valueFrom: + secretKeyRef: + name: feast-az-creds + key: AZURE_BLOB_ACCOUNT_ACCESS_KEY + volumeMounts: + - mountPath: /etc/gcloud/service-account.json + name: service-account + readOnly: true + subPath: service-account.json + volumes: + - name: service-account + secret: + secretName: feast-service-account postsubmits: feast-dev/feast: diff --git a/.prow/plugins.yaml b/.prow/plugins.yaml index bfa55bcf2df..0f89e07be57 100644 --- a/.prow/plugins.yaml +++ b/.prow/plugins.yaml @@ -13,7 +13,23 @@ plugins: - trigger - config-updater - require-matching-label - + - release-note + feast-dev/feast-spark: + - approve + - assign + - help + - hold + - label + - lgtm + - lifecycle + - size + - verify-owners + - wip + - trigger + - config-updater + - require-matching-label + - release-note + config_updater: maps: .prow/config.yaml: @@ -24,6 +40,10 @@ external_plugins: - name: needs-rebase events: - pull_request + feast-dev/feast-spark: + - name: needs-rebase + events: + - pull_request require_matching_label: - missing_label: needs-kind @@ -31,3 +51,8 @@ require_matching_label: repo: feast prs: true regexp: ^kind/ +- missing_label: needs-kind + org: feast-dev + repo: feast-spark + prs: true + regexp: ^kind/ diff --git a/CHANGELOG.md b/CHANGELOG.md index 6f866efce9d..fa6f3018c92 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,130 @@ # Changelog +## [v0.9.0](https://github.com/feast-dev/feast/tree/v0.9.0) (2021-01-28) + +[Full Changelog](https://github.com/feast-dev/feast/compare/v0.8.4...v0.9.0) + +**Implemented enhancements:** + +- Enable user to provide spark job template as input for jobservice deployment [\#1285](https://github.com/feast-dev/feast/pull/1285) ([khorshuheng](https://github.com/khorshuheng)) +- Add feature table name filter to jobs list api [\#1282](https://github.com/feast-dev/feast/pull/1282) ([terryyylim](https://github.com/terryyylim)) +- Report observed value for aggregated checks in pre-ingestion feature validation [\#1278](https://github.com/feast-dev/feast/pull/1278) ([pyalex](https://github.com/pyalex)) +- Add docs page for Azure setup [\#1276](https://github.com/feast-dev/feast/pull/1276) ([jklegar](https://github.com/jklegar)) +- Azure example terraform [\#1274](https://github.com/feast-dev/feast/pull/1274) ([jklegar](https://github.com/jklegar)) + + +**Fixed bugs:** + +- make EMR jar uploader work the same as k8s one [\#1284](https://github.com/feast-dev/feast/pull/1284) ([oavdeev](https://github.com/oavdeev)) +- Don't error when azure vars not set [\#1277](https://github.com/feast-dev/feast/pull/1277) ([jklegar](https://github.com/jklegar)) +- Prevent ingestion job config parser from unwanted fieldMapping transformation [\#1261](https://github.com/feast-dev/feast/pull/1261) ([pyalex](https://github.com/pyalex)) +- Features are not being ingested due to max age overflow [\#1209](https://github.com/feast-dev/feast/pull/1209) ([pyalex](https://github.com/pyalex)) +- Feature Table is not being update when only max\_age was changed [\#1208](https://github.com/feast-dev/feast/pull/1208) ([pyalex](https://github.com/pyalex)) +- Truncate staging timestamps in entities dataset to ms [\#1207](https://github.com/feast-dev/feast/pull/1207) ([pyalex](https://github.com/pyalex)) +- Bump terraform rds module version [\#1204](https://github.com/feast-dev/feast/pull/1204) ([oavdeev](https://github.com/oavdeev)) + + +**Merged pull requests:** + +- Use date partitioning column in FileSource [\#1293](https://github.com/feast-dev/feast/pull/1293) ([pyalex](https://github.com/pyalex)) +- Add EMR CI/CD entrypoint script [\#1290](https://github.com/feast-dev/feast/pull/1290) ([oavdeev](https://github.com/oavdeev)) +- Online serving optimizations [\#1286](https://github.com/feast-dev/feast/pull/1286) ([pyalex](https://github.com/pyalex)) +- Make third party grpc packages recognizable as python module [\#1283](https://github.com/feast-dev/feast/pull/1283) ([khorshuheng](https://github.com/khorshuheng)) +- Report observed values in feature validation as Gauge [\#1280](https://github.com/feast-dev/feast/pull/1280) ([pyalex](https://github.com/pyalex)) +- Keep same amount of partitions after repartitioning in IngestionJob [\#1279](https://github.com/feast-dev/feast/pull/1279) ([pyalex](https://github.com/pyalex)) +- Add request feature counter metric [\#1272](https://github.com/feast-dev/feast/pull/1272) ([terryyylim](https://github.com/terryyylim)) +- Use SEND\_INTERRUPT to cancel EMR jobs [\#1271](https://github.com/feast-dev/feast/pull/1271) ([oavdeev](https://github.com/oavdeev)) +- Fix historical test flakiness [\#1270](https://github.com/feast-dev/feast/pull/1270) ([jklegar](https://github.com/jklegar)) +- Allow https url for spark ingestion jar [\#1266](https://github.com/feast-dev/feast/pull/1266) ([jklegar](https://github.com/jklegar)) +- Add project name to feature validation metric [\#1264](https://github.com/feast-dev/feast/pull/1264) ([pyalex](https://github.com/pyalex)) +- Use dataproc console url instead of gcs for log uri [\#1263](https://github.com/feast-dev/feast/pull/1263) ([khorshuheng](https://github.com/khorshuheng)) +- Make nodes priority \(for redis cluster\) configurable in Serving [\#1260](https://github.com/feast-dev/feast/pull/1260) ([pyalex](https://github.com/pyalex)) +- Enhance job api to return associated feature table and start time [\#1259](https://github.com/feast-dev/feast/pull/1259) ([khorshuheng](https://github.com/khorshuheng)) +- Reporting metrics from validation UDF [\#1256](https://github.com/feast-dev/feast/pull/1256) ([pyalex](https://github.com/pyalex)) +- Allow use the same timestamp column for both created & even timestamp in Historical Retrieval [\#1255](https://github.com/feast-dev/feast/pull/1255) ([pyalex](https://github.com/pyalex)) +- Apply grpc tracing interceptor on Feast SDK [\#1243](https://github.com/feast-dev/feast/pull/1243) ([khorshuheng](https://github.com/khorshuheng)) +- Apply grpc tracing interceptor on online serving [\#1242](https://github.com/feast-dev/feast/pull/1242) ([khorshuheng](https://github.com/khorshuheng)) +- Python UDF in Ingestion being used for feature validation [\#1234](https://github.com/feast-dev/feast/pull/1234) ([pyalex](https://github.com/pyalex)) +- Add spark k8s operator launcher [\#1225](https://github.com/feast-dev/feast/pull/1225) ([oavdeev](https://github.com/oavdeev)) +- Add deadletter/read-from-source metrics to batch and stream ingestion [\#1223](https://github.com/feast-dev/feast/pull/1223) ([terryyylim](https://github.com/terryyylim)) +- Implement AbstractStagingClient for azure blob storage [\#1218](https://github.com/feast-dev/feast/pull/1218) ([jklegar](https://github.com/jklegar)) +- Configurable materialization destination for view in BigQuerySource [\#1201](https://github.com/feast-dev/feast/pull/1201) ([pyalex](https://github.com/pyalex)) +- Update Feast Core list features method [\#1176](https://github.com/feast-dev/feast/pull/1176) ([terryyylim](https://github.com/terryyylim)) +- S3 endpoint configuration \#1169 [\#1172](https://github.com/feast-dev/feast/pull/1172) ([mike0sv](https://github.com/mike0sv)) +- Increase kafka consumer waiting time in e2e tests [\#1268](https://github.com/feast-dev/feast/pull/1268) ([pyalex](https://github.com/pyalex)) +- E2E tests support for jobservice's control loop [\#1267](https://github.com/feast-dev/feast/pull/1267) ([pyalex](https://github.com/pyalex)) +- Optimize memory footprint for Spark Ingestion Job [\#1265](https://github.com/feast-dev/feast/pull/1265) ([pyalex](https://github.com/pyalex)) +- Fix historical test for azure [\#1262](https://github.com/feast-dev/feast/pull/1262) ([jklegar](https://github.com/jklegar)) +- Change azure https to wasbs and add azure creds to spark [\#1258](https://github.com/feast-dev/feast/pull/1258) ([jklegar](https://github.com/jklegar)) +- Docs, fixes and scripts to run e2e tests in minikube [\#1254](https://github.com/feast-dev/feast/pull/1254) ([oavdeev](https://github.com/oavdeev)) +- Fix azure blob storage access in e2e tests [\#1253](https://github.com/feast-dev/feast/pull/1253) ([jklegar](https://github.com/jklegar)) +- Update python version requirements to 3.7 for Dataproc launcher [\#1251](https://github.com/feast-dev/feast/pull/1251) ([pyalex](https://github.com/pyalex)) +- Fix build-ingestion-py-dependencies script [\#1250](https://github.com/feast-dev/feast/pull/1250) ([pyalex](https://github.com/pyalex)) +- Add datadog\(statsd\) client to python package for IngestionJob [\#1249](https://github.com/feast-dev/feast/pull/1249) ([pyalex](https://github.com/pyalex)) +- Add prow job for azure e2e test [\#1244](https://github.com/feast-dev/feast/pull/1244) ([jklegar](https://github.com/jklegar)) +- Azure e2e test [\#1241](https://github.com/feast-dev/feast/pull/1241) ([jklegar](https://github.com/jklegar)) +- Add Feast Serving histogram metrics [\#1240](https://github.com/feast-dev/feast/pull/1240) ([terryyylim](https://github.com/terryyylim)) +- CI should work on python 3.6 [\#1237](https://github.com/feast-dev/feast/pull/1237) ([pyalex](https://github.com/pyalex)) +- Integration test for k8s spark operator support [\#1236](https://github.com/feast-dev/feast/pull/1236) ([oavdeev](https://github.com/oavdeev)) +- Add prow config for spark k8s operator integration testing [\#1235](https://github.com/feast-dev/feast/pull/1235) ([oavdeev-tt](https://github.com/oavdeev-tt)) +- Upgrading spark to 3.0.1 [\#1227](https://github.com/feast-dev/feast/pull/1227) ([pyalex](https://github.com/pyalex)) +- Support TFRecord as one of the output formats for historical feature retrieval [\#1222](https://github.com/feast-dev/feast/pull/1222) ([khorshuheng](https://github.com/khorshuheng)) +- Remove stage\_dataframe from the launcher interface [\#1220](https://github.com/feast-dev/feast/pull/1220) ([oavdeev](https://github.com/oavdeev)) +- Refactor staging client uploader and use it in EMR launcher [\#1219](https://github.com/feast-dev/feast/pull/1219) ([oavdeev](https://github.com/oavdeev)) +- Remove unused EMR code [\#1217](https://github.com/feast-dev/feast/pull/1217) ([oavdeev](https://github.com/oavdeev)) +- Remove job id from ingested row counter metric [\#1216](https://github.com/feast-dev/feast/pull/1216) ([terryyylim](https://github.com/terryyylim)) +- Quickstart link fixed [\#1213](https://github.com/feast-dev/feast/pull/1213) ([szczeles](https://github.com/szczeles)) +- Delete v1 concepts [\#1194](https://github.com/feast-dev/feast/pull/1194) ([terryyylim](https://github.com/terryyylim)) +- Dont write defaults to config [\#1188](https://github.com/feast-dev/feast/pull/1188) ([mike0sv](https://github.com/mike0sv)) +- Refactor tests which utilizes feature sets [\#1186](https://github.com/feast-dev/feast/pull/1186) ([terryyylim](https://github.com/terryyylim)) +- Refactor configurable options and add sphinx docs [\#1174](https://github.com/feast-dev/feast/pull/1174) ([terryyylim](https://github.com/terryyylim)) +- Remove unnecessary Google Auth dependency [\#1170](https://github.com/feast-dev/feast/pull/1170) ([woop](https://github.com/woop)) + + +## [v0.8.2](https://github.com/feast-dev/feast/tree/v0.8.2) (2020-12-01) + +[Full Changelog](https://github.com/feast-dev/feast/compare/v0.8.1...v0.8.2) + +**Implemented enhancements:** + +- Configurable materialization destination for view in BigQuerySource [\#1201](https://github.com/feast-dev/feast/pull/1201) ([pyalex](https://github.com/pyalex)) + +**Fixed bugs:** + +- Fix tag order for release workflow [\#1205](https://github.com/feast-dev/feast/pull/1205) ([terryyylim](https://github.com/terryyylim)) +- Fix Feature Table not updated on new feature addition [\#1197](https://github.com/feast-dev/feast/pull/1197) ([khorshuheng](https://github.com/khorshuheng)) + +**Merged pull requests:** + +- Suppress kafka logs in Ingestion Job [\#1206](https://github.com/feast-dev/feast/pull/1206) ([pyalex](https://github.com/pyalex)) +- Add project name to metrics labels in Ingestion Job [\#1202](https://github.com/feast-dev/feast/pull/1202) ([pyalex](https://github.com/pyalex)) + + +## [v0.8.1](https://github.com/feast-dev/feast/tree/v0.8.1) (2020-11-24) + +[Full Changelog](https://github.com/feast-dev/feast/compare/v0.8.0...v0.8.1) + +**Implemented enhancements:** + +- Expires Redis Keys based on Feature Table Max Age [\#1161](https://github.com/feast-dev/feast/pull/1161) ([khorshuheng](https://github.com/khorshuheng)) +- Jobservice control loop \(based on \#1140\) [\#1156](https://github.com/feast-dev/feast/pull/1156) ([oavdeev](https://github.com/oavdeev)) + +**Fixed bugs:** + +- Lazy metrics initialization \(to correct pick up in executor\) [\#1195](https://github.com/feast-dev/feast/pull/1195) ([pyalex](https://github.com/pyalex)) +- Add missing third\_party folder [\#1185](https://github.com/feast-dev/feast/pull/1185) ([terryyylim](https://github.com/terryyylim)) +- Fix missing name variable instantiation [\#1166](https://github.com/feast-dev/feast/pull/1166) ([terryyylim](https://github.com/terryyylim)) + +**Merged pull requests:** + +- Bump ssh-agent version [\#1175](https://github.com/feast-dev/feast/pull/1175) ([terryyylim](https://github.com/terryyylim)) +- Refactor configurable options and add sphinx docs [\#1174](https://github.com/feast-dev/feast/pull/1174) ([terryyylim](https://github.com/terryyylim)) +- Stabilize flaky e2e tests [\#1173](https://github.com/feast-dev/feast/pull/1173) ([pyalex](https://github.com/pyalex)) +- Fix connection resets in CI for Maven [\#1164](https://github.com/feast-dev/feast/pull/1164) ([woop](https://github.com/woop)) +- Add dataproc executor resource config [\#1160](https://github.com/feast-dev/feast/pull/1160) ([terryyylim](https://github.com/terryyylim)) +- Fix github workflow deprecating env variable [\#1158](https://github.com/feast-dev/feast/pull/1158) ([terryyylim](https://github.com/terryyylim)) +- Ensure consistency of github workflow [\#1157](https://github.com/feast-dev/feast/pull/1157) ([terryyylim](https://github.com/terryyylim)) + + ## [v0.8.0](https://github.com/feast-dev/feast/tree/v0.8.0) (2020-11-10) [Full Changelog](https://github.com/feast-dev/feast/compare/v0.7.1...v0.8.0) diff --git a/Makefile b/Makefile index 985151b5728..c8dc7f0c3a6 100644 --- a/Makefile +++ b/Makefile @@ -17,6 +17,7 @@ ROOT_DIR := $(shell dirname $(realpath $(firstword $(MAKEFILE_LIST)))) PROTO_TYPE_SUBDIRS = core serving types storage PROTO_SERVICE_SUBDIRS = core serving +MVN := mvn ${MAVEN_EXTRA_OPTS} # General @@ -35,28 +36,28 @@ install-ci-dependencies: install-python-ci-dependencies install-go-ci-dependenci # Java install-java-ci-dependencies: - mvn verify clean --fail-never + ${MVN} verify clean --fail-never format-java: - mvn spotless:apply + ${MVN} spotless:apply lint-java: - mvn --no-transfer-progress spotless:check + ${MVN} --no-transfer-progress spotless:check test-java: - mvn --no-transfer-progress test + ${MVN} --no-transfer-progress -DskipITs=true test test-java-integration: - mvn --no-transfer-progress -Dmaven.javadoc.skip=true -Dgpg.skip -DskipUTs=true clean verify + ${MVN} --no-transfer-progress -Dmaven.javadoc.skip=true -Dgpg.skip -DskipUTs=true clean verify test-java-with-coverage: - mvn --no-transfer-progress test jacoco:report-aggregate + ${MVN} --no-transfer-progress -DskipITs=true test jacoco:report-aggregate build-java: - mvn clean verify + ${MVN} clean verify build-java-no-tests: - mvn --no-transfer-progress -Dmaven.javadoc.skip=true -Dgpg.skip -DskipUTs=true -Drevision=${REVISION} clean package + ${MVN} --no-transfer-progress -Dmaven.javadoc.skip=true -Dgpg.skip -DskipUTs=true -DskipITs=true -Drevision=${REVISION} clean package # Python SDK @@ -141,13 +142,13 @@ push-jupyter-docker: docker push $(REGISTRY)/feast-jupyter:$(VERSION) build-core-docker: - docker build -t $(REGISTRY)/feast-core:$(VERSION) -f infra/docker/core/Dockerfile . + docker build --build-arg VERSION=$(VERSION) -t $(REGISTRY)/feast-core:$(VERSION) -f infra/docker/core/Dockerfile . build-jobservice-docker: docker build -t $(REGISTRY)/feast-jobservice:$(VERSION) -f infra/docker/jobservice/Dockerfile . build-serving-docker: - docker build -t $(REGISTRY)/feast-serving:$(VERSION) -f infra/docker/serving/Dockerfile . + docker build --build-arg VERSION=$(VERSION) -t $(REGISTRY)/feast-serving:$(VERSION) -f infra/docker/serving/Dockerfile . build-ci-docker: docker build -t $(REGISTRY)/feast-ci:$(VERSION) -f infra/docker/ci/Dockerfile . @@ -155,6 +156,9 @@ build-ci-docker: build-jupyter-docker: docker build -t $(REGISTRY)/feast-jupyter:$(VERSION) -f infra/docker/jupyter/Dockerfile . +build-local-test-docker: + docker build -t feast:local -f infra/docker/tests/Dockerfile . + # Documentation install-dependencies-proto-docs: diff --git a/README.md b/README.md index 684fde364da..08ce6dec08a 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,7 @@ ## Overview -Feast (Feature Store) is an operational data system for managing and serving machine learning features to models in production. Please see our [documentation](https://docs.feast.dev/) for the motivation behind the project. +Feast (Feature Store) is an operational data system for managing and serving machine learning features to models in production. Please see our [documentation](https://docs.feast.dev/) for more information about the project. ![](docs/.gitbook/assets/feast-architecture-diagrams.svg) @@ -21,7 +21,7 @@ Feast (Feature Store) is an operational data system for managing and serving mac Clone the latest stable version of the [Feast repository](https://github.com/feast-dev/feast/) and navigate to the `infra/docker-compose` sub-directory: ``` -git clone --depth 1 --branch v0.7.0 https://github.com/feast-dev/feast.git +git clone https://github.com/feast-dev/feast.git cd feast/infra/docker-compose cp .env.sample .env ``` @@ -30,16 +30,17 @@ The `.env` file can optionally be configured based on your environment. Bring up Feast: ``` -docker-compose up -d +docker-compose pull && docker-compose up -d ``` +Please wait for the containers to start up. This could take a few minutes since the quickstart contains demo infastructure like Kafka and Jupyter. -The command above will bring up a complete Feast deployment with a [Jupyter Notebook](http://localhost:8888/tree/feast/examples) containing example notebooks. +Once the containers are all running, please connect to the provided [Jupyter Notebook](http://localhost:8888/tree/minimal) containing example notebooks to try out. ## Important resources Please refer to the official documentation at - * [Concepts](https://docs.feast.dev/user-guide/overview) + * [Concepts](https://docs.feast.dev/concepts/overview) * [Installation](https://docs.feast.dev/getting-started) * [Examples](https://github.com/feast-dev/feast/blob/master/examples/) * [Roadmap](https://docs.feast.dev/roadmap) @@ -48,4 +49,67 @@ Please refer to the official documentation at ## Notice -Feast is a community project and is still under active development. Your feedback and contributions are important to us. Please have a look at our [contributing guide](docs/contributing/contributing.md) for details. +Feast is a community project and is still under active development. Your feedback and contributions are important to us. Please have a look at our [contributing guide](https://docs.feast.dev/contributing/contributing) for details. + +## Contributors ✨ + +Thanks goes to these incredible people: + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
diff --git a/common-test/pom.xml b/common-test/pom.xml index 563129cf063..62fd6ae00f2 100644 --- a/common-test/pom.xml +++ b/common-test/pom.xml @@ -106,17 +106,17 @@ org.testcontainers junit-jupiter - 1.14.3 + 1.15.0 org.testcontainers postgresql - 1.14.3 + 1.15.0 org.testcontainers kafka - 1.14.3 + 1.15.0 org.junit.jupiter diff --git a/common-test/src/main/java/feast/common/it/DataGenerator.java b/common-test/src/main/java/feast/common/it/DataGenerator.java index 9fe46bb3ba3..0606c759516 100644 --- a/common-test/src/main/java/feast/common/it/DataGenerator.java +++ b/common-test/src/main/java/feast/common/it/DataGenerator.java @@ -26,21 +26,16 @@ import feast.proto.core.DataFormatProto.StreamFormat.AvroFormat; import feast.proto.core.DataFormatProto.StreamFormat.ProtoFormat; import feast.proto.core.DataSourceProto.DataSource; -import feast.proto.core.DataSourceProto.DataSource.BigQueryOptions; import feast.proto.core.DataSourceProto.DataSource.FileOptions; import feast.proto.core.DataSourceProto.DataSource.KafkaOptions; import feast.proto.core.DataSourceProto.DataSource.KinesisOptions; import feast.proto.core.EntityProto; import feast.proto.core.FeatureProto; import feast.proto.core.FeatureProto.FeatureSpecV2; -import feast.proto.core.FeatureSetProto; import feast.proto.core.FeatureTableProto.FeatureTableSpec; -import feast.proto.core.SourceProto; import feast.proto.core.StoreProto; import feast.proto.serving.ServingAPIProto; import feast.proto.types.ValueProto; -import java.util.Collections; -import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.stream.Collectors; @@ -54,8 +49,6 @@ public class DataGenerator { createStore( "test-store", StoreProto.Store.StoreType.REDIS, ImmutableList.of(defaultSubscription)); - static SourceProto.Source defaultSource = createSource("localhost", "topic"); - public static Triple getDefaultSubscription() { return defaultSubscription; } @@ -64,25 +57,6 @@ public static StoreProto.Store getDefaultStore() { return defaultStore; } - public static SourceProto.Source getDefaultSource() { - return defaultSource; - } - - public static FeatureSetProto.FeatureSet getDefaultFeatureSet() { - return createFeatureSet(DataGenerator.getDefaultSource(), "default", "test"); - } - - public static SourceProto.Source createSource(String server, String topic) { - return SourceProto.Source.newBuilder() - .setType(SourceProto.SourceType.KAFKA) - .setKafkaSourceConfig( - SourceProto.KafkaSourceConfig.newBuilder() - .setBootstrapServers(server) - .setTopic(topic) - .build()) - .build(); - } - public static StoreProto.Store createStore( String name, StoreProto.Store.StoreType type, @@ -107,10 +81,6 @@ public static StoreProto.Store createStore( StoreProto.Store.RedisConfig redisConfig = StoreProto.Store.RedisConfig.newBuilder().build(); return builder.setRedisConfig(redisConfig).build(); - case BIGQUERY: - StoreProto.Store.BigQueryConfig bqConfig = - StoreProto.Store.BigQueryConfig.newBuilder().build(); - return builder.setBigqueryConfig(bqConfig).build(); case REDIS_CLUSTER: StoreProto.Store.RedisClusterConfig redisClusterConfig = StoreProto.Store.RedisClusterConfig.newBuilder().build(); @@ -120,20 +90,6 @@ public static StoreProto.Store createStore( } } - public static FeatureSetProto.FeatureSpec createFeature( - String name, ValueProto.ValueType.Enum valueType, Map labels) { - return FeatureSetProto.FeatureSpec.newBuilder() - .setName(name) - .setValueType(valueType) - .putAllLabels(labels) - .build(); - } - - public static FeatureSetProto.EntitySpec createEntitySpec( - String name, ValueProto.ValueType.Enum valueType) { - return FeatureSetProto.EntitySpec.newBuilder().setName(name).setValueType(valueType).build(); - } - public static EntityProto.EntitySpecV2 createEntitySpecV2( String name, String description, @@ -156,70 +112,6 @@ public static FeatureProto.FeatureSpecV2 createFeatureSpecV2( .build(); } - public static FeatureSetProto.FeatureSet createFeatureSet( - SourceProto.Source source, - String projectName, - String name, - List entities, - List features, - Map labels) { - return FeatureSetProto.FeatureSet.newBuilder() - .setSpec( - FeatureSetProto.FeatureSetSpec.newBuilder() - .setSource(source) - .setName(name) - .setProject(projectName) - .putAllLabels(labels) - .addAllEntities(entities) - .addAllFeatures(features) - .build()) - .build(); - } - - public static FeatureSetProto.FeatureSet createFeatureSet( - SourceProto.Source source, - String projectName, - String name, - Map entities, - Map features, - Map labels) { - return FeatureSetProto.FeatureSet.newBuilder() - .setSpec( - FeatureSetProto.FeatureSetSpec.newBuilder() - .setSource(source) - .setName(name) - .setProject(projectName) - .putAllLabels(labels) - .addAllEntities( - entities.entrySet().stream() - .map(entry -> createEntitySpec(entry.getKey(), entry.getValue())) - .collect(Collectors.toList())) - .addAllFeatures( - features.entrySet().stream() - .map( - entry -> - createFeature( - entry.getKey(), entry.getValue(), Collections.emptyMap())) - .collect(Collectors.toList())) - .build()) - .build(); - } - - public static FeatureSetProto.FeatureSet createFeatureSet( - SourceProto.Source source, - String projectName, - String name, - Map entities, - Map features) { - return createFeatureSet(source, projectName, name, entities, features, new HashMap<>()); - } - - public static FeatureSetProto.FeatureSet createFeatureSet( - SourceProto.Source source, String projectName, String name) { - return createFeatureSet( - source, projectName, name, Collections.emptyMap(), Collections.emptyMap()); - } - // Create a Feature Table spec without DataSources configured. public static FeatureTableSpec createFeatureTableSpec( String name, @@ -242,6 +134,19 @@ public static FeatureTableSpec createFeatureTableSpec( .build()) .collect(Collectors.toList())) .setMaxAge(Duration.newBuilder().setSeconds(3600).build()) + .setBatchSource( + DataSource.newBuilder() + .setEventTimestampColumn("ts") + .setType(DataSource.SourceType.BATCH_FILE) + .setFileOptions( + FileOptions.newBuilder() + .setFileFormat( + FileFormat.newBuilder() + .setParquetFormat(ParquetFormat.newBuilder().build()) + .build()) + .setFileUrl("/dev/null") + .build()) + .build()) .putAllLabels(labels) .build(); } @@ -289,7 +194,8 @@ public static DataSource createBigQueryDataSourceSpec( String bigQueryTableRef, String timestampColumn, String datePartitionColumn) { return DataSource.newBuilder() .setType(DataSource.SourceType.BATCH_BIGQUERY) - .setBigqueryOptions(BigQueryOptions.newBuilder().setTableRef(bigQueryTableRef).build()) + .setBigqueryOptions( + DataSource.BigQueryOptions.newBuilder().setTableRef(bigQueryTableRef).build()) .setEventTimestampColumn(timestampColumn) .setDatePartitionColumn(datePartitionColumn) .build(); @@ -313,6 +219,10 @@ public static ValueProto.Value createEmptyValue() { return ValueProto.Value.newBuilder().build(); } + public static ValueProto.Value createStrValue(String val) { + return ValueProto.Value.newBuilder().setStringVal(val).build(); + } + public static ValueProto.Value createDoubleValue(double value) { return ValueProto.Value.newBuilder().setDoubleVal(value).build(); } diff --git a/common-test/src/main/java/feast/common/it/SimpleCoreClient.java b/common-test/src/main/java/feast/common/it/SimpleCoreClient.java index cffb4f42afa..11fa6715dc8 100644 --- a/common-test/src/main/java/feast/common/it/SimpleCoreClient.java +++ b/common-test/src/main/java/feast/common/it/SimpleCoreClient.java @@ -31,12 +31,6 @@ public SimpleCoreClient(CoreServiceGrpc.CoreServiceBlockingStub stub) { this.stub = stub; } - public CoreServiceProto.ApplyFeatureSetResponse simpleApplyFeatureSet( - FeatureSetProto.FeatureSet featureSet) { - return stub.applyFeatureSet( - CoreServiceProto.ApplyFeatureSetRequest.newBuilder().setFeatureSet(featureSet).build()); - } - public CoreServiceProto.ApplyEntityResponse simpleApplyEntity( String projectName, EntityProto.EntitySpecV2 spec) { return stub.applyEntity( @@ -84,53 +78,6 @@ public List simpleListFeatureTables( .getTablesList(); } - public List simpleListFeatureSets( - String projectName, String featureSetName, Map labels) { - return stub.listFeatureSets( - CoreServiceProto.ListFeatureSetsRequest.newBuilder() - .setFilter( - CoreServiceProto.ListFeatureSetsRequest.Filter.newBuilder() - .setProject(projectName) - .setFeatureSetName(featureSetName) - .putAllLabels(labels) - .build()) - .build()) - .getFeatureSetsList(); - } - - public List simpleListFeatureSets( - String projectName, String featureSetName, FeatureSetProto.FeatureSetStatus status) { - return stub.listFeatureSets( - CoreServiceProto.ListFeatureSetsRequest.newBuilder() - .setFilter( - CoreServiceProto.ListFeatureSetsRequest.Filter.newBuilder() - .setProject(projectName) - .setFeatureSetName(featureSetName) - .setStatus(status) - .build()) - .build()) - .getFeatureSetsList(); - } - - public List simpleListFeatureSets( - String projectName, String featureSetName) { - return simpleListFeatureSets( - projectName, featureSetName, FeatureSetProto.FeatureSetStatus.STATUS_INVALID); - } - - public List simpleListFeatureSets(String featureSetName) { - return simpleListFeatureSets("default", featureSetName); - } - - public FeatureSetProto.FeatureSet simpleGetFeatureSet(String projectName, String name) { - return stub.getFeatureSet( - CoreServiceProto.GetFeatureSetRequest.newBuilder() - .setName(name) - .setProject(projectName) - .build()) - .getFeatureSet(); - } - public EntityProto.Entity simpleGetEntity(String projectName, String name) { return stub.getEntity( CoreServiceProto.GetEntityRequest.newBuilder() @@ -149,20 +96,7 @@ public FeatureTableProto.FeatureTable simpleGetFeatureTable(String projectName, .getTable(); } - public void updateFeatureSetStatus( - String projectName, String name, FeatureSetProto.FeatureSetStatus status) { - stub.updateFeatureSetStatus( - CoreServiceProto.UpdateFeatureSetStatusRequest.newBuilder() - .setReference( - FeatureSetReferenceProto.FeatureSetReference.newBuilder() - .setProject(projectName) - .setName(name) - .build()) - .setStatus(status) - .build()); - } - - public Map simpleListFeatures( + public Map simpleListFeatures( String projectName, Map labels, List entities) { return stub.listFeatures( CoreServiceProto.ListFeaturesRequest.newBuilder() @@ -176,7 +110,7 @@ public Map simpleListFeatures( .getFeaturesMap(); } - public Map simpleListFeatures( + public Map simpleListFeatures( String projectName, String... entities) { return simpleListFeatures(projectName, Collections.emptyMap(), Arrays.asList(entities)); } @@ -200,15 +134,6 @@ public String getFeastCoreVersion() { .getVersion(); } - public FeatureSetProto.FeatureSet getFeatureSet(String projectName, String featureSetName) { - return stub.getFeatureSet( - CoreServiceProto.GetFeatureSetRequest.newBuilder() - .setProject(projectName) - .setName(featureSetName) - .build()) - .getFeatureSet(); - } - public FeatureTableProto.FeatureTable applyFeatureTable( String projectName, FeatureTableSpec spec) { return stub.applyFeatureTable( diff --git a/common-test/src/main/java/feast/common/it/SimpleJcClient.java b/common-test/src/main/java/feast/common/it/SimpleJcClient.java deleted file mode 100644 index 247e3b01b6b..00000000000 --- a/common-test/src/main/java/feast/common/it/SimpleJcClient.java +++ /dev/null @@ -1,43 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.common.it; - -import feast.proto.core.CoreServiceProto; -import feast.proto.core.IngestionJobProto; -import feast.proto.core.JobControllerServiceGrpc; -import java.util.List; - -public class SimpleJcClient { - private final JobControllerServiceGrpc.JobControllerServiceBlockingStub stub; - - public SimpleJcClient(JobControllerServiceGrpc.JobControllerServiceBlockingStub stub) { - this.stub = stub; - } - - public void restartIngestionJob(String jobId) { - stub.restartIngestionJob( - CoreServiceProto.RestartIngestionJobRequest.newBuilder().setId(jobId).build()); - } - - public List listIngestionJobs() { - return stub.listIngestionJobs( - CoreServiceProto.ListIngestionJobsRequest.newBuilder() - .setFilter(CoreServiceProto.ListIngestionJobsRequest.Filter.newBuilder().build()) - .build()) - .getJobsList(); - } -} diff --git a/common-test/src/main/java/feast/common/util/TestUtil.java b/common-test/src/main/java/feast/common/util/TestUtil.java index 142c5e4850d..ee355d3766f 100644 --- a/common-test/src/main/java/feast/common/util/TestUtil.java +++ b/common-test/src/main/java/feast/common/util/TestUtil.java @@ -62,11 +62,12 @@ public static boolean compareFeatureTableSpec(FeatureTableSpec spec, FeatureTabl .toBuilder() .clearFeatures() .addAllFeatures( - spec.getFeaturesList().stream() + otherSpec.getFeaturesList().stream() .sorted(Comparator.comparing(FeatureSpecV2::getName)) .collect(Collectors.toSet())) .clearEntities() - .addAllEntities(spec.getEntitiesList().stream().sorted().collect(Collectors.toSet())) + .addAllEntities( + otherSpec.getEntitiesList().stream().sorted().collect(Collectors.toSet())) .build(); return spec.equals(otherSpec); diff --git a/common/src/main/java/feast/common/logging/entry/LogResource.java b/common/src/main/java/feast/common/logging/entry/LogResource.java index 02e7589f976..1d0345a4042 100644 --- a/common/src/main/java/feast/common/logging/entry/LogResource.java +++ b/common/src/main/java/feast/common/logging/entry/LogResource.java @@ -26,7 +26,7 @@ public abstract class LogResource { public enum ResourceType { JOB, - FEATURE_SET, + FEATURE_TABLE } public abstract ResourceType getType(); diff --git a/common/src/main/java/feast/common/models/Feature.java b/common/src/main/java/feast/common/models/Feature.java deleted file mode 100644 index 1d7fc43ba3c..00000000000 --- a/common/src/main/java/feast/common/models/Feature.java +++ /dev/null @@ -1,55 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.common.models; - -import feast.proto.serving.ServingAPIProto.FeatureReference; - -public class Feature { - - /** - * Accepts FeatureReference object and returns its reference in String - * "featureset_name:feature_name". - * - * @param featureReference {@link FeatureReference} - * @return String format of FeatureReference - */ - public static String getFeatureStringRef(FeatureReference featureReference) { - String ref = featureReference.getName(); - if (!featureReference.getFeatureSet().isEmpty()) { - ref = featureReference.getFeatureSet() + ":" + ref; - } - return ref; - } - - /** - * Accepts FeatureReference object and returns its reference with project included in String, eg. - * "project/featureset_name:feature_name". - * - * @param featureReference {@link FeatureReference} - * @return String format of FeatureReference - */ - public static String getFeatureStringWithProjectRef(FeatureReference featureReference) { - String ref = featureReference.getName(); - if (!featureReference.getFeatureSet().isEmpty()) { - ref = featureReference.getFeatureSet() + ":" + ref; - } - if (!featureReference.getProject().isEmpty()) { - ref = featureReference.getProject() + "/" + ref; - } - return ref; - } -} diff --git a/common/src/main/java/feast/common/models/FeatureSet.java b/common/src/main/java/feast/common/models/FeatureSet.java deleted file mode 100644 index f9db0f744f4..00000000000 --- a/common/src/main/java/feast/common/models/FeatureSet.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.common.models; - -import feast.proto.core.FeatureSetProto.FeatureSetSpec; -import feast.proto.core.FeatureSetReferenceProto.FeatureSetReference; - -public class FeatureSet { - - /** - * Accepts FeatureSetSpec object and returns its reference in String "project/featureset_name". - * - * @param featureSetSpec {@link FeatureSetSpec} - * @return String format of FeatureSetReference - */ - public static String getFeatureSetStringRef(FeatureSetSpec featureSetSpec) { - return String.format("%s/%s", featureSetSpec.getProject(), featureSetSpec.getName()); - } - - /** - * Accepts FeatureSetReference object and returns its reference in String - * "project/featureset_name". - * - * @param featureSetReference {@link FeatureSetReference} - * @return String format of FeatureSetReference - */ - public static String getFeatureSetStringRef(FeatureSetReference featureSetReference) { - return String.format("%s/%s", featureSetReference.getProject(), featureSetReference.getName()); - } -} diff --git a/common/src/main/java/feast/common/models/FeatureSetReference.java b/common/src/main/java/feast/common/models/FeatureSetReference.java deleted file mode 100644 index ea01bf5cf73..00000000000 --- a/common/src/main/java/feast/common/models/FeatureSetReference.java +++ /dev/null @@ -1,75 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.common.models; - -import java.io.Serializable; -import lombok.AllArgsConstructor; -import lombok.Data; - -/** - * FeatureSetReference is key that uniquely defines specific version of FeatureSet or FeatureSetSpec - */ -@Data -@AllArgsConstructor -public class FeatureSetReference implements Serializable { - public static String PROJECT_DEFAULT_NAME = "default"; - - /* Name of project to which this featureSet is assigned */ - private String projectName; - /* Name of FeatureSet */ - private String featureSetName; - /* Version of FeatureSet */ - private Integer version; - - // Empty constructor required for Avro decoding. - @SuppressWarnings("unused") - public FeatureSetReference() {} - - public static FeatureSetReference of(String projectName, String featureSetName, Integer version) { - projectName = projectName.isEmpty() ? PROJECT_DEFAULT_NAME : projectName; - return new FeatureSetReference(projectName, featureSetName, version); - } - - public static FeatureSetReference of(String projectName, String featureSetName) { - return FeatureSetReference.of(projectName, featureSetName, -1); - } - - /** - * Parse string representation of FeatureSetReference that expected to have format - * <ProjectName>/<FeatureSetName>. If project's not given - default will be used. - * - * @param reference string representation - * @return construct {@link FeatureSetReference} - */ - public static FeatureSetReference parse(String reference) { - String[] split = reference.split("/", 2); - if (split.length == 1) { - return FeatureSetReference.of(PROJECT_DEFAULT_NAME, split[0]); - } - - if (split.length > 2) { - throw new RuntimeException( - "FeatureSet reference must have the format /"); - } - - return FeatureSetReference.of(split[0], split[1]); - } - - public String getReference() { - return String.format("%s/%s", getProjectName(), getFeatureSetName()); - } -} diff --git a/common/src/test/java/feast/common/logging/entry/AuditLogEntryTest.java b/common/src/test/java/feast/common/logging/entry/AuditLogEntryTest.java index a332e0be799..cf355e09e4b 100644 --- a/common/src/test/java/feast/common/logging/entry/AuditLogEntryTest.java +++ b/common/src/test/java/feast/common/logging/entry/AuditLogEntryTest.java @@ -22,8 +22,8 @@ import com.google.gson.JsonObject; import com.google.gson.JsonParser; import feast.common.logging.entry.LogResource.ResourceType; -import feast.proto.serving.ServingAPIProto.FeatureReference; -import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesRequest; +import feast.proto.serving.ServingAPIProto.FeatureReferenceV2; +import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesRequestV2; import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesResponse; import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesResponse.FieldValues; import feast.proto.types.ValueProto.Value; @@ -34,13 +34,18 @@ public class AuditLogEntryTest { public List getTestAuditLogs() { - GetOnlineFeaturesRequest requestSpec = - GetOnlineFeaturesRequest.newBuilder() - .setOmitEntitiesInResponse(false) + GetOnlineFeaturesRequestV2 requestSpec = + GetOnlineFeaturesRequestV2.newBuilder() .addAllFeatures( Arrays.asList( - FeatureReference.newBuilder().setName("feature1").build(), - FeatureReference.newBuilder().setName("feature2").build())) + FeatureReferenceV2.newBuilder() + .setFeatureTable("featuretable_1") + .setName("feature1") + .build(), + FeatureReferenceV2.newBuilder() + .setFeatureTable("featuretable_1") + .setName("feature2") + .build())) .build(); GetOnlineFeaturesResponse responseSpec = @@ -48,17 +53,19 @@ public List getTestAuditLogs() { .addAllFieldValues( Arrays.asList( FieldValues.newBuilder() - .putFields("feature", Value.newBuilder().setInt32Val(32).build()) + .putFields( + "featuretable_1:feature_1", Value.newBuilder().setInt32Val(32).build()) .build(), FieldValues.newBuilder() - .putFields("feature2", Value.newBuilder().setInt32Val(64).build()) + .putFields( + "featuretable_1:feature2", Value.newBuilder().setInt32Val(64).build()) .build())) .build(); return Arrays.asList( MessageAuditLogEntry.newBuilder() .setComponent("feast-serving") - .setVersion("0.6") + .setVersion("0.9") .setService("ServingService") .setMethod("getOnlineFeatures") .setRequest(requestSpec) @@ -67,12 +74,9 @@ public List getTestAuditLogs() { .setIdentity("adam@no.such.email") .build(), ActionAuditLogEntry.of( - "core", "0.6", LogResource.of(ResourceType.JOB, "kafka-to-redis"), "CREATE"), + "core", "0.9", LogResource.of(ResourceType.JOB, "kafka-to-redis"), "CREATE"), TransitionAuditLogEntry.of( - "core", - "0.6", - LogResource.of(ResourceType.FEATURE_SET, "project/feature_set"), - "READY")); + "core", "0.9", LogResource.of(ResourceType.FEATURE_TABLE, "featuretable_1"), "READY")); } @Test diff --git a/common/src/test/java/feast/common/models/FeatureSetTest.java b/common/src/test/java/feast/common/models/FeatureSetTest.java deleted file mode 100644 index 52b7dd36ade..00000000000 --- a/common/src/test/java/feast/common/models/FeatureSetTest.java +++ /dev/null @@ -1,97 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.common.models; - -import static org.hamcrest.MatcherAssert.assertThat; -import static org.hamcrest.core.IsEqual.equalTo; - -import feast.proto.core.FeatureSetProto.EntitySpec; -import feast.proto.core.FeatureSetProto.FeatureSetSpec; -import feast.proto.core.FeatureSetProto.FeatureSpec; -import feast.proto.core.FeatureSetReferenceProto; -import feast.proto.types.ValueProto; -import java.util.Arrays; -import java.util.List; -import org.junit.Before; -import org.junit.Test; -import org.tensorflow.metadata.v0.*; - -public class FeatureSetTest { - - private List entitySpecs; - private List featureSpecs; - - @Before - public void setUp() { - // Entity Specs - EntitySpec entitySpec1 = - EntitySpec.newBuilder() - .setName("entity1") - .setValueType(ValueProto.ValueType.Enum.INT64) - .build(); - EntitySpec entitySpec2 = - EntitySpec.newBuilder() - .setName("entity2") - .setValueType(ValueProto.ValueType.Enum.INT64) - .build(); - - // Feature Specs - FeatureSpec featureSpec1 = - FeatureSpec.newBuilder() - .setName("feature1") - .setValueType(ValueProto.ValueType.Enum.INT64) - .setPresence(FeaturePresence.getDefaultInstance()) - .setShape(FixedShape.getDefaultInstance()) - .setDomain("mydomain") - .build(); - FeatureSpec featureSpec2 = - FeatureSpec.newBuilder() - .setName("feature2") - .setValueType(ValueProto.ValueType.Enum.INT64) - .setGroupPresence(FeaturePresenceWithinGroup.getDefaultInstance()) - .setValueCount(ValueCount.getDefaultInstance()) - .setIntDomain(IntDomain.getDefaultInstance()) - .build(); - - entitySpecs = Arrays.asList(entitySpec1, entitySpec2); - featureSpecs = Arrays.asList(featureSpec1, featureSpec2); - } - - @Test - public void shouldReturnFeatureSetStringRef() { - FeatureSetSpec featureSetSpec = - FeatureSetSpec.newBuilder() - .setProject("project1") - .setName("featureSetWithConstraints") - .addAllEntities(entitySpecs) - .addAllFeatures(featureSpecs) - .build(); - - FeatureSetReferenceProto.FeatureSetReference featureSetReference = - FeatureSetReferenceProto.FeatureSetReference.newBuilder() - .setName(featureSetSpec.getName()) - .setProject(featureSetSpec.getProject()) - .build(); - - String actualFeatureSetStringRef1 = FeatureSet.getFeatureSetStringRef(featureSetSpec); - String actualFeatureSetStringRef2 = FeatureSet.getFeatureSetStringRef(featureSetReference); - String expectedFeatureSetStringRef = "project1/featureSetWithConstraints"; - - assertThat(actualFeatureSetStringRef1, equalTo(expectedFeatureSetStringRef)); - assertThat(actualFeatureSetStringRef2, equalTo(expectedFeatureSetStringRef)); - } -} diff --git a/common/src/test/java/feast/common/models/FeaturesTest.java b/common/src/test/java/feast/common/models/FeaturesTest.java index a4426ad21c0..180f7e4e697 100644 --- a/common/src/test/java/feast/common/models/FeaturesTest.java +++ b/common/src/test/java/feast/common/models/FeaturesTest.java @@ -19,81 +19,28 @@ import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.core.IsEqual.equalTo; -import feast.proto.core.FeatureSetProto.EntitySpec; -import feast.proto.core.FeatureSetProto.FeatureSetSpec; -import feast.proto.core.FeatureSetProto.FeatureSpec; -import feast.proto.serving.ServingAPIProto; -import feast.proto.types.ValueProto; -import java.util.Arrays; -import java.util.List; +import feast.proto.serving.ServingAPIProto.FeatureReferenceV2; import org.junit.Before; import org.junit.Test; -import org.tensorflow.metadata.v0.*; public class FeaturesTest { - private List entitySpecs; - private List featureSpecs; + private FeatureReferenceV2 featureReference; @Before public void setUp() { - // Entity Specs - EntitySpec entitySpec1 = - EntitySpec.newBuilder() - .setName("entity1") - .setValueType(ValueProto.ValueType.Enum.INT64) - .build(); - EntitySpec entitySpec2 = - EntitySpec.newBuilder() - .setName("entity2") - .setValueType(ValueProto.ValueType.Enum.INT64) - .build(); - - // Feature Specs - FeatureSpec featureSpec1 = - FeatureSpec.newBuilder() + featureReference = + FeatureReferenceV2.newBuilder() + .setFeatureTable("featuretable_1") .setName("feature1") - .setValueType(ValueProto.ValueType.Enum.INT64) - .setPresence(FeaturePresence.getDefaultInstance()) - .setShape(FixedShape.getDefaultInstance()) - .setDomain("mydomain") .build(); - FeatureSpec featureSpec2 = - FeatureSpec.newBuilder() - .setName("feature2") - .setValueType(ValueProto.ValueType.Enum.INT64) - .setGroupPresence(FeaturePresenceWithinGroup.getDefaultInstance()) - .setValueCount(ValueCount.getDefaultInstance()) - .setIntDomain(IntDomain.getDefaultInstance()) - .build(); - - entitySpecs = Arrays.asList(entitySpec1, entitySpec2); - featureSpecs = Arrays.asList(featureSpec1, featureSpec2); } @Test public void shouldReturnFeatureStringRef() { - FeatureSetSpec featureSetSpec = - FeatureSetSpec.newBuilder() - .setProject("project1") - .setName("featureSetWithConstraints") - .addAllEntities(entitySpecs) - .addAllFeatures(featureSpecs) - .build(); - - ServingAPIProto.FeatureReference featureReference = - ServingAPIProto.FeatureReference.newBuilder() - .setProject(featureSetSpec.getProject()) - .setFeatureSet(featureSetSpec.getName()) - .setName(featureSetSpec.getFeatures(0).getName()) - .build(); - - String actualFeatureStringRef = Feature.getFeatureStringWithProjectRef(featureReference); - String actualFeatureIgnoreProjectStringRef = Feature.getFeatureStringRef(featureReference); - String expectedFeatureStringRef = "project1/featureSetWithConstraints:feature1"; - String expectedFeatureIgnoreProjectStringRef = "featureSetWithConstraints:feature1"; + String actualFeatureStringRef = FeatureV2.getFeatureStringRef(featureReference); + String expectedFeatureStringRef = "featuretable_1:feature1"; assertThat(actualFeatureStringRef, equalTo(expectedFeatureStringRef)); - assertThat(actualFeatureIgnoreProjectStringRef, equalTo(expectedFeatureIgnoreProjectStringRef)); } } diff --git a/core/pom.xml b/core/pom.xml index 97d31268b36..7a34b794db2 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -80,12 +80,6 @@ ${project.version} test - - dev.feast - feast-storage-connector-bigquery - ${project.version} - compile - @@ -312,5 +306,21 @@ 2.27.0 test + + org.apache.avro + avro + 1.8.2 + test + + + com.squareup.okhttp + okhttp + 2.7.4 + test + + + io.grpc + grpc-testing + diff --git a/core/src/main/java/feast/core/config/FeatureStreamConfig.java b/core/src/main/java/feast/core/config/FeatureStreamConfig.java deleted file mode 100644 index cc5e707964f..00000000000 --- a/core/src/main/java/feast/core/config/FeatureStreamConfig.java +++ /dev/null @@ -1,58 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2019 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.core.config; - -import feast.core.config.FeastProperties.StreamProperties; -import feast.core.model.Source; -import feast.proto.core.SourceProto; -import feast.proto.core.SourceProto.KafkaSourceConfig; -import feast.proto.core.SourceProto.SourceType; -import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; - -@Slf4j -@Configuration -public class FeatureStreamConfig { - - @Autowired - @Bean - public Source getDefaultSource(FeastProperties feastProperties) { - StreamProperties streamProperties = feastProperties.getStream(); - SourceType featureStreamType = SourceType.valueOf(streamProperties.getType().toUpperCase()); - switch (featureStreamType) { - case KAFKA: - String bootstrapServers = streamProperties.getOptions().getBootstrapServers(); - String topicName = streamProperties.getOptions().getTopic(); - - KafkaSourceConfig sourceConfig = - KafkaSourceConfig.newBuilder() - .setBootstrapServers(bootstrapServers) - .setTopic(topicName) - .build(); - SourceProto.Source source = - SourceProto.Source.newBuilder() - .setType(featureStreamType) - .setKafkaSourceConfig(sourceConfig) - .build(); - return Source.fromProto(source, true); - default: - throw new RuntimeException("Unsupported source stream, only [KAFKA] is supported"); - } - } -} diff --git a/core/src/main/java/feast/core/config/MonitoringConfig.java b/core/src/main/java/feast/core/config/MonitoringConfig.java index 53c9562c47c..5fc6b8280e5 100644 --- a/core/src/main/java/feast/core/config/MonitoringConfig.java +++ b/core/src/main/java/feast/core/config/MonitoringConfig.java @@ -16,7 +16,7 @@ */ package feast.core.config; -import feast.core.dao.FeatureSetRepository; +import feast.core.dao.FeatureTableRepository; import feast.core.dao.StoreRepository; import feast.core.metrics.collector.FeastResourceCollector; import feast.core.metrics.collector.JVMResourceCollector; @@ -47,18 +47,18 @@ public ServletRegistrationBean metricsServlet() { /** * Register custom Prometheus collector that exports metrics about Feast Resources. * - *

For example: total number of registered feature sets and stores. + *

For example: total number of registered feature tables and stores. * - * @param featureSetRepository {@link FeatureSetRepository} + * @param featureTableRepository {@link FeatureTableRepository} * @param storeRepository {@link StoreRepository} * @return {@link FeastResourceCollector} */ @Bean @Autowired public FeastResourceCollector feastResourceCollector( - FeatureSetRepository featureSetRepository, StoreRepository storeRepository) { + FeatureTableRepository featureTableRepository, StoreRepository storeRepository) { FeastResourceCollector collector = - new FeastResourceCollector(featureSetRepository, storeRepository); + new FeastResourceCollector(featureTableRepository, storeRepository); collector.register(); return collector; } diff --git a/core/src/main/java/feast/core/controller/CoreServiceRestController.java b/core/src/main/java/feast/core/controller/CoreServiceRestController.java index 3e3224220a9..f782c172cd6 100644 --- a/core/src/main/java/feast/core/controller/CoreServiceRestController.java +++ b/core/src/main/java/feast/core/controller/CoreServiceRestController.java @@ -16,31 +16,18 @@ */ package feast.core.controller; -import com.google.protobuf.InvalidProtocolBufferException; -import com.google.protobuf.Timestamp; import feast.core.config.FeastProperties; import feast.core.model.Project; import feast.core.service.ProjectService; import feast.core.service.SpecService; -import feast.core.service.StatsService; import feast.proto.core.CoreServiceProto.GetFeastCoreVersionResponse; -import feast.proto.core.CoreServiceProto.GetFeatureStatisticsRequest; -import feast.proto.core.CoreServiceProto.GetFeatureStatisticsRequest.Builder; -import feast.proto.core.CoreServiceProto.GetFeatureStatisticsResponse; import feast.proto.core.CoreServiceProto.ListEntitiesRequest; import feast.proto.core.CoreServiceProto.ListEntitiesResponse; -import feast.proto.core.CoreServiceProto.ListFeatureSetsRequest; -import feast.proto.core.CoreServiceProto.ListFeatureSetsResponse; import feast.proto.core.CoreServiceProto.ListFeatureTablesRequest; import feast.proto.core.CoreServiceProto.ListFeatureTablesResponse; import feast.proto.core.CoreServiceProto.ListFeaturesRequest; import feast.proto.core.CoreServiceProto.ListFeaturesResponse; import feast.proto.core.CoreServiceProto.ListProjectsResponse; -import java.io.IOException; -import java.time.LocalDate; -import java.time.LocalTime; -import java.time.ZoneOffset; -import java.time.format.DateTimeFormatter; import java.util.Arrays; import java.util.List; import java.util.Optional; @@ -64,18 +51,13 @@ public class CoreServiceRestController { private final FeastProperties feastProperties; private SpecService specService; - private StatsService statsService; private ProjectService projectService; @Autowired public CoreServiceRestController( - FeastProperties feastProperties, - SpecService specService, - StatsService statsService, - ProjectService projectService) { + FeastProperties feastProperties, SpecService specService, ProjectService projectService) { this.feastProperties = feastProperties; this.specService = specService; - this.statsService = statsService; this.projectService = projectService; } @@ -84,34 +66,13 @@ public CoreServiceRestController( * * @return (200 OK) Returns {@link GetFeastCoreVersionResponse} in JSON. */ - @RequestMapping(value = "/v1/version", method = RequestMethod.GET) + @RequestMapping(value = "/v2/version", method = RequestMethod.GET) public GetFeastCoreVersionResponse getVersion() { GetFeastCoreVersionResponse response = GetFeastCoreVersionResponse.newBuilder().setVersion(feastProperties.getVersion()).build(); return response; } - /** - * GET /feature-sets : Retrieve a list of Feature Sets according to filtering parameters of Feast - * project name and feature set name. If none matches, an empty JSON response is returned. - * - * @param project Request Parameter: Name of feast project to search in. If set to "*" - * , all existing projects will be filtered. However, asterisk can NOT be - * combined with other strings (for example "merchant_*") to use as wildcard to - * filter feature sets. - * @param name Request Parameter: Feature set name. If set to "*", filter * all feature sets by - * default. Asterisk can be used as wildcard to filter * feature sets. - * @return (200 OK) Return {@link ListFeatureSetsResponse} in JSON. - */ - @RequestMapping(value = "/v1/feature-sets", method = RequestMethod.GET) - public ListFeatureSetsResponse listFeatureSets( - @RequestParam(defaultValue = Project.DEFAULT_NAME) String project, @RequestParam String name) - throws InvalidProtocolBufferException { - ListFeatureSetsRequest.Filter.Builder filterBuilder = - ListFeatureSetsRequest.Filter.newBuilder().setProject(project).setFeatureSetName(name); - return specService.listFeatureSets(filterBuilder.build()); - } - /** * GET /features : List Features based on project and entities. * @@ -119,12 +80,12 @@ public ListFeatureSetsResponse listFeatureSets( * to. At least one entity is required. For example, if entity1 and entity2 * are given, then all features returned (if any) will belong to BOTH * entities. - * @param project (Optional) Request Parameter: A single project where the feature set of all + * @param project (Optional) Request Parameter: A single project where the feature table of all * features returned is under. If not provided, the default project will be used, usually * default. * @return (200 OK) Return {@link ListFeaturesResponse} in JSON. */ - @RequestMapping(value = "/v1/features", method = RequestMethod.GET) + @RequestMapping(value = "/v2/features", method = RequestMethod.GET) public ListFeaturesResponse listFeatures( @RequestParam String[] entities, @RequestParam(required = false) Optional project) { ListFeaturesRequest.Filter.Builder filterBuilder = @@ -133,64 +94,12 @@ public ListFeaturesResponse listFeatures( return specService.listFeatures(filterBuilder.build()); } - /** - * GET /feature-statistics : Fetches statistics for a dataset speficied by the parameters. Either - * both (start_date, end_date) need to be given or ingestion_ids are required. If both are given, - * (start_date, end_date) will be ignored. - * - * @param ingestionIds Request Parameter: List of ingestion IDs. If missing, both startDate and - * endDate should be provided. - * @param startDate Request Parameter: UTC+0 starting date (inclusive) in the ISO format, from - * 0001-01-01 to 9999-12-31. Time given will be ignored. This - * parameter will be ignored if any ingestionIds is provided. - * @param endDate Request Parameter: UTC+0 ending date (exclusive) in the ISO format, from - * 0001-01-01 to 9999-12-31. Time given will be ignored. This parameter - * will be ignored if any ingestionIds is provided. - * @param store Request Parameter: The name of the historical store used in Feast Serving. Online - * store is not allowed. - * @param featureSetId Request Parameter: Feature set ID, which has the form of - * project/feature_set_name. - * @param forceRefresh Request Parameter: whether to override the values in the cache. Accepts - * true, false. - * @param features (Optional) Request Parameter: List of features. If none provided, all features - * in the feature set will be used for statistics. - * @return (200 OK) Returns {@link GetFeatureStatisticsResponse} in JSON. - */ - @RequestMapping(value = "/v1/feature-statistics", method = RequestMethod.GET) - public GetFeatureStatisticsResponse getFeatureStatistics( - @RequestParam(name = "feature_set_id") String featureSetId, - @RequestParam(required = false) Optional features, - @RequestParam String store, - @RequestParam(name = "start_date", required = false) Optional startDate, - @RequestParam(name = "end_date", required = false) Optional endDate, - @RequestParam(name = "ingestion_ids", required = false) Optional ingestionIds, - @RequestParam(name = "force_refresh") boolean forceRefresh) - throws IOException { - - Builder requestBuilder = - GetFeatureStatisticsRequest.newBuilder() - .setForceRefresh(forceRefresh) - .setFeatureSetId(featureSetId) - .setStore(store); - - // set optional request parameters if they are provided - features.ifPresent(theFeatures -> requestBuilder.addAllFeatures(Arrays.asList(theFeatures))); - startDate.ifPresent( - startDateStr -> requestBuilder.setStartDate(utcTimeStringToTimestamp(startDateStr))); - endDate.ifPresent( - endDateStr -> requestBuilder.setEndDate(utcTimeStringToTimestamp(endDateStr))); - ingestionIds.ifPresent( - theIngestionIds -> requestBuilder.addAllIngestionIds(Arrays.asList(theIngestionIds))); - - return statsService.getFeatureStatistics(requestBuilder.build()); - } - /** * GET /projects : Get the list of existing feast projects. * * @return (200 OK) Returns {@link ListProjectsResponse} in JSON. */ - @RequestMapping(value = "/v1/projects", method = RequestMethod.GET) + @RequestMapping(value = "/v2/projects", method = RequestMethod.GET) public ListProjectsResponse listProjects() { List projects = projectService.listProjects(); return ListProjectsResponse.newBuilder() @@ -227,11 +136,4 @@ public ListFeatureTablesResponse listFeatureTables( ListFeatureTablesRequest.Filter.newBuilder().setProject(project); return specService.listFeatureTables(filterBuilder.build()); } - - private Timestamp utcTimeStringToTimestamp(String utcTimeString) { - long epochSecond = - LocalDate.parse(utcTimeString, DateTimeFormatter.ISO_DATE) - .toEpochSecond(LocalTime.MIN, ZoneOffset.UTC); - return Timestamp.newBuilder().setSeconds(epochSecond).setNanos(0).build(); - } } diff --git a/core/src/main/java/feast/core/controller/exception/handler/RestResponseEntityExceptionHandler.java b/core/src/main/java/feast/core/controller/exception/handler/RestResponseEntityExceptionHandler.java index 53fe1bcd453..ef27e4aee8a 100644 --- a/core/src/main/java/feast/core/controller/exception/handler/RestResponseEntityExceptionHandler.java +++ b/core/src/main/java/feast/core/controller/exception/handler/RestResponseEntityExceptionHandler.java @@ -35,8 +35,8 @@ public class RestResponseEntityExceptionHandler extends ResponseEntityExceptionH /** * Handles the case when a request object (such as {@link - * feast.proto.core.CoreServiceProto.GetFeatureSetRequest}) or a response object (such as {@link - * feast.proto.core.CoreServiceProto.GetFeatureSetResponse} is malformed. + * feast.proto.core.CoreServiceProto.GetFeatureTableRequest}) or a response object (such as {@link + * feast.proto.core.CoreServiceProto.GetFeatureTableResponse} is malformed. * * @param ex the {@link InvalidProtocolBufferException} that occurred. * @param request the {@link WebRequest} that caused this exception. diff --git a/core/src/main/java/feast/core/dao/FeatureSetRepository.java b/core/src/main/java/feast/core/dao/FeatureSetRepository.java deleted file mode 100644 index 38a690b0d6c..00000000000 --- a/core/src/main/java/feast/core/dao/FeatureSetRepository.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2019 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.core.dao; - -import feast.core.model.FeatureSet; -import feast.proto.core.FeatureSetProto; -import java.util.List; -import org.springframework.data.jpa.repository.JpaRepository; - -/** JPA repository supplying FeatureSet objects keyed by id. */ -public interface FeatureSetRepository extends JpaRepository { - - long count(); - - // Find single feature set by project and name - FeatureSet findFeatureSetByNameAndProject_Name(String name, String project); - - // find all feature sets and order by name - List findAllByOrderByNameAsc(); - - // find all feature sets matching the given name pattern with a specific project. - List findAllByNameLikeAndProject_NameOrderByNameAsc(String name, String project_name); - - // find all feature sets matching the given name pattern and project pattern - List findAllByNameLikeAndProject_NameLikeOrderByNameAsc( - String name, String project_name); - - // find all feature sets matching given status - List findAllByStatus(FeatureSetProto.FeatureSetStatus status); -} diff --git a/core/src/main/java/feast/core/dao/FeatureStatisticsRepository.java b/core/src/main/java/feast/core/dao/FeatureStatisticsRepository.java deleted file mode 100644 index 4046295b75e..00000000000 --- a/core/src/main/java/feast/core/dao/FeatureStatisticsRepository.java +++ /dev/null @@ -1,31 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2019 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.core.dao; - -import feast.core.model.Feature; -import feast.core.model.FeatureStatistics; -import java.util.Date; -import java.util.Optional; -import org.springframework.data.jpa.repository.JpaRepository; - -/** JPA repository supplying Statistics objects for features keyed by id. */ -public interface FeatureStatisticsRepository extends JpaRepository { - Optional findFeatureStatisticsByFeatureAndDatasetId( - Feature feature, String datasetId); - - Optional findFeatureStatisticsByFeatureAndDate(Feature feature, Date date); -} diff --git a/core/src/main/java/feast/core/dao/SourceRepository.java b/core/src/main/java/feast/core/dao/SourceRepository.java deleted file mode 100644 index 1cf02dbcf82..00000000000 --- a/core/src/main/java/feast/core/dao/SourceRepository.java +++ /dev/null @@ -1,26 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2019 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.core.dao; - -import feast.core.model.Source; -import feast.proto.core.SourceProto.SourceType; -import org.springframework.data.jpa.repository.JpaRepository; - -/** JPA repository supplying Source objects keyed by id. */ -public interface SourceRepository extends JpaRepository { - Source findFirstByTypeAndConfigOrderByIdAsc(SourceType type, String config); -} diff --git a/core/src/main/java/feast/core/exception/TopicExistsException.java b/core/src/main/java/feast/core/exception/TopicExistsException.java deleted file mode 100644 index abd4937c712..00000000000 --- a/core/src/main/java/feast/core/exception/TopicExistsException.java +++ /dev/null @@ -1,32 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2019 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.core.exception; - -/** Exception thrown when creation of a topic in the stream fails because it already exists. */ -public class TopicExistsException extends RuntimeException { - public TopicExistsException() { - super(); - } - - public TopicExistsException(String message) { - super(message); - } - - public TopicExistsException(String message, Throwable cause) { - super(message, cause); - } -} diff --git a/core/src/main/java/feast/core/grpc/CoreServiceImpl.java b/core/src/main/java/feast/core/grpc/CoreServiceImpl.java index 5193bc63b24..efdf0fc778e 100644 --- a/core/src/main/java/feast/core/grpc/CoreServiceImpl.java +++ b/core/src/main/java/feast/core/grpc/CoreServiceImpl.java @@ -16,7 +16,6 @@ */ package feast.core.grpc; -import com.google.protobuf.InvalidProtocolBufferException; import feast.common.auth.service.AuthorizationService; import feast.common.logging.interceptors.GrpcMessageInterceptor; import feast.core.config.FeastProperties; @@ -25,11 +24,9 @@ import feast.core.model.Project; import feast.core.service.ProjectService; import feast.core.service.SpecService; -import feast.core.service.StatsService; import feast.proto.core.CoreServiceGrpc.CoreServiceImplBase; import feast.proto.core.CoreServiceProto.*; import feast.proto.core.EntityProto.EntitySpecV2; -import feast.proto.core.FeatureSetProto.FeatureSet; import io.grpc.Status; import io.grpc.StatusRuntimeException; import io.grpc.stub.StreamObserver; @@ -49,7 +46,6 @@ public class CoreServiceImpl extends CoreServiceImplBase { private final FeastProperties feastProperties; private SpecService specService; - private StatsService statsService; private ProjectService projectService; private final AuthorizationService authorizationService; @@ -57,13 +53,11 @@ public class CoreServiceImpl extends CoreServiceImplBase { public CoreServiceImpl( SpecService specService, ProjectService projectService, - StatsService statsService, FeastProperties feastProperties, AuthorizationService authorizationService) { this.specService = specService; this.projectService = projectService; this.feastProperties = feastProperties; - this.statsService = statsService; this.authorizationService = authorizationService; } @@ -83,20 +77,6 @@ public void getFeastCoreVersion( } } - @Override - public void getFeatureSet( - GetFeatureSetRequest request, StreamObserver responseObserver) { - try { - GetFeatureSetResponse response = specService.getFeatureSet(request); - responseObserver.onNext(response); - responseObserver.onCompleted(); - } catch (RetrievalException | StatusRuntimeException | InvalidProtocolBufferException e) { - log.error("Exception has occurred in GetFeatureSet method: ", e); - responseObserver.onError( - Status.INTERNAL.withDescription(e.getMessage()).withCause(e).asRuntimeException()); - } - } - @Override public void getEntity( GetEntityRequest request, StreamObserver responseObserver) { @@ -122,20 +102,6 @@ public void getEntity( } } - @Override - public void listFeatureSets( - ListFeatureSetsRequest request, StreamObserver responseObserver) { - try { - ListFeatureSetsResponse response = specService.listFeatureSets(request.getFilter()); - responseObserver.onNext(response); - responseObserver.onCompleted(); - } catch (RetrievalException | IllegalArgumentException | InvalidProtocolBufferException e) { - log.error("Exception has occurred in ListFeatureSet method: ", e); - responseObserver.onError( - Status.INTERNAL.withDescription(e.getMessage()).withCause(e).asRuntimeException()); - } - } - /** Retrieve a list of features */ @Override public void listFeatures( @@ -188,48 +154,6 @@ public void listEntities( } } - @Override - public void getFeatureStatistics( - GetFeatureStatisticsRequest request, - StreamObserver responseObserver) { - try { - GetFeatureStatisticsResponse response = statsService.getFeatureStatistics(request); - responseObserver.onNext(response); - responseObserver.onCompleted(); - } catch (IllegalArgumentException e) { - log.error("Illegal arguments provided to GetFeatureStatistics method: ", e); - responseObserver.onError( - Status.INVALID_ARGUMENT - .withDescription(e.getMessage()) - .withCause(e) - .asRuntimeException()); - } catch (RetrievalException e) { - log.error("Unable to fetch feature set requested in GetFeatureStatistics method: ", e); - responseObserver.onError( - Status.NOT_FOUND.withDescription(e.getMessage()).withCause(e).asRuntimeException()); - } catch (Exception e) { - log.error("Exception has occurred in GetFeatureStatistics method: ", e); - responseObserver.onError( - Status.INTERNAL.withDescription(e.getMessage()).withCause(e).asRuntimeException()); - } - } - - @Override - public void updateFeatureSetStatus( - UpdateFeatureSetStatusRequest request, - StreamObserver responseObserver) { - try { - UpdateFeatureSetStatusResponse response = specService.updateFeatureSetStatus(request); - - responseObserver.onNext(response); - responseObserver.onCompleted(); - } catch (Exception e) { - log.error("Exception has occurred in UpdateFeatureSetStatus method: ", e); - responseObserver.onError( - Status.INTERNAL.withDescription(e.getMessage()).withCause(e).asRuntimeException()); - } - } - @Override public void listStores( ListStoresRequest request, StreamObserver responseObserver) { @@ -279,40 +203,6 @@ public void applyEntity( } } - @Override - public void applyFeatureSet( - ApplyFeatureSetRequest request, StreamObserver responseObserver) { - - String projectId = null; - - try { - FeatureSet featureSet = request.getFeatureSet(); - projectId = SpecService.resolveProjectName(featureSet.getSpec().getProject()); - authorizationService.authorizeRequest(SecurityContextHolder.getContext(), projectId); - ApplyFeatureSetResponse response = specService.applyFeatureSet(featureSet); - responseObserver.onNext(response); - responseObserver.onCompleted(); - } catch (org.hibernate.exception.ConstraintViolationException e) { - log.error( - "Unable to persist this feature set due to a constraint violation. Please ensure that" - + " field names are unique within the project namespace: ", - e); - responseObserver.onError( - Status.ALREADY_EXISTS.withDescription(e.getMessage()).withCause(e).asRuntimeException()); - } catch (AccessDeniedException e) { - log.info(String.format("User prevented from accessing project: %s", projectId)); - responseObserver.onError( - Status.PERMISSION_DENIED - .withDescription(e.getMessage()) - .withCause(e) - .asRuntimeException()); - } catch (Exception e) { - log.error("Exception has occurred in ApplyFeatureSet method: ", e); - responseObserver.onError( - Status.INTERNAL.withDescription(e.getMessage()).withCause(e).asRuntimeException()); - } - } - @Override public void updateStore( UpdateStoreRequest request, StreamObserver responseObserver) { diff --git a/core/src/main/java/feast/core/metrics/collector/FeastResourceCollector.java b/core/src/main/java/feast/core/metrics/collector/FeastResourceCollector.java index b79ea5a3c3c..3064a25b8f7 100644 --- a/core/src/main/java/feast/core/metrics/collector/FeastResourceCollector.java +++ b/core/src/main/java/feast/core/metrics/collector/FeastResourceCollector.java @@ -16,7 +16,7 @@ */ package feast.core.metrics.collector; -import feast.core.dao.FeatureSetRepository; +import feast.core.dao.FeatureTableRepository; import feast.core.dao.StoreRepository; import io.prometheus.client.Collector; import io.prometheus.client.GaugeMetricFamily; @@ -26,16 +26,16 @@ /** * FeastResourceCollector exports metrics about Feast Resources. * - *

For example: total number of registered feature sets and stores. + *

For example: total number of registered feature tables and stores. */ public class FeastResourceCollector extends Collector { - private final FeatureSetRepository featureSetRepository; + private final FeatureTableRepository featureTableRepository; private final StoreRepository storeRepository; public FeastResourceCollector( - FeatureSetRepository featureSetRepository, StoreRepository storeRepository) { - this.featureSetRepository = featureSetRepository; + FeatureTableRepository featureTableRepository, StoreRepository storeRepository) { + this.featureTableRepository = featureTableRepository; this.storeRepository = storeRepository; } @@ -45,8 +45,8 @@ public List collect() { samples.add( new GaugeMetricFamily( "feast_core_feature_set_total", - "Total number of registered feature sets", - featureSetRepository.count())); + "Total number of registered feature tables", + featureTableRepository.count())); samples.add( new GaugeMetricFamily( "feast_core_store_total", diff --git a/core/src/main/java/feast/core/model/Entity.java b/core/src/main/java/feast/core/model/Entity.java deleted file mode 100644 index a5fd8c1b05d..00000000000 --- a/core/src/main/java/feast/core/model/Entity.java +++ /dev/null @@ -1,77 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2019 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.core.model; - -import feast.proto.core.FeatureSetProto.EntitySpec; -import feast.proto.types.ValueProto.ValueType; -import java.util.Objects; -import javax.persistence.*; -import lombok.Getter; -import lombok.Setter; - -/** Feast entity object. Contains name and type of the entity. */ -@Getter -@Setter -@javax.persistence.Entity -@Table( - name = "entities", - uniqueConstraints = @UniqueConstraint(columnNames = {"name", "feature_set_id"})) -public class Entity { - - @Id @GeneratedValue private Long id; - - private String name; - - @ManyToOne(fetch = FetchType.LAZY) - private FeatureSet featureSet; - - /** Data type of the entity. String representation of {@link ValueType} * */ - private String type; - - public Entity() {} - - public Entity(String name, ValueType.Enum type) { - this.setName(name); - this.setType(type.toString()); - } - - public static Entity fromProto(EntitySpec entitySpec) { - Entity entity = new Entity(entitySpec.getName(), entitySpec.getValueType()); - return entity; - } - - public EntitySpec toProto() { - return EntitySpec.newBuilder().setName(name).setValueType(ValueType.Enum.valueOf(type)).build(); - } - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - Entity entity = (Entity) o; - return getName().equals(entity.getName()) && getType().equals(entity.getType()); - } - - @Override - public int hashCode() { - return Objects.hash(super.hashCode(), getName(), getType()); - } -} diff --git a/core/src/main/java/feast/core/model/Feature.java b/core/src/main/java/feast/core/model/Feature.java deleted file mode 100644 index b387f3403bb..00000000000 --- a/core/src/main/java/feast/core/model/Feature.java +++ /dev/null @@ -1,298 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2019 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.core.model; - -import com.google.protobuf.InvalidProtocolBufferException; -import feast.core.util.TypeConversion; -import feast.proto.core.FeatureSetProto.FeatureSpec; -import feast.proto.core.FeatureSetProto.FeatureSpec.Builder; -import feast.proto.types.ValueProto.ValueType; -import java.util.Arrays; -import java.util.Map; -import java.util.Objects; -import javax.persistence.*; -import javax.persistence.Entity; -import lombok.Getter; -import lombok.Setter; -import org.tensorflow.metadata.v0.*; - -/** - * Feature belonging to a featureset. Contains name, type as well as domain metadata about the - * feature. - */ -@Getter -@Setter -@Entity -@Table( - name = "features", - uniqueConstraints = @UniqueConstraint(columnNames = {"name", "feature_set_id"})) -public class Feature { - - @Id @GeneratedValue private Long id; - - private String name; - - @ManyToOne(fetch = FetchType.LAZY) - private FeatureSet featureSet; - - /** Data type of the feature. String representation of {@link ValueType} * */ - private String type; - - // Labels for this feature - @Column(name = "labels", columnDefinition = "text") - private String labels; - - // Presence constraints (refer to proto feast.core.FeatureSet.FeatureSpec) - // Only one of them can be set. - private byte[] presence; - private byte[] groupPresence; - - // Shape type (refer to proto feast.core.FeatureSet.FeatureSpec) - // Only one of them can be set. - private byte[] shape; - private byte[] valueCount; - - // Domain info for the values (refer to proto feast.core.FeatureSet.FeatureSpec) - // Only one of them can be set. - private String domain; - private byte[] intDomain; - private byte[] floatDomain; - private byte[] stringDomain; - private byte[] boolDomain; - private byte[] structDomain; - private byte[] naturalLanguageDomain; - private byte[] imageDomain; - private byte[] midDomain; - private byte[] urlDomain; - private byte[] timeDomain; - private byte[] timeOfDayDomain; - - public Feature() {} - // Whether this feature has been archived. A archived feature cannot be - // retrieved from or written to. - private boolean archived = false; - - public Feature(String name, ValueType.Enum type) { - this.setName(name); - this.setType(type.toString()); - } - - /** - * Return a boolean to facilitate streaming elements on the basis of given predicate. - * - * @param labelsFilter contain labels that should be attached to Feature - * @return boolean True if Feature contains all labels in the labelsFilter - */ - public boolean hasAllLabels(Map labelsFilter) { - Map featureLabelsMap = this.getLabels(); - for (String key : labelsFilter.keySet()) { - if (!featureLabelsMap.containsKey(key) - || !featureLabelsMap.get(key).equals(labelsFilter.get(key))) { - return false; - } - } - return true; - } - - public static Feature fromProto(FeatureSpec featureSpec) { - Feature feature = new Feature(featureSpec.getName(), featureSpec.getValueType()); - feature.labels = TypeConversion.convertMapToJsonString(featureSpec.getLabelsMap()); - feature.updateSchema(featureSpec); - return feature; - } - - public FeatureSpec toProto() throws InvalidProtocolBufferException { - Builder featureSpecBuilder = - FeatureSpec.newBuilder().setName(getName()).setValueType(ValueType.Enum.valueOf(getType())); - - if (getPresence() != null) { - featureSpecBuilder.setPresence(FeaturePresence.parseFrom(getPresence())); - } else if (getGroupPresence() != null) { - featureSpecBuilder.setGroupPresence(FeaturePresenceWithinGroup.parseFrom(getGroupPresence())); - } - - if (getShape() != null) { - featureSpecBuilder.setShape(FixedShape.parseFrom(getShape())); - } else if (getValueCount() != null) { - featureSpecBuilder.setValueCount(ValueCount.parseFrom(getValueCount())); - } - - if (getDomain() != null) { - featureSpecBuilder.setDomain(getDomain()); - } else if (getIntDomain() != null) { - featureSpecBuilder.setIntDomain(IntDomain.parseFrom(getIntDomain())); - } else if (getFloatDomain() != null) { - featureSpecBuilder.setFloatDomain(FloatDomain.parseFrom(getFloatDomain())); - } else if (getStringDomain() != null) { - featureSpecBuilder.setStringDomain(StringDomain.parseFrom(getStringDomain())); - } else if (getBoolDomain() != null) { - featureSpecBuilder.setBoolDomain(BoolDomain.parseFrom(getBoolDomain())); - } else if (getStructDomain() != null) { - featureSpecBuilder.setStructDomain(StructDomain.parseFrom(getStructDomain())); - } else if (getNaturalLanguageDomain() != null) { - featureSpecBuilder.setNaturalLanguageDomain( - NaturalLanguageDomain.parseFrom(getNaturalLanguageDomain())); - } else if (getImageDomain() != null) { - featureSpecBuilder.setImageDomain(ImageDomain.parseFrom(getImageDomain())); - } else if (getMidDomain() != null) { - featureSpecBuilder.setMidDomain(MIDDomain.parseFrom(getMidDomain())); - } else if (getUrlDomain() != null) { - featureSpecBuilder.setUrlDomain(URLDomain.parseFrom(getUrlDomain())); - } else if (getTimeDomain() != null) { - featureSpecBuilder.setTimeDomain(TimeDomain.parseFrom(getTimeDomain())); - } else if (getTimeOfDayDomain() != null) { - featureSpecBuilder.setTimeOfDayDomain(TimeOfDayDomain.parseFrom(getTimeOfDayDomain())); - } - - if (getLabels() != null) { - featureSpecBuilder.putAllLabels(getLabels()); - } - return featureSpecBuilder.build(); - } - - private void updateSchema(FeatureSpec featureSpec) { - switch (featureSpec.getPresenceConstraintsCase()) { - case PRESENCE: - setPresence(featureSpec.getPresence().toByteArray()); - break; - case GROUP_PRESENCE: - setGroupPresence(featureSpec.getGroupPresence().toByteArray()); - break; - case PRESENCECONSTRAINTS_NOT_SET: - break; - } - - switch (featureSpec.getShapeTypeCase()) { - case SHAPE: - setShape(featureSpec.getShape().toByteArray()); - break; - case VALUE_COUNT: - setValueCount(featureSpec.getValueCount().toByteArray()); - break; - case SHAPETYPE_NOT_SET: - break; - } - - switch (featureSpec.getDomainInfoCase()) { - case DOMAIN: - setDomain(featureSpec.getDomain()); - break; - case INT_DOMAIN: - setIntDomain(featureSpec.getIntDomain().toByteArray()); - break; - case FLOAT_DOMAIN: - setFloatDomain(featureSpec.getFloatDomain().toByteArray()); - break; - case STRING_DOMAIN: - setStringDomain(featureSpec.getStringDomain().toByteArray()); - break; - case BOOL_DOMAIN: - setBoolDomain(featureSpec.getBoolDomain().toByteArray()); - break; - case STRUCT_DOMAIN: - setStructDomain(featureSpec.getStructDomain().toByteArray()); - break; - case NATURAL_LANGUAGE_DOMAIN: - setNaturalLanguageDomain(featureSpec.getNaturalLanguageDomain().toByteArray()); - break; - case IMAGE_DOMAIN: - setImageDomain(featureSpec.getImageDomain().toByteArray()); - break; - case MID_DOMAIN: - setMidDomain(featureSpec.getMidDomain().toByteArray()); - break; - case URL_DOMAIN: - setUrlDomain(featureSpec.getUrlDomain().toByteArray()); - break; - case TIME_DOMAIN: - setTimeDomain(featureSpec.getTimeDomain().toByteArray()); - break; - case TIME_OF_DAY_DOMAIN: - setTimeOfDayDomain(featureSpec.getTimeOfDayDomain().toByteArray()); - break; - case DOMAININFO_NOT_SET: - break; - } - } - - /** Archive this feature. */ - public void archive() { - this.archived = true; - } - - /** - * Update the feature object with a valid feature spec. - * - * @param featureSpec {@link FeatureSpec} containing schema changes. - */ - public void updateFromProto(FeatureSpec featureSpec) { - if (isArchived()) { - throw new IllegalArgumentException( - String.format( - "You are attempting to create a feature %s that was previously archived. This isn't allowed. Please create a new feature with a different name.", - featureSpec.getName())); - } - if (ValueType.Enum.valueOf(type) != featureSpec.getValueType()) { - throw new IllegalArgumentException( - String.format( - "You are attempting to change the type of feature %s from %s to %s. This isn't allowed. Please create a new feature.", - featureSpec.getName(), type, featureSpec.getValueType())); - } - this.setLabels(TypeConversion.convertMapToJsonString(featureSpec.getLabelsMap())); - updateSchema(featureSpec); - } - - public Map getLabels() { - return TypeConversion.convertJsonStringToMap(this.labels); - } - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - Feature feature = (Feature) o; - return getName().equals(feature.getName()) - && getType().equals(feature.getType()) - && isArchived() == (feature.isArchived()) - && Objects.equals(getLabels(), feature.getLabels()) - && Arrays.equals(getPresence(), feature.getPresence()) - && Arrays.equals(getGroupPresence(), feature.getGroupPresence()) - && Arrays.equals(getShape(), feature.getShape()) - && Arrays.equals(getValueCount(), feature.getValueCount()) - && Objects.equals(getDomain(), feature.getDomain()) - && Arrays.equals(getIntDomain(), feature.getIntDomain()) - && Arrays.equals(getFloatDomain(), feature.getFloatDomain()) - && Arrays.equals(getStringDomain(), feature.getStringDomain()) - && Arrays.equals(getBoolDomain(), feature.getBoolDomain()) - && Arrays.equals(getStructDomain(), feature.getStructDomain()) - && Arrays.equals(getNaturalLanguageDomain(), feature.getNaturalLanguageDomain()) - && Arrays.equals(getImageDomain(), feature.getImageDomain()) - && Arrays.equals(getMidDomain(), feature.getMidDomain()) - && Arrays.equals(getUrlDomain(), feature.getUrlDomain()) - && Arrays.equals(getTimeDomain(), feature.getTimeDomain()) - && Arrays.equals(getTimeDomain(), feature.getTimeOfDayDomain()); - } - - @Override - public int hashCode() { - return Objects.hash(super.hashCode(), getName(), getType(), getLabels()); - } -} diff --git a/core/src/main/java/feast/core/model/FeatureSet.java b/core/src/main/java/feast/core/model/FeatureSet.java deleted file mode 100644 index 023708d6a99..00000000000 --- a/core/src/main/java/feast/core/model/FeatureSet.java +++ /dev/null @@ -1,463 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2019 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.core.model; - -import com.google.common.collect.Sets; -import com.google.protobuf.Duration; -import com.google.protobuf.InvalidProtocolBufferException; -import com.google.protobuf.Timestamp; -import feast.core.util.TypeConversion; -import feast.proto.core.FeatureSetProto; -import feast.proto.core.FeatureSetProto.*; -import feast.proto.serving.ServingAPIProto.FeatureReference; -import java.util.*; -import java.util.stream.Collectors; -import javax.persistence.*; -import lombok.Getter; -import lombok.Setter; -import org.apache.commons.lang3.builder.HashCodeBuilder; -import org.tensorflow.metadata.v0.*; - -@Getter -@Setter -@javax.persistence.Entity -@Table( - name = "feature_sets", - uniqueConstraints = @UniqueConstraint(columnNames = {"name", "project_name"})) -public class FeatureSet extends AbstractTimestampEntity { - - @Id @GeneratedValue private long id; - - // Name of the featureSet - @Column(name = "name", nullable = false) - private String name; - - // Project that this featureSet belongs to - @ManyToOne(fetch = FetchType.LAZY) - @JoinColumn(name = "project_name") - private Project project; - - // Max allowed staleness for features in this featureSet. - @Column(name = "max_age") - private long maxAgeSeconds; - - // Entity fields inside this feature set - @OneToMany( - mappedBy = "featureSet", - cascade = CascadeType.ALL, - fetch = FetchType.EAGER, - orphanRemoval = true) - private Set entities; - - // Feature fields inside this feature set - @OneToMany( - mappedBy = "featureSet", - cascade = CascadeType.ALL, - fetch = FetchType.EAGER, - orphanRemoval = true) - private Set features; - - // Source on which feature rows can be found - @ManyToOne(cascade = CascadeType.ALL, fetch = FetchType.EAGER) - @JoinColumn(name = "source_id", referencedColumnName = "pk") - private Source source; - - @Deprecated - @Column(name = "source") - private String deprecatedSource; - - // Status of the feature set - @Enumerated(EnumType.STRING) - @Column(name = "status") - private FeatureSetStatus status; - - // User defined metadata - @Column(name = "labels", columnDefinition = "text") - private String labels; - - @Column(name = "version", columnDefinition = "integer default 0") - private int version; - - public FeatureSet() { - super(); - } - - public FeatureSet( - String name, - String project, - long maxAgeSeconds, - List entities, - List features, - Source source, - Map labels, - FeatureSetStatus status) { - this.maxAgeSeconds = maxAgeSeconds; - this.source = source; - this.status = status; - this.entities = new HashSet<>(); - this.features = new HashSet<>(); - this.name = name; - this.project = new Project(project); - this.labels = TypeConversion.convertMapToJsonString(labels); - addEntities(entities); - addFeatures(features); - } - - public void setName(String name) { - this.name = name; - } - - private String getProjectName() { - if (getProject() != null) { - return getProject().getName(); - } else { - return ""; - } - } - - /** - * Return a boolean to facilitate streaming elements on the basis of given predicate. - * - * @param entitiesFilter contain entities that should be attached to the FeatureSet - * @return boolean True if FeatureSet contains all entities in the entitiesFilter - */ - public boolean hasAllEntities(List entitiesFilter) { - List allEntitiesName = - this.getEntities().stream().map(entity -> entity.getName()).collect(Collectors.toList()); - return allEntitiesName.equals(entitiesFilter); - } - - /** - * Returns a map of Feature references and Features if FeatureSet's Feature contains all labels in - * the labelsFilter - * - * @param labelsFilter contain labels that should be attached to FeatureSet's features - * @return Map of Feature references and Features - */ - public Map getFeaturesByRef(Map labelsFilter) { - Map validFeaturesMap = new HashMap<>(); - List validFeatures; - if (labelsFilter.size() > 0) { - validFeatures = filterFeaturesByAllLabels(this.getFeatures(), labelsFilter); - for (Feature feature : validFeatures) { - FeatureReference featureRef = - FeatureReference.newBuilder() - .setProject(this.getProjectName()) - .setFeatureSet(this.getName()) - .setName(feature.getName()) - .build(); - validFeaturesMap.put(renderFeatureRef(featureRef), feature); - } - return validFeaturesMap; - } - for (Feature feature : this.getFeatures()) { - FeatureReference featureRef = - FeatureReference.newBuilder() - .setProject(this.getProjectName()) - .setFeatureSet(this.getName()) - .setName(feature.getName()) - .build(); - validFeaturesMap.put(renderFeatureRef(featureRef), feature); - } - return validFeaturesMap; - } - - /** - * Returns a list of Features if FeatureSet's Feature contains all labels in labelsFilter - * - * @param labelsFilter contain labels that should be attached to FeatureSet's features - * @return List of Features - */ - public static List filterFeaturesByAllLabels( - Set features, Map labelsFilter) { - List validFeatures = - features.stream() - .filter(feature -> feature.hasAllLabels(labelsFilter)) - .collect(Collectors.toList()); - - return validFeatures; - } - - /** - * Render a feature reference as string. - * - * @param featureReference to render as string - * @return string representation of feature reference. - */ - public static String renderFeatureRef(FeatureReference featureReference) { - String refStr = - featureReference.getProject() - + "/" - + featureReference.getFeatureSet() - + ":" - + featureReference.getName(); - - return refStr; - } - - /** - * Return a boolean to facilitate streaming elements on the basis of given predicate. - * - * @param labelsFilter labels contain key-value mapping for labels attached to the FeatureSet - * @return boolean True if FeatureSet contains all labels in the labelsFilter - */ - public boolean hasAllLabels(Map labelsFilter) { - Map featureSetLabelsMap = this.getLabelsMap(); - for (String key : labelsFilter.keySet()) { - if (!featureSetLabelsMap.containsKey(key) - || !featureSetLabelsMap.get(key).equals(labelsFilter.get(key))) { - return false; - } - } - return true; - } - - public void setProject(Project project) { - this.project = project; - } - - public int incVersion() { - return ++version; - } - - public static FeatureSet fromProto(FeatureSetProto.FeatureSet featureSetProto) { - FeatureSetSpec featureSetSpec = featureSetProto.getSpec(); - Source source = Source.fromProto(featureSetSpec.getSource()); - - List featureSpecs = new ArrayList<>(); - for (FeatureSpec featureSpec : featureSetSpec.getFeaturesList()) { - featureSpecs.add(Feature.fromProto(featureSpec)); - } - - List entitySpecs = new ArrayList<>(); - for (EntitySpec entitySpec : featureSetSpec.getEntitiesList()) { - entitySpecs.add(Entity.fromProto(entitySpec)); - } - - return new FeatureSet( - featureSetProto.getSpec().getName(), - featureSetProto.getSpec().getProject(), - featureSetSpec.getMaxAge().getSeconds(), - entitySpecs, - featureSpecs, - source, - featureSetProto.getSpec().getLabelsMap(), - featureSetProto.getMeta().getStatus()); - } - - // Updates the existing feature set from a proto. - public void updateFromProto(FeatureSetProto.FeatureSet featureSetProto) - throws InvalidProtocolBufferException { - FeatureSetSpec spec = featureSetProto.getSpec(); - if (this.toProto().getSpec().equals(spec)) { - return; - } - - // 1. validate - // 1a. check no change to identifiers - if (!name.equals(spec.getName())) { - throw new IllegalArgumentException( - String.format("Given feature set name %s does not match name %s.", spec.getName(), name)); - } - if (!project.getName().equals(spec.getProject())) { - throw new IllegalArgumentException( - String.format( - "You are attempting to change the project of feature set %s from %s to %s. This isn't allowed. Please create a new feature set under the desired project.", - spec.getName(), project, spec.getProject())); - } - - Set existingEntities = - entities.stream().map(Entity::toProto).collect(Collectors.toSet()); - - // 1b. check no change to entities - if (!Sets.newHashSet(spec.getEntitiesList()).equals(existingEntities)) { - throw new IllegalArgumentException( - String.format( - "You are attempting to change the entities of this feature set: Given set of entities \n{%s}\n does not match existing set of entities\n {%s}. This isn't allowed. Please create a new feature set. ", - spec.getEntitiesList(), existingEntities)); - } - - // 2. Update max age, source and labels. - this.maxAgeSeconds = spec.getMaxAge().getSeconds(); - this.source = Source.fromProto(spec.getSource()); - this.setLabels(TypeConversion.convertMapToJsonString(spec.getLabelsMap())); - - Map updatedFeatures = - spec.getFeaturesList().stream().collect(Collectors.toMap(FeatureSpec::getName, fs -> fs)); - - // 3. Tombstone features that are gone, update features that have changed - for (Feature existingFeature : features) { - String existingFeatureName = existingFeature.getName(); - FeatureSpec updatedFeatureSpec = updatedFeatures.get(existingFeatureName); - if (updatedFeatureSpec == null) { - existingFeature.archive(); - } else { - existingFeature.updateFromProto(updatedFeatureSpec); - updatedFeatures.remove(existingFeatureName); - } - } - - // 4. Add new features - for (FeatureSpec featureSpec : updatedFeatures.values()) { - Feature newFeature = Feature.fromProto(featureSpec); - addFeature(newFeature); - } - } - - public void addEntities(List entities) { - for (Entity entity : entities) { - addEntity(entity); - } - } - - public void addEntity(Entity entity) { - entity.setFeatureSet(this); - entities.add(entity); - } - - public void addFeatures(List features) { - for (Feature feature : features) { - addFeature(feature); - } - } - - public void addFeature(Feature feature) { - feature.setFeatureSet(this); - features.add(feature); - } - - public FeatureSetProto.FeatureSet toProto() throws InvalidProtocolBufferException { - List entitySpecs = new ArrayList<>(); - for (Entity entityField : entities) { - entitySpecs.add(entityField.toProto()); - } - - List featureSpecs = new ArrayList<>(); - for (Feature featureField : features) { - if (!featureField.isArchived()) { - featureSpecs.add(featureField.toProto()); - } - } - - FeatureSetMeta.Builder meta = - FeatureSetMeta.newBuilder() - .setCreatedTimestamp( - Timestamp.newBuilder().setSeconds(super.getCreated().getTime() / 1000L)) - .setStatus(status); - - FeatureSetSpec.Builder spec = - FeatureSetSpec.newBuilder() - .setName(getName()) - .setProject(project.getName()) - .setMaxAge(Duration.newBuilder().setSeconds(maxAgeSeconds)) - .addAllEntities(entitySpecs) - .addAllFeatures(featureSpecs) - .putAllLabels(TypeConversion.convertJsonStringToMap(labels)) - .setSource(source.toProto()) - .setVersion(version); - - return FeatureSetProto.FeatureSet.newBuilder().setMeta(meta).setSpec(spec).build(); - } - - public Map getLabelsMap() { - return TypeConversion.convertJsonStringToMap(this.getLabels()); - } - - @Override - public int hashCode() { - HashCodeBuilder hcb = new HashCodeBuilder(); - hcb.append(project.getName()); - hcb.append(getName()); - return hcb.toHashCode(); - } - - @Override - public boolean equals(Object obj) { - if (this == obj) { - return true; - } - if (!(obj instanceof FeatureSet)) { - return false; - } - - FeatureSet other = (FeatureSet) obj; - if (!getName().equals(other.getName())) { - return false; - } - - if (!getLabels().equals(other.getLabels())) { - return false; - } - - if (!project.getName().equals(other.project.getName())) { - return false; - } - - if (!source.equalTo(other.getSource())) { - return false; - } - - if (maxAgeSeconds != other.maxAgeSeconds) { - return false; - } - - if (version != other.version) { - return false; - } - - // Create a map of all fields in this feature set - Map entitiesMap = new HashMap<>(); - Map featuresMap = new HashMap<>(); - - for (Entity e : entities) { - entitiesMap.putIfAbsent(e.getName(), e); - } - - for (Feature f : features) { - featuresMap.putIfAbsent(f.getName(), f); - } - - // Ensure map size is consistent with existing fields - if (entitiesMap.size() != other.getEntities().size()) { - return false; - } - if (featuresMap.size() != other.getFeatures().size()) { - return false; - } - - // Ensure the other entities and features exist in the field map - for (Entity e : other.getEntities()) { - if (!entitiesMap.containsKey(e.getName())) { - return false; - } - if (!e.equals(entitiesMap.get(e.getName()))) { - return false; - } - } - - for (Feature f : other.getFeatures()) { - if (!featuresMap.containsKey(f.getName())) { - return false; - } - if (!f.equals(featuresMap.get(f.getName()))) { - return false; - } - } - - return true; - } -} diff --git a/core/src/main/java/feast/core/model/FeatureStatistics.java b/core/src/main/java/feast/core/model/FeatureStatistics.java deleted file mode 100644 index e06e7d9a6f3..00000000000 --- a/core/src/main/java/feast/core/model/FeatureStatistics.java +++ /dev/null @@ -1,243 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.core.model; - -import com.google.protobuf.InvalidProtocolBufferException; -import java.io.*; -import java.util.Date; -import java.util.List; -import javax.persistence.*; -import javax.persistence.Entity; -import lombok.Getter; -import lombok.NoArgsConstructor; -import lombok.Setter; -import org.tensorflow.metadata.v0.*; - -@NoArgsConstructor -@Getter -@Setter -@Entity -@Table( - name = "feature_statistics", - indexes = { - @Index(name = "idx_feature_statistics_feature", columnList = "feature_id"), - @Index(name = "idx_feature_statistics_dataset_id", columnList = "datasetId"), - @Index(name = "idx_feature_statistics_date", columnList = "date"), - }) -public class FeatureStatistics { - @Id - @GeneratedValue(strategy = GenerationType.AUTO) - private int id; - - @ManyToOne private Feature feature; - - // Only one of these fields should be populated. - private String datasetId; - private Date date; - - // General statistics - private String featureType; - private long count; - private long numMissing; - private long minNumValues; - private long maxNumValues; - private float avgNumValues; - private long totalNumValues; - private byte[] numValuesHistogram; - - // Numeric statistics - private double mean; - private double stdev; - private long zeroes; - private double min; - private double max; - private double median; - private byte[] numericValueHistogram; - private byte[] numericValueQuantiles; - - // String statistics - @Column(name = "n_unique") - private long unique; - - private float averageLength; - private byte[] rankHistogram; - private byte[] topValues; - - // Byte statistics - private float minBytes; - private float maxBytes; - private float avgBytes; - - // Instantiates a Statistics object from a tensorflow metadata FeatureNameStatistics object and a - // dataset ID. - public static FeatureStatistics createForDataset( - Feature feature, FeatureNameStatistics featureNameStatistics, String datasetId) - throws IOException { - FeatureStatistics featureStatistics = FeatureStatistics.fromProto(featureNameStatistics); - featureStatistics.setFeature(feature); - featureStatistics.setDatasetId(datasetId); - return featureStatistics; - } - - // Instantiates a Statistics object from a tensorflow metadata FeatureNameStatistics object and a - // date. - public static FeatureStatistics createForDate( - Feature feature, FeatureNameStatistics featureNameStatistics, Date date) throws IOException { - FeatureStatistics featureStatistics = FeatureStatistics.fromProto(featureNameStatistics); - featureStatistics.setDate(date); - featureStatistics.setFeature(feature); - return featureStatistics; - } - - public FeatureNameStatistics toProto() throws InvalidProtocolBufferException { - FeatureNameStatistics.Builder featureNameStatisticsBuilder = - FeatureNameStatistics.newBuilder() - .setType(FeatureNameStatistics.Type.valueOf(featureType)) - .setPath(Path.newBuilder().addStep(feature.getName())); - CommonStatistics commonStatistics = - CommonStatistics.newBuilder() - .setNumNonMissing(count - numMissing) - .setNumMissing(numMissing) - .setMaxNumValues(maxNumValues) - .setMinNumValues(minNumValues) - .setTotNumValues(totalNumValues) - .setNumValuesHistogram(Histogram.parseFrom(numValuesHistogram)) - .build(); - - switch (featureNameStatisticsBuilder.getType()) { - case INT: - case FLOAT: - NumericStatistics numStats = - NumericStatistics.newBuilder() - .setCommonStats(commonStatistics) - .setMean(mean) - .setStdDev(stdev) - .setNumZeros(zeroes) - .setMin(min) - .setMax(max) - .setMedian(median) - .addHistograms(Histogram.parseFrom(numericValueHistogram)) - .addHistograms(Histogram.parseFrom(numericValueQuantiles)) - .build(); - featureNameStatisticsBuilder.setNumStats(numStats); - break; - case STRING: - StringStatistics.Builder stringStats = - StringStatistics.newBuilder() - .setCommonStats(commonStatistics) - .setUnique(unique) - .setAvgLength(averageLength); - if (rankHistogram == null) { - stringStats.setRankHistogram(RankHistogram.getDefaultInstance()); - } else { - stringStats.setRankHistogram(RankHistogram.parseFrom(rankHistogram)); - } - try (ByteArrayInputStream bis = new ByteArrayInputStream(topValues)) { - ObjectInputStream ois = new ObjectInputStream(bis); - List freqAndValueList = - (List) ois.readObject(); - stringStats.addAllTopValues(freqAndValueList); - } catch (IOException | ClassNotFoundException e) { - throw new InvalidProtocolBufferException( - "Failed to parse field: StringStatistics.TopValues. Check if the value is malformed."); - } - featureNameStatisticsBuilder.setStringStats(stringStats); - break; - case BYTES: - BytesStatistics bytesStats = - BytesStatistics.newBuilder() - .setCommonStats(commonStatistics) - .setAvgNumBytes(avgBytes) - .setMinNumBytes(minBytes) - .setMaxNumBytes(maxBytes) - .build(); - featureNameStatisticsBuilder.setBytesStats(bytesStats); - break; - case STRUCT: - StructStatistics structStats = - StructStatistics.newBuilder().setCommonStats(commonStatistics).build(); - featureNameStatisticsBuilder.setStructStats(structStats); - break; - } - return featureNameStatisticsBuilder.build(); - } - - private static FeatureStatistics fromProto(FeatureNameStatistics featureNameStatistics) - throws IOException, IllegalArgumentException { - FeatureStatistics featureStatistics = new FeatureStatistics(); - featureStatistics.setFeatureType(featureNameStatistics.getType().toString()); - CommonStatistics commonStats; - switch (featureNameStatistics.getType()) { - case FLOAT: - case INT: - NumericStatistics numStats = featureNameStatistics.getNumStats(); - commonStats = numStats.getCommonStats(); - featureStatistics.setMean(numStats.getMean()); - featureStatistics.setStdev(numStats.getStdDev()); - featureStatistics.setZeroes(numStats.getNumZeros()); - featureStatistics.setMin(numStats.getMin()); - featureStatistics.setMax(numStats.getMax()); - featureStatistics.setMedian(numStats.getMedian()); - for (Histogram histogram : numStats.getHistogramsList()) { - switch (histogram.getType()) { - case STANDARD: - featureStatistics.setNumericValueHistogram(histogram.toByteArray()); - case QUANTILES: - featureStatistics.setNumericValueQuantiles(histogram.toByteArray()); - default: - // invalid type, dropping the values - } - } - break; - case STRING: - StringStatistics stringStats = featureNameStatistics.getStringStats(); - commonStats = stringStats.getCommonStats(); - featureStatistics.setUnique(stringStats.getUnique()); - featureStatistics.setAverageLength(stringStats.getAvgLength()); - featureStatistics.setRankHistogram(stringStats.getRankHistogram().toByteArray()); - try (ByteArrayOutputStream bos = new ByteArrayOutputStream()) { - ObjectOutputStream oos = new ObjectOutputStream(bos); - oos.writeObject(stringStats.getTopValuesList()); - featureStatistics.setTopValues(bos.toByteArray()); - } - break; - case BYTES: - BytesStatistics bytesStats = featureNameStatistics.getBytesStats(); - commonStats = bytesStats.getCommonStats(); - featureStatistics.setUnique(bytesStats.getUnique()); - featureStatistics.setMaxBytes(bytesStats.getMaxNumBytes()); - featureStatistics.setMinBytes(bytesStats.getMinNumBytes()); - featureStatistics.setAvgBytes(bytesStats.getAvgNumBytes()); - break; - case STRUCT: - StructStatistics structStats = featureNameStatistics.getStructStats(); - commonStats = structStats.getCommonStats(); - break; - default: - throw new IllegalArgumentException("Feature statistics provided were of unknown type."); - } - featureStatistics.setCount(commonStats.getNumMissing() + commonStats.getNumNonMissing()); - featureStatistics.setNumMissing(commonStats.getNumMissing()); - featureStatistics.setMinNumValues(commonStats.getMinNumValues()); - featureStatistics.setMaxNumValues(commonStats.getMaxNumValues()); - featureStatistics.setAvgNumValues(commonStats.getAvgNumValues()); - featureStatistics.setTotalNumValues(commonStats.getTotNumValues()); - featureStatistics.setNumValuesHistogram(commonStats.getNumValuesHistogram().toByteArray()); - - return featureStatistics; - } -} diff --git a/core/src/main/java/feast/core/model/FeatureTable.java b/core/src/main/java/feast/core/model/FeatureTable.java index 8c62df64867..b442d57b583 100644 --- a/core/src/main/java/feast/core/model/FeatureTable.java +++ b/core/src/main/java/feast/core/model/FeatureTable.java @@ -16,6 +16,8 @@ */ package feast.core.model; +import static feast.common.models.FeatureV2.getFeatureStringRef; + import com.google.common.hash.Hashing; import com.google.protobuf.Duration; import com.google.protobuf.Timestamp; @@ -25,6 +27,7 @@ import feast.proto.core.FeatureProto.FeatureSpecV2; import feast.proto.core.FeatureTableProto; import feast.proto.core.FeatureTableProto.FeatureTableSpec; +import feast.proto.serving.ServingAPIProto; import java.util.*; import java.util.stream.Collectors; import javax.persistence.CascadeType; @@ -73,7 +76,7 @@ public class FeatureTable extends AbstractTimestampEntity { private Set features; // Entites to associate the features defined in this FeatureTable with - @ManyToMany + @ManyToMany(fetch = FetchType.EAGER) @JoinTable( name = "feature_tables_entities_v2", joinColumns = @JoinColumn(name = "feature_table_id"), @@ -263,6 +266,72 @@ private static Set resolveEntities( .collect(Collectors.toSet()); } + /** + * Return a boolean to indicate if FeatureTable contains all specified entities. + * + * @param entitiesFilter contain entities that should be attached to the FeatureTable + * @return boolean True if FeatureTable contains all entities in the entitiesFilter + */ + public boolean hasAllEntities(List entitiesFilter) { + Set allEntitiesName = + this.getEntities().stream().map(entity -> entity.getName()).collect(Collectors.toSet()); + return allEntitiesName.equals(new HashSet<>(entitiesFilter)); + } + + /** + * Returns a map of Feature references and Features if FeatureTable's Feature contains all labels + * in the labelsFilter + * + * @param labelsFilter contain labels that should be attached to FeatureTable's features + * @return Map of Feature references and Features + */ + public Map getFeaturesByLabels(Map labelsFilter) { + Map validFeaturesMap; + List validFeatures; + if (labelsFilter.size() > 0) { + validFeatures = filterFeaturesByAllLabels(this.getFeatures(), labelsFilter); + validFeaturesMap = getFeaturesRefToFeaturesMap(validFeatures); + return validFeaturesMap; + } + validFeaturesMap = getFeaturesRefToFeaturesMap(List.copyOf(this.getFeatures())); + return validFeaturesMap; + } + + /** + * Returns map for accessing features using their respective feature reference. + * + * @param features List of features to insert to map. + * @return Map of featureRef:feature. + */ + private Map getFeaturesRefToFeaturesMap(List features) { + Map validFeaturesMap = new HashMap<>(); + for (FeatureV2 feature : features) { + ServingAPIProto.FeatureReferenceV2 featureRef = + ServingAPIProto.FeatureReferenceV2.newBuilder() + .setFeatureTable(this.getName()) + .setName(feature.getName()) + .build(); + validFeaturesMap.put(getFeatureStringRef(featureRef), feature); + } + return validFeaturesMap; + } + + /** + * Returns a list of Features if FeatureTable's Feature contains all labels in labelsFilter + * + * @param labelsFilter contain labels that should be attached to FeatureTable's features + * @return List of Features + */ + public static List filterFeaturesByAllLabels( + Set features, Map labelsFilter) { + List validFeatures = + features.stream() + .filter(feature -> feature.hasAllLabels(labelsFilter)) + .collect(Collectors.toList()); + + return validFeatures; + } + /** * Determine whether a FeatureTable has all the specified labels. * @@ -340,9 +409,9 @@ public boolean equals(Object o) { return getName().equals(other.getName()) && getProject().equals(other.getProject()) && getLabelsJSON().equals(other.getLabelsJSON()) - && getFeatures().containsAll(other.getFeatures()) - && getEntities().containsAll(other.getEntities()) - && getMaxAgeSecs() == getMaxAgeSecs() + && getFeatures().equals(other.getFeatures()) + && getEntities().equals(other.getEntities()) + && getMaxAgeSecs() == other.getMaxAgeSecs() && Optional.ofNullable(getBatchSource()).equals(Optional.ofNullable(other.getBatchSource())) && Optional.ofNullable(getStreamSource()) .equals(Optional.ofNullable(other.getStreamSource())); diff --git a/core/src/main/java/feast/core/model/FeatureV2.java b/core/src/main/java/feast/core/model/FeatureV2.java index e10d51647ce..f25e951efc7 100644 --- a/core/src/main/java/feast/core/model/FeatureV2.java +++ b/core/src/main/java/feast/core/model/FeatureV2.java @@ -106,6 +106,23 @@ public void updateFromProto(FeatureSpecV2 spec) { this.labelsJSON = TypeConversion.convertMapToJsonString(spec.getLabelsMap()); } + /** + * Return a boolean to indicate if Feature contains all specified labels. + * + * @param labelsFilter contain labels that should be attached to Feature + * @return boolean True if Feature contains all labels in the labelsFilter + */ + public boolean hasAllLabels(Map labelsFilter) { + Map featureLabelsMap = TypeConversion.convertJsonStringToMap(getLabelsJSON()); + for (String key : labelsFilter.keySet()) { + if (!featureLabelsMap.containsKey(key) + || !featureLabelsMap.get(key).equals(labelsFilter.get(key))) { + return false; + } + } + return true; + } + @Override public int hashCode() { return Objects.hash(getName(), getType(), getLabelsJSON()); diff --git a/core/src/main/java/feast/core/model/Project.java b/core/src/main/java/feast/core/model/Project.java index e516f5868c4..2d60d5e0e07 100644 --- a/core/src/main/java/feast/core/model/Project.java +++ b/core/src/main/java/feast/core/model/Project.java @@ -45,13 +45,6 @@ public class Project { @Column(name = "archived", nullable = false) private boolean archived; - @OneToMany( - cascade = CascadeType.ALL, - fetch = FetchType.EAGER, - orphanRemoval = true, - mappedBy = "project") - private Set featureSets; - @OneToMany( cascade = CascadeType.ALL, fetch = FetchType.EAGER, @@ -72,16 +65,10 @@ public Project() { public Project(String name) { this.name = name; - this.featureSets = new HashSet<>(); this.entities = new HashSet<>(); this.featureTables = new HashSet<>(); } - public void addFeatureSet(FeatureSet featureSet) { - featureSet.setProject(this); - featureSets.add(featureSet); - } - public void addEntity(EntityV2 entity) { entity.setProject(this); entities.add(entity); diff --git a/core/src/main/java/feast/core/model/Source.java b/core/src/main/java/feast/core/model/Source.java deleted file mode 100644 index 9ccbf5d490c..00000000000 --- a/core/src/main/java/feast/core/model/Source.java +++ /dev/null @@ -1,208 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2019 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.core.model; - -import com.google.protobuf.TextFormat; -import feast.proto.core.SourceProto; -import feast.proto.core.SourceProto.KafkaSourceConfig; -import feast.proto.core.SourceProto.Source.Builder; -import feast.proto.core.SourceProto.SourceType; -import io.grpc.Status; -import java.util.Objects; -import javax.persistence.*; -import javax.persistence.Entity; -import lombok.AllArgsConstructor; -import lombok.Getter; -import lombok.Setter; - -@Entity -@Getter -@Setter -@AllArgsConstructor -@Table(name = "sources") -public class Source { - - /** Source Id. Internal use only, do not use to identify the source. */ - @Id - @GeneratedValue - @Column(name = "pk") - private Integer id; - - @Deprecated - @Column(name = "id") - private String deprecatedId; - - @Deprecated - @Column(name = "bootstrap_servers") - private String bootstrapServers; - - @Deprecated - @Column(name = "topics") - private String topics; - - /** Type of the source */ - @Enumerated(EnumType.STRING) - @Column(name = "type", nullable = false) - private SourceType type; - - /** Configuration object specific to each source type */ - @Column(name = "config") - private String config; - - @Column(name = "is_default") - private boolean isDefault; - - public Source() { - super(); - } - - public String getConfig() { - if ((config == null || config.isEmpty()) && bootstrapServers != null && topics != null) { - config = - KafkaSourceConfig.newBuilder() - .setBootstrapServers(bootstrapServers) - .setTopic(topics) - .build() - .toString(); - } - - return config; - } - - /** - * Construct a source facade object from a given proto object. - * - * @param sourceSpec SourceProto.Source object - * @param isDefault Whether to return the default source object if the source was not defined by - * the user - * @return Source facade object - */ - public static Source fromProto(SourceProto.Source sourceSpec, boolean isDefault) { - - if (sourceSpec.equals(SourceProto.Source.getDefaultInstance())) { - Source source = new Source(); - source.setDefault(true); - return source; - } - - Source source = new Source(); - source.setType(sourceSpec.getType()); - - switch (sourceSpec.getType()) { - case KAFKA: - if (sourceSpec.getKafkaSourceConfig().getBootstrapServers().isEmpty() - || sourceSpec.getKafkaSourceConfig().getTopic().isEmpty()) { - throw Status.INVALID_ARGUMENT - .withDescription( - "Unsupported source options. Kafka source requires bootstrap servers and topic to be specified.") - .asRuntimeException(); - } - source.setConfig(sourceSpec.getKafkaSourceConfig().toString()); - break; - case UNRECOGNIZED: - default: - throw Status.INVALID_ARGUMENT - .withDescription("Unsupported source type. Only [KAFKA] is supported.") - .asRuntimeException(); - } - - source.setDefault(isDefault); - return source; - } - - /** - * Construct a source facade object from a given proto object. - * - * @param sourceSpec SourceProto.Source object - * @return Source facade object - */ - public static Source fromProto(SourceProto.Source sourceSpec) { - return fromProto(sourceSpec, false); - } - - /** - * Convert this object to its equivalent proto object. - * - * @return SourceProto.Source - */ - public SourceProto.Source toProto() { - Builder builder = SourceProto.Source.newBuilder().setType(this.getType()); - - switch (this.getType()) { - case KAFKA: - KafkaSourceConfig.Builder kafkaSourceConfig = KafkaSourceConfig.newBuilder(); - try { - com.google.protobuf.TextFormat.getParser().merge(this.getConfig(), kafkaSourceConfig); - } catch (TextFormat.ParseException e) { - throw new RuntimeException( - String.format( - "Unable to deserialize source configuration from String to KafkaSourceConfig: %s", - this.getConfig()), - e); - } - return builder.setKafkaSourceConfig(kafkaSourceConfig).build(); - case INVALID: - case UNRECOGNIZED: - default: - throw new RuntimeException( - String.format( - "Unable to build Source from configuration and type: %s %s", - this.getConfig(), this.getType())); - } - } - - /** - * Override equality for sources. Sources are compared based on their type and type-specific - * options. - * - * @param other other Source - * @return boolean equal - */ - public boolean equalTo(Source other) { - if ((this.getType() == null || !this.getType().equals(other.getType()))) { - return false; - } - - return this.getConfig().equals(other.getConfig()); - } - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - Source source = (Source) o; - return this.equalTo(source); - } - - @Override - public int hashCode() { - return Objects.hash(getType(), getConfig()); - } - - /** - * Returns the type of this Source in String format - * - * @return Source type in String format - */ - public String getTypeString() { - return this.getType().getValueDescriptor().getName(); - } -} diff --git a/core/src/main/java/feast/core/model/Store.java b/core/src/main/java/feast/core/model/Store.java index 9288217e74f..7a8f9a6d61a 100644 --- a/core/src/main/java/feast/core/model/Store.java +++ b/core/src/main/java/feast/core/model/Store.java @@ -21,9 +21,7 @@ import com.google.protobuf.InvalidProtocolBufferException; import feast.proto.core.StoreProto; -import feast.proto.core.StoreProto.Store.BigQueryConfig; import feast.proto.core.StoreProto.Store.Builder; -import feast.proto.core.StoreProto.Store.CassandraConfig; import feast.proto.core.StoreProto.Store.RedisClusterConfig; import feast.proto.core.StoreProto.Store.RedisConfig; import feast.proto.core.StoreProto.Store.StoreType; @@ -81,12 +79,6 @@ public static Store fromProto(StoreProto.Store storeProto) throws IllegalArgumen case REDIS: config = storeProto.getRedisConfig().toByteArray(); break; - case BIGQUERY: - config = storeProto.getBigqueryConfig().toByteArray(); - break; - case CASSANDRA: - config = storeProto.getCassandraConfig().toByteArray(); - break; case REDIS_CLUSTER: config = storeProto.getRedisClusterConfig().toByteArray(); break; @@ -108,12 +100,6 @@ public StoreProto.Store toProto() throws InvalidProtocolBufferException { case REDIS: RedisConfig redisConfig = RedisConfig.parseFrom(config); return storeProtoBuilder.setRedisConfig(redisConfig).build(); - case BIGQUERY: - BigQueryConfig bqConfig = BigQueryConfig.parseFrom(config); - return storeProtoBuilder.setBigqueryConfig(bqConfig).build(); - case CASSANDRA: - CassandraConfig cassConfig = CassandraConfig.parseFrom(config); - return storeProtoBuilder.setCassandraConfig(cassConfig).build(); case REDIS_CLUSTER: RedisClusterConfig redisClusterConfig = RedisClusterConfig.parseFrom(config); return storeProtoBuilder.setRedisClusterConfig(redisClusterConfig).build(); diff --git a/core/src/main/java/feast/core/service/SpecService.java b/core/src/main/java/feast/core/service/SpecService.java index 9f6569fb827..4a35d3ef3a4 100644 --- a/core/src/main/java/feast/core/service/SpecService.java +++ b/core/src/main/java/feast/core/service/SpecService.java @@ -16,38 +16,28 @@ */ package feast.core.service; -import static feast.common.models.Store.isSubscribedToFeatureSet; import static feast.core.validators.Matchers.checkValidCharacters; import static feast.core.validators.Matchers.checkValidCharactersAllowAsterisk; import com.google.protobuf.InvalidProtocolBufferException; import feast.core.dao.EntityRepository; -import feast.core.dao.FeatureSetRepository; import feast.core.dao.FeatureTableRepository; import feast.core.dao.ProjectRepository; import feast.core.dao.StoreRepository; -import feast.core.exception.RegistrationException; import feast.core.exception.RetrievalException; import feast.core.model.*; import feast.core.validators.EntityValidator; -import feast.core.validators.FeatureSetValidator; import feast.core.validators.FeatureTableValidator; import feast.proto.core.CoreServiceProto.ApplyEntityResponse; -import feast.proto.core.CoreServiceProto.ApplyFeatureSetResponse; -import feast.proto.core.CoreServiceProto.ApplyFeatureSetResponse.Status; import feast.proto.core.CoreServiceProto.ApplyFeatureTableRequest; import feast.proto.core.CoreServiceProto.ApplyFeatureTableResponse; import feast.proto.core.CoreServiceProto.DeleteFeatureTableRequest; import feast.proto.core.CoreServiceProto.GetEntityRequest; import feast.proto.core.CoreServiceProto.GetEntityResponse; -import feast.proto.core.CoreServiceProto.GetFeatureSetRequest; -import feast.proto.core.CoreServiceProto.GetFeatureSetResponse; import feast.proto.core.CoreServiceProto.GetFeatureTableRequest; import feast.proto.core.CoreServiceProto.GetFeatureTableResponse; import feast.proto.core.CoreServiceProto.ListEntitiesRequest; import feast.proto.core.CoreServiceProto.ListEntitiesResponse; -import feast.proto.core.CoreServiceProto.ListFeatureSetsRequest; -import feast.proto.core.CoreServiceProto.ListFeatureSetsResponse; import feast.proto.core.CoreServiceProto.ListFeatureTablesRequest; import feast.proto.core.CoreServiceProto.ListFeatureTablesResponse; import feast.proto.core.CoreServiceProto.ListFeaturesRequest; @@ -55,18 +45,12 @@ import feast.proto.core.CoreServiceProto.ListStoresRequest; import feast.proto.core.CoreServiceProto.ListStoresResponse; import feast.proto.core.CoreServiceProto.ListStoresResponse.Builder; -import feast.proto.core.CoreServiceProto.UpdateFeatureSetStatusRequest; -import feast.proto.core.CoreServiceProto.UpdateFeatureSetStatusResponse; import feast.proto.core.CoreServiceProto.UpdateStoreRequest; import feast.proto.core.CoreServiceProto.UpdateStoreResponse; import feast.proto.core.EntityProto; -import feast.proto.core.FeatureSetProto; -import feast.proto.core.FeatureSetProto.FeatureSetStatus; import feast.proto.core.FeatureTableProto.FeatureTableSpec; -import feast.proto.core.SourceProto; import feast.proto.core.StoreProto; import feast.proto.core.StoreProto.Store.Subscription; -import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.NoSuchElementException; @@ -86,26 +70,20 @@ public class SpecService { private final EntityRepository entityRepository; - private final FeatureSetRepository featureSetRepository; private final FeatureTableRepository tableRepository; private final ProjectRepository projectRepository; private final StoreRepository storeRepository; - private final Source defaultSource; @Autowired public SpecService( EntityRepository entityRepository, - FeatureSetRepository featureSetRepository, FeatureTableRepository tableRepository, StoreRepository storeRepository, - ProjectRepository projectRepository, - Source defaultSource) { + ProjectRepository projectRepository) { this.entityRepository = entityRepository; - this.featureSetRepository = featureSetRepository; this.tableRepository = tableRepository; this.storeRepository = storeRepository; this.projectRepository = projectRepository; - this.defaultSource = defaultSource; } /** @@ -143,142 +121,6 @@ public GetEntityResponse getEntity(GetEntityRequest request) { return response; } - /** - * Get a feature set matching the feature name and version and project. The feature set name and - * project are required, but version can be omitted by providing 0 for its value. If the version - * is omitted, the latest feature set will be provided. If the project is omitted, the default - * would be used. - * - * @param request: GetFeatureSetRequest Request containing filter parameters. - * @return Returns a GetFeatureSetResponse containing a feature set.. - */ - public GetFeatureSetResponse getFeatureSet(GetFeatureSetRequest request) - throws InvalidProtocolBufferException { - FeatureSet featureSet = getFeatureSet(request.getProject(), request.getName()); - - return GetFeatureSetResponse.newBuilder().setFeatureSet(featureSet.toProto()).build(); - } - - private FeatureSet getFeatureSet(String projectName, String featureSetName) { - // Validate input arguments - checkValidCharacters(featureSetName, "featureset"); - - if (featureSetName.isEmpty()) { - throw new IllegalArgumentException("No feature set name provided"); - } - // Autofill default project if project is not specified - if (projectName.isEmpty()) { - projectName = Project.DEFAULT_NAME; - } - - FeatureSet featureSet; - - featureSet = - featureSetRepository.findFeatureSetByNameAndProject_Name(featureSetName, projectName); - - if (featureSet == null) { - throw new RetrievalException( - String.format("Feature set with name \"%s\" could not be found.", featureSetName)); - } - return featureSet; - } - - /** - * Return a list of feature sets matching the feature set name, project and labels provided in the - * filter. All fields are required. Use '*' in feature set name and project, and empty map in - * labels in order to return all feature sets in all projects. - * - *

Project name can be explicitly provided, or an asterisk can be provided to match all - * projects. It is not possible to provide a combination of asterisks/wildcards and text. If the - * project name is omitted, the default project would be used. - * - *

The feature set name in the filter accepts an asterisk as a wildcard. All matching feature - * sets will be returned. Regex is not supported. Explicitly defining a feature set name is not - * possible if a project name is not set explicitly - * - *

The labels in the filter accepts a map. All feature sets which contain every provided label - * will be returned. - * - * @param filter filter containing the desired featureSet name - * @return ListFeatureSetsResponse with list of featureSets found matching the filter - */ - public ListFeatureSetsResponse listFeatureSets(ListFeatureSetsRequest.Filter filter) - throws InvalidProtocolBufferException { - String name = filter.getFeatureSetName(); - String project = filter.getProject(); - Map labelsFilter = filter.getLabelsMap(); - FeatureSetStatus statusFilter = filter.getStatus(); - - if (name.isEmpty()) { - throw new IllegalArgumentException( - "Invalid listFeatureSetRequest, missing arguments. Must provide feature set name:"); - } - - checkValidCharactersAllowAsterisk(name, "featureset"); - checkValidCharactersAllowAsterisk(project, "project"); - - // Autofill default project if project not specified - if (project.isEmpty()) { - project = Project.DEFAULT_NAME; - } - - List featureSets = new ArrayList() {}; - - if (project.contains("*")) { - // Matching a wildcard project - if (name.contains("*")) { - featureSets = - featureSetRepository.findAllByNameLikeAndProject_NameLikeOrderByNameAsc( - name.replace('*', '%'), project.replace('*', '%')); - } else { - throw new IllegalArgumentException( - String.format( - "Invalid listFeatureSetRequest. Feature set name must be set to " - + "\"*\" if the project name and feature set name aren't set explicitly: \n%s", - filter.toString())); - } - } else if (!project.contains("*")) { - // Matching a specific project - if (name.contains("*")) { - // Find all feature sets matching a pattern in a specific project - featureSets = - featureSetRepository.findAllByNameLikeAndProject_NameOrderByNameAsc( - name.replace('*', '%'), project); - - } else if (!name.contains("*")) { - // Find a specific feature set in a specific project - FeatureSet featureSet = - featureSetRepository.findFeatureSetByNameAndProject_Name(name, project); - if (featureSet != null) { - featureSets.add(featureSet); - } - } - } else { - throw new IllegalArgumentException( - String.format( - "Invalid listFeatureSetRequest. Project name cannot be a pattern. It may only be" - + "a specific project name or an asterisk: \n%s", - filter.toString())); - } - - ListFeatureSetsResponse.Builder response = ListFeatureSetsResponse.newBuilder(); - if (featureSets.size() > 0) { - featureSets = - featureSets.stream() - .filter( - featureSet -> - statusFilter.equals(FeatureSetStatus.STATUS_INVALID) - || featureSet.getStatus().equals(statusFilter)) - .filter(featureSet -> featureSet.hasAllLabels(labelsFilter)) - .collect(Collectors.toList()); - for (FeatureSet featureSet : featureSets) { - response.addFeatureSets(featureSet.toProto()); - } - } - - return response.build(); - } - /** * Return a map of feature references and features matching the project, labels and entities * provided in the filter. All fields are required. @@ -297,46 +139,37 @@ public ListFeatureSetsResponse listFeatureSets(ListFeatureSetsRequest.Filter fil * filter */ public ListFeaturesResponse listFeatures(ListFeaturesRequest.Filter filter) { - try { - String project = filter.getProject(); - List entities = filter.getEntitiesList(); - Map labels = filter.getLabelsMap(); + String project = filter.getProject(); + List entities = filter.getEntitiesList(); + Map labels = filter.getLabelsMap(); - checkValidCharactersAllowAsterisk(project, "project"); + checkValidCharactersAllowAsterisk(project, "project"); - // Autofill default project if project not specified - if (project.isEmpty()) { - project = Project.DEFAULT_NAME; - } + // Autofill default project if project not specified + if (project.isEmpty()) { + project = Project.DEFAULT_NAME; + } - // Currently defaults to all FeatureSets - List featureSets = - featureSetRepository.findAllByNameLikeAndProject_NameOrderByNameAsc("%", project); - // TODO: List features in Feature Tables. - - ListFeaturesResponse.Builder response = ListFeaturesResponse.newBuilder(); - if (entities.size() > 0) { - featureSets = - featureSets.stream() - .filter(featureSet -> featureSet.hasAllEntities(entities)) - .collect(Collectors.toList()); - } + // Currently defaults to all FeatureTables + List featureTables = tableRepository.findAllByProject_Name(project); - Map featuresMap; - for (FeatureSet featureSet : featureSets) { - featuresMap = featureSet.getFeaturesByRef(labels); - for (Map.Entry entry : featuresMap.entrySet()) { - response.putFeatures(entry.getKey(), entry.getValue().toProto()); - } - } + ListFeaturesResponse.Builder response = ListFeaturesResponse.newBuilder(); + if (entities.size() > 0) { + featureTables = + featureTables.stream() + .filter(featureTable -> featureTable.hasAllEntities(entities)) + .collect(Collectors.toList()); + } - return response.build(); - } catch (InvalidProtocolBufferException e) { - throw io.grpc.Status.NOT_FOUND - .withDescription("Unable to retrieve features") - .withCause(e) - .asRuntimeException(); + Map featuresMap; + for (FeatureTable featureTable : featureTables) { + featuresMap = featureTable.getFeaturesByLabels(labels); + for (Map.Entry entry : featuresMap.entrySet()) { + response.putFeatures(entry.getKey(), entry.getValue().toProto()); + } } + + return response.build(); } /** @@ -385,18 +218,6 @@ public ListEntitiesResponse listEntities(ListEntitiesRequest.Filter filter) { return response.build(); } - /** Update FeatureSet's status by given FeatureSetReference and new status */ - public UpdateFeatureSetStatusResponse updateFeatureSetStatus( - UpdateFeatureSetStatusRequest request) { - FeatureSet featureSet = - getFeatureSet(request.getReference().getProject(), request.getReference().getName()); - - featureSet.setStatus(request.getStatus()); - featureSetRepository.saveAndFlush(featureSet); - - return UpdateFeatureSetStatusResponse.newBuilder().build(); - } - /** * Get stores matching the store name provided in the filter. If the store name is not provided, * the method will return all stores currently registered to Feast. @@ -487,109 +308,6 @@ public ApplyEntityResponse applyEntity( return response; } - /** - * Creates or updates a feature set in the repository. - * - *

This function is idempotent. If no changes are detected in the incoming featureSet's schema, - * this method will update the incoming featureSet spec with the latest version stored in the - * repository, and return that. If project is not specified in the given featureSet, will assign - * the featureSet to the'default' project. - * - * @param newFeatureSet Feature set that will be created or updated. - */ - @Transactional - public ApplyFeatureSetResponse applyFeatureSet(FeatureSetProto.FeatureSet newFeatureSet) - throws InvalidProtocolBufferException { - // Autofill default project if not specified - if (newFeatureSet.getSpec().getProject().isEmpty()) { - newFeatureSet = - newFeatureSet - .toBuilder() - .setSpec(newFeatureSet.getSpec().toBuilder().setProject(Project.DEFAULT_NAME).build()) - .build(); - } - - String projectName = newFeatureSet.getSpec().getProject(); - String featureSetName = newFeatureSet.getSpec().getName(); - List isSubscribedToStores = new ArrayList<>() {}; - for (Store store : storeRepository.findAll()) { - List subscriptionList = store.getSubscriptions(); - boolean isSubscribed = - isSubscribedToFeatureSet(subscriptionList, projectName, featureSetName); - isSubscribedToStores.add(isSubscribed); - } - // Only throw error if FeatureSet is not subscribed by ALL stores - if (!isSubscribedToStores.isEmpty() - && isSubscribedToStores.stream().allMatch(x -> x == false)) { - throw new RegistrationException( - String.format( - "The supplied Project and FeatureSet, %s/%s is either not subscribed or blacklisted and is not available for registration. " - + "Please ask your administrator to update subscription in store configuration on serving layer.", - projectName, featureSetName)); - } - - // Validate incoming feature set - FeatureSetValidator.validateSpec(newFeatureSet); - - // Find project or create new one if it does not exist - String project_name = newFeatureSet.getSpec().getProject(); - Project project = - projectRepository - .findById(newFeatureSet.getSpec().getProject()) - .orElse(new Project(project_name)); - - // Ensure that the project retrieved from repository is not archived - if (project.isArchived()) { - throw new IllegalArgumentException(String.format("Project is archived: %s", project_name)); - } - - // Set source to default if not set in proto - if (newFeatureSet.getSpec().getSource() == SourceProto.Source.getDefaultInstance()) { - newFeatureSet = - newFeatureSet - .toBuilder() - .setSpec( - newFeatureSet.getSpec().toBuilder().setSource(defaultSource.toProto()).build()) - .build(); - } - - // Retrieve existing FeatureSet - FeatureSet featureSet = - featureSetRepository.findFeatureSetByNameAndProject_Name( - newFeatureSet.getSpec().getName(), project_name); - - Status status; - if (featureSet == null) { - // Create new feature set since it doesn't exist - newFeatureSet = newFeatureSet.toBuilder().setSpec(newFeatureSet.getSpec()).build(); - featureSet = FeatureSet.fromProto(newFeatureSet); - status = Status.CREATED; - } else { - // If the featureSet remains unchanged, we do nothing. - if (featureSet.toProto().getSpec().equals(newFeatureSet.getSpec())) { - return ApplyFeatureSetResponse.newBuilder() - .setFeatureSet(featureSet.toProto()) - .setStatus(Status.NO_CHANGE) - .build(); - } - featureSet.updateFromProto(newFeatureSet); - status = Status.UPDATED; - } - - featureSet.incVersion(); - - // Persist the FeatureSet object - featureSet.setStatus(FeatureSetStatus.STATUS_PENDING); - project.addFeatureSet(featureSet); - projectRepository.saveAndFlush(project); - - // Build ApplyFeatureSetResponse - return ApplyFeatureSetResponse.newBuilder() - .setFeatureSet(featureSet.toProto()) - .setStatus(status) - .build(); - } - /** * Resolves the project name by returning name if given, autofilling default project otherwise. * diff --git a/core/src/main/java/feast/core/service/StatsService.java b/core/src/main/java/feast/core/service/StatsService.java deleted file mode 100644 index e790e4e97eb..00000000000 --- a/core/src/main/java/feast/core/service/StatsService.java +++ /dev/null @@ -1,640 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.core.service; - -import static java.lang.Math.*; - -import com.google.common.annotations.VisibleForTesting; -import com.google.protobuf.InvalidProtocolBufferException; -import com.google.protobuf.Timestamp; -import feast.core.dao.FeatureSetRepository; -import feast.core.dao.FeatureStatisticsRepository; -import feast.core.dao.StoreRepository; -import feast.core.exception.RetrievalException; -import feast.core.model.*; -import feast.core.model.Feature; -import feast.proto.core.CoreServiceProto.GetFeatureStatisticsRequest; -import feast.proto.core.CoreServiceProto.GetFeatureStatisticsResponse; -import feast.proto.core.StoreProto; -import feast.proto.core.StoreProto.Store.StoreType; -import feast.storage.api.statistics.FeatureStatistics; -import feast.storage.api.statistics.StatisticsRetriever; -import feast.storage.connectors.bigquery.statistics.BigQueryStatisticsRetriever; -import java.io.IOException; -import java.time.Instant; -import java.util.*; -import java.util.stream.Collectors; -import lombok.extern.slf4j.Slf4j; -import org.joda.time.DateTime; -import org.joda.time.DateTimeZone; -import org.joda.time.format.DateTimeFormat; -import org.joda.time.format.DateTimeFormatter; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Service; -import org.springframework.transaction.annotation.Transactional; -import org.tensorflow.metadata.v0.*; -import org.tensorflow.metadata.v0.FeatureNameStatistics.Builder; - -/** Facilitates the retrieval of feature set statistics from historical stores. */ -@Slf4j -@Service -public class StatsService { - - private StoreRepository storeRepository; - private FeatureStatisticsRepository featureStatisticsRepository; - private FeatureSetRepository featureSetRepository; - - @Autowired - public StatsService( - StoreRepository storeRepository, - FeatureStatisticsRepository featureStatisticsRepository, - FeatureSetRepository featureSetRepository) { - this.storeRepository = storeRepository; - this.featureStatisticsRepository = featureStatisticsRepository; - this.featureSetRepository = featureSetRepository; - } - - /** - * Get {@link DatasetFeatureStatistics} for the requested feature set in the provided datasets or - * date range for the store provided. The {@link DatasetFeatureStatistics} will contain a list of - * {@link FeatureNameStatistics} for each feature requested. Results retrieved will be cached - * indefinitely. To force Feast to recompute the statistics, set forceRefresh to true. - * - *

Only one of ingestionIds or startDate/endDate should be provided. If both are provided, the - * former will be used over the latter. - * - *

If multiple datasetIds or if the date ranges over a few days, statistics will be retrieved - * for each single unit (dataset id or day) and results aggregated across that set. As a result of - * this, in such a scenario, statistics that cannot be aggregated will be dropped. This includes - * all histograms and quantiles, unique values, and top value counts. - * - * @param request {@link GetFeatureStatisticsRequest} containing feature set name, subset of - * features, dataset ids or date range, and store to retrieve the data from. - * @return {@link GetFeatureStatisticsResponse} containing {@link DatasetFeatureStatistics} with - * the feature statistics requested. - * @throws IOException - */ - @Transactional - public GetFeatureStatisticsResponse getFeatureStatistics(GetFeatureStatisticsRequest request) - throws IOException { - - // Validate the request - validateRequest(request); - - // Get the stats retriever for the store requested - StatisticsRetriever statisticsRetriever = getStatisticsRetriever(request.getStore()); - - // 1. Retrieve the feature set spec from the db - FeatureSet featureSet = getFeatureSet(request.getFeatureSetId()); - if (featureSet == null) { - throw new IllegalArgumentException( - String.format( - "Illegal request. Unable to find feature set %s", request.getFeatureSetId())); - } - - // 2. Filter out the features requested by the user. If none are provided, - // use all features in the feature set. - List features = request.getFeaturesList(); - if (features.size() == 0) { - features = - featureSet.getFeatures().stream() - .filter(feature -> !feature.isArchived()) - .map(Feature::getName) - .collect(Collectors.toList()); - } - - // 3. Retrieve the statistics from the StatsRetriever. - List> featureNameStatisticsList = new ArrayList<>(); - if (request.getIngestionIdsCount() == 0) { - Timestamp endDate = request.getEndDate(); - Timestamp startDate = request.getStartDate(); - // If no dataset provided, retrieve by date - - long timestamp = startDate.getSeconds(); - while (timestamp < endDate.getSeconds()) { - List featureNameStatistics = - getFeatureNameStatisticsByDate( - statisticsRetriever, featureSet, features, timestamp, request.getForceRefresh()); - if (featureNameStatistics.size() != 0) { - featureNameStatisticsList.add(featureNameStatistics); - } - timestamp += 86400; // advance by a day - } - if (featureNameStatisticsList.size() == 0) { - DateTimeFormatter fmt = DateTimeFormat.forPattern("yyyy-MM-dd"); - DateTime startDateTime = new DateTime(startDate.getSeconds() * 1000, DateTimeZone.UTC); - DateTime endDateTime = new DateTime(endDate.getSeconds() * 1000, DateTimeZone.UTC); - throw new RetrievalException( - String.format( - "Unable to find any data over provided dates [%s, %s)", - fmt.print(startDateTime), fmt.print(endDateTime))); - } - } else { - // else, retrieve by dataset - for (String datasetId : request.getIngestionIdsList()) { - List featureNameStatistics = - getFeatureNameStatisticsByDataset( - statisticsRetriever, featureSet, features, datasetId, request.getForceRefresh()); - if (featureNameStatistics.size() != 0) { - featureNameStatisticsList.add(featureNameStatistics); - } - } - if (featureNameStatisticsList.size() == 0) { - throw new RetrievalException( - String.format( - "Unable to find any data over provided data sets %s", - request.getIngestionIdsList())); - } - } - - // Merge statistics values across days/datasets - List featureNameStatistics = mergeStatistics(featureNameStatisticsList); - long totalCount = getTotalCount(featureNameStatistics.get(0)); - return GetFeatureStatisticsResponse.newBuilder() - .setDatasetFeatureStatisticsList( - DatasetFeatureStatisticsList.newBuilder() - .addDatasets( - DatasetFeatureStatistics.newBuilder() - .setNumExamples(totalCount) - .addAllFeatures(featureNameStatistics))) - .build(); - } - - /** - * Get {@link FeatureNameStatistics} by dataset id. - * - * @param statisticsRetriever {@link StatisticsRetriever} corresponding to the store to get the - * data from. - * @param featureSet {@link FeatureSet} requested by the user - * @param features features to retrieve - * @param datasetId dataset id to subset the data by - * @param forceRefresh whether to override the values in the cache - * @return {@link FeatureNameStatistics} for the data within the dataset id provided - * @throws IOException - */ - private List getFeatureNameStatisticsByDataset( - StatisticsRetriever statisticsRetriever, - FeatureSet featureSet, - List features, - String datasetId, - boolean forceRefresh) - throws IOException { - List featureNameStatistics = new ArrayList<>(); - List featuresMissingStats = new ArrayList<>(); - Map featureNameToFeature = - featureSet.getFeatures().stream().collect(Collectors.toMap(Feature::getName, f -> f)); - - // For each feature requested, check if statistics already exist in the cache - // If not refreshing data in the cache, retrieve the cached data and add it to the - // list of FeatureNameStatistics for this dataset. - // Else, add to the list of features we still need to retrieve statistics for. - for (String featureName : features) { - Feature feature = featureNameToFeature.get(featureName); - Optional cachedFeatureStatistics = Optional.empty(); - if (!forceRefresh) { - cachedFeatureStatistics = - featureStatisticsRepository.findFeatureStatisticsByFeatureAndDatasetId( - feature, datasetId); - } - if (cachedFeatureStatistics.isPresent()) { - featureNameStatistics.add(cachedFeatureStatistics.get().toProto()); - } else { - featuresMissingStats.add(featureName); - } - } - - // Retrieve the balance of statistics after checking the cache, and add it to the - // list of FeatureNameStatistics. - if (featuresMissingStats.size() > 0) { - FeatureStatistics featureSetStatistics = - statisticsRetriever.getFeatureStatistics( - featureSet.toProto().getSpec(), featuresMissingStats, datasetId); - - // Persist the newly retrieved statistics in the cache. - for (FeatureNameStatistics stat : featureSetStatistics.getFeatureNameStatistics()) { - if (isEmpty(stat)) { - continue; - } - Feature feature = featureNameToFeature.get(stat.getName()); - feast.core.model.FeatureStatistics featureStatistics = - feast.core.model.FeatureStatistics.createForDataset(feature, stat, datasetId); - Optional existingRecord = - featureStatisticsRepository.findFeatureStatisticsByFeatureAndDatasetId( - featureStatistics.getFeature(), datasetId); - existingRecord.ifPresent(statistics -> featureStatistics.setId(statistics.getId())); - featureStatisticsRepository.save(featureStatistics); - featureNameStatistics.add(stat); - } - } - return featureNameStatistics; - } - - /** - * Get {@link FeatureNameStatistics} by date. - * - * @param statisticsRetriever {@link StatisticsRetriever} corresponding to the store to get the - * data from. - * @param featureSet {@link FeatureSet} requested by the user - * @param features features to retrieve - * @param timestamp timestamp of the date to subset the data - * @param forceRefresh whether to override the values in the cache - * @return {@link FeatureNameStatistics} for the data within the dataset id provided - * @throws IOException - */ - private List getFeatureNameStatisticsByDate( - StatisticsRetriever statisticsRetriever, - FeatureSet featureSet, - List features, - long timestamp, - boolean forceRefresh) - throws IOException { - Date date = Date.from(Instant.ofEpochSecond(timestamp)); - List featureNameStatistics = new ArrayList<>(); - List featuresMissingStats = new ArrayList<>(); - Map featureNameToFeature = - featureSet.getFeatures().stream().collect(Collectors.toMap(Feature::getName, f -> f)); - - // For each feature requested, check if statistics already exist in the cache - // If not refreshing data in the cache, retrieve the cached data and add it to the - // list of FeatureNameStatistics for this date. - // Else, add to the list of features we still need to retrieve statistics for. - for (String featureName : features) { - Feature feature = featureNameToFeature.get(featureName); - Optional cachedFeatureStatistics = Optional.empty(); - if (!forceRefresh) { - cachedFeatureStatistics = - featureStatisticsRepository.findFeatureStatisticsByFeatureAndDate(feature, date); - } - if (cachedFeatureStatistics.isPresent()) { - featureNameStatistics.add(cachedFeatureStatistics.get().toProto()); - } else { - featuresMissingStats.add(featureName); - } - } - - // Retrieve the balance of statistics after checking the cache, and add it to the - // list of FeatureNameStatistics. - if (featuresMissingStats.size() > 0) { - FeatureStatistics featureSetStatistics = - statisticsRetriever.getFeatureStatistics( - featureSet.toProto().getSpec(), - featuresMissingStats, - Timestamp.newBuilder().setSeconds(timestamp).build()); - - // Persist the newly retrieved statistics in the cache. - for (FeatureNameStatistics stat : featureSetStatistics.getFeatureNameStatistics()) { - if (isEmpty(stat)) { - continue; - } - Feature feature = featureNameToFeature.get(stat.getName()); - feast.core.model.FeatureStatistics featureStatistics = - feast.core.model.FeatureStatistics.createForDate(feature, stat, date); - Optional existingRecord = - featureStatisticsRepository.findFeatureStatisticsByFeatureAndDate( - featureStatistics.getFeature(), date); - existingRecord.ifPresent(statistics -> featureStatistics.setId(statistics.getId())); - featureStatisticsRepository.save(featureStatistics); - featureNameStatistics.add(stat); - } - } - return featureNameStatistics; - } - - /** - * Get the {@link StatisticsRetriever} corresponding to the store name provided. - * - * @param storeName name of the store to retrieve statistics from - * @return {@link StatisticsRetriever} - */ - StatisticsRetriever getStatisticsRetriever(String storeName) - throws InvalidProtocolBufferException { - Store store = - storeRepository - .findById(storeName) - .orElseThrow( - () -> - new RetrievalException( - String.format("Could not find store with name %s", storeName))); - StoreProto.Store storeProto = store.toProto(); - if (storeProto.getType() != StoreType.BIGQUERY) { - throw new IllegalArgumentException( - String.format( - "Invalid store %s with type %s specified. Batch statistics are only supported for BigQuery stores", - store.getName(), store.getType())); - } - return BigQueryStatisticsRetriever.create(storeProto.getBigqueryConfig()); - } - - private FeatureSet getFeatureSet(String featureSetId) { - String[] split = featureSetId.split("/"); - String project; - String featureSetName; - if (split.length == 1) { - project = Project.DEFAULT_NAME; - featureSetName = split[0]; - } else { - project = split[0]; - featureSetName = split[1]; - } - FeatureSet featureSet = - featureSetRepository.findFeatureSetByNameAndProject_Name(featureSetName, project); - return featureSet; - } - - /** - * Merge feature statistics by name. This method is used to merge statistics retrieved over - * multiple days or datasets. - * - * @param featureNameStatistics {@link FeatureNameStatistics} retrieved from the store - * @return Merged list of {@link FeatureNameStatistics} by name - */ - @VisibleForTesting - public List mergeStatistics( - List> featureNameStatistics) { - List unnestedList = new ArrayList<>(); - - featureNameStatistics.forEach(unnestedList::addAll); - Map> groupByPath = - unnestedList.stream() - .collect(Collectors.groupingBy(FeatureNameStatistics::getPath, Collectors.toList())); - - List merged = new ArrayList<>(); - for (Path key : groupByPath.keySet()) { - List featureNameStatisticsForKey = groupByPath.get(key); - if (featureNameStatisticsForKey.size() == 1) { - merged.add(featureNameStatisticsForKey.get(0)); - } else { - switch (featureNameStatisticsForKey.get(0).getType()) { - case INT: - case FLOAT: - merged.add(mergeNumStatistics(featureNameStatisticsForKey)); - break; - case STRING: - merged.add(mergeCategoricalStatistics(groupByPath.get(key))); - break; - case BYTES: - merged.add(mergeByteStatistics(groupByPath.get(key))); - break; - case STRUCT: - merged.add(mergeStructStats(groupByPath.get(key))); - break; - default: - throw new IllegalArgumentException( - "Statistics are only supported for string, boolean, bytes and numeric features"); - } - } - } - return merged; - } - - private FeatureNameStatistics mergeStructStats( - List featureNameStatisticsList) { - Builder mergedFeatureNameStatistics = - FeatureNameStatistics.newBuilder() - .setPath(featureNameStatisticsList.get(0).getPath()) - .setType(featureNameStatisticsList.get(0).getType()); - - long totalCount = 0; - long missingCount = 0; - long totalNumValues = 0; - long maxNumValues = - featureNameStatisticsList.get(0).getStructStats().getCommonStats().getMaxNumValues(); - long minNumValues = - featureNameStatisticsList.get(0).getStructStats().getCommonStats().getMinNumValues(); - - for (FeatureNameStatistics featureNameStatistics : featureNameStatisticsList) { - StructStatistics structStats = featureNameStatistics.getStructStats(); - totalCount += structStats.getCommonStats().getNumNonMissing(); - missingCount += structStats.getCommonStats().getNumMissing(); - totalNumValues += - structStats.getCommonStats().getAvgNumValues() - * structStats.getCommonStats().getNumNonMissing(); - maxNumValues = max(maxNumValues, structStats.getCommonStats().getMaxNumValues()); - minNumValues = min(minNumValues, structStats.getCommonStats().getMinNumValues()); - } - - StructStatistics mergedStructStatistics = - StructStatistics.newBuilder() - .setCommonStats( - CommonStatistics.newBuilder() - .setTotNumValues(totalNumValues) - .setNumNonMissing(totalCount) - .setAvgNumValues((float) totalNumValues / totalCount) - .setMaxNumValues(maxNumValues) - .setMinNumValues(minNumValues) - .setNumMissing(missingCount)) - .build(); - - return mergedFeatureNameStatistics.setStructStats(mergedStructStatistics).build(); - } - - private FeatureNameStatistics mergeNumStatistics( - List featureNameStatisticsList) { - Builder mergedFeatureNameStatistics = - FeatureNameStatistics.newBuilder() - .setPath(featureNameStatisticsList.get(0).getPath()) - .setType(featureNameStatisticsList.get(0).getType()); - - FeatureNameStatistics first = featureNameStatisticsList.remove(0); - double max = first.getNumStats().getMax(); - double min = first.getNumStats().getMin(); - double var = pow(first.getNumStats().getStdDev(), 2); - long totalCount = first.getNumStats().getCommonStats().getNumNonMissing(); - double totalVal = totalCount * first.getNumStats().getMean(); - long missingCount = first.getNumStats().getCommonStats().getNumMissing(); - long zeroes = first.getNumStats().getNumZeros(); - - for (FeatureNameStatistics featureNameStatistics : featureNameStatisticsList) { - NumericStatistics numStats = featureNameStatistics.getNumStats(); - max = max(numStats.getMax(), max); - min = min(numStats.getMin(), min); - long count = numStats.getCommonStats().getNumNonMissing(); - double sampleVar = pow(numStats.getStdDev(), 2); - float aggMean = (float) totalVal / totalCount; - var = getVar(var, totalCount, aggMean, sampleVar, count, numStats.getMean()); - totalVal += numStats.getMean() * count; - totalCount += count; - missingCount += numStats.getCommonStats().getNumMissing(); - zeroes += numStats.getNumZeros(); - } - NumericStatistics mergedNumericStatistics = - NumericStatistics.newBuilder() - .setMax(max) - .setMin(min) - .setMean(totalVal / totalCount) - .setNumZeros(zeroes) - .setStdDev(sqrt(var)) - .setCommonStats( - CommonStatistics.newBuilder() - .setTotNumValues(totalCount) - .setNumNonMissing(totalCount) - .setAvgNumValues(1) - .setMaxNumValues(1) - .setMinNumValues(1) - .setNumMissing(missingCount)) - .build(); - return mergedFeatureNameStatistics.setNumStats(mergedNumericStatistics).build(); - } - - // Aggregation of sample variance follows the formula described here: - // https://www.tandfonline.com/doi/abs/10.1080/00031305.2014.966589 - private double getVar( - double s1Var, long s1Count, double s1Mean, double s2Var, long s2Count, double s2Mean) { - long totalCount = s1Count + s2Count; - return ((s1Count - 1) * s1Var - + (s2Count - 1) * s2Var - + ((float) s1Count * s2Count / totalCount) * pow(s1Mean - s2Mean, 2)) - / (s1Count + s2Count - 1); - } - - private FeatureNameStatistics mergeCategoricalStatistics( - List featureNameStatisticsList) { - Builder mergedFeatureNameStatistics = - FeatureNameStatistics.newBuilder() - .setPath(featureNameStatisticsList.get(0).getPath()) - .setType(featureNameStatisticsList.get(0).getType()); - long totalCount = 0; - long missingCount = 0; - long totalLen = 0; - for (FeatureNameStatistics featureNameStatistics : featureNameStatisticsList) { - StringStatistics stringStats = featureNameStatistics.getStringStats(); - totalCount += stringStats.getCommonStats().getNumNonMissing(); - missingCount += stringStats.getCommonStats().getNumMissing(); - totalLen += stringStats.getAvgLength() * stringStats.getCommonStats().getNumNonMissing(); - } - StringStatistics mergedStringStatistics = - StringStatistics.newBuilder() - .setAvgLength((float) totalLen / totalCount) - .setCommonStats( - CommonStatistics.newBuilder() - .setTotNumValues(totalCount) - .setNumNonMissing(totalCount) - .setAvgNumValues(1) - .setMaxNumValues(1) - .setMinNumValues(1) - .setNumMissing(missingCount)) - .build(); - return mergedFeatureNameStatistics.setStringStats(mergedStringStatistics).build(); - } - - private FeatureNameStatistics mergeByteStatistics( - List featureNameStatisticsList) { - Builder mergedFeatureNameStatistics = - FeatureNameStatistics.newBuilder() - .setPath(featureNameStatisticsList.get(0).getPath()) - .setType(featureNameStatisticsList.get(0).getType()); - - long totalCount = 0; - long missingCount = 0; - float totalNumBytes = 0; - float maxNumBytes = featureNameStatisticsList.get(0).getBytesStats().getMaxNumBytes(); - float minNumBytes = featureNameStatisticsList.get(0).getBytesStats().getMinNumBytes(); - - for (FeatureNameStatistics featureNameStatistics : featureNameStatisticsList) { - BytesStatistics bytesStats = featureNameStatistics.getBytesStats(); - totalCount += bytesStats.getCommonStats().getNumNonMissing(); - missingCount += bytesStats.getCommonStats().getNumMissing(); - totalNumBytes += bytesStats.getAvgNumBytes() * bytesStats.getCommonStats().getNumNonMissing(); - maxNumBytes = max(maxNumBytes, bytesStats.getMaxNumBytes()); - minNumBytes = min(minNumBytes, bytesStats.getMinNumBytes()); - } - - BytesStatistics mergedBytesStatistics = - BytesStatistics.newBuilder() - .setAvgNumBytes(totalNumBytes / totalCount) - .setMinNumBytes(minNumBytes) - .setMaxNumBytes(maxNumBytes) - .setCommonStats( - CommonStatistics.newBuilder() - .setTotNumValues(totalCount) - .setNumNonMissing(totalCount) - .setAvgNumValues(1) - .setMaxNumValues(1) - .setMinNumValues(1) - .setNumMissing(missingCount)) - .build(); - - return mergedFeatureNameStatistics.setBytesStats(mergedBytesStatistics).build(); - } - - private long getTotalCount(FeatureNameStatistics featureNameStatistics) { - CommonStatistics commonStats; - switch (featureNameStatistics.getType()) { - case STRUCT: - commonStats = featureNameStatistics.getStructStats().getCommonStats(); - break; - case STRING: - commonStats = featureNameStatistics.getStringStats().getCommonStats(); - break; - case BYTES: - commonStats = featureNameStatistics.getBytesStats().getCommonStats(); - break; - case FLOAT: - case INT: - commonStats = featureNameStatistics.getNumStats().getCommonStats(); - break; - default: - throw new RuntimeException("Unable to extract dataset size; Invalid type provided"); - } - return commonStats.getNumNonMissing() + commonStats.getNumMissing(); - } - - private void validateRequest(GetFeatureStatisticsRequest request) { - if (request.getIngestionIdsCount() == 0) { - Timestamp startDate = request.getStartDate(); - Timestamp endDate = request.getEndDate(); - if (!request.hasStartDate() || !request.hasEndDate()) { - throw new IllegalArgumentException( - "Invalid request. Either provide dataset ids to retrieve statistics over, or a start date and end date."); - } - if (endDate.getSeconds() < startDate.getSeconds()) { - throw new IllegalArgumentException( - String.format( - "Invalid request. Start timestamp %d is greater than the end timestamp %d", - startDate.getSeconds(), endDate.getSeconds())); - } - } - } - - private boolean isEmpty(FeatureNameStatistics featureNameStatistics) { - switch (featureNameStatistics.getType()) { - case STRUCT: - return featureNameStatistics - .getStructStats() - .getCommonStats() - .equals(CommonStatistics.getDefaultInstance()); - case STRING: - return featureNameStatistics - .getStringStats() - .getCommonStats() - .equals(CommonStatistics.getDefaultInstance()); - case BYTES: - return featureNameStatistics - .getBytesStats() - .getCommonStats() - .equals(CommonStatistics.getDefaultInstance()); - case FLOAT: - case INT: - return featureNameStatistics - .getNumStats() - .getCommonStats() - .equals(CommonStatistics.getDefaultInstance()); - default: - return true; - } - } -} diff --git a/core/src/main/java/feast/core/validators/FeatureSetValidator.java b/core/src/main/java/feast/core/validators/FeatureSetValidator.java deleted file mode 100644 index 8787d75f69f..00000000000 --- a/core/src/main/java/feast/core/validators/FeatureSetValidator.java +++ /dev/null @@ -1,86 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2019 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.core.validators; - -import static feast.core.validators.Matchers.checkValidCharacters; - -import com.google.common.collect.Sets; -import feast.proto.core.FeatureSetProto.EntitySpec; -import feast.proto.core.FeatureSetProto.FeatureSet; -import feast.proto.core.FeatureSetProto.FeatureSpec; -import java.util.Arrays; -import java.util.HashSet; -import java.util.List; -import java.util.stream.Collectors; -import org.apache.commons.lang3.StringUtils; - -public class FeatureSetValidator { - - private static List reservedNames = - Arrays.asList("created_timestamp", "event_timestamp", "ingestion_id", "job_id"); - - public static void validateSpec(FeatureSet featureSet) { - if (featureSet.getSpec().getProject().isEmpty()) { - throw new IllegalArgumentException("Project name must be provided"); - } - if (featureSet.getSpec().getName().isEmpty()) { - throw new IllegalArgumentException("Feature set name must be provided"); - } - if (featureSet.getSpec().getLabelsMap().containsKey("")) { - throw new IllegalArgumentException("Feature set label keys must not be empty"); - } - - checkValidCharacters(featureSet.getSpec().getProject(), "project"); - checkValidCharacters(featureSet.getSpec().getName(), "featureset"); - checkUniqueColumns( - featureSet.getSpec().getEntitiesList(), featureSet.getSpec().getFeaturesList()); - checkReservedColumns(featureSet.getSpec().getFeaturesList()); - for (EntitySpec entitySpec : featureSet.getSpec().getEntitiesList()) { - checkValidCharacters(entitySpec.getName(), "entity"); - } - for (FeatureSpec featureSpec : featureSet.getSpec().getFeaturesList()) { - checkValidCharacters(featureSpec.getName(), "feature"); - if (featureSpec.getLabelsMap().containsKey("")) { - throw new IllegalArgumentException("Feature label keys must not be empty"); - } - } - } - - private static void checkUniqueColumns( - List entitySpecs, List featureSpecs) { - List names = entitySpecs.stream().map(EntitySpec::getName).collect(Collectors.toList()); - featureSpecs.stream().map(f -> names.add(f.getName())); - HashSet nameSet = Sets.newHashSet(names); - if (nameSet.size() != names.size()) { - throw new IllegalArgumentException( - String.format("fields within a featureset must be unique.")); - } - } - - private static void checkReservedColumns(List featureSpecs) { - String reservedNamesString = StringUtils.join(reservedNames, ", "); - for (FeatureSpec featureSpec : featureSpecs) { - if (reservedNames.contains(featureSpec.getName())) { - throw new IllegalArgumentException( - String.format( - "Reserved feature names have been used, which are not allowed. These names include %s." - + "You've just used an invalid name, %s.", - reservedNamesString, featureSpec.getName())); - } - } - } -} diff --git a/core/src/test/java/feast/core/auth/CoreServiceAuthTest.java b/core/src/test/java/feast/core/auth/CoreServiceAuthTest.java index c6f88f3f630..4bfa084e155 100644 --- a/core/src/test/java/feast/core/auth/CoreServiceAuthTest.java +++ b/core/src/test/java/feast/core/auth/CoreServiceAuthTest.java @@ -23,34 +23,23 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; +import avro.shaded.com.google.common.collect.ImmutableMap; import com.google.protobuf.InvalidProtocolBufferException; import feast.common.auth.authorization.AuthorizationProvider; import feast.common.auth.authorization.AuthorizationResult; import feast.common.auth.config.SecurityProperties; import feast.common.auth.service.AuthorizationService; +import feast.common.it.DataGenerator; import feast.core.config.FeastProperties; import feast.core.dao.ProjectRepository; import feast.core.grpc.CoreServiceImpl; -import feast.core.model.Entity; -import feast.core.model.Feature; -import feast.core.model.FeatureSet; -import feast.core.model.Source; import feast.core.service.ProjectService; import feast.core.service.SpecService; -import feast.core.service.StatsService; -import feast.proto.core.CoreServiceProto.ApplyFeatureSetRequest; -import feast.proto.core.CoreServiceProto.ApplyFeatureSetResponse; -import feast.proto.core.FeatureSetProto; -import feast.proto.core.FeatureSetProto.FeatureSetStatus; -import feast.proto.core.SourceProto; -import feast.proto.core.SourceProto.KafkaSourceConfig; -import feast.proto.core.SourceProto.SourceType; -import feast.proto.types.ValueProto.ValueType.Enum; +import feast.proto.core.CoreServiceProto.ApplyEntityRequest; +import feast.proto.core.CoreServiceProto.ApplyEntityResponse; +import feast.proto.core.EntityProto; +import feast.proto.types.ValueProto; import io.grpc.internal.testing.StreamRecorder; -import java.sql.Date; -import java.time.Instant; -import java.util.Arrays; -import java.util.HashMap; import org.junit.Test; import org.mockito.Mock; import org.mockito.MockitoAnnotations; @@ -66,7 +55,6 @@ public class CoreServiceAuthTest { @Mock private SpecService specService; @Mock private ProjectRepository projectRepository; @Mock private AuthorizationProvider authProvider; - @Mock private StatsService statsService; public CoreServiceAuthTest() { MockitoAnnotations.initMocks(this); @@ -80,13 +68,11 @@ public CoreServiceAuthTest() { projectService = new ProjectService(projectRepository); AuthorizationService authService = new AuthorizationService(feastProperties.getSecurity(), authProvider); - coreService = - new CoreServiceImpl( - specService, projectService, statsService, feastProperties, authService); + coreService = new CoreServiceImpl(specService, projectService, feastProperties, authService); } @Test - public void shouldNotApplyFeatureSetIfNotProjectMember() throws InvalidProtocolBufferException { + public void shouldNotApplyEntityIfNotProjectMember() throws InvalidProtocolBufferException { String project = "project1"; Authentication auth = mock(Authentication.class); @@ -98,22 +84,23 @@ public void shouldNotApplyFeatureSetIfNotProjectMember() throws InvalidProtocolB .when(authProvider) .checkAccessToProject(anyString(), any(Authentication.class)); - StreamRecorder responseObserver = StreamRecorder.create(); - FeatureSetProto.FeatureSet incomingFeatureSet = newDummyFeatureSet("f2", 1, project).toProto(); + StreamRecorder responseObserver = StreamRecorder.create(); + EntityProto.EntitySpecV2 incomingEntitySpec = + DataGenerator.createEntitySpecV2( + "entity1", + "Entity 1 description", + ValueProto.ValueType.Enum.STRING, + ImmutableMap.of("label_key", "label_value")); - FeatureSetProto.FeatureSetSpec incomingFeatureSetSpec = - incomingFeatureSet.getSpec().toBuilder().build(); - FeatureSetProto.FeatureSet spec = - FeatureSetProto.FeatureSet.newBuilder().setSpec(incomingFeatureSetSpec).build(); - ApplyFeatureSetRequest request = - ApplyFeatureSetRequest.newBuilder().setFeatureSet(spec).build(); + ApplyEntityRequest request = + ApplyEntityRequest.newBuilder().setProject(project).setSpec(incomingEntitySpec).build(); - coreService.applyFeatureSet(request, responseObserver); + coreService.applyEntity(request, responseObserver); assertEquals("PERMISSION_DENIED: Access Denied", responseObserver.getError().getMessage()); } @Test - public void shouldApplyFeatureSetIfProjectMember() throws InvalidProtocolBufferException { + public void shouldApplyEntityIfProjectMember() throws InvalidProtocolBufferException { String project = "project1"; Authentication auth = mock(Authentication.class); @@ -124,42 +111,16 @@ public void shouldApplyFeatureSetIfProjectMember() throws InvalidProtocolBufferE .when(authProvider) .checkAccessToProject(anyString(), any(Authentication.class)); - StreamRecorder responseObserver = StreamRecorder.create(); - FeatureSetProto.FeatureSet incomingFeatureSet = newDummyFeatureSet("f2", 1, project).toProto(); - FeatureSetProto.FeatureSetSpec incomingFeatureSetSpec = - incomingFeatureSet.getSpec().toBuilder().build(); - FeatureSetProto.FeatureSet spec = - FeatureSetProto.FeatureSet.newBuilder().setSpec(incomingFeatureSetSpec).build(); - ApplyFeatureSetRequest request = - ApplyFeatureSetRequest.newBuilder().setFeatureSet(spec).build(); - - coreService.applyFeatureSet(request, responseObserver); - } - - private FeatureSet newDummyFeatureSet(String name, int version, String project) { - Feature feature = new Feature("feature", Enum.INT64); - Entity entity = new Entity("entity", Enum.STRING); - SourceProto.Source sourceSpec = - SourceProto.Source.newBuilder() - .setType(SourceType.KAFKA) - .setKafkaSourceConfig( - KafkaSourceConfig.newBuilder() - .setBootstrapServers("kafka:9092") - .setTopic("my-topic") - .build()) - .build(); - Source defaultSource = Source.fromProto(sourceSpec); - FeatureSet fs = - new FeatureSet( - name, - project, - 100L, - Arrays.asList(entity), - Arrays.asList(feature), - defaultSource, - new HashMap(), - FeatureSetStatus.STATUS_READY); - fs.setCreated(Date.from(Instant.ofEpochSecond(10L))); - return fs; + StreamRecorder responseObserver = StreamRecorder.create(); + EntityProto.EntitySpecV2 incomingEntitySpec = + DataGenerator.createEntitySpecV2( + "entity1", + "Entity 1 description", + ValueProto.ValueType.Enum.STRING, + ImmutableMap.of("label_key", "label_value")); + ApplyEntityRequest request = + ApplyEntityRequest.newBuilder().setProject(project).setSpec(incomingEntitySpec).build(); + + coreService.applyEntity(request, responseObserver); } } diff --git a/core/src/test/java/feast/core/auth/CoreServiceAuthenticationIT.java b/core/src/test/java/feast/core/auth/CoreServiceAuthenticationIT.java index d6f13fdb55d..77cae43cd87 100644 --- a/core/src/test/java/feast/core/auth/CoreServiceAuthenticationIT.java +++ b/core/src/test/java/feast/core/auth/CoreServiceAuthenticationIT.java @@ -18,6 +18,7 @@ import static org.junit.jupiter.api.Assertions.*; +import avro.shaded.com.google.common.collect.ImmutableMap; import com.github.tomakehurst.wiremock.client.WireMock; import com.github.tomakehurst.wiremock.junit.WireMockClassRule; import com.nimbusds.jose.JOSEException; @@ -28,6 +29,7 @@ import feast.core.auth.infra.JwtHelper; import feast.core.config.FeastProperties; import feast.proto.core.*; +import feast.proto.types.ValueProto; import io.grpc.CallCredentials; import io.grpc.Channel; import io.grpc.ManagedChannelBuilder; @@ -57,8 +59,6 @@ public class CoreServiceAuthenticationIT extends BaseIT { private static JwtHelper jwtHelper = new JwtHelper(); - static String project = "myproject"; - static String subject = "random@example.com"; static String subjectClaim = "sub"; @ClassRule public static WireMockClassRule wireMockRule = new WireMockClassRule(JWKS_PORT); @@ -126,54 +126,40 @@ public void shouldGetVersionFromFeastCoreAlways() { * Core as anonymous users. They are not forced to authenticate. */ @Test - public void shouldAllowUnauthenticatedFeatureSetListing() { - FeatureSetProto.FeatureSet expectedFeatureSet = DataGenerator.getDefaultFeatureSet(); - insecureApiClient.simpleApplyFeatureSet(expectedFeatureSet); - - List listFeatureSetsResponse = - insecureApiClient.simpleListFeatureSets("*"); - FeatureSetProto.FeatureSet actualFeatureSet = listFeatureSetsResponse.get(0); - - assert listFeatureSetsResponse.size() == 1; - assertEquals( - actualFeatureSet.getSpec().getProject(), expectedFeatureSet.getSpec().getProject()); - assertEquals( - actualFeatureSet.getSpec().getProject(), expectedFeatureSet.getSpec().getProject()); + public void shouldAllowUnauthenticatedEntityApplyAndListing() { + String project = "default"; + EntityProto.EntitySpecV2 expectedEntitySpec = + DataGenerator.createEntitySpecV2( + "entity1", + "Entity 1 description", + ValueProto.ValueType.Enum.STRING, + ImmutableMap.of("label_key", "label_value")); + insecureApiClient.simpleApplyEntity(project, expectedEntitySpec); + + List listEntitiesResponse = insecureApiClient.simpleListEntities(project); + EntityProto.Entity actualEntity = listEntitiesResponse.get(0); + + assert listEntitiesResponse.size() == 1; + assertEquals(actualEntity.getSpec().getName(), expectedEntitySpec.getName()); } @Test - public void shouldAllowAuthenticatedFeatureSetListing() { + public void shouldAllowAuthenticatedEntityApplyAndListing() { SimpleCoreClient secureApiClient = getSecureApiClient("AuthenticatedUserWithoutAuthorization@example.com"); - FeatureSetProto.FeatureSet expectedFeatureSet = DataGenerator.getDefaultFeatureSet(); - secureApiClient.simpleApplyFeatureSet(expectedFeatureSet); - List listFeatureSetsResponse = - secureApiClient.simpleListFeatureSets("*"); - FeatureSetProto.FeatureSet actualFeatureSet = listFeatureSetsResponse.get(0); - - assert listFeatureSetsResponse.size() == 1; - assertEquals( - actualFeatureSet.getSpec().getProject(), expectedFeatureSet.getSpec().getProject()); - assertEquals( - actualFeatureSet.getSpec().getProject(), expectedFeatureSet.getSpec().getProject()); - } - - @Test - void canApplyFeatureSetIfAuthenticated() { - SimpleCoreClient secureApiClient = - getSecureApiClient("AuthenticatedUserWithoutAuthorization@example.com"); - FeatureSetProto.FeatureSet expectedFeatureSet = - DataGenerator.createFeatureSet(DataGenerator.getDefaultSource(), "project_1", "test_1"); - - secureApiClient.simpleApplyFeatureSet(expectedFeatureSet); - - FeatureSetProto.FeatureSet actualFeatureSet = - secureApiClient.simpleGetFeatureSet("project_1", "test_1"); - - assertEquals( - expectedFeatureSet.getSpec().getProject(), actualFeatureSet.getSpec().getProject()); - assertEquals(expectedFeatureSet.getSpec().getName(), actualFeatureSet.getSpec().getName()); - assertEquals(expectedFeatureSet.getSpec().getSource(), actualFeatureSet.getSpec().getSource()); + String project = "default"; + EntityProto.EntitySpecV2 expectedEntitySpec = + DataGenerator.createEntitySpecV2( + "entity1", + "Entity 1 description", + ValueProto.ValueType.Enum.STRING, + ImmutableMap.of("label_key", "label_value")); + secureApiClient.simpleApplyEntity(project, expectedEntitySpec); + List listEntitiesResponse = insecureApiClient.simpleListEntities(project); + EntityProto.Entity actualEntity = listEntitiesResponse.get(0); + + assert listEntitiesResponse.size() == 1; + assertEquals(actualEntity.getSpec().getName(), expectedEntitySpec.getName()); } @TestConfiguration diff --git a/core/src/test/java/feast/core/auth/CoreServiceAuthorizationIT.java b/core/src/test/java/feast/core/auth/CoreServiceAuthorizationIT.java index 584fcc3854a..41faee7f712 100644 --- a/core/src/test/java/feast/core/auth/CoreServiceAuthorizationIT.java +++ b/core/src/test/java/feast/core/auth/CoreServiceAuthorizationIT.java @@ -19,6 +19,7 @@ import static org.junit.jupiter.api.Assertions.*; import static org.testcontainers.containers.wait.strategy.Wait.forHttp; +import avro.shaded.com.google.common.collect.ImmutableMap; import com.github.tomakehurst.wiremock.client.WireMock; import com.github.tomakehurst.wiremock.junit.WireMockClassRule; import com.google.protobuf.InvalidProtocolBufferException; @@ -30,7 +31,8 @@ import feast.core.auth.infra.JwtHelper; import feast.core.config.FeastProperties; import feast.proto.core.CoreServiceGrpc; -import feast.proto.core.FeatureSetProto; +import feast.proto.core.EntityProto; +import feast.proto.types.ValueProto; import io.grpc.CallCredentials; import io.grpc.Channel; import io.grpc.ManagedChannelBuilder; @@ -153,8 +155,13 @@ public static void globalSetUp(@Value("${grpc.server.port}") int port) { @BeforeEach public void setUp() { SimpleCoreClient secureApiClient = getSecureApiClient(subjectIsAdmin); - FeatureSetProto.FeatureSet expectedFeatureSet = DataGenerator.getDefaultFeatureSet(); - secureApiClient.simpleApplyFeatureSet(expectedFeatureSet); + EntityProto.EntitySpecV2 expectedEntitySpec = + DataGenerator.createEntitySpecV2( + "entity1", + "Entity 1 description", + ValueProto.ValueType.Enum.STRING, + ImmutableMap.of("label_key", "label_value")); + secureApiClient.simpleApplyEntity(project, expectedEntitySpec); } @AfterAll @@ -176,12 +183,12 @@ public void shouldGetVersionFromFeastCoreAlways() { } @Test - public void shouldNotAllowUnauthenticatedFeatureSetListing() { + public void shouldNotAllowUnauthenticatedEntityListing() { Exception exception = assertThrows( StatusRuntimeException.class, () -> { - insecureApiClient.simpleListFeatureSets("8"); + insecureApiClient.simpleListEntities("8"); }); String expectedMessage = "UNAUTHENTICATED: Authentication failed"; @@ -190,32 +197,37 @@ public void shouldNotAllowUnauthenticatedFeatureSetListing() { } @Test - public void shouldAllowAuthenticatedFeatureSetListing() { + public void shouldAllowAuthenticatedEntityListing() { SimpleCoreClient secureApiClient = getSecureApiClient("AuthenticatedUserWithoutAuthorization@example.com"); - FeatureSetProto.FeatureSet expectedFeatureSet = DataGenerator.getDefaultFeatureSet(); - List listFeatureSetsResponse = - secureApiClient.simpleListFeatureSets("*"); - FeatureSetProto.FeatureSet actualFeatureSet = listFeatureSetsResponse.get(0); - - assert listFeatureSetsResponse.size() == 1; - assertEquals( - actualFeatureSet.getSpec().getProject(), expectedFeatureSet.getSpec().getProject()); - assertEquals( - actualFeatureSet.getSpec().getProject(), expectedFeatureSet.getSpec().getProject()); + EntityProto.EntitySpecV2 expectedEntitySpec = + DataGenerator.createEntitySpecV2( + "entity1", + "Entity 1 description", + ValueProto.ValueType.Enum.STRING, + ImmutableMap.of("label_key", "label_value")); + List listEntitiesResponse = secureApiClient.simpleListEntities("myproject"); + EntityProto.Entity actualEntity = listEntitiesResponse.get(0); + + assert listEntitiesResponse.size() == 1; + assertEquals(actualEntity.getSpec().getName(), expectedEntitySpec.getName()); } @Test - void cantApplyFeatureSetIfNotProjectMember() throws InvalidProtocolBufferException { + void cantApplyEntityIfNotProjectMember() throws InvalidProtocolBufferException { String userName = "random_user@example.com"; SimpleCoreClient secureApiClient = getSecureApiClient(userName); - FeatureSetProto.FeatureSet expectedFeatureSet = - DataGenerator.createFeatureSet(DataGenerator.getDefaultSource(), project, "test_5"); + EntityProto.EntitySpecV2 expectedEntitySpec = + DataGenerator.createEntitySpecV2( + "entity1", + "Entity 1 description", + ValueProto.ValueType.Enum.STRING, + ImmutableMap.of("label_key", "label_value")); StatusRuntimeException exception = assertThrows( StatusRuntimeException.class, - () -> secureApiClient.simpleApplyFeatureSet(expectedFeatureSet)); + () -> secureApiClient.simpleApplyEntity(project, expectedEntitySpec)); String expectedMessage = String.format( @@ -225,37 +237,39 @@ void cantApplyFeatureSetIfNotProjectMember() throws InvalidProtocolBufferExcepti } @Test - void canApplyFeatureSetIfProjectMember() { + void canApplyEntityIfProjectMember() { SimpleCoreClient secureApiClient = getSecureApiClient(subjectInProject); - FeatureSetProto.FeatureSet expectedFeatureSet = - DataGenerator.createFeatureSet(DataGenerator.getDefaultSource(), project, "test_6"); + EntityProto.EntitySpecV2 expectedEntitySpec = + DataGenerator.createEntitySpecV2( + "entity_6", + "Entity 1 description", + ValueProto.ValueType.Enum.STRING, + ImmutableMap.of("label_key", "label_value")); - secureApiClient.simpleApplyFeatureSet(expectedFeatureSet); + secureApiClient.simpleApplyEntity(project, expectedEntitySpec); - FeatureSetProto.FeatureSet actualFeatureSet = - secureApiClient.simpleGetFeatureSet(project, "test_6"); + EntityProto.Entity actualEntity = secureApiClient.simpleGetEntity(project, "entity_6"); - assertEquals( - expectedFeatureSet.getSpec().getProject(), actualFeatureSet.getSpec().getProject()); - assertEquals(expectedFeatureSet.getSpec().getName(), actualFeatureSet.getSpec().getName()); - assertEquals(expectedFeatureSet.getSpec().getSource(), actualFeatureSet.getSpec().getSource()); + assertEquals(expectedEntitySpec.getName(), actualEntity.getSpec().getName()); + assertEquals(expectedEntitySpec.getValueType(), actualEntity.getSpec().getValueType()); } @Test - void canApplyFeatureSetIfAdmin() { + void canApplyEntityIfAdmin() { SimpleCoreClient secureApiClient = getSecureApiClient(subjectIsAdmin); - FeatureSetProto.FeatureSet expectedFeatureSet = - DataGenerator.createFeatureSet(DataGenerator.getDefaultSource(), "any_project", "test_2"); + EntityProto.EntitySpecV2 expectedEntitySpec = + DataGenerator.createEntitySpecV2( + "entity_7", + "Entity 1 description", + ValueProto.ValueType.Enum.STRING, + ImmutableMap.of("label_key", "label_value")); - secureApiClient.simpleApplyFeatureSet(expectedFeatureSet); + secureApiClient.simpleApplyEntity(project, expectedEntitySpec); - FeatureSetProto.FeatureSet actualFeatureSet = - secureApiClient.simpleGetFeatureSet("any_project", "test_2"); + EntityProto.Entity actualEntity = secureApiClient.simpleGetEntity(project, "entity_7"); - assertEquals( - expectedFeatureSet.getSpec().getProject(), actualFeatureSet.getSpec().getProject()); - assertEquals(expectedFeatureSet.getSpec().getName(), actualFeatureSet.getSpec().getName()); - assertEquals(expectedFeatureSet.getSpec().getSource(), actualFeatureSet.getSpec().getSource()); + assertEquals(expectedEntitySpec.getName(), actualEntity.getSpec().getName()); + assertEquals(expectedEntitySpec.getValueType(), actualEntity.getSpec().getValueType()); } @TestConfiguration diff --git a/core/src/test/java/feast/core/controller/CoreServiceRestIT.java b/core/src/test/java/feast/core/controller/CoreServiceRestIT.java index ea295918494..f26ce8a343f 100644 --- a/core/src/test/java/feast/core/controller/CoreServiceRestIT.java +++ b/core/src/test/java/feast/core/controller/CoreServiceRestIT.java @@ -25,13 +25,10 @@ import feast.common.it.BaseIT; import feast.common.it.DataGenerator; import feast.common.it.SimpleCoreClient; -import feast.core.model.Project; import feast.proto.core.CoreServiceGrpc; import feast.proto.core.EntityProto; -import feast.proto.core.FeatureSetProto.FeatureSet; import feast.proto.core.FeatureTableProto; import feast.proto.types.ValueProto; -import feast.proto.types.ValueProto.ValueType.Enum; import io.grpc.ManagedChannel; import io.grpc.ManagedChannelBuilder; import io.restassured.RestAssured; @@ -71,7 +68,7 @@ public static void globalSetUp(@Value("${grpc.server.port}") int port) { @Test public void getVersion() { - String uriString = UriComponentsBuilder.fromPath("/api/v1/version").toUriString(); + String uriString = UriComponentsBuilder.fromPath("/api/v2/version").toUriString(); get(uriString) .then() .log() @@ -85,7 +82,7 @@ public void getVersion() { @Test public void listProjects() { // should get 2 projects - String uriString = UriComponentsBuilder.fromPath("/api/v1/projects").toUriString(); + String uriString = UriComponentsBuilder.fromPath("/api/v2/projects").toUriString(); String responseBody = get(uriString) .then() @@ -98,90 +95,14 @@ public void listProjects() { .getBody() .asString(); List projectList = JsonPath.from(responseBody).getList("projects"); - assertEquals(projectList, List.of("default", "merchant")); - } - - // list feature sets - @Test - public void listFeatureSets() { - // project = default - // name = merchant_ratings - // getting a specific feature set - String uri1 = - UriComponentsBuilder.fromPath("/api/v1/feature-sets") - .queryParam("project", "default") - .queryParam("name", "merchant_ratings") - .buildAndExpand() - .toString(); - String responseBody = - get(uri1) - .then() - .log() - .everything() - .assertThat() - .contentType(ContentType.JSON) - .extract() - .response() - .getBody() - .asString(); - List featureSetList = JsonPath.from(responseBody).getList("featureSets"); - assertEquals(featureSetList.size(), 1); - - // project = * - // name = *merchant_ratings - // should have two feature sets named *merchant_ratings - String uri2 = - UriComponentsBuilder.fromPath("/api/v1/feature-sets") - .queryParam("project", "*") - .queryParam("name", "*merchant_ratings") - .buildAndExpand() - .toString(); - responseBody = - get(uri2) - .then() - .log() - .everything() - .assertThat() - .contentType(ContentType.JSON) - .extract() - .response() - .getBody() - .asString(); - featureSetList = JsonPath.from(responseBody).getList("featureSets"); - assertEquals(featureSetList.size(), 2); - - // project = * - // name = * - // should have three feature sets - String uri3 = - UriComponentsBuilder.fromPath("/api/v1/feature-sets") - .queryParam("project", "*") - .queryParam("name", "*") - .buildAndExpand() - .toString(); - responseBody = - get(uri3) - .then() - .log() - .everything() - .assertThat() - .contentType(ContentType.JSON) - .extract() - .response() - .getBody() - .asString(); - featureSetList = JsonPath.from(responseBody).getList("featureSets"); - assertEquals(featureSetList.size(), 3); + assertEquals(projectList, List.of("default")); } @Test public void listFeatures() { - // entities = [merchant_id] - // project = default - // should return 4 features String uri1 = - UriComponentsBuilder.fromPath("/api/v1/features") - .queryParam("entities", "merchant_id") + UriComponentsBuilder.fromPath("/api/v2/features") + .queryParam("entities", "entity1", "entity2") .buildAndExpand() .toString(); get(uri1) @@ -190,15 +111,12 @@ public void listFeatures() { .everything() .assertThat() .contentType(ContentType.JSON) - .body("features", aMapWithSize(4)); + .body("features", aMapWithSize(2)); - // entities = [merchant_id] - // project = merchant - // should return 2 features String uri2 = - UriComponentsBuilder.fromPath("/api/v1/features") - .queryParam("entities", "merchant_id") - .queryParam("project", "merchant") + UriComponentsBuilder.fromPath("/api/v2/features") + .queryParam("entities", "entity1", "entity2") + .queryParam("project", "default") .buildAndExpand() .toString(); get(uri2) @@ -256,36 +174,6 @@ public void listFeatureTables() { @BeforeEach private void createSpecs() { - // Apply feature sets - FeatureSet merchantFeatureSet = - DataGenerator.createFeatureSet( - DataGenerator.getDefaultSource(), - Project.DEFAULT_NAME, - "merchant_ratings", - ImmutableMap.of("merchant_id", Enum.STRING), - ImmutableMap.of("average_rating", Enum.DOUBLE, "total_ratings", Enum.INT64)); - apiClient.simpleApplyFeatureSet(merchantFeatureSet); - - FeatureSet anotherMerchantFeatureSet = - DataGenerator.createFeatureSet( - DataGenerator.getDefaultSource(), - Project.DEFAULT_NAME, - "another_merchant_ratings", - ImmutableMap.of("merchant_id", Enum.STRING), - ImmutableMap.of( - "another_average_rating", Enum.DOUBLE, - "another_total_ratings", Enum.INT64)); - apiClient.simpleApplyFeatureSet(anotherMerchantFeatureSet); - - FeatureSet yetAnotherMerchantFeatureSet = - DataGenerator.createFeatureSet( - DataGenerator.getDefaultSource(), - "merchant", - "yet_another_merchant_feature_set", - ImmutableMap.of("merchant_id", Enum.STRING), - ImmutableMap.of("merchant_prop1", Enum.BOOL, "merchant_prop2", Enum.FLOAT)); - apiClient.simpleApplyFeatureSet(yetAnotherMerchantFeatureSet); - // Apply entities EntityProto.EntitySpecV2 entitySpec1 = DataGenerator.createEntitySpecV2( diff --git a/core/src/test/java/feast/core/logging/CoreLoggingIT.java b/core/src/test/java/feast/core/logging/CoreLoggingIT.java index 0ca9eda32cc..0f137b46393 100644 --- a/core/src/test/java/feast/core/logging/CoreLoggingIT.java +++ b/core/src/test/java/feast/core/logging/CoreLoggingIT.java @@ -35,7 +35,7 @@ import feast.proto.core.CoreServiceGrpc.CoreServiceBlockingStub; import feast.proto.core.CoreServiceGrpc.CoreServiceFutureStub; import feast.proto.core.CoreServiceProto.GetFeastCoreVersionRequest; -import feast.proto.core.CoreServiceProto.ListFeatureSetsRequest; +import feast.proto.core.CoreServiceProto.ListFeatureTablesRequest; import feast.proto.core.CoreServiceProto.ListStoresRequest; import feast.proto.core.CoreServiceProto.ListStoresResponse; import feast.proto.core.CoreServiceProto.UpdateStoreRequest; @@ -123,19 +123,15 @@ public void shouldProduceMessageAuditLogsOnCall() @Test public void shouldProduceMessageAuditLogsOnError() throws InterruptedException { // Send a bad request which should cause Core to error - ListFeatureSetsRequest request = - ListFeatureSetsRequest.newBuilder() - .setFilter( - ListFeatureSetsRequest.Filter.newBuilder() - .setProject("*") - .setFeatureSetName("nop") - .build()) + ListFeatureTablesRequest request = + ListFeatureTablesRequest.newBuilder() + .setFilter(ListFeatureTablesRequest.Filter.newBuilder().setProject("*").build()) .build(); boolean hasExpectedException = false; Code statusCode = null; try { - coreService.listFeatureSets(request); + coreService.listFeatureTables(request); } catch (StatusRuntimeException e) { hasExpectedException = true; statusCode = e.getStatus().getCode(); @@ -146,7 +142,7 @@ public void shouldProduceMessageAuditLogsOnError() throws InterruptedException { Thread.sleep(1000); // Pull message audit logs logs from test log appender List logJsonObjects = - parseMessageJsonLogObjects(testAuditLogAppender.getLogs(), "ListFeatureSets"); + parseMessageJsonLogObjects(testAuditLogAppender.getLogs(), "ListFeatureTables"); assertEquals(1, logJsonObjects.size()); JsonObject logJsonObject = logJsonObjects.get(0); diff --git a/core/src/test/java/feast/core/model/FeatureSetTest.java b/core/src/test/java/feast/core/model/FeatureSetTest.java deleted file mode 100644 index 270dc3f3bcd..00000000000 --- a/core/src/test/java/feast/core/model/FeatureSetTest.java +++ /dev/null @@ -1,205 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.core.model; - -import static org.hamcrest.CoreMatchers.containsString; -import static org.hamcrest.CoreMatchers.equalTo; -import static org.junit.Assert.assertThat; - -import com.google.protobuf.Duration; -import com.google.protobuf.InvalidProtocolBufferException; -import feast.proto.core.FeatureSetProto; -import feast.proto.core.FeatureSetProto.EntitySpec; -import feast.proto.core.FeatureSetProto.FeatureSetSpec; -import feast.proto.core.FeatureSetProto.FeatureSetStatus; -import feast.proto.core.FeatureSetProto.FeatureSpec; -import feast.proto.core.SourceProto; -import feast.proto.core.SourceProto.KafkaSourceConfig; -import feast.proto.core.SourceProto.SourceType; -import feast.proto.types.ValueProto.ValueType.Enum; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.ExpectedException; -import org.tensorflow.metadata.v0.IntDomain; - -public class FeatureSetTest { - @Rule public final ExpectedException expectedException = ExpectedException.none(); - - private FeatureSetProto.FeatureSet oldFeatureSetProto; - - @Before - public void setUp() { - SourceProto.Source oldSource = - SourceProto.Source.newBuilder() - .setType(SourceType.KAFKA) - .setKafkaSourceConfig( - KafkaSourceConfig.newBuilder() - .setBootstrapServers("kafka:9092") - .setTopic("mytopic")) - .build(); - - oldFeatureSetProto = - FeatureSetProto.FeatureSet.newBuilder() - .setSpec( - FeatureSetSpec.newBuilder() - .setName("featureSet") - .setProject("project") - .setMaxAge(Duration.newBuilder().setSeconds(100)) - .setSource(oldSource) - .addFeatures( - FeatureSpec.newBuilder().setName("feature1").setValueType(Enum.INT64)) - .addFeatures( - FeatureSpec.newBuilder().setName("feature2").setValueType(Enum.STRING)) - .addEntities( - EntitySpec.newBuilder().setName("entity").setValueType(Enum.STRING)) - .build()) - .build(); - } - - @Test - public void shouldUpdateFromProto() throws InvalidProtocolBufferException { - SourceProto.Source newSource = - SourceProto.Source.newBuilder() - .setType(SourceType.KAFKA) - .setKafkaSourceConfig( - KafkaSourceConfig.newBuilder() - .setBootstrapServers("kafka:9092") - .setTopic("mytopic-changed")) - .build(); - - FeatureSetProto.FeatureSet newFeatureSetProto = - FeatureSetProto.FeatureSet.newBuilder() - .setSpec( - FeatureSetSpec.newBuilder() - .setName("featureSet") - .setProject("project") - .setMaxAge(Duration.newBuilder().setSeconds(101)) - .setSource(newSource) - .addFeatures( - FeatureSpec.newBuilder() - .setName("feature1") - .setValueType(Enum.INT64) - .setIntDomain(IntDomain.newBuilder().setMax(10).setMin(0))) - .addFeatures( - FeatureSpec.newBuilder().setName("feature3").setValueType(Enum.STRING)) - .addEntities( - EntitySpec.newBuilder().setName("entity").setValueType(Enum.STRING)) - .build()) - .build(); - - FeatureSet actual = FeatureSet.fromProto(oldFeatureSetProto); - actual.updateFromProto(newFeatureSetProto); - - FeatureSet expected = FeatureSet.fromProto(newFeatureSetProto); - Feature archivedFeature = - Feature.fromProto( - FeatureSpec.newBuilder().setName("feature2").setValueType(Enum.STRING).build()); - archivedFeature.setArchived(true); - expected.addFeature(archivedFeature); - assertThat(actual, equalTo(expected)); - } - - @Test - public void shouldNotUpdateIfNoChange() throws InvalidProtocolBufferException { - FeatureSet actual = FeatureSet.fromProto(oldFeatureSetProto); - actual.setStatus(FeatureSetStatus.STATUS_READY); - actual.updateFromProto(oldFeatureSetProto); - - FeatureSet expected = FeatureSet.fromProto(oldFeatureSetProto); - expected.setStatus(FeatureSetStatus.STATUS_READY); - - assertThat(actual, equalTo(expected)); - } - - @Test - public void shouldThrowExceptionIfUpdateWithEntitiesChanged() - throws InvalidProtocolBufferException { - SourceProto.Source newSource = - SourceProto.Source.newBuilder() - .setType(SourceType.KAFKA) - .setKafkaSourceConfig( - KafkaSourceConfig.newBuilder() - .setBootstrapServers("kafka:9092") - .setTopic("mytopic-changed")) - .build(); - - FeatureSetProto.FeatureSet newFeatureSetProto = - FeatureSetProto.FeatureSet.newBuilder() - .setSpec( - FeatureSetSpec.newBuilder() - .setName("featureSet") - .setProject("project") - .setMaxAge(Duration.newBuilder().setSeconds(101)) - .setSource(newSource) - .addFeatures( - FeatureSpec.newBuilder() - .setName("feature1") - .setValueType(Enum.INT64) - .setIntDomain(IntDomain.newBuilder().setMax(10).setMin(0))) - .addFeatures( - FeatureSpec.newBuilder().setName("feature3").setValueType(Enum.STRING)) - .addEntities(EntitySpec.newBuilder().setName("entity").setValueType(Enum.FLOAT)) - .build()) - .build(); - - expectedException.expect(IllegalArgumentException.class); - expectedException.expectMessage(containsString("does not match existing set of entities")); - FeatureSet existingFeatureSet = FeatureSet.fromProto(oldFeatureSetProto); - existingFeatureSet.updateFromProto(newFeatureSetProto); - } - - @Test - public void shouldThrowExceptionIfUpdateWithFeatureTypesChanged() - throws InvalidProtocolBufferException { - SourceProto.Source newSource = - SourceProto.Source.newBuilder() - .setType(SourceType.KAFKA) - .setKafkaSourceConfig( - KafkaSourceConfig.newBuilder() - .setBootstrapServers("kafka:9092") - .setTopic("mytopic-changed")) - .build(); - - FeatureSetProto.FeatureSet newFeatureSetProto = - FeatureSetProto.FeatureSet.newBuilder() - .setSpec( - FeatureSetSpec.newBuilder() - .setName("featureSet") - .setProject("project") - .setMaxAge(Duration.newBuilder().setSeconds(101)) - .setSource(newSource) - .addFeatures( - FeatureSpec.newBuilder() - .setName("feature1") - .setValueType(Enum.INT64) - .setIntDomain(IntDomain.newBuilder().setMax(10).setMin(0))) - .addFeatures( - FeatureSpec.newBuilder().setName("feature2").setValueType(Enum.FLOAT)) - .addEntities( - EntitySpec.newBuilder().setName("entity").setValueType(Enum.STRING)) - .build()) - .build(); - - expectedException.expect(IllegalArgumentException.class); - expectedException.expectMessage( - containsString( - "You are attempting to change the type of feature feature2 from STRING to FLOAT.")); - FeatureSet existingFeatureSet = FeatureSet.fromProto(oldFeatureSetProto); - existingFeatureSet.updateFromProto(newFeatureSetProto); - } -} diff --git a/core/src/test/java/feast/core/service/SpecServiceIT.java b/core/src/test/java/feast/core/service/SpecServiceIT.java index 8d56de606b0..8851d875ae8 100644 --- a/core/src/test/java/feast/core/service/SpecServiceIT.java +++ b/core/src/test/java/feast/core/service/SpecServiceIT.java @@ -17,15 +17,12 @@ package feast.core.service; import static com.jayway.jsonassert.impl.matcher.IsMapContainingKey.hasKey; -import static org.hamcrest.CoreMatchers.allOf; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.beans.HasPropertyWithValue.hasProperty; import static org.hamcrest.collection.IsCollectionWithSize.hasSize; -import static org.hamcrest.collection.IsMapContaining.hasEntry; import static org.hamcrest.collection.IsMapWithSize.aMapWithSize; import static org.hamcrest.core.IsEqual.equalTo; import static org.hamcrest.core.IsIterableContaining.hasItem; -import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; import static org.junit.jupiter.api.Assertions.assertThrows; @@ -42,7 +39,7 @@ import io.grpc.ManagedChannelBuilder; import io.grpc.StatusRuntimeException; import java.util.*; -import org.apache.commons.lang3.StringUtils; +import java.util.stream.IntStream; import org.apache.commons.lang3.tuple.Triple; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; @@ -50,7 +47,6 @@ import org.junit.jupiter.api.Test; import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.test.context.SpringBootTest; -import org.tensorflow.metadata.v0.*; import org.testcontainers.shaded.com.google.common.collect.ImmutableList; @SpringBootTest @@ -67,9 +63,11 @@ public static void globalSetUp(@Value("${grpc.server.port}") int port) { apiClient = new SimpleCoreClient(stub); } + private FeatureTableProto.FeatureTableSpec example1; + private FeatureTableProto.FeatureTableSpec example2; + @BeforeEach public void initState() { - SourceProto.Source source = DataGenerator.getDefaultSource(); EntityProto.EntitySpecV2 entitySpec1 = DataGenerator.createEntitySpecV2( @@ -85,8 +83,8 @@ public void initState() { ImmutableMap.of("label_key2", "label_value2")); apiClient.simpleApplyEntity("default", entitySpec1); apiClient.simpleApplyEntity("default", entitySpec2); - apiClient.applyFeatureTable( - "default", + + example1 = DataGenerator.createFeatureTableSpec( "featuretable1", Arrays.asList("entity1", "entity2"), @@ -101,7 +99,27 @@ public void initState() { .toBuilder() .setBatchSource( DataGenerator.createFileDataSourceSpec("file:///path/to/file", "ts_col", "")) - .build()); + .build(); + + example2 = + DataGenerator.createFeatureTableSpec( + "featuretable2", + Arrays.asList("entity1", "entity2"), + new HashMap<>() { + { + put("feature3", ValueProto.ValueType.Enum.STRING); + put("feature4", ValueProto.ValueType.Enum.FLOAT); + } + }, + 7200, + ImmutableMap.of("feat_key4", "feat_value4")) + .toBuilder() + .setBatchSource( + DataGenerator.createFileDataSourceSpec("file:///path/to/file", "ts_col", "")) + .build(); + + apiClient.applyFeatureTable("default", example1); + apiClient.applyFeatureTable("default", example2); apiClient.simpleApplyEntity( "project1", DataGenerator.createEntitySpecV2( @@ -109,137 +127,9 @@ public void initState() { "Entity 3 description", ValueProto.ValueType.Enum.STRING, ImmutableMap.of("label_key2", "label_value2"))); - apiClient.simpleApplyFeatureSet( - DataGenerator.createFeatureSet( - source, - "default", - "fs1", - ImmutableMap.of("id", ValueProto.ValueType.Enum.STRING), - ImmutableMap.of("total", ValueProto.ValueType.Enum.INT64))); - apiClient.simpleApplyFeatureSet( - DataGenerator.createFeatureSet( - source, - "default", - "fs2", - ImmutableMap.of("user_id", ValueProto.ValueType.Enum.STRING), - ImmutableMap.of("sum", ValueProto.ValueType.Enum.INT64))); - apiClient.simpleApplyFeatureSet( - DataGenerator.createFeatureSet( - source, - "project1", - "fs3", - ImmutableList.of( - DataGenerator.createEntitySpec("user_id", ValueProto.ValueType.Enum.STRING)), - ImmutableList.of( - DataGenerator.createFeature( - "feature1", ValueProto.ValueType.Enum.INT32, Collections.emptyMap()), - DataGenerator.createFeature( - "feature2", ValueProto.ValueType.Enum.INT32, Collections.emptyMap())), - Collections.emptyMap())); - apiClient.simpleApplyFeatureSet( - DataGenerator.createFeatureSet( - source, - "project1", - "fs4", - ImmutableList.of( - DataGenerator.createEntitySpec("customer_id", ValueProto.ValueType.Enum.STRING)), - ImmutableList.of( - DataGenerator.createFeature( - "feature2", - ValueProto.ValueType.Enum.INT32, - ImmutableMap.of("app", "feast", "version", "one"))), - ImmutableMap.of("label", "some"))); - apiClient.simpleApplyFeatureSet( - DataGenerator.createFeatureSet( - source, - "project1", - "fs5", - ImmutableList.of( - DataGenerator.createEntitySpec("customer_id", ValueProto.ValueType.Enum.STRING)), - ImmutableList.of( - DataGenerator.createFeature( - "feature3", - ValueProto.ValueType.Enum.INT32, - ImmutableMap.of("app", "feast", "version", "two"))), - Collections.emptyMap())); - apiClient.simpleApplyFeatureSet(DataGenerator.createFeatureSet(source, "default", "new_fs")); apiClient.updateStore(DataGenerator.getDefaultStore()); } - @Nested - class ListFeatureSets { - - @Test - public void shouldGetAllFeatureSetsIfOnlyWildcardsProvided() { - List featureSets = apiClient.simpleListFeatureSets("*", "*"); - - assertThat(featureSets, hasSize(6)); - } - - @Test - public void shouldGetAllFeatureSetsMatchingNameWithWildcardSearch() { - List featureSets = - apiClient.simpleListFeatureSets("default", "fs*"); - - assertThat(featureSets, hasSize(2)); - assertThat(featureSets, hasItem(hasProperty("spec", hasProperty("name", equalTo("fs1"))))); - assertThat(featureSets, hasItem(hasProperty("spec", hasProperty("name", equalTo("fs2"))))); - } - - @Test - public void shouldFilterFeatureSetsByNameAndProject() { - List featureSets = - apiClient.simpleListFeatureSets("project1", "fs3"); - - assertThat(featureSets, hasItem(hasProperty("spec", hasProperty("name", equalTo("fs3"))))); - } - - @Test - public void shouldFilterFeatureSetsByStatus() { - apiClient.updateFeatureSetStatus( - "project1", "fs3", FeatureSetProto.FeatureSetStatus.STATUS_READY); - - apiClient.updateFeatureSetStatus( - "project1", "fs4", FeatureSetProto.FeatureSetStatus.STATUS_READY); - - List featureSets = - apiClient.simpleListFeatureSets("*", "*", FeatureSetProto.FeatureSetStatus.STATUS_READY); - - assertThat(featureSets, hasSize(2)); - assertThat(featureSets, hasItem(hasProperty("spec", hasProperty("name", equalTo("fs3"))))); - assertThat(featureSets, hasItem(hasProperty("spec", hasProperty("name", equalTo("fs4"))))); - - assertThat( - apiClient.simpleListFeatureSets( - "default", "*", FeatureSetProto.FeatureSetStatus.STATUS_PENDING), - hasSize(3)); - } - - @Test - public void shouldFilterFeatureSetsByLabels() { - List featureSets = - apiClient.simpleListFeatureSets("project1", "*", ImmutableMap.of("label", "some")); - - assertThat(featureSets, hasSize(1)); - assertThat(featureSets, hasItem(hasProperty("spec", hasProperty("name", equalTo("fs4"))))); - } - - @Test - public void shouldUseDefaultProjectIfProjectUnspecified() { - List featureSets = apiClient.simpleListFeatureSets("", "*"); - - assertThat(featureSets, hasSize(3)); - assertThat(featureSets, hasItem(hasProperty("spec", hasProperty("name", equalTo("fs1"))))); - assertThat(featureSets, hasItem(hasProperty("spec", hasProperty("name", equalTo("fs2"))))); - assertThat(featureSets, hasItem(hasProperty("spec", hasProperty("name", equalTo("new_fs"))))); - } - - @Test - public void shouldThrowExceptionGivenMissingFeatureSetName() { - assertThrows(StatusRuntimeException.class, () -> apiClient.simpleListFeatureSets("", "")); - } - } - @Nested class ListEntities { @Test @@ -312,10 +202,13 @@ public void shouldUseDefaultProjectIfProjectUnspecified() { List featureTables = apiClient.simpleListFeatureTables(filter); - assertThat(featureTables, hasSize(1)); + assertThat(featureTables, hasSize(2)); assertThat( featureTables, hasItem(hasProperty("spec", hasProperty("name", equalTo("featuretable1"))))); + assertThat( + featureTables, + hasItem(hasProperty("spec", hasProperty("name", equalTo("featuretable2"))))); } @Test @@ -338,393 +231,6 @@ public void shouldThrowExceptionGivenWildcardProject() { } } - @Nested - class ApplyFeatureSet { - @Test - public void shouldThrowExceptionGivenReservedFeatureName() { - List reservedNames = - Arrays.asList("created_timestamp", "event_timestamp", "ingestion_id", "job_id"); - String reservedNamesString = StringUtils.join(reservedNames, ", "); - - StatusRuntimeException exc = - assertThrows( - StatusRuntimeException.class, - () -> - apiClient.simpleApplyFeatureSet( - DataGenerator.createFeatureSet( - DataGenerator.getDefaultSource(), - "project", - "name", - ImmutableMap.of("entity", ValueProto.ValueType.Enum.STRING), - ImmutableMap.of("event_timestamp", ValueProto.ValueType.Enum.STRING)))); - - assertThat( - exc.getMessage(), - equalTo( - String.format( - "INTERNAL: Reserved feature names have been used, which are not allowed. These names include %s." - + "You've just used an invalid name, %s.", - reservedNamesString, "event_timestamp"))); - } - - @Test - public void shouldThrowExceptionGivenFeatureSetWithDash() { - StatusRuntimeException exc = - assertThrows( - StatusRuntimeException.class, - () -> - apiClient.simpleApplyFeatureSet( - DataGenerator.createFeatureSet( - DataGenerator.getDefaultSource(), - "project", - "dash-name", - ImmutableMap.of("entity", ValueProto.ValueType.Enum.STRING), - ImmutableMap.of("test_string", ValueProto.ValueType.Enum.STRING)))); - - assertThat( - exc.getMessage(), - equalTo( - String.format( - "INTERNAL: invalid value for %s resource, %s: %s", - "featureset", - "dash-name", - "argument must only contain alphanumeric characters and underscores."))); - } - - @Test - public void shouldReturnFeatureSetIfFeatureSetHasNotChanged() { - FeatureSetProto.FeatureSet featureSet = apiClient.getFeatureSet("default", "fs1"); - - CoreServiceProto.ApplyFeatureSetResponse response = - apiClient.simpleApplyFeatureSet(featureSet); - - assertThat( - response.getStatus(), equalTo(CoreServiceProto.ApplyFeatureSetResponse.Status.NO_CHANGE)); - assertThat( - response.getFeatureSet().getSpec().getVersion(), - equalTo(featureSet.getSpec().getVersion())); - } - - @Test - public void shouldApplyFeatureSetIfNotExists() { - FeatureSetProto.FeatureSet featureSet = - DataGenerator.createFeatureSet( - DataGenerator.getDefaultSource(), - "default", - "new", - ImmutableMap.of("id", ValueProto.ValueType.Enum.STRING), - ImmutableMap.of("feature", ValueProto.ValueType.Enum.STRING)); - - CoreServiceProto.ApplyFeatureSetResponse response = - apiClient.simpleApplyFeatureSet(featureSet); - - assertThat( - response.getFeatureSet().getSpec(), - equalTo( - featureSet - .getSpec() - .toBuilder() - .setVersion(1) - .setMaxAge(Duration.newBuilder().build()) - .build())); - assertThat( - response.getStatus(), equalTo(CoreServiceProto.ApplyFeatureSetResponse.Status.CREATED)); - } - - @Test - public void shouldUpdateAndSaveFeatureSetIfAlreadyExists() { - CoreServiceProto.ApplyFeatureSetResponse response = - apiClient.simpleApplyFeatureSet( - DataGenerator.createFeatureSet( - DataGenerator.getDefaultSource(), - "default", - "fs1", - ImmutableMap.of("id", ValueProto.ValueType.Enum.STRING), - ImmutableMap.of( - "total", ValueProto.ValueType.Enum.INT64, - "subtotal", ValueProto.ValueType.Enum.INT64))); - - assertThat( - response.getFeatureSet().getSpec().getFeaturesList(), - hasItem(hasProperty("name", equalTo("subtotal")))); - assertThat( - response.getStatus(), equalTo(CoreServiceProto.ApplyFeatureSetResponse.Status.UPDATED)); - assertThat(response.getFeatureSet().getSpec().getVersion(), equalTo(2)); - } - - @Test - public void shouldAcceptPresenceShapeAndDomainConstraints() { - List entitySpecs = new ArrayList<>(); - entitySpecs.add( - FeatureSetProto.EntitySpec.newBuilder() - .setName("entity1") - .setValueType(ValueProto.ValueType.Enum.INT64) - .build()); - entitySpecs.add( - FeatureSetProto.EntitySpec.newBuilder() - .setName("entity2") - .setValueType(ValueProto.ValueType.Enum.INT64) - .build()); - entitySpecs.add( - FeatureSetProto.EntitySpec.newBuilder() - .setName("entity3") - .setValueType(ValueProto.ValueType.Enum.FLOAT) - .build()); - entitySpecs.add( - FeatureSetProto.EntitySpec.newBuilder() - .setName("entity4") - .setValueType(ValueProto.ValueType.Enum.STRING) - .build()); - entitySpecs.add( - FeatureSetProto.EntitySpec.newBuilder() - .setName("entity5") - .setValueType(ValueProto.ValueType.Enum.BOOL) - .build()); - - List featureSpecs = new ArrayList<>(); - featureSpecs.add( - FeatureSetProto.FeatureSpec.newBuilder() - .setName("feature1") - .setValueType(ValueProto.ValueType.Enum.INT64) - .setPresence(FeaturePresence.getDefaultInstance()) - .setShape(FixedShape.getDefaultInstance()) - .setDomain("mydomain") - .build()); - featureSpecs.add( - FeatureSetProto.FeatureSpec.newBuilder() - .setName("feature2") - .setValueType(ValueProto.ValueType.Enum.INT64) - .setGroupPresence(FeaturePresenceWithinGroup.getDefaultInstance()) - .setValueCount(ValueCount.getDefaultInstance()) - .setIntDomain(IntDomain.getDefaultInstance()) - .build()); - featureSpecs.add( - FeatureSetProto.FeatureSpec.newBuilder() - .setName("feature3") - .setValueType(ValueProto.ValueType.Enum.FLOAT) - .setPresence(FeaturePresence.getDefaultInstance()) - .setValueCount(ValueCount.getDefaultInstance()) - .setFloatDomain(FloatDomain.getDefaultInstance()) - .build()); - featureSpecs.add( - FeatureSetProto.FeatureSpec.newBuilder() - .setName("feature4") - .setValueType(ValueProto.ValueType.Enum.STRING) - .setPresence(FeaturePresence.getDefaultInstance()) - .setValueCount(ValueCount.getDefaultInstance()) - .setStringDomain(StringDomain.getDefaultInstance()) - .build()); - featureSpecs.add( - FeatureSetProto.FeatureSpec.newBuilder() - .setName("feature5") - .setValueType(ValueProto.ValueType.Enum.BOOL) - .setPresence(FeaturePresence.getDefaultInstance()) - .setValueCount(ValueCount.getDefaultInstance()) - .setBoolDomain(BoolDomain.getDefaultInstance()) - .build()); - - FeatureSetProto.FeatureSetSpec featureSetSpec = - FeatureSetProto.FeatureSetSpec.newBuilder() - .setProject("project1") - .setName("featureSetWithConstraints") - .addAllEntities(entitySpecs) - .addAllFeatures(featureSpecs) - .build(); - FeatureSetProto.FeatureSet featureSet = - FeatureSetProto.FeatureSet.newBuilder().setSpec(featureSetSpec).build(); - - CoreServiceProto.ApplyFeatureSetResponse applyFeatureSetResponse = - apiClient.simpleApplyFeatureSet(featureSet); - FeatureSetProto.FeatureSetSpec appliedFeatureSetSpec = - applyFeatureSetResponse.getFeatureSet().getSpec(); - - // appliedEntitySpecs needs to be sorted because the list returned by specService may not - // follow the order in the request - List appliedEntitySpecs = - new ArrayList<>(appliedFeatureSetSpec.getEntitiesList()); - appliedEntitySpecs.sort(Comparator.comparing(FeatureSetProto.EntitySpec::getName)); - - // appliedFeatureSpecs needs to be sorted because the list returned by specService may not - // follow the order in the request - List appliedFeatureSpecs = - new ArrayList<>(appliedFeatureSetSpec.getFeaturesList()); - appliedFeatureSpecs.sort(Comparator.comparing(FeatureSetProto.FeatureSpec::getName)); - - assertEquals(appliedEntitySpecs, entitySpecs); - assertEquals(appliedFeatureSpecs, featureSpecs); - } - - @Test - public void shouldUpdateFeatureSetWhenConstraintsAreUpdated() { - CoreServiceProto.ApplyFeatureSetResponse response = - apiClient.simpleApplyFeatureSet( - DataGenerator.createFeatureSet( - DataGenerator.getDefaultSource(), - "default", - "fs1", - ImmutableList.of( - FeatureSetProto.EntitySpec.newBuilder() - .setName("id") - .setValueType(ValueProto.ValueType.Enum.STRING) - .build()), - ImmutableList.of( - FeatureSetProto.FeatureSpec.newBuilder() - .setName("total") - .setValueType(ValueProto.ValueType.Enum.INT64) - .setIntDomain(IntDomain.newBuilder().setMin(0).setMax(100).build()) - .build()), - Collections.emptyMap())); - - assertThat( - response.getStatus(), equalTo(CoreServiceProto.ApplyFeatureSetResponse.Status.UPDATED)); - assertThat( - response.getFeatureSet().getSpec().getFeaturesList(), - hasItem( - hasProperty( - "intDomain", equalTo(IntDomain.newBuilder().setMin(0).setMax(100).build())))); - } - - @Test - public void shouldCreateProjectWhenNotAlreadyExists() { - CoreServiceProto.ApplyFeatureSetResponse response = - apiClient.simpleApplyFeatureSet( - DataGenerator.createFeatureSet( - DataGenerator.getDefaultSource(), - "new_project", - "new_fs", - ImmutableMap.of("id", ValueProto.ValueType.Enum.STRING), - ImmutableMap.of("total", ValueProto.ValueType.Enum.INT64))); - - assertThat( - response.getStatus(), equalTo(CoreServiceProto.ApplyFeatureSetResponse.Status.CREATED)); - assertThat(response.getFeatureSet().getSpec().getProject(), equalTo("new_project")); - } - - @Test - public void shouldUsedDefaultProjectIfUnspecified() { - CoreServiceProto.ApplyFeatureSetResponse response = - apiClient.simpleApplyFeatureSet( - DataGenerator.createFeatureSet( - DataGenerator.getDefaultSource(), - "", - "some", - ImmutableMap.of("id", ValueProto.ValueType.Enum.STRING), - ImmutableMap.of("total", ValueProto.ValueType.Enum.INT64))); - - assertThat( - response.getStatus(), equalTo(CoreServiceProto.ApplyFeatureSetResponse.Status.CREATED)); - assertThat(response.getFeatureSet().getSpec().getProject(), equalTo("default")); - } - - @Test - public void shouldFailWhenProjectIsArchived() { - apiClient.createProject("archived"); - apiClient.archiveProject("archived"); - - StatusRuntimeException exc = - assertThrows( - StatusRuntimeException.class, - () -> - apiClient.simpleApplyFeatureSet( - DataGenerator.createFeatureSet( - DataGenerator.getDefaultSource(), - "archived", - "fs", - ImmutableMap.of("id", ValueProto.ValueType.Enum.STRING), - ImmutableMap.of("total", ValueProto.ValueType.Enum.INT64)))); - assertThat(exc.getMessage(), equalTo("INTERNAL: Project is archived: archived")); - } - - @Test - public void shouldAcceptFeatureLabels() { - CoreServiceProto.ApplyFeatureSetResponse response = - apiClient.simpleApplyFeatureSet( - DataGenerator.createFeatureSet( - DataGenerator.getDefaultSource(), - "default", - "some", - ImmutableList.of( - FeatureSetProto.EntitySpec.newBuilder() - .setName("id") - .setValueType(ValueProto.ValueType.Enum.STRING) - .build()), - ImmutableList.of( - FeatureSetProto.FeatureSpec.newBuilder() - .setName("feature1") - .setValueType(ValueProto.ValueType.Enum.INT64) - .putAllLabels(ImmutableMap.of("type", "integer")) - .build(), - FeatureSetProto.FeatureSpec.newBuilder() - .setName("feature2") - .setValueType(ValueProto.ValueType.Enum.STRING) - .putAllLabels(ImmutableMap.of("type", "string")) - .build()), - Collections.emptyMap())); - - assertThat( - response.getStatus(), equalTo(CoreServiceProto.ApplyFeatureSetResponse.Status.CREATED)); - assertThat( - response.getFeatureSet().getSpec().getFeaturesList(), - hasItem( - allOf( - hasProperty("name", equalTo("feature1")), - hasProperty("labelsMap", hasEntry("type", "integer"))))); - assertThat( - response.getFeatureSet().getSpec().getFeaturesList(), - hasItem( - allOf( - hasProperty("name", equalTo("feature2")), - hasProperty("labelsMap", hasEntry("type", "string"))))); - } - - @Test - public void shouldUpdateLabels() { - CoreServiceProto.ApplyFeatureSetResponse response = - apiClient.simpleApplyFeatureSet( - DataGenerator.createFeatureSet( - DataGenerator.getDefaultSource(), - "project1", - "fs4", - ImmutableList.of( - DataGenerator.createEntitySpec( - "customer_id", ValueProto.ValueType.Enum.STRING)), - ImmutableList.of( - DataGenerator.createFeature( - "feature2", - ValueProto.ValueType.Enum.INT32, - ImmutableMap.of("app", "feast", "version", "two"))), - ImmutableMap.of("label", "some"))); - - assertThat( - response.getStatus(), equalTo(CoreServiceProto.ApplyFeatureSetResponse.Status.UPDATED)); - assertThat( - response.getFeatureSet().getSpec().getFeaturesList(), - hasItem( - allOf( - hasProperty("name", equalTo("feature2")), - hasProperty("labelsMap", hasEntry("version", "two"))))); - } - - @Test - public void shouldAcceptFeatureSetLabels() { - CoreServiceProto.ApplyFeatureSetResponse response = - apiClient.simpleApplyFeatureSet( - DataGenerator.createFeatureSet( - DataGenerator.getDefaultSource(), - "", - "some", - ImmutableList.of( - DataGenerator.createEntitySpec( - "customer_id", ValueProto.ValueType.Enum.STRING)), - ImmutableList.of(), - ImmutableMap.of("label", "some"))); - - assertThat( - response.getStatus(), equalTo(CoreServiceProto.ApplyFeatureSetResponse.Status.CREATED)); - assertThat(response.getFeatureSet().getSpec().getLabelsMap(), hasEntry("label", "some")); - } - } - @Nested class ApplyEntity { @Test @@ -884,20 +390,6 @@ public void shouldDoNothingIfNoChange() { } } - @Nested - class GetFeatureSet { - @Test - public void shouldThrowExceptionGivenMissingFeatureSet() { - StatusRuntimeException exc = - assertThrows( - StatusRuntimeException.class, () -> apiClient.getFeatureSet("default", "unknown")); - - assertThat( - exc.getMessage(), - equalTo("INTERNAL: Feature set with name \"unknown\" could not be found.")); - } - } - @Nested class GetEntity { @Test @@ -938,26 +430,10 @@ public void shouldThrowExceptionGivenNoSuchFeatureTable() { @Test public void shouldReturnFeatureTableIfExists() { - FeatureTableSpec featureTableSpec = - DataGenerator.createFeatureTableSpec( - "featuretable1", - Arrays.asList("entity1", "entity2"), - new HashMap<>() { - { - put("feature1", ValueProto.ValueType.Enum.STRING); - put("feature2", ValueProto.ValueType.Enum.FLOAT); - } - }, - 7200, - ImmutableMap.of("feat_key2", "feat_value2")) - .toBuilder() - .setBatchSource( - DataGenerator.createFileDataSourceSpec("file:///path/to/file", "ts_col", "")) - .build(); FeatureTableProto.FeatureTable featureTable = apiClient.simpleGetFeatureTable("default", "featuretable1"); - assertTrue(TestUtil.compareFeatureTableSpec(featureTable.getSpec(), featureTableSpec)); + assertTrue(TestUtil.compareFeatureTableSpec(featureTable.getSpec(), example1)); } } @@ -1005,66 +481,64 @@ class ListFeatures { @Test public void shouldFilterFeaturesByEntitiesAndLabels() { // Case 1: Only filter by entities - Map result1 = - apiClient.simpleListFeatures("project1", "user_id"); + Map result1 = + apiClient.simpleListFeatures("default", "entity1", "entity2"); - assertThat(result1, aMapWithSize(2)); - assertThat(result1, hasKey(equalTo("project1/fs3:feature1"))); - assertThat(result1, hasKey(equalTo("project1/fs3:feature2"))); + assertThat(result1, aMapWithSize(4)); + assertThat(result1, hasKey(equalTo("featuretable1:feature1"))); + assertThat(result1, hasKey(equalTo("featuretable1:feature2"))); + assertThat(result1, hasKey(equalTo("featuretable2:feature3"))); + assertThat(result1, hasKey(equalTo("featuretable2:feature4"))); // Case 2: Filter by entities and labels - Map result2 = + Map result2 = apiClient.simpleListFeatures( - "project1", - ImmutableMap.of("app", "feast", "version", "one"), - ImmutableList.of("customer_id")); + "default", + ImmutableMap.of("feat_key2", "feat_value2"), + ImmutableList.of("entity1", "entity2")); - assertThat(result2, aMapWithSize(1)); - assertThat(result2, hasKey(equalTo("project1/fs4:feature2"))); + assertThat(result2, aMapWithSize(2)); + assertThat(result2, hasKey(equalTo("featuretable1:feature1"))); + assertThat(result2, hasKey(equalTo("featuretable1:feature2"))); // Case 3: Filter by labels - Map result3 = + Map result3 = apiClient.simpleListFeatures( - "project1", ImmutableMap.of("app", "feast"), Collections.emptyList()); + "default", ImmutableMap.of("feat_key4", "feat_value4"), Collections.emptyList()); assertThat(result3, aMapWithSize(2)); - assertThat(result3, hasKey(equalTo("project1/fs4:feature2"))); - assertThat(result3, hasKey(equalTo("project1/fs5:feature3"))); + assertThat(result3, hasKey(equalTo("featuretable2:feature3"))); + assertThat(result3, hasKey(equalTo("featuretable2:feature4"))); // Case 4: Filter by nothing, except project - Map result4 = + Map result4 = apiClient.simpleListFeatures("project1", ImmutableMap.of(), Collections.emptyList()); - assertThat(result4, aMapWithSize(4)); - assertThat(result4, hasKey(equalTo("project1/fs3:feature1"))); - assertThat(result4, hasKey(equalTo("project1/fs3:feature1"))); - assertThat(result4, hasKey(equalTo("project1/fs4:feature2"))); - assertThat(result4, hasKey(equalTo("project1/fs5:feature3"))); + assertThat(result4, aMapWithSize(0)); // Case 5: Filter by nothing; will use default project - Map result5 = + Map result5 = apiClient.simpleListFeatures("", ImmutableMap.of(), Collections.emptyList()); - assertThat(result5, aMapWithSize(2)); - assertThat(result5, hasKey(equalTo("default/fs1:total"))); - assertThat(result5, hasKey(equalTo("default/fs2:sum"))); + assertThat(result5, aMapWithSize(4)); + assertThat(result5, hasKey(equalTo("featuretable1:feature1"))); + assertThat(result5, hasKey(equalTo("featuretable1:feature2"))); + assertThat(result5, hasKey(equalTo("featuretable2:feature3"))); + assertThat(result5, hasKey(equalTo("featuretable2:feature4"))); + + // Case 6: Filter by mismatched entity + Map result6 = + apiClient.simpleListFeatures("default", ImmutableMap.of(), ImmutableList.of("entity1")); + assertThat(result6, aMapWithSize(0)); } } @Nested public class ApplyFeatureTable { private FeatureTableSpec getTestSpec() { - return DataGenerator.createFeatureTableSpec( - "ft", - List.of("entity1", "entity2"), - Map.of( - "feature1", ValueProto.ValueType.Enum.INT64, - "feature2", ValueProto.ValueType.Enum.FLOAT), - 3600, - Map.of()) + return example1 .toBuilder() - .setBatchSource( - DataGenerator.createFileDataSourceSpec("file:///path/to/file", "ts_col", "")) + .setName("apply_test") .setStreamSource( DataGenerator.createKafkaDataSourceSpec( "localhost:9092", "topic", "class.path", "ts_col")) @@ -1084,22 +558,17 @@ public void shouldUpdateExistingTableWithValidSpec() { FeatureTableProto.FeatureTable table = apiClient.applyFeatureTable("default", getTestSpec()); FeatureTableSpec updatedSpec = - DataGenerator.createFeatureTableSpec( - "ft", - List.of("entity1", "entity2"), - Map.of( - "feature2", ValueProto.ValueType.Enum.FLOAT, - "feature3", ValueProto.ValueType.Enum.INT64, - "feature4", ValueProto.ValueType.Enum.INT64), - 2100, - Map.of("test", "labels")) + getTestSpec() .toBuilder() + .clearFeatures() + .addFeatures( + DataGenerator.createFeatureSpecV2( + "feature5", ValueProto.ValueType.Enum.FLOAT, ImmutableMap.of())) .setStreamSource( - DataGenerator.createFileDataSourceSpec("file:///path/to/file", "ts_col", "")) - .setBatchSource( DataGenerator.createKafkaDataSourceSpec( - "localhost:9092", "topic", "class.path", "ts_col")) + "localhost:9092", "new_topic", "new.class", "ts_col")) .build(); + FeatureTableProto.FeatureTable updatedTable = apiClient.applyFeatureTable("default", updatedSpec); @@ -1109,22 +578,21 @@ public void shouldUpdateExistingTableWithValidSpec() { @Test public void shouldUpdateFeatureTableOnEntityChange() { - List entities = Arrays.asList("entity1", "entity2"); FeatureTableProto.FeatureTableSpec updatedSpec = - DataGenerator.createFeatureTableSpec( - "featuretable1", - Arrays.asList("entity1"), - new HashMap<>() { - { - put("feature1", ValueProto.ValueType.Enum.STRING); - put("feature2", ValueProto.ValueType.Enum.FLOAT); - } - }, - 7200, - ImmutableMap.of("feat_key2", "feat_value2")) + getTestSpec().toBuilder().clearEntities().addEntities("entity1").build(); + + FeatureTableProto.FeatureTable updatedTable = + apiClient.applyFeatureTable("default", updatedSpec); + + assertTrue(TestUtil.compareFeatureTableSpec(updatedTable.getSpec(), updatedSpec)); + } + + @Test + public void shouldUpdateFeatureTableOnMaxAgeChange() { + FeatureTableProto.FeatureTableSpec updatedSpec = + getTestSpec() .toBuilder() - .setBatchSource( - DataGenerator.createFileDataSourceSpec("file:///path/to/file", "ts_col", "")) + .setMaxAge(Duration.newBuilder().setSeconds(600).build()) .build(); FeatureTableProto.FeatureTable updatedTable = @@ -1135,21 +603,35 @@ public void shouldUpdateFeatureTableOnEntityChange() { @Test public void shouldUpdateFeatureTableOnFeatureTypeChange() { + int featureIdx = + IntStream.range(0, getTestSpec().getFeaturesCount()) + .filter(i -> getTestSpec().getFeatures(i).getName().equals("feature2")) + .findFirst() + .orElse(-1); + FeatureTableProto.FeatureTableSpec updatedSpec = - DataGenerator.createFeatureTableSpec( - "featuretable1", - Arrays.asList("entity1", "entity2"), - new HashMap<>() { - { - put("feature1", ValueProto.ValueType.Enum.STRING); - put("feature2", ValueProto.ValueType.Enum.STRING_LIST); - } - }, - 7200, - ImmutableMap.of("feat_key2", "feat_value2")) + getTestSpec() .toBuilder() - .setBatchSource( - DataGenerator.createFileDataSourceSpec("file:///path/to/file", "ts_col", "")) + .setFeatures( + featureIdx, + DataGenerator.createFeatureSpecV2( + "feature2", ValueProto.ValueType.Enum.STRING_LIST, ImmutableMap.of())) + .build(); + + FeatureTableProto.FeatureTable updatedTable = + apiClient.applyFeatureTable("default", updatedSpec); + + assertTrue(TestUtil.compareFeatureTableSpec(updatedTable.getSpec(), updatedSpec)); + } + + @Test + public void shouldUpdateFeatureTableOnFeatureAddition() { + FeatureTableProto.FeatureTableSpec updatedSpec = + getTestSpec() + .toBuilder() + .addFeatures( + DataGenerator.createFeatureSpecV2( + "feature6", ValueProto.ValueType.Enum.FLOAT, ImmutableMap.of())) .build(); FeatureTableProto.FeatureTable updatedTable = @@ -1177,6 +659,7 @@ public void shouldErrorOnMissingBatchSource() { 3600, Map.of()) .toBuilder() + .clearBatchSource() .build(); StatusRuntimeException exc = @@ -1350,6 +833,7 @@ public void shouldReturnNoTables() { CoreServiceProto.ListFeatureTablesRequest.Filter filter = CoreServiceProto.ListFeatureTablesRequest.Filter.newBuilder() .setProject("default") + .putLabels("feat_key2", "feat_value2") .build(); List featureTables = apiClient.simpleListFeatureTables(filter); diff --git a/core/src/test/java/feast/core/service/StatsServiceTest.java b/core/src/test/java/feast/core/service/StatsServiceTest.java deleted file mode 100644 index 276d8df4b7c..00000000000 --- a/core/src/test/java/feast/core/service/StatsServiceTest.java +++ /dev/null @@ -1,384 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.core.service; - -import static org.hamcrest.Matchers.equalTo; -import static org.junit.Assert.assertThat; -import static org.mockito.Mockito.*; -import static org.mockito.MockitoAnnotations.initMocks; - -import com.google.protobuf.Timestamp; -import feast.core.dao.FeatureSetRepository; -import feast.core.dao.FeatureStatisticsRepository; -import feast.core.dao.StoreRepository; -import feast.core.model.Project; -import feast.core.model.Store; -import feast.proto.core.CoreServiceProto.GetFeatureStatisticsRequest; -import feast.proto.core.StoreProto; -import feast.proto.core.StoreProto.Store.BigQueryConfig; -import feast.proto.core.StoreProto.Store.StoreType; -import feast.storage.connectors.bigquery.statistics.BigQueryStatisticsRetriever; -import java.io.IOException; -import java.util.Arrays; -import java.util.Optional; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.ExpectedException; -import org.mockito.Mock; -import org.tensorflow.metadata.v0.*; -import org.tensorflow.metadata.v0.FeatureNameStatistics.Type; - -public class StatsServiceTest { - - private StatsService statsService; - @Mock private FeatureStatisticsRepository featureStatisticsRepository; - @Mock private StoreRepository storeRepository; - @Mock private FeatureSetRepository featureSetRepository; - - @Rule public final ExpectedException expectedException = ExpectedException.none(); - - @Before - public void setUp() { - initMocks(this); - statsService = - new StatsService(storeRepository, featureStatisticsRepository, featureSetRepository); - } - - @Test - public void shouldThrowExceptionIfNeitherDatesNorDatasetsProvided() throws IOException { - GetFeatureStatisticsRequest request = GetFeatureStatisticsRequest.newBuilder().build(); - - expectedException.expect(IllegalArgumentException.class); - expectedException.expectMessage( - "Invalid request. Either provide dataset ids to retrieve statistics over, or a start date and end date."); - statsService.getFeatureStatistics(request); - } - - @Test - public void shouldThrowExceptionIfInvalidDatesProvided() throws IOException { - GetFeatureStatisticsRequest request = - GetFeatureStatisticsRequest.newBuilder() - .setStartDate(Timestamp.newBuilder().setSeconds(1)) - .setEndDate(Timestamp.newBuilder().setSeconds(0)) - .build(); - - expectedException.expect(IllegalArgumentException.class); - expectedException.expectMessage( - "Invalid request. Start timestamp 1 is greater than the end timestamp 0"); - statsService.getFeatureStatistics(request); - } - - @Test - public void shouldThrowExceptionIfInvalidStoreProvided() throws IOException { - GetFeatureStatisticsRequest request = - GetFeatureStatisticsRequest.newBuilder() - .setStartDate(Timestamp.newBuilder().setSeconds(0)) - .setEndDate(Timestamp.newBuilder().setSeconds(1)) - .setStore("redis") - .build(); - - when(storeRepository.findById("redis")) - .thenReturn( - Optional.of( - Store.fromProto( - StoreProto.Store.newBuilder() - .setName("redis") - .setType(StoreType.REDIS) - .build()))); - - expectedException.expect(IllegalArgumentException.class); - expectedException.expectMessage( - "Invalid store redis with type REDIS specified. Batch statistics are only supported for BigQuery stores"); - statsService.getFeatureStatistics(request); - } - - @Test - public void shouldThrowExceptionIfFeatureSetNotFound() throws IOException { - GetFeatureStatisticsRequest request = - GetFeatureStatisticsRequest.newBuilder() - .setStartDate(Timestamp.newBuilder().setSeconds(0)) - .setEndDate(Timestamp.newBuilder().setSeconds(1)) - .setStore("bigquery") - .setFeatureSetId("my_feature_set") - .build(); - - StoreProto.Store storeProto = - StoreProto.Store.newBuilder() - .setName("bigquery") - .setType(StoreType.BIGQUERY) - .setBigqueryConfig( - BigQueryConfig.newBuilder().setProjectId("project").setDatasetId("dataset")) - .build(); - when(storeRepository.findById("bigquery")).thenReturn(Optional.of(Store.fromProto(storeProto))); - when(featureSetRepository.findFeatureSetByNameAndProject_Name( - "my_feature_set", Project.DEFAULT_NAME)) - .thenReturn(null); - - statsService = spy(statsService); - BigQueryStatisticsRetriever retriever = mock(BigQueryStatisticsRetriever.class); - doReturn(retriever).when(statsService).getStatisticsRetriever(storeProto.getName()); - - expectedException.expect(IllegalArgumentException.class); - expectedException.expectMessage("Illegal request. Unable to find feature set my_feature_set"); - statsService.getFeatureStatistics(request); - } - - @Test - public void shouldAggregateNumericStatistics() { - FeatureNameStatistics stat1 = - FeatureNameStatistics.newBuilder() - .setNumStats( - NumericStatistics.newBuilder() - .setMax(20) - .setMin(1) - .setMean(6) - .setNumZeros(0) - .setStdDev(7.90569415) - .setCommonStats( - CommonStatistics.newBuilder() - .setTotNumValues(5) - .setNumNonMissing(5) - .setAvgNumValues(1) - .setMaxNumValues(1) - .setMinNumValues(1) - .setNumMissing(0))) - .setType(Type.INT) - .setPath(Path.newBuilder().addStep("feature").build()) - .build(); - FeatureNameStatistics stat2 = - FeatureNameStatistics.newBuilder() - .setNumStats( - NumericStatistics.newBuilder() - .setMax(10) - .setMin(0) - .setMean(4) - .setNumZeros(1) - .setStdDev(3.807886553) - .setCommonStats( - CommonStatistics.newBuilder() - .setTotNumValues(5) - .setNumNonMissing(5) - .setAvgNumValues(1) - .setMaxNumValues(1) - .setMinNumValues(1) - .setNumMissing(1))) - .setPath(Path.newBuilder().addStep("feature").build()) - .setType(Type.INT) - .build(); - - FeatureNameStatistics expected = - FeatureNameStatistics.newBuilder() - .setNumStats( - NumericStatistics.newBuilder() - .setMax(20) - .setMin(0) - .setMean(5) - .setNumZeros(1) - .setStdDev(5.944184833146219) - .setCommonStats( - CommonStatistics.newBuilder() - .setTotNumValues(10) - .setNumNonMissing(10) - .setAvgNumValues(1) - .setMaxNumValues(1) - .setMinNumValues(1) - .setNumMissing(1))) - .setPath(Path.newBuilder().addStep("feature").build()) - .setType(Type.INT) - .build(); - - assertThat( - statsService.mergeStatistics(Arrays.asList(Arrays.asList(stat1, stat2))), - equalTo(Arrays.asList(expected))); - } - - @Test - public void shouldAggregateCategoricalStatistics() { - FeatureNameStatistics stat1 = - FeatureNameStatistics.newBuilder() - .setStringStats( - StringStatistics.newBuilder() - .setCommonStats( - CommonStatistics.newBuilder() - .setTotNumValues(5) - .setNumNonMissing(5) - .setAvgNumValues(1) - .setMaxNumValues(1) - .setMinNumValues(1) - .setNumMissing(0)) - .setUnique(4) - .setAvgLength(6)) - .setType(Type.STRING) - .setPath(Path.newBuilder().addStep("feature").build()) - .build(); - FeatureNameStatistics stat2 = - FeatureNameStatistics.newBuilder() - .setStringStats( - StringStatistics.newBuilder() - .setCommonStats( - CommonStatistics.newBuilder() - .setTotNumValues(5) - .setNumNonMissing(5) - .setAvgNumValues(1) - .setMaxNumValues(1) - .setMinNumValues(1) - .setNumMissing(1)) - .setUnique(4) - .setAvgLength(4)) - .setType(Type.STRING) - .setPath(Path.newBuilder().addStep("feature").build()) - .build(); - - FeatureNameStatistics expected = - FeatureNameStatistics.newBuilder() - .setStringStats( - StringStatistics.newBuilder() - .setCommonStats( - CommonStatistics.newBuilder() - .setTotNumValues(10) - .setNumNonMissing(10) - .setAvgNumValues(1) - .setMaxNumValues(1) - .setMinNumValues(1) - .setNumMissing(1)) - .setAvgLength(5)) - .setType(Type.STRING) - .setPath(Path.newBuilder().addStep("feature").build()) - .build(); - assertThat( - statsService.mergeStatistics(Arrays.asList(Arrays.asList(stat1, stat2))), - equalTo(Arrays.asList(expected))); - } - - @Test - public void shouldAggregateBytesStatistics() { - FeatureNameStatistics stat1 = - FeatureNameStatistics.newBuilder() - .setBytesStats( - BytesStatistics.newBuilder() - .setCommonStats( - CommonStatistics.newBuilder() - .setTotNumValues(5) - .setNumNonMissing(5) - .setAvgNumValues(1) - .setMaxNumValues(1) - .setMinNumValues(1) - .setNumMissing(0)) - .setUnique(4) - .setAvgNumBytes(6) - .setMaxNumBytes(10) - .setMinNumBytes(0)) - .setType(Type.BYTES) - .setPath(Path.newBuilder().addStep("feature").build()) - .build(); - FeatureNameStatistics stat2 = - FeatureNameStatistics.newBuilder() - .setBytesStats( - BytesStatistics.newBuilder() - .setCommonStats( - CommonStatistics.newBuilder() - .setTotNumValues(5) - .setNumNonMissing(5) - .setAvgNumValues(1) - .setMaxNumValues(1) - .setMinNumValues(1) - .setNumMissing(1)) - .setUnique(4) - .setAvgNumBytes(4) - .setMaxNumBytes(20) - .setMinNumBytes(1)) - .setType(Type.BYTES) - .setPath(Path.newBuilder().addStep("feature").build()) - .build(); - - FeatureNameStatistics expected = - FeatureNameStatistics.newBuilder() - .setBytesStats( - BytesStatistics.newBuilder() - .setCommonStats( - CommonStatistics.newBuilder() - .setTotNumValues(10) - .setNumNonMissing(10) - .setAvgNumValues(1) - .setMaxNumValues(1) - .setMinNumValues(1) - .setNumMissing(1)) - .setAvgNumBytes(5) - .setMaxNumBytes(20) - .setMinNumBytes(0)) - .setType(Type.BYTES) - .setPath(Path.newBuilder().addStep("feature").build()) - .build(); - assertThat( - statsService.mergeStatistics(Arrays.asList(Arrays.asList(stat1, stat2))), - equalTo(Arrays.asList(expected))); - } - - @Test - public void shouldAggregateStructStatistics() { - FeatureNameStatistics stat1 = - FeatureNameStatistics.newBuilder() - .setStructStats( - StructStatistics.newBuilder() - .setCommonStats( - CommonStatistics.newBuilder() - .setTotNumValues(5) - .setNumNonMissing(5) - .setAvgNumValues(1) - .setMaxNumValues(1) - .setMinNumValues(1) - .setNumMissing(0))) - .setType(Type.STRUCT) - .setPath(Path.newBuilder().addStep("feature").build()) - .build(); - FeatureNameStatistics stat2 = - FeatureNameStatistics.newBuilder() - .setStructStats( - StructStatistics.newBuilder() - .setCommonStats( - CommonStatistics.newBuilder() - .setTotNumValues(5) - .setNumNonMissing(5) - .setAvgNumValues(1) - .setMaxNumValues(1) - .setMinNumValues(1) - .setNumMissing(1))) - .setType(Type.STRUCT) - .setPath(Path.newBuilder().addStep("feature").build()) - .build(); - - FeatureNameStatistics expected = - FeatureNameStatistics.newBuilder() - .setStructStats( - StructStatistics.newBuilder() - .setCommonStats( - CommonStatistics.newBuilder() - .setTotNumValues(10) - .setNumNonMissing(10) - .setAvgNumValues(1) - .setMaxNumValues(1) - .setMinNumValues(1) - .setNumMissing(1))) - .setType(Type.STRUCT) - .setPath(Path.newBuilder().addStep("feature").build()) - .build(); - assertThat( - statsService.mergeStatistics(Arrays.asList(Arrays.asList(stat1, stat2))), - equalTo(Arrays.asList(expected))); - } -} diff --git a/core/src/test/java/feast/core/validators/FeatureSetValidatorTest.java b/core/src/test/java/feast/core/validators/FeatureSetValidatorTest.java deleted file mode 100644 index 155a52d1005..00000000000 --- a/core/src/test/java/feast/core/validators/FeatureSetValidatorTest.java +++ /dev/null @@ -1,87 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.core.validators; - -import feast.proto.core.FeatureSetProto; -import feast.proto.types.ValueProto; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.ExpectedException; - -public class FeatureSetValidatorTest { - - @Rule public final ExpectedException expectedException = ExpectedException.none(); - - @Test - public void shouldThrowExceptionForFeatureLabelsWithAnEmptyKey() { - Map featureLabels = - new HashMap<>() { - { - put("", "empty_key"); - } - }; - - List featureSpecs = new ArrayList<>(); - featureSpecs.add( - FeatureSetProto.FeatureSpec.newBuilder() - .setName("feature1") - .setValueType(ValueProto.ValueType.Enum.INT64) - .putAllLabels(featureLabels) - .build()); - - FeatureSetProto.FeatureSetSpec featureSetSpec = - FeatureSetProto.FeatureSetSpec.newBuilder() - .setProject("project1") - .setName("featureSetWithConstraints") - .addAllFeatures(featureSpecs) - .build(); - FeatureSetProto.FeatureSet featureSet = - FeatureSetProto.FeatureSet.newBuilder().setSpec(featureSetSpec).build(); - - expectedException.expect(IllegalArgumentException.class); - expectedException.expectMessage("Feature label keys must not be empty"); - FeatureSetValidator.validateSpec(featureSet); - } - - @Test - public void shouldThrowExceptionForFeatureSetLabelsWithAnEmptyKey() { - - Map featureSetLabels = - new HashMap<>() { - { - put("", "empty_key"); - } - }; - - FeatureSetProto.FeatureSetSpec featureSetSpec = - FeatureSetProto.FeatureSetSpec.newBuilder() - .setProject("project1") - .setName("featureSetWithConstraints") - .putAllLabels(featureSetLabels) - .build(); - FeatureSetProto.FeatureSet featureSet = - FeatureSetProto.FeatureSet.newBuilder().setSpec(featureSetSpec).build(); - - expectedException.expect(IllegalArgumentException.class); - expectedException.expectMessage("Feature set label keys must not be empty"); - FeatureSetValidator.validateSpec(featureSet); - } -} diff --git a/core/src/test/java/feast/core/validators/MatchersTest.java b/core/src/test/java/feast/core/validators/MatchersTest.java index fdf4d0469d2..17332123d21 100644 --- a/core/src/test/java/feast/core/validators/MatchersTest.java +++ b/core/src/test/java/feast/core/validators/MatchersTest.java @@ -31,13 +31,13 @@ public class MatchersTest { @Test public void checkUpperSnakeCaseShouldPassForLegitUpperSnakeCase() { String in = "REDIS_DB"; - checkUpperSnakeCase(in, "featureset"); + checkUpperSnakeCase(in, "featuretable"); } @Test public void checkUpperSnakeCaseShouldPassForLegitUpperSnakeCaseWithNumbers() { String in = "REDIS1"; - checkUpperSnakeCase(in, "featureset"); + checkUpperSnakeCase(in, "featuretable"); } @Test @@ -46,11 +46,11 @@ public void checkUpperSnakeCaseShouldThrowIllegalArgumentExceptionWithFieldForIn exception.expectMessage( Strings.lenientFormat( "invalid value for %s resource, %s: %s", - "featureset", + "featuretable", "redis", "argument must be in upper snake case, and cannot include any special characters.")); String in = "redis"; - checkUpperSnakeCase(in, "featureset"); + checkUpperSnakeCase(in, "featuretable"); } @Test diff --git a/datatypes/java/README.md b/datatypes/java/README.md index 0996af74b1c..c926926f209 100644 --- a/datatypes/java/README.md +++ b/datatypes/java/README.md @@ -16,7 +16,7 @@ Dependency Coordinates dev.feast datatypes-java - 0.8.0 + 0.9.0 ``` diff --git a/docs/.gitbook/assets/architecture.png b/docs/.gitbook/assets/arch.png similarity index 100% rename from docs/.gitbook/assets/architecture.png rename to docs/.gitbook/assets/arch.png diff --git a/docs/.gitbook/assets/basic-architecture-diagram (1).svg b/docs/.gitbook/assets/basic-architecture-diagram (3) (3) (1).svg similarity index 100% rename from docs/.gitbook/assets/basic-architecture-diagram (1).svg rename to docs/.gitbook/assets/basic-architecture-diagram (3) (3) (1).svg diff --git a/docs/.gitbook/assets/basic-architecture-diagram (2).svg b/docs/.gitbook/assets/basic-architecture-diagram (3) (3) (2).svg similarity index 100% rename from docs/.gitbook/assets/basic-architecture-diagram (2).svg rename to docs/.gitbook/assets/basic-architecture-diagram (3) (3) (2).svg diff --git a/docs/.gitbook/assets/basic-architecture-diagram (3).svg b/docs/.gitbook/assets/basic-architecture-diagram (3) (3) (3).svg similarity index 100% rename from docs/.gitbook/assets/basic-architecture-diagram (3).svg rename to docs/.gitbook/assets/basic-architecture-diagram (3) (3) (3).svg diff --git a/docs/.gitbook/assets/basic-architecture-diagram.svg b/docs/.gitbook/assets/basic-architecture-diagram (3) (3).svg similarity index 100% rename from docs/.gitbook/assets/basic-architecture-diagram.svg rename to docs/.gitbook/assets/basic-architecture-diagram (3) (3).svg diff --git a/docs/.gitbook/assets/blank-diagram-4 (1).svg b/docs/.gitbook/assets/blank-diagram-4 (4) (4) (1).svg similarity index 100% rename from docs/.gitbook/assets/blank-diagram-4 (1).svg rename to docs/.gitbook/assets/blank-diagram-4 (4) (4) (1).svg diff --git a/docs/.gitbook/assets/blank-diagram-4 (2).svg b/docs/.gitbook/assets/blank-diagram-4 (4) (4) (2).svg similarity index 100% rename from docs/.gitbook/assets/blank-diagram-4 (2).svg rename to docs/.gitbook/assets/blank-diagram-4 (4) (4) (2).svg diff --git a/docs/.gitbook/assets/blank-diagram-4 (3).svg b/docs/.gitbook/assets/blank-diagram-4 (4) (4) (3).svg similarity index 100% rename from docs/.gitbook/assets/blank-diagram-4 (3).svg rename to docs/.gitbook/assets/blank-diagram-4 (4) (4) (3).svg diff --git a/docs/.gitbook/assets/blank-diagram-4.svg b/docs/.gitbook/assets/blank-diagram-4 (4) (4) (4).svg similarity index 100% rename from docs/.gitbook/assets/blank-diagram-4.svg rename to docs/.gitbook/assets/blank-diagram-4 (4) (4) (4).svg diff --git a/docs/.gitbook/assets/blank-diagram-4 (4) (4).svg b/docs/.gitbook/assets/blank-diagram-4 (4) (4).svg new file mode 100644 index 00000000000..fb5e0659e55 --- /dev/null +++ b/docs/.gitbook/assets/blank-diagram-4 (4) (4).svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/docs/.gitbook/assets/concept_hierarchy.png b/docs/.gitbook/assets/concept_hierarchy (1) (1).png similarity index 100% rename from docs/.gitbook/assets/concept_hierarchy.png rename to docs/.gitbook/assets/concept_hierarchy (1) (1).png diff --git a/docs/.gitbook/assets/concept_hierarchy (1).png b/docs/.gitbook/assets/concept_hierarchy (1).png new file mode 100644 index 00000000000..f5cf59ad673 Binary files /dev/null and b/docs/.gitbook/assets/concept_hierarchy (1).png differ diff --git a/docs/.gitbook/assets/feast-architecture-diagrams.svg b/docs/.gitbook/assets/feast-architecture-diagrams (1) (1) (1).svg similarity index 100% rename from docs/.gitbook/assets/feast-architecture-diagrams.svg rename to docs/.gitbook/assets/feast-architecture-diagrams (1) (1) (1).svg diff --git a/docs/.gitbook/assets/feast-architecture-diagrams (1) (1).svg b/docs/.gitbook/assets/feast-architecture-diagrams (1) (1).svg new file mode 100644 index 00000000000..7335c131c44 --- /dev/null +++ b/docs/.gitbook/assets/feast-architecture-diagrams (1) (1).svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/docs/.gitbook/assets/feast-docs-overview-diagram-2 (1).svg b/docs/.gitbook/assets/feast-docs-overview-diagram-2 (5) (1) (1).svg similarity index 100% rename from docs/.gitbook/assets/feast-docs-overview-diagram-2 (1).svg rename to docs/.gitbook/assets/feast-docs-overview-diagram-2 (5) (1) (1).svg diff --git a/docs/.gitbook/assets/feast-docs-overview-diagram-2 (2).svg b/docs/.gitbook/assets/feast-docs-overview-diagram-2 (5) (1) (2).svg similarity index 100% rename from docs/.gitbook/assets/feast-docs-overview-diagram-2 (2).svg rename to docs/.gitbook/assets/feast-docs-overview-diagram-2 (5) (1) (2).svg diff --git a/docs/.gitbook/assets/feast-docs-overview-diagram-2 (3) (1).svg b/docs/.gitbook/assets/feast-docs-overview-diagram-2 (5) (1) (3).svg similarity index 100% rename from docs/.gitbook/assets/feast-docs-overview-diagram-2 (3) (1).svg rename to docs/.gitbook/assets/feast-docs-overview-diagram-2 (5) (1) (3).svg diff --git a/docs/.gitbook/assets/feast-docs-overview-diagram-2 (3).svg b/docs/.gitbook/assets/feast-docs-overview-diagram-2 (5) (1) (4).svg similarity index 100% rename from docs/.gitbook/assets/feast-docs-overview-diagram-2 (3).svg rename to docs/.gitbook/assets/feast-docs-overview-diagram-2 (5) (1) (4).svg diff --git a/docs/.gitbook/assets/feast-docs-overview-diagram-2 (4).svg b/docs/.gitbook/assets/feast-docs-overview-diagram-2 (5) (1) (5).svg similarity index 100% rename from docs/.gitbook/assets/feast-docs-overview-diagram-2 (4).svg rename to docs/.gitbook/assets/feast-docs-overview-diagram-2 (5) (1) (5).svg diff --git a/docs/.gitbook/assets/feast-docs-overview-diagram-2 (5).svg b/docs/.gitbook/assets/feast-docs-overview-diagram-2 (5) (1) (6).svg similarity index 100% rename from docs/.gitbook/assets/feast-docs-overview-diagram-2 (5).svg rename to docs/.gitbook/assets/feast-docs-overview-diagram-2 (5) (1) (6).svg diff --git a/docs/.gitbook/assets/feast-docs-overview-diagram-2.svg b/docs/.gitbook/assets/feast-docs-overview-diagram-2 (5) (1).svg similarity index 100% rename from docs/.gitbook/assets/feast-docs-overview-diagram-2.svg rename to docs/.gitbook/assets/feast-docs-overview-diagram-2 (5) (1).svg diff --git a/docs/.gitbook/assets/feast-on-aws-3- (1).png b/docs/.gitbook/assets/feast-on-aws-3- (1).png new file mode 100644 index 00000000000..e6de77dde9b Binary files /dev/null and b/docs/.gitbook/assets/feast-on-aws-3- (1).png differ diff --git a/docs/.gitbook/assets/image (2) (1).png b/docs/.gitbook/assets/image (2) (3) (3) (1).png similarity index 100% rename from docs/.gitbook/assets/image (2) (1).png rename to docs/.gitbook/assets/image (2) (3) (3) (1).png diff --git a/docs/.gitbook/assets/image (2) (2).png b/docs/.gitbook/assets/image (2) (3) (3) (2).png similarity index 100% rename from docs/.gitbook/assets/image (2) (2).png rename to docs/.gitbook/assets/image (2) (3) (3) (2).png diff --git a/docs/.gitbook/assets/image (2) (3).png b/docs/.gitbook/assets/image (2) (3) (3) (3).png similarity index 100% rename from docs/.gitbook/assets/image (2) (3).png rename to docs/.gitbook/assets/image (2) (3) (3) (3).png diff --git a/docs/.gitbook/assets/image (2).png b/docs/.gitbook/assets/image (2) (3) (3).png similarity index 100% rename from docs/.gitbook/assets/image (2).png rename to docs/.gitbook/assets/image (2) (3) (3).png diff --git a/docs/.gitbook/assets/image (3) (1).png b/docs/.gitbook/assets/image (3) (2) (2) (4) (1).png similarity index 100% rename from docs/.gitbook/assets/image (3) (1).png rename to docs/.gitbook/assets/image (3) (2) (2) (4) (1).png diff --git a/docs/.gitbook/assets/image (3) (2) (1).png b/docs/.gitbook/assets/image (3) (2) (2) (4) (2).png similarity index 100% rename from docs/.gitbook/assets/image (3) (2) (1).png rename to docs/.gitbook/assets/image (3) (2) (2) (4) (2).png diff --git a/docs/.gitbook/assets/image (3) (2) (2).png b/docs/.gitbook/assets/image (3) (2) (2) (4) (3).png similarity index 100% rename from docs/.gitbook/assets/image (3) (2) (2).png rename to docs/.gitbook/assets/image (3) (2) (2) (4) (3).png diff --git a/docs/.gitbook/assets/image (3) (2).png b/docs/.gitbook/assets/image (3) (2) (2) (4) (4).png similarity index 100% rename from docs/.gitbook/assets/image (3) (2).png rename to docs/.gitbook/assets/image (3) (2) (2) (4) (4).png diff --git a/docs/.gitbook/assets/image (3).png b/docs/.gitbook/assets/image (3) (2) (2) (4).png similarity index 100% rename from docs/.gitbook/assets/image (3).png rename to docs/.gitbook/assets/image (3) (2) (2) (4).png diff --git a/docs/.gitbook/assets/image (4) (1).png b/docs/.gitbook/assets/image (4) (1).png new file mode 100644 index 00000000000..cd77f27cc45 Binary files /dev/null and b/docs/.gitbook/assets/image (4) (1).png differ diff --git a/docs/.gitbook/assets/image (4).png b/docs/.gitbook/assets/image (4).png new file mode 100644 index 00000000000..cd77f27cc45 Binary files /dev/null and b/docs/.gitbook/assets/image (4).png differ diff --git a/docs/.gitbook/assets/image (6) (1).png b/docs/.gitbook/assets/image (6) (1).png new file mode 100644 index 00000000000..49670e20054 Binary files /dev/null and b/docs/.gitbook/assets/image (6) (1).png differ diff --git a/docs/.gitbook/assets/image (6).png b/docs/.gitbook/assets/image (6).png new file mode 100644 index 00000000000..49670e20054 Binary files /dev/null and b/docs/.gitbook/assets/image (6).png differ diff --git a/docs/.gitbook/assets/point_in_time_join (1).png b/docs/.gitbook/assets/point_in_time_join (1) (2) (1).png similarity index 100% rename from docs/.gitbook/assets/point_in_time_join (1).png rename to docs/.gitbook/assets/point_in_time_join (1) (2) (1).png diff --git a/docs/.gitbook/assets/point_in_time_join.png b/docs/.gitbook/assets/point_in_time_join (1) (2) (2).png similarity index 100% rename from docs/.gitbook/assets/point_in_time_join.png rename to docs/.gitbook/assets/point_in_time_join (1) (2) (2).png diff --git a/docs/.gitbook/assets/point_in_time_join (1) (2).png b/docs/.gitbook/assets/point_in_time_join (1) (2).png new file mode 100644 index 00000000000..331a090d719 Binary files /dev/null and b/docs/.gitbook/assets/point_in_time_join (1) (2).png differ diff --git a/docs/.gitbook/assets/rsz_untitled23 (1).jpg b/docs/.gitbook/assets/rsz_untitled23 (2) (2) (1).jpg similarity index 100% rename from docs/.gitbook/assets/rsz_untitled23 (1).jpg rename to docs/.gitbook/assets/rsz_untitled23 (2) (2) (1).jpg diff --git a/docs/.gitbook/assets/rsz_untitled23.jpg b/docs/.gitbook/assets/rsz_untitled23 (2) (2) (2).jpg similarity index 100% rename from docs/.gitbook/assets/rsz_untitled23.jpg rename to docs/.gitbook/assets/rsz_untitled23 (2) (2) (2).jpg diff --git a/docs/.gitbook/assets/rsz_untitled23 (2) (2).jpg b/docs/.gitbook/assets/rsz_untitled23 (2) (2).jpg new file mode 100644 index 00000000000..b92ec6fed72 Binary files /dev/null and b/docs/.gitbook/assets/rsz_untitled23 (2) (2).jpg differ diff --git a/docs/.gitbook/assets/statistics-sources (1) (1) (1).png b/docs/.gitbook/assets/statistics-sources (2).png similarity index 100% rename from docs/.gitbook/assets/statistics-sources (1) (1) (1).png rename to docs/.gitbook/assets/statistics-sources (2).png diff --git a/docs/.gitbook/assets/statistics-sources (1) (1).png b/docs/.gitbook/assets/statistics-sources (3).png similarity index 100% rename from docs/.gitbook/assets/statistics-sources (1) (1).png rename to docs/.gitbook/assets/statistics-sources (3).png diff --git a/docs/.gitbook/assets/statistics-sources (1) (2).png b/docs/.gitbook/assets/statistics-sources (4).png similarity index 100% rename from docs/.gitbook/assets/statistics-sources (1) (2).png rename to docs/.gitbook/assets/statistics-sources (4).png diff --git a/docs/.gitbook/assets/untitled-25-1- (1).jpg b/docs/.gitbook/assets/untitled-25-1- (2) (2) (1).jpg similarity index 100% rename from docs/.gitbook/assets/untitled-25-1- (1).jpg rename to docs/.gitbook/assets/untitled-25-1- (2) (2) (1).jpg diff --git a/docs/.gitbook/assets/untitled-25-1-.jpg b/docs/.gitbook/assets/untitled-25-1- (2) (2) (2).jpg similarity index 100% rename from docs/.gitbook/assets/untitled-25-1-.jpg rename to docs/.gitbook/assets/untitled-25-1- (2) (2) (2).jpg diff --git a/docs/.gitbook/assets/untitled-25-1- (2) (2).jpg b/docs/.gitbook/assets/untitled-25-1- (2) (2).jpg new file mode 100644 index 00000000000..93d010406bd Binary files /dev/null and b/docs/.gitbook/assets/untitled-25-1- (2) (2).jpg differ diff --git a/docs/README.md b/docs/README.md index e84deb4ae77..cb68aecb48f 100644 --- a/docs/README.md +++ b/docs/README.md @@ -4,7 +4,7 @@ Feast \(**Fea**ture **St**ore\) is an operational data system for managing and serving machine learning features to models in production. -![](.gitbook/assets/feast-architecture-diagrams.svg) +![](.gitbook/assets/feast-architecture-diagrams%20%281%29%20%281%29.svg) ### Problems Feast Solves @@ -50,7 +50,7 @@ The best way to learn Feast is to use it. Head over to our [Quickstart](quicksta * [Getting Started](getting-started/) provides guides on [Installing Feast](getting-started/install-feast/) and [Connecting to Feast](getting-started/connect-to-feast/). * [Concepts](./) describes all important Feast API concepts. -* [User guide](user-guide/data-ingestion.md) provides guidance on completing Feast workflows. +* [User guide](user-guide/define-and-ingest-features.md) provides guidance on completing Feast workflows. * [Examples](https://github.com/feast-dev/feast/tree/master/examples) contains a Jupyter notebook that you can run on your Feast deployment. * [Advanced](advanced/troubleshooting.md) contains information about both advanced and operational aspects of Feast. * [Reference](reference/api/) contains detailed API and design documents for advanced users. diff --git a/docs/SUMMARY.md b/docs/SUMMARY.md index 81c35f4e9eb..6d76d3f7174 100644 --- a/docs/SUMMARY.md +++ b/docs/SUMMARY.md @@ -6,13 +6,15 @@ * [Install Feast](getting-started/install-feast/README.md) * [Kubernetes \(with Helm\)](getting-started/install-feast/kubernetes-with-helm.md) * [Amazon EKS \(with Terraform\)](getting-started/install-feast/kubernetes-amazon-eks-with-terraform.md) + * [Azure AKS \(with Terraform\)](getting-started/install-feast/kubernetes-azure-aks-with-terraform.md) + * [Google Cloud GKE \(with Terraform\)](getting-started/install-feast/google-cloud-gke-with-terraform.md) * [Connect to Feast](getting-started/connect-to-feast/README.md) * [Python SDK](getting-started/connect-to-feast/python-sdk.md) - * [Feast CLI](getting-started/connect-to-feast/connecting-to-feast.md) + * [Feast CLI](getting-started/connect-to-feast/feast-cli.md) * [Learn Feast](getting-started/learn-feast.md) * [Roadmap](roadmap.md) * [Changelog](https://github.com/feast-dev/feast/blob/master/CHANGELOG.md) -* [Community](getting-help.md) +* [Community](community.md) ## Concepts @@ -21,12 +23,14 @@ * [Entities](concepts/entities.md) * [Sources](concepts/sources.md) * [Feature Tables](concepts/feature-tables.md) -* [Feature References](concepts/feature-references.md) +* [Stores](concepts/stores.md) +* [Glossary](concepts/glossary.md) ## User Guide -* [Getting data into Feast](user-guide/data-ingestion.md) -* [Getting training features](user-guide/feature-retrieval.md) +* [Overview](user-guide/overview.md) +* [Define and ingest features](user-guide/define-and-ingest-features.md) +* [Getting training features](user-guide/getting-training-features.md) * [Getting online features](user-guide/getting-online-features.md) ## Tutorials @@ -44,14 +48,15 @@ ## Reference * [API Reference](reference/api/README.md) - * [Core gRPC API](https://api.docs.feast.dev/grpc/feast.core.pb.html) - * [Serving gRPC API](https://api.docs.feast.dev/grpc/feast.serving.pb.html) - * [gRPC Types](https://api.docs.feast.dev/grpc/feast.types.pb.html) -* [Configuration Reference](reference/configuration-reference/README.md) * [Go SDK](https://godoc.org/github.com/feast-dev/feast/sdk/go) -* [Metrics Reference](reference/metrics-reference/README.md) * [Java SDK](https://javadoc.io/doc/dev.feast/feast-sdk) + * [Core gRPC API](https://api.docs.feast.dev/grpc/feast.core.pb.html) * [Python SDK](https://api.docs.feast.dev/python/) + * [Serving gRPC API](https://api.docs.feast.dev/grpc/feast.serving.pb.html) + * [gRPC Types](https://api.docs.feast.dev/grpc/feast.types.pb.html) +* [Configuration Reference](reference/configuration-reference.md) +* [Feast and Spark](reference/feast-and-spark.md) +* [Metrics Reference](reference/metrics-reference.md) * [Limitations](reference/limitations.md) ## Contributing diff --git a/docs/advanced/audit-logging.md b/docs/advanced/audit-logging.md index 269f063c597..7f80f105ef9 100644 --- a/docs/advanced/audit-logging.md +++ b/docs/advanced/audit-logging.md @@ -1,5 +1,9 @@ # Audit Logging +{% hint style="warning" %} +This page applies to Feast 0.7. The content may be out of date for Feast 0.8+ +{% endhint %} + ## Introduction Feast provides audit logging functionality in order to debug problems and to trace the lineage of events. @@ -44,7 +48,7 @@ Audit Logs produced by Feast are written to the console similar to normal logs b "service": "CoreService", "component": "feast-core", "id": "45329ea9-0d48-46c5-b659-4604f6193711", - "version": "0.8.0" + "version": "0.9.0" }, "hostname": "feast.core" "timestamp": "2020-10-20T04:45:24Z", diff --git a/docs/advanced/metrics.md b/docs/advanced/metrics.md index 57b9ebfc059..fd457b5120d 100644 --- a/docs/advanced/metrics.md +++ b/docs/advanced/metrics.md @@ -1,5 +1,9 @@ # Metrics +{% hint style="warning" %} +This page applies to Feast 0.7. The content may be out of date for Feast 0.8+ +{% endhint %} + ### Overview Feast Components export metrics that can provide insight into Feast behavior: @@ -7,7 +11,7 @@ Feast Components export metrics that can provide insight into Feast behavior: * [Feast Ingestion Jobs can be configured to push metrics into StatsD](metrics.md#2-exporting-feast-metrics-to-prometheus) * [Prometheus can be configured to scrape metrics from Feast Core and Serving.](metrics.md#2-exporting-feast-metrics-to-prometheus) -See the [Metrics Reference ](../reference/metrics-reference/)for documentation on metrics are exported by Feast. +See the [Metrics Reference ](../reference/metrics-reference.md)for documentation on metrics are exported by Feast. {% hint style="info" %} Feast Job Controller currently does not export any metrics on its own. However its `application.yml` is used to configure metrics export for ingestion jobs. @@ -51,7 +55,7 @@ server: ### Further Reading -See the [Metrics Reference ](../reference/metrics-reference/)for documentation on metrics are exported by Feast. +See the [Metrics Reference ](../reference/metrics-reference.md)for documentation on metrics are exported by Feast. ## diff --git a/docs/advanced/security.md b/docs/advanced/security.md index 937dadb3bdc..feb7b978ac9 100644 --- a/docs/advanced/security.md +++ b/docs/advanced/security.md @@ -5,14 +5,12 @@ description: 'Secure Feast with SSL/TLS, Authentication and Authorization.' # Security {% hint style="warning" %} -Currently, Security functionality applies only to Feast Core and Feast Online Serving. - -Security for Historical Serving will become available once offline storage is introduced in Feast 0.9. +This page applies to Feast 0.7. The content may be out of date for Feast 0.8+ {% endhint %} ### Overview -![Overview of Feast's Security Methods.](../.gitbook/assets/untitled-25-1-.jpg) +![Overview of Feast's Security Methods.](../.gitbook/assets/untitled-25-1-%20%282%29%20%282%29%20%281%29.jpg) Feast supports the following security methods: @@ -22,11 +20,11 @@ Feast supports the following security methods: [Important considerations when integrating Authentication/Authorization](security.md#5-authentication-and-authorization). -## **1. SSL/TLS** +### **SSL/TLS** Feast supports SSL/TLS encrypted inter-service communication among Feast Core, Feast Online Serving, and Feast SDKs. -### Configuring SSL/TLS on Feast Core and Feast Serving +#### Configuring SSL/TLS on Feast Core and Feast Serving The following properties configure SSL/TLS. These properties are located in their corresponding `application.yml`files: @@ -38,9 +36,9 @@ The following properties configure SSL/TLS. These properties are located in thei > Read more on enabling SSL/TLS in the[ gRPC starter docs.](https://yidongnan.github.io/grpc-spring-boot-starter/en/server/security.html#enable-transport-layer-security) -### Configuring SSL/TLS on Python SDK/CLI +#### Configuring SSL/TLS on Python SDK/CLI -To enable SSL/TLS in the [Feast Python SDK](https://api.docs.feast.dev/python/#feast.client.Client) or [Feast CLI](../getting-started/connect-to-feast/connecting-to-feast.md), set the config options via `feast config`: +To enable SSL/TLS in the [Feast Python SDK](https://api.docs.feast.dev/python/#feast.client.Client) or [Feast CLI](../getting-started/connect-to-feast/feast-cli.md), set the config options via `feast config`: | Configuration Option | Description | | :--- | :--- | @@ -53,7 +51,7 @@ To enable SSL/TLS in the [Feast Python SDK](https://api.docs.feast.dev/python/#f The Python SDK automatically uses SSL/TLS when connecting to Feast Core and Feast Online Serving via port 443. {% endhint %} -### Configuring SSL/TLS on Go SDK +#### Configuring SSL/TLS on Go SDK Configure SSL/TLS on the [Go SDK](https://godoc.org/github.com/feast-dev/feast/sdk/go) by passing configuration via `SecurityConfig`: @@ -69,7 +67,7 @@ cli, err := feast.NewSecureGrpcClient("localhost", 6566, feast.SecurityConfig{ | `EnableTLS` | Enables SSL/TLS functionality when connecting to Feast if `true` | | `TLSCertPath` | Optional. Provides the path of the root certificate used to verify Feast Service's identity. If omitted, uses system certificates. | -### Configuring SSL/TLS on **Java** SDK +#### Configuring SSL/TLS on **Java** SDK Configure SSL/TLS on the [Feast Java SDK](https://javadoc.io/doc/dev.feast/feast-sdk) by passing configuration via `SecurityConfig`: @@ -86,7 +84,7 @@ FeastClient client = FeastClient.createSecure("localhost", 6566, | `setTLSEnabled()` | Enables SSL/TLS functionality when connecting to Feast if `true` | | `setCertificatesPath()` | Optional. Set the path of the root certificate used to verify Feast Service's identity. If omitted, uses system certificates. | -## **2. Authentication** +### **Authentication** {% hint style="warning" %} To prevent man in the middle attacks, we recommend that SSL/TLS be implemented prior to authentication. @@ -118,7 +116,7 @@ Behind the scenes, Feast Core and Feast Online Serving authenticate by: * Validates token's authenticity using the JWK retrieved from the `jwkEndpointURI` -### **Authenticating Serving with Feast Core** +#### **Authenticating Serving with Feast Core** Feast Online Serving communicates with Feast Core during normal operation. When both authentication and authorization are enabled on Feast Core, Feast Online Serving is forced to authenticate its requests to Feast Core. Otherwise, Feast Online Serving produces an Authentication failure error when connecting to Feast Core. @@ -186,9 +184,9 @@ OAuth Provider makes an OAuth [client credentials](https://auth0.com/docs/flows/ {% endtab %} {% endtabs %} -### **Enabling Authentication in Python SDK/CLI** +#### **Enabling Authentication in Python SDK/CLI** -Configure the [Feast Python SDK](https://api.docs.feast.dev/python/) and [Feast CLI](../getting-started/connect-to-feast/connecting-to-feast.md) to use authentication via `feast config`: +Configure the [Feast Python SDK](https://api.docs.feast.dev/python/) and [Feast CLI](../getting-started/connect-to-feast/feast-cli.md) to use authentication via `feast config`: ```python $ feast config set enable_auth true @@ -262,7 +260,7 @@ OAuth Provider makes an OAuth [client credentials](https://auth0.com/docs/flows/ {% endtab %} {% endtabs %} -### **Enabling Authentication in Go SDK** +#### **Enabling Authentication in Go SDK** Configure the [Feast Java SDK](https://javadoc.io/doc/dev.feast/feast-sdk/latest/com/gojek/feast/package-summary.html) to use authentication by specifying the credential via `SecurityConfig`: @@ -340,7 +338,7 @@ cred := feast.NewOAuthCredential("localhost:6566", "client_id", "secret", "https {% endtab %} {% endtabs %} -### **Enabling Authentication in Java SDK** +#### **Enabling Authentication in Java SDK** Configure the [Feast Java SDK](https://javadoc.io/doc/dev.feast/feast-sdk/latest/com/gojek/feast/package-summary.html) to use authentication by setting credentials via `SecurityConfig`: @@ -438,7 +436,7 @@ CallCredentials credentials = new OAuthCredentials(Map.of( {% endtab %} {% endtabs %} -## 3. Authorization +### Authorization {% hint style="info" %} Authorization requires that authentication be configured to obtain a user identity for use in authorizing requests. @@ -449,9 +447,9 @@ Authorization provides access control to FeatureTables and/or Features based on * Create and/or Update a Feature Table in the Project. * Retrieve Feature Values for Features in that Project. -### **Authorization API/Server** +#### **Authorization API/Server** -![Feast Authorization Flow](../.gitbook/assets/rsz_untitled23.jpg) +![Feast Authorization Flow](../.gitbook/assets/rsz_untitled23%20%282%29%20%282%29.jpg) Feast delegates Authorization grants to an external Authorization Server that implements the [Authorization Open API specification](https://github.com/feast-dev/feast/blob/master/common/src/main/resources/api.yaml). @@ -471,7 +469,7 @@ Authorization can be configured for Feast Core and Feast Online Serving via prop This example of the [Authorization Server with Keto](https://github.com/feast-dev/feast-keto-auth-server) can be used as a reference implementation for implementing an Authorization Server that Feast supports. {% endhint %} -## **4. Authentication & Authorization** +### **Authentication & Authorization** When using Authentication & Authorization, consider: diff --git a/docs/advanced/troubleshooting.md b/docs/advanced/troubleshooting.md index 8936a6b77c7..1060466d300 100644 --- a/docs/advanced/troubleshooting.md +++ b/docs/advanced/troubleshooting.md @@ -1,10 +1,14 @@ # Troubleshooting -If at any point in time you cannot resolve a problem, please see the [Community](../getting-help.md) section for reaching out to the Feast community. +{% hint style="warning" %} +This page applies to Feast 0.7. The content may be out of date for Feast 0.8+ +{% endhint %} -## How can I verify that all services are operational? +If at any point in time you cannot resolve a problem, please see the [Community](../community.md) section for reaching out to the Feast community. -### Docker Compose +### How can I verify that all services are operational? + +#### Docker Compose The containers should be in an `up` state: @@ -12,7 +16,7 @@ The containers should be in an `up` state: docker ps ``` -### Google Kubernetes Engine +#### Google Kubernetes Engine All services should either be in a `RUNNING` state or `COMPLETED`state: @@ -20,7 +24,7 @@ All services should either be in a `RUNNING` state or `COMPLETED`state: kubectl get pods ``` -## How can I verify that I can connect to all services? +### How can I verify that I can connect to all services? First locate the the host and port of the Feast Services. @@ -59,7 +63,7 @@ export FEAST_HISTORICAL_SERVING_URL=${FEAST_IP}:32092 `netcat`, `telnet`, or even `curl` can be used to test whether all services are available and ports are open, but `grpc_cli` is the most powerful. It can be installed from [here](https://github.com/grpc/grpc/blob/master/doc/command_line_tool.md). -### Testing Connectivity From Feast Services: +#### Testing Connectivity From Feast Services: Use `grpc_cli` to test connetivity by listing the gRPC methods exposed by Feast services: @@ -79,7 +83,7 @@ grpc_cli ls ${FEAST_HISTORICAL_SERVING_URL} feast.serving.ServingService grpc_cli ls ${FEAST_ONLINE_SERVING_URL} feast.serving.ServingService ``` -## How can I print logs from the Feast Services? +### How can I print logs from the Feast Services? Feast will typically have three services that you need to monitor if something goes wrong. @@ -90,7 +94,7 @@ Feast will typically have three services that you need to monitor if something g In order to print the logs from these services, please run the commands below. -### Docker Compose +#### Docker Compose Use `docker-compose logs` to obtain Feast component logs: @@ -110,7 +114,7 @@ docker logs -f feast_historical_serving_1 docker logs -f feast_online_serving_1 ``` -### Google Kubernetes Engine +#### Google Kubernetes Engine Use `kubectl logs` to obtain Feast component logs: diff --git a/docs/advanced/upgrading.md b/docs/advanced/upgrading.md index 8e1c426584c..3c7b95d5441 100644 --- a/docs/advanced/upgrading.md +++ b/docs/advanced/upgrading.md @@ -1,8 +1,8 @@ # Upgrading Feast -## Migration v0.6 to v0.7 +### Migration from v0.6 to v0.7 -### Feast Core Validation changes +#### Feast Core Validation changes In v0.7, Feast Core no longer accepts starting with number \(0-9\) and using dash in names for: @@ -20,9 +20,9 @@ Feast now prevents feature sets from being applied if no store is subscribed to * Ensure that a store is configured to subscribe to the Feature Set before applying the Feature Set. -### Feast Core's Job Coordinator is now Feast Job Controller +#### Feast Core's Job Coordinator is now Feast Job Controller -In v0.7, Feast Core's Job Coordinator has been decoupled from Feast Core and runs as a separate Feast Job Controller application. See its [Configuration reference](../reference/configuration-reference/#2-feast-core-serving-and-job-controller) for how to configure Feast Job Controller. +In v0.7, Feast Core's Job Coordinator has been decoupled from Feast Core and runs as a separate Feast Job Controller application. See its [Configuration reference](../reference/configuration-reference.md#2-feast-core-serving-and-job-controller) for how to configure Feast Job Controller. **Ingestion Job API** @@ -42,15 +42,15 @@ Users of Ingestion Job via Python SDK \(ie `feast ingest-jobs list` or `client.s * `ingest_job()`methods only: Create a new separate [Job Controller client](https://github.com/feast-dev/feast/blob/master/sdk/python/feast/contrib/job_controller/client.py) to connect to the job controller and call `ingest_job()` methods using the new client. * Configure the Feast Job Controller endpoint url via `jobcontroller_url` config option. -### Configuration Properties Changes +#### Configuration Properties Changes * Rename `feast.jobs.consolidate-jobs-per-source property` to `feast.jobs.controller.consolidate-jobs-per-sources` * Rename`feast.security.authorization.options.subjectClaim` to `feast.security.authentication.options.subjectClaim` * Rename `feast.logging.audit.messageLoggingEnabled` to `feast.audit.messageLogging.enabled` -## Migration v0.5 to v0.6 +### Migration from v0.5 to v0.6 -### Database schema +#### Database schema In Release 0.6 we introduced [Flyway](https://flywaydb.org/) to handle schema migrations in PostgreSQL. Flyway is integrated into `core` and for now on all migrations will be run automatically on `core` start. It uses table `flyway_schema_history` in the same database \(also created automatically\) to keep track of already applied migrations. So no specific maintenance should be needed. @@ -94,9 +94,9 @@ Minor changes: has now `version` and `delivery_status`. -## Migrate v0.4 to v0.6 +### Migration from v0.4 to v0.6 -### Database +#### Database For all versions earlier than 0.5 seamless migration is not feasible due to earlier breaking changes and creation of new database will be required. diff --git a/docs/getting-help.md b/docs/community.md similarity index 63% rename from docs/getting-help.md rename to docs/community.md index e9ead181caa..38fd35bc0ee 100644 --- a/docs/getting-help.md +++ b/docs/community.md @@ -1,5 +1,9 @@ # Community +{% hint style="success" %} +**Office Hours:** Have a question, feature request, idea, or just looking to speak to a real person? Come and join the [Feast Office Hours](https://tinyurl.com/feast-office-hours) on Friday and chat to a Feast contributor! +{% endhint %} + ### Links & Resources * [Slack](https://kubeflow.slack.com/messages/CE0L8T267): We use the channel [\#Feast](https://kubeflow.slack.com/messages/CE0L8T267) in [kubeflow.slack.com](https://join.slack.com/t/kubeflow/shared_invite/zt-cpr020z4-PfcAue_2nw67~iIDy7maAQ). Feel free to ask questions or say hello! @@ -11,19 +15,26 @@ * User surveys and meeting minutes. * Slide decks of conferences our contributors have spoken at. * [Feast GitHub Repository](https://github.com/feast-dev/feast/): Find the complete Feast codebase on GitHub. +* [Feast Linux Foundation Wiki](https://wiki.lfaidata.foundation/display/FEAST/Feast+Home): Our LFAI wiki page contains links to resources for contributors and maintainers. ### How can I get help? -* **Slack:** Need to speak to a human? Come ask a question in our Slack channel \(link above\) +* **Slack:** Need to speak to a human? Come ask a question in our Slack channel \(link above\). * **GitHub Issues:** Found a bug or need a feature? [Create an issue on GitHub](https://github.com/feast-dev/feast/issues/new). * **StackOverflow:** Need to ask a question on how to use Feast? We also monitor and respond to [StackOverflow](https://stackoverflow.com/questions/tagged/feast). -### Community Call +### Community Calls + +We have a user and contributor community call every two weeks \(Asia & US friendly\). + +#### Frequency \(every 2 weeks\) -We have a community call every 2 weeks. Alternating between two times: +* **Asia \(UTC+08:00\):** Wednesday 10:00 am to 10:30 am. +* **US West Coast \(PT\):** Tuesday 18:00 pm to 18:30 pm. -* 11 am \(UTC + 8\) -* 5 pm \(UTC + 8\) +#### Links -Join the [feast-dev](getting-help.md#feast-development) mailing list to receive a Google calendar invitation. +* Calendar: [Feast Community Calendar \(Linux Foundation\)](https://wiki.lfaidata.foundation/pages/viewpage.action?pageId=30408973) +* Zoom: [https://zoom.us/j/6325193230](https://zoom.us/j/6325193230) +* Meeting notes: [https://bit.ly/feast-notes](https://bit.ly/feast-notes%20) diff --git a/docs/concepts/architecture.md b/docs/concepts/architecture.md index eab6941d8e7..2617b4cd2fa 100644 --- a/docs/concepts/architecture.md +++ b/docs/concepts/architecture.md @@ -1,36 +1,51 @@ # Architecture -![Feast high-level flow](../.gitbook/assets/blank-diagram-4.svg) - -### **Feast Core** - -Feast Core is the central management service of a Feast deployment. It's role is to: - -* Allow users to create [entities](entities.md). -* Allow users to create features through the creation of [feature tables](feature-tables.md). -* Act as a source of truth and central registry of feature tables. - -### **Feast Ingestion** - -Before you ingest data into Feast, first register one or more entity, then register feature tables. These [feature tables](feature-tables.md) tell Feast where to find their data and how to ingest it. The feature tables also describe the characteristics of the data for validation purposes. After a feature table is registered, you can start a Spark job to populate a store with data from the defined source in the feature table specification. - -To ensure stores are populated with data, you must publish the data to a [source](sources.md). Currently, Feast supports a few batch and stream sources. Feast users \(or pipelines\) ingest batch data through the [Feast Python SDK](../getting-started/connect-to-feast/python-sdk.md) using its `ingest()` method. The SDK publishes the data into the batch source specified for the feature table's batch source. - -Streaming systems can also ingest data into Feast. This is done by publishing to the correct stream source from the feature table specification in the expected format. The topic and brokers can be found on the feature table's stream source if specified during registration. - -### **Stores** - -Stores are nothing more than databases used to store feature data. Feast loads data into stores through an ingestion process, after which the data can be served through the [Feast Online Serving API](https://api.docs.feast.dev/grpc/feast.serving.pb.html). Stores are documented in the following section. - -### **Feast Online Serving** - -`Feast Online Serving` is the data-access layer through which end users and production systems retrieve feature data. Each `Serving` instance is backed by a [store](). - -Because Feast supports multiple store types \(online, historical\), multiple instances of a deployed `Feast Online Serving` is common: those for online serving and those for historical. This means Feast allows for any number of `Feast Online Serving` deployments, presenting the possibility to use a `Feast Online Serving` deployment per production system, with its own stores and population jobs. - -`Feast Online Serving` deployments subscribe to all feature data, consuming all features known to a `Feast Core` deployment. - -Feature retrieval \(and feature references\) are documented in more detail in subsequent sections. - -{% page-ref page="../user-guide/feature-retrieval.md" %} +![](../.gitbook/assets/image%20%286%29%20%281%29.png) + +### Sequence description + +1. **Log Raw Events:** Production backend applications are configured to emit internal state changes as events to a stream. +2. **Create Stream Features:** Stream processing systems like Flink, Spark, and Beam are used to transform and refine events and to produce features that are logged back to the stream. +3. **Log Streaming Features:** Both raw and refined events are logged into a data lake or batch storage location. +4. **Create Batch Features:** ELT/ETL systems like Spark and SQL are used to transform data in the batch store. +5. **Define and Ingest Features:** The Feast user defines [feature tables](feature-tables.md) based on the features available in batch and streaming sources and publish these definitions to Feast Core. +6. **Poll Feature Definitions:** The Feast Job Service polls for new or changed feature definitions. +7. **Start Ingestion Jobs:** Every new feature table definition results in a new ingestion job being provisioned \(see limitations\). +8. **Batch Ingestion:** Batch ingestion jobs are short-lived jobs that load data from batch sources into either an offline or online store \(see limitations\). +9. **Stream Ingestion:** Streaming ingestion jobs are long-lived jobs that load data from stream sources into online stores. A stream source and batch source on a feature table must have the same features/fields. +10. **Model Training:** A model training pipeline is launched. It uses the Feast Python SDK to retrieve a training dataset and trains a model. +11. **Get Historical Features:** Feast exports a point-in-time correct training dataset based on the list of features and entity DataFrame provided by the model training pipeline. +12. **Deploy Model:** The trained model binary \(and list of features\) are deployed into a model serving system. +13. **Get Prediction:** A backend system makes a request for a prediction from the model serving service. +14. **Retrieve Online Features:** The model serving service makes a request to the Feast Online Serving service for online features using a Feast SDK. +15. **Return Prediction:** The model serving service makes a prediction using the returned features and returns the outcome. + +{% hint style="warning" %} +Limitations + +* Feast 0.8 has no offline store. Batch retrieval is direct from source. We plan to implement an optional offline store in Feast 0.9 +* Only Redis is supported for online storage. +* Batch ingestion jobs must be triggered from your own scheduler like Airflow. Streaming ingestion jobs are automatically launched by the Feast Job Service. +{% endhint %} + +### Components: + +A complete Feast deployment contains the following components: + +* **Feast Core:** Acts as the central registry for feature and entity definitions in Feast. +* **Feast Job Service:** Manages data processing jobs that load data from sources into stores, and jobs that export training datasets. +* **Feast Online Serving:** Provides low-latency access to feature values in an online store. +* **Feast Python SDK:** The primary user facing SDK. Used to: + * Manage feature definitions with Feast Core. + * Launch jobs through the Feast Job Service. + * Retrieve training datasets. + * Retrieve online features. +* **Online Store:** The online store is a database that stores only the latest feature values for each entity entity. The online store can be populated by either batch ingestion jobs \(in the case the user has no streaming source\), or can be populated by a streaming ingestion job from a streaming source. Feast Online Serving looks up feature values from the online store. +* **Offline Store:** The offline store persists batch data that has been ingested into Feast. This data is used for producing training datasets. + +Please see the [configuration reference](../reference/configuration-reference.md#overview) for more details on configuring these components. + +{% hint style="info" %} +Java and Go SDKs are also available for online feature retrieval. See [API Reference](../reference/api/). +{% endhint %} diff --git a/docs/concepts/concepts.md b/docs/concepts/concepts.md deleted file mode 100644 index e54bd488d6a..00000000000 --- a/docs/concepts/concepts.md +++ /dev/null @@ -1,124 +0,0 @@ -# Concepts - -## Architecture - -![Logical diagram of a typical Feast deployment](../.gitbook/assets/basic-architecture-diagram%20%282%29.svg) - -The core components of a Feast deployment are - -* **Feast Core:** Feast Core is a centralized service that acts as the authority on features within an organization. Typically there is only one "Core" deployment per organization, with all feature management happening through it. -* **Feast Ingestion Jobs:** Feast ingestion jobs retrieve feature data from user defined data sources and populate serving stores with this feature data. These jobs are managed by Feast Core. Data can either be sources from existing sources \(like [Kafka](https://kafka.apache.org/)\), or it can be loaded into Feast through its API. -* **Feast Serving:** Feast Serving is the data access layer through which end users and production systems retrieve feature data. Each Serving store is backed by one or more databases. These databases are updated by the Feast ingestion jobs. There are two types of stores: batch and online. Batch stores hold large volumes historical data, while online stores only hold the latest feature values. - -## Data Model - -### Feature Set - -User data is typically in the form of dataframes, tables in data warehouses, or events on a stream. These data sources are loaded into Feast in order to serve features for model training or serving. - -Feature sets allow for groups of fields in these data sources to be ingested and stored together. This allows for efficient storage and logical namespacing of data. - -When data is loaded from these sources, each field in the feature set must be found in every record of the data source. Fields from these data sources must be either a timestamp, an entity, or a feature. - -{% hint style="info" %} -Feature sets are a grouping of feature sets based on how they are loaded into Feast. They ensure that data is efficiently stored during ingestion. Feature sets are not a grouping of features for retrieval of features. During retrieval it is possible to retrieve feature values from any number of feature sets. -{% endhint %} - -#### Customer Transactions Example - -Below is an example of a basic `customer transactions` feature set that has been exported to YAML: - -{% tabs %} -{% tab title="customer\_transactions\_feature\_set.yaml" %} -```yaml -name: customer_transactions -kind: feature_set -entities: -- name: customer_id - valueType: INT64 -features: -- name: daily_transactions - valueType: FLOAT -- name: total_transactions - valueType: FLOAT - maxAge: 3600s -``` -{% endtab %} -{% endtabs %} - -The dataframe below \(`customer_data.csv`\) contains the features and entities of the above feature set - -| datetime | customer\_id | daily\_transactions | total\_tra**nsactions** | -| :--- | :--- | :--- | :--- | -| 2019-01-01 01:00:00 | 20001 | 5.0 | 14.0 | -| 2019-01-01 01:00:00 | 20002 | 2.6 | 43.0 | -| 2019-01-01 01:00:00 | 20003 | 4.1 | 154.0 | -| 2019-01-01 01:00:00 | 20004 | 3.4 | 74.0 | - -In order to ingest feature data into Feast for this specific feature set: - -```python -# Load dataframe -customer_df = pd.read_csv("customer_data.csv") - -# Create feature set from YAML (using YAML is optional) -cust_trans_fs = FeatureSet.from_yaml("customer_transactions_feature_set.yaml") - -# Load feature data into Feast for this specific feature set -client.ingest(cust_trans_fs, customer_data) -``` - -### Feature - -A feature is an individual measurable property or characteristic of a phenomenon being observed. Features are the most important concepts within a feature store. Feature data is used both as input to models during training and when models are served in production. - -In the context of Feast, features are values that are associated with either one or more entities over time. In Feast, these values are either primitives or lists of primitives. Each feature can also have additional information attached to it. For example whether it is a categorical feature or numerical. - -{% hint style="info" %} -Features in Feast are defined within Feature Sets and are not treated as standalone concepts. -{% endhint %} - -### Entity - -An entity type is any object in an organization that needs to be modeled and on which information should be stored. Entity types are usually recognizable concepts, either concrete or abstract, such as persons, places, things, or events which have relevance to the modeled system. - -An entity is an instance of an entity type. - -* Examples of entity types in the context of ride-hailing and food delivery: `customer`, `order`, `driver`, `restaurant`, `dish`, `area`. -* A specific driver, for example a driver with ID `D011234` would be an entity of the entity type `driver` - -An entity is the object on which features are observed. For example we could have a feature `total_trips_24h` on the driver `D01123` with a feature value of `11`. - -In the context of Feast, entities are important because they are used as keys when looking up feature values. Entities are also used when joining feature values between different feature sets in order to build one large data set to train a model, or to serve a model. - -{% hint style="info" %} -Entities in Feast are defined within Feature Sets and are not treated as standalone concepts. -{% endhint %} - -### Types - -Feast supports the following types for feature values - -* BYTES -* STRING -* INT32 -* INT64 -* DOUBLE -* FLOAT -* BOOL -* BYTES\_LIST -* STRING\_LIST -* INT32\_LIST -* INT64\_LIST -* DOUBLE\_LIST -* FLOAT\_LIST -* BOOL\_LIST - -## Glossary - -| Term | Description | -| :--- | :--- | -| Feast deployment | A complete Feast system as it is deployed. Consists out of a single Feast Core deployment and one or more Feast Serving deployments. | -| Feast Core | The centralized service which acts as a registry and authority of features. Organizations should only deploy a single Feast Core instance. Feast Core also manages the ingestion of feature data and population of Feast Serving data stores. | -| Feast Serving | Feast Serving is a service used to access both online and batch feature data. Feast Serving deployments are backed by one or more databases. | - diff --git a/docs/concepts/entities.md b/docs/concepts/entities.md index 3691f711431..64f09f575ad 100644 --- a/docs/concepts/entities.md +++ b/docs/concepts/entities.md @@ -1,67 +1,64 @@ # Entities -### Overview +## Overview -An entity is any domain object that can be modelled and about which information can be stored. Entities are usually recognisable concepts, either concrete or abstract, such as persons, places, things, or events which have relevance to the modelled system. +An entity is any domain object that can be modeled and about which information can be stored. Entities are usually recognizable concepts, either concrete or abstract, such as persons, places, things, or events. -* Examples of entity types in the context of ride-hailing and food delivery: `customer`, `order`, `driver`, `restaurant`, `dish`, `area`. -* A specific driver, for example a driver with ID `D011234` would be an entity of the entity type `driver` +Examples of entities in the context of ride-hailing and food delivery: `customer`, `order`, `driver`, `restaurant`, `dish`, `area`. -An entity is the domain object on which features are observed. For example, we could have a feature `total_trips_24h` for driver `D011234` with a feature value of `11`. +Entities are important in the context of feature stores since features are always properties of a specific entity. For example, we could have a feature `total_trips_24h` for driver `D011234` with a feature value of `11`. -Entities are important for Feast because they are used as keys when searching for feature values. Entities are also used when joining feature values from different feature tables to build a large data set that is used to train or serve models. +Feast uses entities in the following way: -### Structure of an Entity +* Entities serve as the keys used to look up features for producing training datasets and online feature values. +* Entities serve as a natural grouping of features in a feature table. A feature table must belong to an entity \(which could be a composite entity\) + +## Structure of an Entity When creating an entity specification, consider the following fields: -* **name**: Name of the entity -* **description**: Description of the entity -* **value\_type**: Value type of the entity -* **labels**: User-defined metadata +* **Name**: Name of the entity +* **Description**: Description of the entity +* **Value Type**: Value type of the entity. Feast will attempt to coerce entity columns in your data sources into this type. +* **Labels**: Labels are maps that allow users to attach their own metadata to entities A valid entity specification is shown below: ```python -from feast import Entity, ValueType - -# Create a customer entity customer = Entity( - "customer_id", - "Customer id for ride customer", - ValueType.INT64 + name="customer_id", + description="Customer id for ride customer", + value_type=ValueType.INT64, + labels={} ) ``` -### Working with an Entity +## Working with an Entity -Creating an Entity: +### Creating an Entity: ```python # Create a customer entity customer_entity = Entity(name="customer_id", description="ID of car customer") -client.apply_entity(customer_entity) +client.apply(customer_entity) ``` -Updating an Entity: +### Updating an Entity: ```python # Update a customer entity customer_entity = client.get_entity("customer_id") customer_entity.description = "ID of bike customer" -client.apply_entity(customer_entity) +client.apply(customer_entity) ``` Permitted changes include: * The entity's description and labels -{% hint style="warning" %} -You **cannot** change the following: - -* Project or name of an entity -* Types of entity -{% endhint %} +The following changes are note permitted: -Visit [EntitySpec](https://api.docs.feast.dev/grpc/feast.core.pb.html#EntitySpecV2) for the entity-specification API. +* Project +* Name of an entity +* Type diff --git a/docs/concepts/feature-references.md b/docs/concepts/feature-references.md deleted file mode 100644 index 4f712a24c45..00000000000 --- a/docs/concepts/feature-references.md +++ /dev/null @@ -1,36 +0,0 @@ -# Feature References - -## Overview - -In Feast, each feature can be uniquely addressed through a feature reference. A feature reference is composed of the following components: - -* Feature Table name -* Feature name - -## Structure of a Feature Reference - -A string based feature reference takes on the following format: - -`:` - -```python -# Feature references -feature_refs = [ - "driver_trips:average_daily_rides", - "driver_trips:maximum_daily_rides", - "driver_trips:rating", -] -``` - -Feature references only apply to a single `project`. Features cannot be retrieved across projects in a single request. - -## Working with a Feature Reference - -#### Feature Retrieval - -Feature retrieval \(or serving\) is the process of retrieving either historical features or online features from Feast, for the purposes of training or serving a model. - -Feast attempts to unify the process of retrieving features in both the historical and online case. It does this through the creation of feature references. One of the major advantages of using Feast is that you have a single semantic reference to a feature. These feature references can then be stored alongside your model and loaded into a serving layer where it can be used for online feature retrieval. - -More information about how to perform feature retrieval for historical and online features can be found in the sections under **User Guide**. - diff --git a/docs/concepts/feature-tables.md b/docs/concepts/feature-tables.md index 470be7a755e..a27e0c000b2 100644 --- a/docs/concepts/feature-tables.md +++ b/docs/concepts/feature-tables.md @@ -6,33 +6,25 @@ Feature tables are both a schema and a logical means of grouping features, data Feature tables serve the following purposes: -* They are a means for defining the location and properties of data [sources](sources.md). -* They are used to create within Feast a database-level structure for the storage of feature values. -* The data sources described within feature tables enable Feast to ingest and store features within Feast. -* They ensure data is efficiently stored during [ingestion](../user-guide/data-ingestion.md). +* Feature tables are a means for defining the location and properties of data [sources](sources.md). +* Feature tables are used to create within Feast a database-level structure for the storage of feature values. +* The data sources described within feature tables allow Feast to find and ingest feature data into stores within Feast. +* Feature tables ensure data is efficiently stored during [ingestion](../user-guide/define-and-ingest-features.md) by providing a grouping mechanism of features values that occur on the same event timestamp. {% hint style="info" %} -Feast does not yet apply feature transformations. Transformations are currently expected to happen before data is ingested into Feast. The data sources described within feature tables should reference feature values in their already computed form. +Feast does not yet apply feature transformations. Transformations are currently expected to happen before data is ingested into Feast. The data sources described within feature tables should reference feature values in their already transformed form. {% endhint %} ### Features -A feature is an individual measurable property or characteristic of an observable phenomenon. For example, in a bank, a feature could be `total_foreign_transactions_24h` for a specific class of credit cards the bank issues. Feature data is the input both for training models, and for models served in production. +A feature is an individual measurable property observed on an entity. For example the amount of transactions \(feature\) a customer \(entity\) has completed. Features are used for both model training and scoring \(batch, online\). -{% hint style="info" %} -Features are the most important concepts within a feature store. -{% endhint %} - -In Feast, features are values that are associated with one or more [entities](entities.md). These values are either primitives or lists of primitives. Each feature can also have additional information attached to it. - -You define a feature by providing a name and value type. In our example, we use a name and value type that might be used in a ride-hailing company: +Features are defined as part of feature tables. Since Feast does not apply transformations, a feature is basically a schema that only contains a name and a type: ```python avg_daily_ride = Feature("average_daily_rides", ValueType.FLOAT) ``` -Features act purely as a schema within feature tables. Feature tables and features act as normal database tables and columns. - Visit [FeatureSpec](https://api.docs.feast.dev/grpc/feast.core.pb.html#FeatureSpecV2) for the complete feature specification API. ## Structure of a Feature Table @@ -41,20 +33,20 @@ Feature tables contain the following fields: * **Name:** Name of feature table. This name must be unique within a project. * **Entities:** List of [entities](entities.md) to associate with the features defined in this feature table. Entities are used as lookup keys when retrieving features from a feature table. -* **Features:** List of features within this feature table. +* **Features:** List of features within a feature table. * **Labels:** Labels are arbitrary key-value properties that can be defined by users. -* **Max age:** Max age affect the retrieval of features from a feature table. Age is measured as the duration of time between the event timestamp of a feature and the lookup time on an entity key used to retrieve the feature. Feature values outside max age will be returned as unset values. Max age allows for eviction of keys from online stores and limits the amount of scanning for historical feature values during retrieval. -* **Batch Source:** The batch data source from which you can ingest feature values into Feast. Visit [Sources](sources.md) to learn more about them. -* **Stream Source:** The streaming data source from which you can ingest streaming feature values into Feast. Visit [Sources](sources.md) to learn more about them. +* **Max age:** Max age affect the retrieval of features from a feature table. Age is measured as the duration of time between the event timestamp of a feature and the lookup time on an [entity key](glossary.md#entity-key) used to retrieve the feature. Feature values outside max age will be returned as unset values. Max age allows for eviction of keys from online stores and limits the amount of historical scanning required for historical feature values during retrieval. +* **Batch Source:** The batch data source from which Feast will ingest feature values into stores. This can either be used to back-fill stores before switching over to a streaming source, or it can be used as the primary source of data for a feature table. Visit [Sources](sources.md) to learn more about batch sources. +* **Stream Source:** The streaming data source from which you can ingest streaming feature values into Feast. Streaming sources must be paired with a batch source containing the same feature values. A streaming source is only used to populate online stores. The batch equivalent source that is paired with a streaming source is used during the generation of historical feature datasets. Visit [Sources](sources.md) to learn more about stream sources. -Here is a ride-hailing example of a valid feature-table specification: +Here is a ride-hailing example of a valid feature table specification: {% tabs %} {% tab title="driver\_trips\_feature\_table.py" %} ```python from feast import BigQuerySource, FeatureTable, Feature, ValueType +from google.protobuf.duration_pb2 import Duration -# Create an empty feature table driver_ft = FeatureTable( name="driver_trips", entities=["driver_id"], @@ -62,7 +54,7 @@ driver_ft = FeatureTable( Feature("average_daily_rides", ValueType.FLOAT), Feature("rating", ValueType.FLOAT) ], - max_age=14400, + max_age=Duration(seconds=3600), labels={ "team": "driver_matching" }, @@ -79,13 +71,11 @@ driver_ft = FeatureTable( {% endtab %} {% endtabs %} -When you register a feature table, at a minimum specify a batch source to populate the feature table. Stream sources are optional. They are used to stream feature values into online stores. - By default, Feast assumes that features specified in the feature-table specification corresponds one-to-one to the fields found in the sources. All features defined in a feature table should be available in the defined sources. -However, if the names of the fields in the batch source are different from the names of features, you can use `field_mappings` to ensure the names correspond. +Field mappings can be used to map features defined in Feast to fields as they occur in data sources. -In the example feature-specification table above, we use `field_mappings` to ensure the field named `rating` in the batch source is mapped to the feature named `driver_rating`. +In the example feature-specification table above, we use field mappings to ensure the feature named `rating` in the batch source is mapped to the field named `driver_rating`. ## Working with a Feature Table @@ -93,34 +83,32 @@ In the example feature-specification table above, we use `field_mappings` to ens ```python driver_ft = FeatureTable(...) -client.apply_feature_table(driver_ft) +client.apply(driver_ft) ``` #### Updating a Feature Table -Feature table definitions may need to change over time to reflect more accurately your use case. In our ride-hailing example below, we update the max age: - ```python driver_ft = FeatureTable() -client.apply_feature_table(driver_ft) +client.apply(driver_ft) driver_ft.labels = {"team": "marketplace"} -client.apply_feature_table(driver_ft) +client.apply(driver_ft) ``` -Feast currently supports the following changes to feature tables: +#### Feast currently supports the following changes to feature tables: * Adding new features. -* Deleting existing features -* Changing the feature table's source, max age, and labels. +* Removing features. +* Updating source, max age, and labels. {% hint style="warning" %} Deleted features are archived, rather than removed completely. Importantly, new features cannot use the names of these deleted features. {% endhint %} -Feast currently does not support the following changes to feature tables: +#### Feast currently does not support the following changes to feature tables: * Changes to the project or name of a feature table. * Changes to entities related to a feature table. diff --git a/docs/concepts/glossary.md b/docs/concepts/glossary.md new file mode 100644 index 00000000000..2c521e57ac8 --- /dev/null +++ b/docs/concepts/glossary.md @@ -0,0 +1,36 @@ +# Glossary + +#### **Entity key** + +The combination of entities that uniquely identify a row. For example a feature table with the composite entity of \(customer, country\) might have an entity key of \(1001, 5\). They key is used during lookups of feature values and for deduplicating historical rows. + +#### Entity timestamp + +The timestamp on which an event occurred. The entity timestamp could describe the event time at which features were calculated, or it could describe the event timestamps at which outcomes were observed. + +Entity timestamps are commonly found on the entity dataframe and associated with the target variable \(outcome\) that needs to be predicted. These timestamps are the target on which point-in-time joins should be made. + +#### Entity rows + +A combination of a single [entity key ](glossary.md#entity-key)and a single [entity timestamp](glossary.md#entity-timestamp). + +#### Entity dataframe + +A collection of [entity rows](glossary.md#entity-rows). This dataframe is enriched with feature values before being used for model training. + +#### Feature References + +Feature references uniquely identify feature values throughout Feast. Feature references can either be defined as objects or as strings. + +The structure of a feature reference in string form is as follows: + +`feature_table:feature` + +Example: + +`drivers_stream:unique_drivers` + +Feature references are unique within a project. It is not possible to reference \(or retrieve\) features from multiple projects at the same time. + +\*\*\*\* + diff --git a/docs/concepts/overview.md b/docs/concepts/overview.md index 0becb135531..6ac4bceb80e 100644 --- a/docs/concepts/overview.md +++ b/docs/concepts/overview.md @@ -1,64 +1,21 @@ # Overview -## Using Feast - -Feast is the bridge between your ML models and data. Feast enables your team to: - -1. Create feature specifications to manage features, and load data that you want managed -2. Retrieve historical features for training models -3. Retrieve online features for serving models - -{% hint style="info" %} -Feast currently does not apply feature transformations to data. -{% endhint %} - -### Creating and managing features - -Feature creators model the data within their organization into Feast through the creation of [feature tables](feature-tables.md). - -Feature tables are both a schema and a means of identifying data sources for features. They allow Feast to know how to interpret your data, and optionally where to find it. Feature tables allow you to define domain [entities](entities.md) along with the features that are available on these entities. Feature tables also allow you to define schemas that describe properties of the respective data, which in turn can be used for validation purposes. - -After you register a feature table, Feast creates the relevant schemas to store feature data within its feature [stores](). These stores are then populated by [ingestion jobs](../user-guide/data-ingestion.md) that ingest data from data [sources](sources.md). The now data-rich stores enable Feast to provide access to features for training and serving. Alternatively, you can [ingest](../user-guide/data-ingestion.md) data into Feast instead of using an external source. - -Visit [feature tables](feature-tables.md) to learn more about them. - -### Retrieving historical features during training - -Historical retrieval uses [feature references](../user-guide/feature-retrieval.md) through the[ Feast SDK](https://api.docs.feast.dev/python/) to retrieve historical features. For historical serving, Feast requires that you provide the entities and timestamps for the corresponding feature data. Feast produces a point-in-time correct dataset using the requested features. These features can be requested from an unlimited number of feature sets. - -{% hint style="info" %} -For historical serving, Feast stores all historical values. -{% endhint %} - -Stores supported: [BigQuery](https://cloud.google.com/bigquery) - -### Retrieving online features during serving - -Online retrieval uses feature references through the [Feast Online Serving API](https://api.docs.feast.dev/grpc/feast.serving.pb.html) to retrieve online features. Online serving allows for very low latency requests to feature data at very high throughput. - -{% hint style="info" %} -During online serving, Feast stores **only** the latest values for each feature. -{% endhint %} - -Stores supported: [Redis](https://redis.io/), [Redis Cluster](https://redis.io/topics/cluster-tutorial) - -## Concept Hierarchy - -![](../.gitbook/assets/concept_hierarchy.png) - -Feast resources are arranged in the above hierarchy, with projects grouping one or more [entities](entities.md), which in turn groups [feature tables](feature-tables.md). These feature tables consist of [data sources](sources.md) and multiple features. +### Concepts -The logical grouping of these resources is important for namespacing and retrieval. Retrieval requires referencing individual features through feature references. These references uniquely identify a feature within a Feast deployment. +[Entities](entities.md) are objects in an organization like customers, transactions, and drivers, products, etc. -### Concepts +[Sources](sources.md) are external sources of data where feature data can be found. -[Entities](entities.md) are objects in an organization that model a specific construct. Examples of these include customers, transactions, and drivers. +[Feature Tables](feature-tables.md) are objects that define logical groupings of features, data sources, and other related metadata. -[Sources](sources.md) are either internal or external data sources where feature data can be found. +### Concept Hierarchy -[Feature Tables](feature-tables.md) are schemas that define logical groupings of features, data sources, and other related metadata. +![](../.gitbook/assets/image%20%284%29%20%281%29.png) -[Stores]() are databases that maintain feature data that gets served to models during training or inference. +Feast contains the following core concepts: -[Ingestion](../user-guide/data-ingestion.md) is the process of loading data into Feast. +* **Projects:** Serve as a top level namespace for all Feast resources. Each project is a completely independent environment in Feast. Users can only work in a single project at a time. +* **Entities:** Entities are the objects in an organization on which features occur. They map to your business domain \(users, products, transactions, locations\). +* **Feature Tables:** Defines a group of features that occur on a specific entity. +* **Features:** Individual feature within a feature table. diff --git a/docs/concepts/sources.md b/docs/concepts/sources.md index 1069e8dfaa7..e032b2f5950 100644 --- a/docs/concepts/sources.md +++ b/docs/concepts/sources.md @@ -2,25 +2,33 @@ ### Overview -A `source` is a data source that can be used to find feature data. Users define sources as part of [feature tables](feature-tables.md). Currently, Feast supports the following source types: +Sources are descriptions of external feature data and are registered to Feast as part of [feature tables](feature-tables.md). Once registered, Feast can ingest feature data from these sources into stores. -* Batch Source - * File - * [BigQuery](https://cloud.google.com/bigquery) -* Stream Source - * [Kafka](https://kafka.apache.org/) - * [Kinesis](https://aws.amazon.com/kinesis/) +Currently, Feast supports the following source types: -### Structure of a Source +#### Batch Source + +* File \(as in Spark\): Parquet and CSV files supported. +* BigQuery + +#### Stream Source + +* Kafka +* Kinesis -For both batch and stream sources, the following configurations are **necessary**: +The following encodings are supported on streams -* **created\_timestamp\_column**: Name of column containing timestamp when data is created. -* **event\_timestamp\_column**: Name of column containing timestamp when event data occurred. +* Avro +* Protobuf -When configuring data source options, see the [Feast Python API documentation](https://api.docs.feast.dev/python/) for more details. +### Structure of a Source + +For both batch and stream sources, the following configurations are necessary: -Some valid source specifications are shown below: +* **Event timestamp column**: Name of column containing timestamp when event data occurred. Used during point-in-time join of feature values to [entity timestamps](glossary.md#entity-timestamp). +* **Created timestamp column**: Name of column containing timestamp when data is created. Used to deduplicate data when multiple copies of the same [entity key](glossary.md#entity-key) is ingested. + +Example data source specifications: {% tabs %} {% tab title="batch\_sources.py" %} @@ -30,7 +38,7 @@ from feast.data_format import ParquetFormat batch_file_source = FileSource( file_format=ParquetFormat(), - file_url="file://feast/*", + file_url="file:///feast/customer.parquet", event_timestamp_column="event_timestamp", created_timestamp_column="created_timestamp", ) @@ -55,15 +63,11 @@ stream_kafka_source = KafkaSource( The [Feast Python API documentation](https://api.docs.feast.dev/python/) provides more information about options to specify for the above sources. -{% hint style="info" %} -When creating a Feature Table for use in training datasets, specify a batch source already containing materialized data. -{% endhint %} - ### Working with a Source #### Creating a Source -Sources are required when specifying a [feature table](feature-tables.md): +Sources are defined as part of [feature tables](feature-tables.md): ```python batch_bigquery_source = BigQuerySource( diff --git a/docs/concepts/stores.md b/docs/concepts/stores.md new file mode 100644 index 00000000000..3695f6c37d0 --- /dev/null +++ b/docs/concepts/stores.md @@ -0,0 +1,26 @@ +# Stores + +In Feast, a store is a database that is populated with feature data that will ultimately be served to models. + +### Offline \(Historical\) Store + +The offline store maintains historical copies of feature values. These features are grouped and stored in feature tables. During retrieval of historical data, features are queries from these feature tables in order to produce training datasets. + +{% hint style="warning" %} +Feast 0.8 does not support offline storage. Support will be added in Feast 0.9. +{% endhint %} + +### Online Store + +The online store maintains only the latest values for a specific feature. + +* Feature values are stored based on their [entity keys](glossary.md#entity-key) +* Feast currently supports Redis as an online store. +* Online stores are meant for very high throughput writes from ingestion jobs and very low latency access to features during online serving. + +{% hint style="info" %} +Feast only supports a single online store in production +{% endhint %} + + + diff --git a/docs/contributing/adding-a-new-store-1.md b/docs/contributing/adding-a-new-store-1.md deleted file mode 100644 index 56c06b9b4b6..00000000000 --- a/docs/contributing/adding-a-new-store-1.md +++ /dev/null @@ -1,87 +0,0 @@ -# Adding a New Store - -The following guide will explain the process of adding a new store through the introduction of a storage connector. - -## 1. Storage API - -Feast has an external module where storage interfaces are defined: [Storage API](https://github.com/gojek/feast/tree/master/storage/api/src/main/java/feast/storage/api) - -Feast interacts with a store at three points . - -1. **During initialization:** Store configuration is loaded into memory by Feast Serving and synchronized with Feast Core -2. **During ingestion of feature data.** [writer interfaces](https://github.com/gojek/feast/tree/master/storage/api/src/main/java/feast/storage/api/writer) are used by the Apache Beam ingestion jobs in order to populate stores \(historical or online\). -3. **During retrieval of feature data:** [Retrieval interfaces](https://github.com/gojek/feast/tree/master/storage/api/src/main/java/feast/storage/api/retriever) are used by Feast Serving in order to read data from stores in order to create training datasets or to serve online data. - -All three of these components should be implemented in order to have a complete storage connector. - -## 2. Adding a Storage Connector - -### 2.1 Initialization and configuration - -Stores are configured in Feast Serving. Feast Serving publishes its store configuration to Feast Core, after which Feast Core can start ingestion/population jobs to populate it. - -Store configuration is always in the form of a map<String, String>. The keys and configuration for stores are defined in [protos](https://github.com/gojek/feast/blob/master/protos/feast/core/Store.proto). This must be added in order to define a new store - -Then the store must be configured to be loaded through Feast Serving. The above configuration is loaded through [FeastProperties.java](https://github.com/gojek/feast/blob/a1937c374a4e39b7a75d828e7b7c3b87a64d9d6e/serving/src/main/java/feast/serving/config/FeastProperties.java#L175). - -Once configuration is loaded, the store will then be instantiated. - -* Feast Core: The [StoreUtil.java](https://github.com/gojek/feast/blob/master/ingestion/src/main/java/feast/ingestion/utils/StoreUtil.java#L85) instantiates new stores for the purposes of feature ingestion. -* Feast Serving: The [ServingServiceConfig](https://github.com/gojek/feast/blob/a1937c374a4e39b7a75d828e7b7c3b87a64d9d6e/serving/src/main/java/feast/serving/config/ServingServiceConfig.java#L56) instantiates new stores for the purposes of retrieval - -{% hint style="info" %} -In the future we plan to provide a plugin interface for adding stores. -{% endhint %} - -### 2.2 Feature Ingestion \(Writer\) - -Feast creates and manages ingestion/population jobs that stream in data from upstream data sources. Currently Feast only supports Kafka as a data source, meaning these jobs are all long running. Batch ingestion \(from users\) results in data being pushed to Kafka topics after which they are picked up by these "population" jobs and written to stores. - -In order for ingestion to succeed, the destination store must be writable. This means that Feast must be able to create the appropriate tables/schemas in the store and also write data from the population job into the store. - -Currently Feast Core starts and manages these population jobs that ingest data into stores \(although we are planning to move this responsibility to the serving layer\). Feast Core starts an [Apache Beam](https://beam.apache.org/) job which synchronously runs migrations on the destination store and subsequently starts consuming [FeatureRows](https://github.com/gojek/feast/blob/master/protos/feast/types/FeatureRow.proto) from Kafka and writing it into stores using a [writer](https://github.com/gojek/feast/tree/master/storage/api/src/main/java/feast/storage/api/writer). - -Below is a "happy path" of a batch ingestion process which includes a blocking step at the Python SDK. - -![](https://user-images.githubusercontent.com/6728866/74807906-91e73c00-5324-11ea-8ba5-2b43c7c5282b.png) - - - -The complete ingestion flow is executed by a [FeatureSink](https://github.com/gojek/feast/blob/master/storage/api/src/main/java/feast/storage/api/writer/FeatureSink.java). Two methods should be implemented - -* [prepareWrite\(\)](https://github.com/gojek/feast/blob/a1937c374a4e39b7a75d828e7b7c3b87a64d9d6e/storage/api/src/main/java/feast/storage/api/writer/FeatureSink.java#L45): Sets up storage backend for writing/ingestion. This method will be called once during pipeline initialisation. Typically this is used to apply schemas. -* [writer\(\)](https://github.com/gojek/feast/blob/a1937c374a4e39b7a75d828e7b7c3b87a64d9d6e/storage/api/src/main/java/feast/storage/api/writer/FeatureSink.java#L53): Retrieves an Apache Beam PTransform that is used to write data to this store. - -### 2.2 Feature Serving \(Retriever\) - -Feast Serving can serve both historical/batch features and online features. Depending on the store that is being added, you should implement either a historical/batch store or an online storage. - -#### 2.2.1 Historical Serving - -The historical serving interface is defined through the [HistoricalRetriever](https://github.com/gojek/feast/blob/master/storage/api/src/main/java/feast/storage/api/retriever/HistoricalRetriever.java) interface. Historical retrieval is an asynchronous process. The client submits a request for a dataset to be produced, and polls until it is ready. - -![High-level flow for batch retrieval](https://user-images.githubusercontent.com/6728866/74797157-702a8c80-5305-11ea-8901-bf6f4eb075f9.png) - -The current implementation of batch retrieval starts and ends with a file \(dataset\) in a Google Cloud Storage bucket. The user ingests an entity dataset. This dataset is loaded into a store \(BigQuery0, joined to features in a point-in-time correct way, then exported again to the bucket. - -Additionally, we have also implemented a [batch retrieval method ](https://github.com/gojek/feast/blob/a1937c374a4e39b7a75d828e7b7c3b87a64d9d6e/sdk/python/feast/client.py#L509)in the Python SDK. Depending on the means through which this new store will export data, this client may have to change. At the very least it would change if Google Cloud Storage isn't used as the staging bucket. - -The means through which you implement the export/import of data into the store will depend on your store. - -#### 2.2.2 Online Serving - -In the case of online serving it is necessary to implement an [OnlineRetriever](https://github.com/gojek/feast/blob/master/storage/api/src/main/java/feast/storage/api/retriever/OnlineRetriever.java). This online retriever will read rows directly and synchronously from an online database. The exact encoding strategy you use to store your data in the store would be defined in the FeatureSink. The OnlineRetriever is expected to read and decode those rows. - -## 3. Storage Connectors Examples - -Feast currently provides support for the following storage types - -Historical storage - -* [BigQuery](https://github.com/gojek/feast/tree/master/storage/connectors/bigquery) - -Online storage - -* [Redis](https://github.com/gojek/feast/tree/master/storage/connectors/redis) -* [Redis Cluster](https://github.com/gojek/feast/tree/master/storage/connectors/rediscluster) - diff --git a/docs/contributing/contributing.md b/docs/contributing/contributing.md index 2be097d6ffd..d9378ca6847 100644 --- a/docs/contributing/contributing.md +++ b/docs/contributing/contributing.md @@ -1,10 +1,10 @@ # Contribution Process -We use [RFCs](https://en.wikipedia.org/wiki/Request_for_Comments) and [GitHub issues](https://github.com/gojek/feast/issues) to communicate development ideas. The simplest way to contribute to Feast is to leave comments in our [RFCs](https://drive.google.com/drive/u/0/folders/1Lj1nIeRB868oZvKTPLYqAvKQ4O0BksjY) in the [Feast Google Drive](https://drive.google.com/drive/u/0/folders/0AAe8j7ZK3sxSUk9PVA) or our GitHub issues. You will need to join our [Google Group](../getting-help.md) in order to get access. +We use [RFCs](https://en.wikipedia.org/wiki/Request_for_Comments) and [GitHub issues](https://github.com/feast-dev/feast/issues) to communicate development ideas. The simplest way to contribute to Feast is to leave comments in our [RFCs](https://drive.google.com/drive/u/0/folders/1Lj1nIeRB868oZvKTPLYqAvKQ4O0BksjY) in the [Feast Google Drive](https://drive.google.com/drive/u/0/folders/0AAe8j7ZK3sxSUk9PVA) or our GitHub issues. You will need to join our [Google Group](../community.md) in order to get access. -We follow a process of [lazy consensus](http://community.apache.org/committers/lazyConsensus.html). If you believe you know what the project needs then just start development. If you are unsure about which direction to take with development then please communicate your ideas through a GitHub issue or through our [Slack Channel](../getting-help.md) before starting development. +We follow a process of [lazy consensus](http://community.apache.org/committers/lazyConsensus.html). If you believe you know what the project needs then just start development. If you are unsure about which direction to take with development then please communicate your ideas through a GitHub issue or through our [Slack Channel](../community.md) before starting development. -Please [submit a PR ](https://github.com/gojek/feast/pulls)to the master branch of the Feast repository once you are ready to submit your contribution. Code submission to Feast \(including submission from project maintainers\) require review and approval from maintainers or code owners. +Please [submit a PR ](https://github.com/feast-dev/feast/pulls)to the master branch of the Feast repository once you are ready to submit your contribution. Code submission to Feast \(including submission from project maintainers\) require review and approval from maintainers or code owners. PRs that are submitted by the general public need to be identified as `ok-to-test`. Once enabled, [Prow](https://github.com/kubernetes/test-infra/tree/master/prow) will run a range of tests to verify the submission, after which community members will help to review the pull request. diff --git a/docs/contributing/development-guide.md b/docs/contributing/development-guide.md index 9c9ec2bdf51..b52c66c1e3d 100644 --- a/docs/contributing/development-guide.md +++ b/docs/contributing/development-guide.md @@ -18,6 +18,7 @@ The following software is required for Feast development * Java SE Development Kit 11 * Python version 3.6 \(or above\) and pip * [Maven](https://maven.apache.org/install.html) version 3.6.x +* PySpark 2.4.2 ### **Services** @@ -108,10 +109,10 @@ Feast Serving has a dependency on Feast Core, thus always start Feast Core first ```bash # Start Feast Core locally -java -jar core/target/feast-core-0.8.0-exec.jar +java -jar core/target/feast-core-0.9.0-exec.jar # Start Feast Serving locally -java -jar serving/target/feast-serving-0.8.0-exec.jar +java -jar serving/target/feast-serving-0.9.0-exec.jar ``` Test whether Feast Core, Feast Serving are started and running correctly: @@ -122,8 +123,8 @@ feast version --core-url="localhost:6565" --serving-url="localhost:6566" ```javascript { - 'serving': {'url': 'localhost:6566', 'version': '0.8.0'}, - 'core': {'url': 'localhost:6565', 'version': '0.8.0'} + 'serving': {'url': 'localhost:6566', 'version': '0.9.0'}, + 'core': {'url': 'localhost:6565', 'version': '0.9.0'} } ``` diff --git a/docs/coverage/java/pom.xml b/docs/coverage/java/pom.xml index 007f8060764..21a75debd2c 100644 --- a/docs/coverage/java/pom.xml +++ b/docs/coverage/java/pom.xml @@ -47,12 +47,6 @@ ${project.version} - - dev.feast - feast-storage-connector-bigquery - ${project.version} - - dev.feast feast-storage-connector-redis diff --git a/docs/getting-started/connect-to-feast/README.md b/docs/getting-started/connect-to-feast/README.md index 214cf89fa3b..4333359f902 100644 --- a/docs/getting-started/connect-to-feast/README.md +++ b/docs/getting-started/connect-to-feast/README.md @@ -19,7 +19,7 @@ The Feast CLI is a command line implementation of the Feast Python SDK. * Ingest data into Feast * Manage ingestion jobs -{% page-ref page="connecting-to-feast.md" %} +{% page-ref page="feast-cli.md" %} ### Online Serving Clients diff --git a/docs/getting-started/connect-to-feast/connecting-to-feast.md b/docs/getting-started/connect-to-feast/feast-cli.md similarity index 100% rename from docs/getting-started/connect-to-feast/connecting-to-feast.md rename to docs/getting-started/connect-to-feast/feast-cli.md diff --git a/docs/getting-started/install-feast/README.md b/docs/getting-started/install-feast/README.md index c358c2e12ae..7fa1948aa8d 100644 --- a/docs/getting-started/install-feast/README.md +++ b/docs/getting-started/install-feast/README.md @@ -12,3 +12,15 @@ This guide installs Feast into an AWS environment using Terraform. The Terraform {% page-ref page="kubernetes-amazon-eks-with-terraform.md" %} +### Azure AKS \(with Terraform\) + +This guide installs Feast into an Azure environment using Terraform. The Terraform script is opinionated and intended to allow you to start quickly. + +{% page-ref page="kubernetes-azure-aks-with-terraform.md" %} + +### Google Cloud GKE \(with Terraform\) + +This guide installs Feast into a Google Cloud environment using Terraform. The Terraform script is opinionated and intended to allow you to start quickly. + +{% page-ref page="google-cloud-gke-with-terraform.md" %} + diff --git a/docs/getting-started/install-feast/google-cloud-gke-with-terraform.md b/docs/getting-started/install-feast/google-cloud-gke-with-terraform.md new file mode 100644 index 00000000000..a3252cf0bbb --- /dev/null +++ b/docs/getting-started/install-feast/google-cloud-gke-with-terraform.md @@ -0,0 +1,52 @@ +# Google Cloud GKE \(with Terraform\) + +### Overview + +This guide installs Feast on GKE using our [reference Terraform configuration](https://github.com/feast-dev/feast/tree/master/infra/terraform/gcp). + +{% hint style="info" %} +The Terraform configuration used here is a greenfield installation that neither assumes anything about, nor integrates with, existing resources in your GCP account. The Terraform configuration presents an easy way to get started, but you may want to customize this set up before using Feast in production. +{% endhint %} + +This Terraform configuration creates the following resources: + +* GKE cluster +* Feast services running on GKE +* Google Memorystore \(Redis\) as online store +* Dataproc cluster +* Kafka running on GKE, exposed to the dataproc cluster via internal load balancer + +### 1. Requirements + +* Install [Terraform](https://www.terraform.io/) > = 0.12 \(tested with 0.13.3\) +* Install [Helm](https://helm.sh/docs/intro/install/) \(tested with v3.3.4\) +* GCP [authentication](https://cloud.google.com/docs/authentication) and sufficient [privilege](https://cloud.google.com/iam/docs/understanding-roles) to create the resources listed above. + +### 2. Configure Terraform + +Create a `.tfvars` file under`feast/infra/terraform/gcp`. Name the file. In our example, we use `my_feast.tfvars`. You can see the full list of configuration variables in `variables.tf`. Sample configurations are provided below: + +{% code title="my\_feast.tfvars" %} +```typescript +gcp_project_name = "kf-feast" +name_prefix = "feast-0-8" +region = "asia-east1" +gke_machine_type = "n1-standard-2" +network = "default" +subnetwork = "default" +dataproc_staging_bucket = "feast-dataproc" +``` +{% endcode %} + +### 3. Apply + +After completing the configuration, initialize Terraform and apply: + +```bash +$ cd feast/infra/terraform/gcp +$ terraform init +$ terraform apply -var-file=my_feast.tfvars +``` + + + diff --git a/docs/getting-started/install-feast/kubernetes-amazon-eks-with-terraform.md b/docs/getting-started/install-feast/kubernetes-amazon-eks-with-terraform.md index d3058913d1f..232c16f193f 100644 --- a/docs/getting-started/install-feast/kubernetes-amazon-eks-with-terraform.md +++ b/docs/getting-started/install-feast/kubernetes-amazon-eks-with-terraform.md @@ -17,7 +17,7 @@ This Terraform configuration creates the following resources: * Amazon EMR cluster to run Spark \(3x spot m4.xlarge\) * Staging S3 bucket to store temporary data -![](../../.gitbook/assets/feast-on-aws-3-.png) +![](../../.gitbook/assets/feast-on-aws-3-%20%281%29.png) ### 1. Requirements diff --git a/docs/getting-started/install-feast/kubernetes-azure-aks-with-terraform.md b/docs/getting-started/install-feast/kubernetes-azure-aks-with-terraform.md new file mode 100644 index 00000000000..b97d8d008c7 --- /dev/null +++ b/docs/getting-started/install-feast/kubernetes-azure-aks-with-terraform.md @@ -0,0 +1,63 @@ +# Azure AKS \(with Terraform\) + +### Overview + +This guide installs Feast on Azure using our [reference Terraform configuration](https://github.com/feast-dev/feast/tree/master/infra/terraform/azure). + +{% hint style="info" %} +The Terraform configuration used here is a greenfield installation that neither assumes anything about, nor integrates with, existing resources in your Azure account. The Terraform configuration presents an easy way to get started, but you may want to customize this set up before using Feast in production. +{% endhint %} + +This Terraform configuration creates the following resources: + +* Kubernetes cluster on Azure AKS +* Kafka managed by HDInsight +* Postgres database for Feast metadata, running as a pod on AKS +* Redis cluster, using Azure Cache for Redis +* [spark-on-k8s-operator](https://github.com/GoogleCloudPlatform/spark-on-k8s-operator) to run Spark +* Staging Azure blob storage container to store temporary data + +### 1. Requirements + +* Create an Azure account and [configure credentials locally](https://docs.microsoft.com/en-us/cli/azure/install-azure-cli) +* Install [Terraform](https://www.terraform.io/) \(tested with 0.13.5\) +* Install [Helm](https://helm.sh/docs/intro/install/) \(tested with v3.4.2\) + +### 2. Configure Terraform + +Create a `.tfvars` file under`feast/infra/terraform/azure`. Name the file. In our example, we use `my_feast.tfvars`. You can see the full list of configuration variables in `variables.tf`. At a minimum, you need to set `name_prefix` and `resource_group`: + +{% code title="my\_feast.tfvars" %} +```typescript +name_prefix = "feast" +resource_group = "Feast" # pre-existing resource group +``` +{% endcode %} + +### 3. Apply + +After completing the configuration, initialize Terraform and apply: + +```bash +$ cd feast/infra/terraform/azure +$ terraform init +$ terraform apply -var-file=my_feast.tfvars +``` + +### 4. Connect to Feast using Jupyter + +After all pods are running, connect to the Jupyter Notebook Server running in the cluster. + +To connect to the remote Feast server you just created, forward a port from the remote k8s cluster to your local machine. + +```bash +kubectl port-forward $(kubectl get pod -o custom-columns=:metadata.name | grep jupyter) 8888:8888 +``` + +```text +Forwarding from 127.0.0.1:8888 -> 8888 +Forwarding from [::1]:8888 -> 8888 +``` + +You can now connect to the bundled Jupyter Notebook Server at `localhost:8888` and follow the example Jupyter notebook. + diff --git a/docs/getting-started/install-feast/kubernetes-with-helm.md b/docs/getting-started/install-feast/kubernetes-with-helm.md index 1f38a8ba50c..4c69efd0a8e 100644 --- a/docs/getting-started/install-feast/kubernetes-with-helm.md +++ b/docs/getting-started/install-feast/kubernetes-with-helm.md @@ -64,5 +64,6 @@ You can now connect to the bundled Jupyter Notebook Server at `localhost:8888` a * [Feast Concepts](../../concepts/overview.md) * [Feast Examples/Tutorials](https://github.com/feast-dev/feast/tree/master/examples) * [Feast Helm Chart Documentation](https://github.com/feast-dev/feast/blob/master/infra/charts/feast/README.md) -* [Configuring Feast components](../../reference/configuration-reference/) +* [Configuring Feast components](../../reference/configuration-reference.md) +* [Feast and Spark](../../reference/feast-and-spark.md) diff --git a/docs/getting-started/learn-feast.md b/docs/getting-started/learn-feast.md index 004b1075aaa..10f2eb6d291 100644 --- a/docs/getting-started/learn-feast.md +++ b/docs/getting-started/learn-feast.md @@ -3,7 +3,7 @@ Explore the following resources to learn more about Feast: * [Concepts](../) describes all important Feast API concepts. -* [User guide](../user-guide/data-ingestion.md) provides guidance on completing Feast workflows. +* [User guide](../user-guide/define-and-ingest-features.md) provides guidance on completing Feast workflows. * [Examples](https://github.com/feast-dev/feast/tree/master/examples) contains Jupyter notebooks that you can run on your Feast deployment. * [Advanced](../advanced/troubleshooting.md) contains information about both advanced and operational aspects of Feast. * [Reference](../reference/api/) contains detailed API and design documents for advanced users. diff --git a/docs/getting-started/quickstart.md b/docs/getting-started/quickstart.md deleted file mode 100644 index ce315dbf27f..00000000000 --- a/docs/getting-started/quickstart.md +++ /dev/null @@ -1,56 +0,0 @@ -# Quickstart - -## Overview - -This guide will give a walkthrough on deploying Feast using Docker Compose, which allows the user to quickly explore the functionalities in Feast with minimal infrastructure setup. It includes a built in Jupyter Notebook Server that is preloaded with PySpark and Feast SDK, as well as Feast example notebooks to get you started. - -## 0. Requirements - -* [Docker Compose](https://docs.docker.com/compose/install/) should be installed. -* Optional dependancies: - * a [GCP service account](https://cloud.google.com/iam/docs/creating-managing-service-account-keys) that has access to [Google Cloud Storage](https://cloud.google.com/storage). - -## 1. Set up environment - -Clone the latest stable version of the [Feast repository](https://github.com/gojek/feast/) and setup before we deploy: - -```text -git clone https://github.com/feast-dev/feast.git -cd feast/infra/docker-compose -cp .env.sample .env -``` - -## 2. Start Feast Services - -Start the Feast services. Make sure that the following ports are free on the host machines: 6565, 6566, 8888, 9094, 5432. Alternatively, change the port mapping to use a different port on the host. - -```javascript -docker-compose up -d -``` - -{% hint style="info" %} -The Docker Compose deployment will take some time fully startup: - -* During this time Feast Serving container may restart, which should be automatically corrected after Feast Core is up and ready. -* If container restarts do not stop after 10 minutes, check the docker compose log to see if there is any error that prevents Feast Core from starting successfully. -{% endhint %} - -Once deployed, you should be able to connect at `localhost:8888` to the bundled Jupyter Notebook Server and follow the example notebooks: - -{% embed url="http://localhost:8888/tree?" caption="" %} - -## 3. Optional dependancies - -### 3.1 Set up Google Cloud Platform - -The example Jupyter notebook does not require any GCP dependancies by default. If you would like to modify the example such that a GCP service is required \(eg. Google Cloud Storage\), you would need to set up a [service account](https://cloud.google.com/iam/docs/creating-managing-service-accounts) that is associated with the notebook. Make sure that the service account has sufficient privileges to access the required GCP services. - -Once the service account is created, download the associated JSON key file and copy the file to the path configured in `.env` , under `GCP_SERVICE_ACCOUNT` . - -## 4. Further Reading - -* [Feast Concepts](../concepts/overview.md) -* [Feast Examples/Tutorials](https://github.com/feast-dev/feast/tree/master/examples) -* [Configuring Feast Components](../reference/configuration-reference.md) -* [Configuration Reference](https://app.gitbook.com/@feast/s/docs/v/master/reference/configuration-reference) - diff --git a/docs/installation/docker-compose.md b/docs/installation/docker-compose.md deleted file mode 100644 index 3c8c50862e3..00000000000 --- a/docs/installation/docker-compose.md +++ /dev/null @@ -1,112 +0,0 @@ -# Docker Compose - -### Overview - -This guide will bring Feast up using Docker Compose. This will allow you to: - -* Create, register, and manage feature sets -* Ingest feature data into Feast -* Retrieve features for online serving -* Retrieve features for batch serving \(only if using Google Cloud Platform\) - -This guide is split into three parts: - -1. Setting up your environment -2. Starting Feast with **online serving support only** \(does not require GCP\). -3. Starting Feast with support for **both online and batch** serving \(requires GCP\) - -{% hint style="info" %} -The docker compose setup uses Direct Runner for the Apache Beam jobs that populate data stores. Running Beam with the Direct Runner means it does not need a dedicated runner like Flink or Dataflow, but this comes at the cost of performance. We recommend the use of a dedicated runner when running Feast with very large workloads. -{% endhint %} - -### 0. Requirements - -* [Docker compose](https://docs.docker.com/compose/install/) must be installed. -* The following list of TCP ports must be free: - * 6565, 6566, 8888, and 9094. - * Alternatively it is possible to modify port mappings in `/docker-compose/docker-compose.yml`. -* \(for batch serving only\) For batch serving you will also need a [GCP service account key](https://cloud.google.com/iam/docs/creating-managing-service-account-keys) that has access to [Google Cloud Storage](https://cloud.google.com/storage) and [BigQuery](https://cloud.google.com/bigquery). -* \(for batch serving only\) [Google Cloud SDK ](https://cloud.google.com/sdk/install)installed, authenticated, and configured to the project you will use. - -## 1. Set up environment - -Clone the [Feast repository](https://github.com/feast-dev/feast/) and navigate to the `docker-compose` sub-directory: - -```bash -git clone https://github.com/feast-dev/feast.git && \ -cd feast && export FEAST_HOME_DIR=$(pwd) && \ -cd infra/docker-compose -``` - -Make a copy of the `.env.sample` file: - -```bash -cp .env.sample .env -``` - -## 2. Docker Compose for Online Serving Only - -### 2.1 Start Feast \(without batch retrieval support\) - -If you do not require batch serving, then its possible to simply bring up Feast: - -```javascript -docker-compose up -d -``` - -A Jupyter Notebook environment is now available to use Feast: - -[http://localhost:8888/tree/feast/examples](http://localhost:8888/tree/feast/examples) - -## 3. Docker Compose for Online and Batch Serving - -{% hint style="info" %} -Batch serving requires Google Cloud Storage to function, specifically Google Cloud Storage \(GCP\) and BigQuery. -{% endhint %} - -### 3.1 Set up Google Cloud Platform - -Create a [service account ](https://cloud.google.com/iam/docs/creating-managing-service-accounts)from the GCP console and copy it to the `infra/docker-compose/gcp-service-accounts` folder: - -```javascript -cp my-service-account.json ${FEAST_HOME_DIR}/infra/docker-compose/gcp-service-accounts -``` - -Create a Google Cloud Storage bucket. Make sure that your service account above has read/write permissions to this bucket: - -```bash -gsutil mb gs://my-feast-staging-bucket -``` - -### 3.2 Configure .env - -Configure the `.env` file based on your environment. At the very least you have to modify: - -| Parameter | Description | -| :--- | :--- | -| FEAST\_CORE\_GCP\_SERVICE\_ACCOUNT\_KEY | This should be your service account file name, for example `key.json`. | -| FEAST\_HISTORICAL\_SERVING\_GCP\_SERVICE\_ACCOUNT\_KEY | This should be your service account file name, for example `key.json` | -| FEAST\_JUPYTER\_GCP\_SERVICE\_ACCOUNT\_KEY | This should be your service account file name, for example `key.json` | -| FEAST\_JOB\_STAGING\_LOCATION | Google Cloud Storage bucket that Feast will use to stage data exports and batch retrieval requests, for example `gs://your-gcs-bucket/staging` | - -### 3.3 Configure .bq-store.yml - -We will also need to configure the `bq-store.yml` file inside `infra/docker-compose/serving/` to configure the BigQuery storage configuration as well as the feature sets that the store subscribes to. At a minimum you will need to set: - -| Parameter | Description | -| :--- | :--- | -| bigquery\_config.project\_id | This is you [GCP project Id](https://cloud.google.com/resource-manager/docs/creating-managing-projects). | -| bigquery\_config.dataset\_id | This is the name of the BigQuery dataset that tables will be created in. Each feature set will have one table in BigQuery. | - -### 3.4 Start Feast \(with batch retrieval support\) - -Start Feast: - -```javascript -docker-compose up -d -``` - -A Jupyter Notebook environment is now available to use Feast: - -[http://localhost:8888/tree/feast/examples](http://localhost:8888/tree/feast/examples) - diff --git a/docs/installation/gke.md b/docs/installation/gke.md deleted file mode 100644 index 66041887786..00000000000 --- a/docs/installation/gke.md +++ /dev/null @@ -1,211 +0,0 @@ -# Google Kubernetes Engine \(GKE\) - -### Overview - -This guide will install Feast into a Kubernetes cluster on GCP. It assumes that all of your services will run within a single Kubernetes cluster. Once Feast is installed you will be able to: - -* Define and register features. -* Load feature data from both batch and streaming sources. -* Retrieve features for model training. -* Retrieve features for online serving. - -{% hint style="info" %} -This guide requires [Google Cloud Platform](https://cloud.google.com/) for installation. - -* [BigQuery](https://cloud.google.com/bigquery/) is used for storing historical features. -* [Google Cloud Storage](https://cloud.google.com/storage/) is used for intermediate data storage. -{% endhint %} - -## 0. Requirements - -1. [Google Cloud SDK ](https://cloud.google.com/sdk/install)installed, authenticated, and configured to the project you will use. -2. [Kubectl](https://kubernetes.io/docs/tasks/tools/install-kubectl/) installed. -3. [Helm](https://helm.sh/3) \(2.16.0 or greater\) installed on your local machine with Tiller installed in your cluster. Helm 3 has not been tested yet. - -## 1. Set up GCP - -First define the environmental variables that we will use throughout this installation. Please customize these to reflect your environment. - -```bash -export FEAST_GCP_PROJECT_ID=my-gcp-project -export FEAST_GCP_REGION=us-central1 -export FEAST_GCP_ZONE=us-central1-a -export FEAST_BIGQUERY_DATASET_ID=feast -export FEAST_GCS_BUCKET=${FEAST_GCP_PROJECT_ID}_feast_bucket -export FEAST_GKE_CLUSTER_NAME=feast -export FEAST_SERVICE_ACCOUNT_NAME=feast-sa -``` - -Create a Google Cloud Storage bucket for Feast to stage batch data exports: - -```bash -gsutil mb gs://${FEAST_GCS_BUCKET} -``` - -Create the service account that Feast will run as: - -```bash -gcloud iam service-accounts create ${FEAST_SERVICE_ACCOUNT_NAME} - -gcloud projects add-iam-policy-binding ${FEAST_GCP_PROJECT_ID} \ - --member serviceAccount:${FEAST_SERVICE_ACCOUNT_NAME}@${FEAST_GCP_PROJECT_ID}.iam.gserviceaccount.com \ - --role roles/editor - -gcloud iam service-accounts keys create key.json --iam-account \ -${FEAST_SERVICE_ACCOUNT_NAME}@${FEAST_GCP_PROJECT_ID}.iam.gserviceaccount.com -``` - -## 2. Set up a Kubernetes \(GKE\) cluster - -{% hint style="warning" %} -Provisioning a GKE cluster can expose your services publicly. This guide does not cover securing access to the cluster. -{% endhint %} - -Create a GKE cluster: - -```bash -gcloud container clusters create ${FEAST_GKE_CLUSTER_NAME} \ - --machine-type n1-standard-4 -``` - -Create a secret in the GKE cluster based on your local key `key.json`: - -```bash -kubectl create secret generic feast-gcp-service-account --from-file=key.json -``` - -For this guide we will use `NodePort` for exposing Feast services. In order to do so, we must find an External IP of at least one GKE node. This should be a public IP. - -```bash -export FEAST_IP=$(kubectl describe nodes | grep ExternalIP | awk '{print $2}' | head -n 1) -export FEAST_CORE_URL=${FEAST_IP}:32090 -export FEAST_ONLINE_SERVING_URL=${FEAST_IP}:32091 -export FEAST_HISTORICAL_SERVING_URL=${FEAST_IP}:32092 -``` - -Add firewall rules to open up ports on your Google Cloud Platform project: - -```bash -gcloud compute firewall-rules create feast-core-port --allow tcp:32090 -gcloud compute firewall-rules create feast-online-port --allow tcp:32091 -gcloud compute firewall-rules create feast-batch-port --allow tcp:32092 -gcloud compute firewall-rules create feast-redis-port --allow tcp:32101 -gcloud compute firewall-rules create feast-kafka-ports --allow tcp:31090-31095 -``` - -## 3. Set up Helm - -Run the following command to provide Tiller with authorization to install Feast: - -```bash -kubectl apply -f - < + + + Feast Setting + Value + + + + + +

+

SPARK_LAUNCHER +

+ + "k8s" + + + + +

+

SPARK_K8S_NAMESPACE +

+ + The name of the Kubernetes namespace to run Spark jobs in. This should + match the value of sparkJobNamespace set on spark-on-k8s-operator + Helm chart. Typically this is also the namespace Feast itself will run + in. + + + SPARK_STAGING_LOCATION + + S3 URL to use as a staging location, must be readable and writable by + Feast. Use s3a:// prefix here. Ex.: s3a://some-bucket/some-prefix + + + + + +Lastly, make sure that the service account used by Feast has permissions to manage Spark Application resources. This depends on your k8s setup, but typically you'd need to configure a Role and a RoleBinding like the one below: + +```text +cat < +rules: +- apiGroups: ["sparkoperator.k8s.io"] + resources: ["sparkapplications"] + verbs: ["create", "delete", "deletecollection", "get", "list", "update", "watch", "patch"] +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: RoleBinding +metadata: + name: use-spark-operator + namespace: +roleRef: + kind: Role + name: use-spark-operator + apiGroup: rbac.authorization.k8s.io +subjects: + - kind: ServiceAccount + name: default +EOF +``` + +### Option 2. Use GCP and Dataproc + +If you're running Feast in Google Cloud, you can use Dataproc, a managed Spark platform. To configure Feast to use it, set the following options in Feast config: + +| Feast Setting | Value | +| :--- | :--- | +| `SPARK_LAUNCHER` | `"dataproc"` | +| `DATAPROC_CLUSTER_NAME` | Dataproc cluster name | +| `DATAPROC_PROJECT` | Dataproc project name | +| `SPARK_STAGING_LOCATION` | GCS URL to use as a staging location, must be readable and writable by Feast. Ex.: `gs://some-bucket/some-prefix` | + +See [Feast documentation](https://api.docs.feast.dev/python/#module-feast.constants) for more configuration options for Dataproc. + +### Option 3. Use AWS and EMR + +If you're running Feast in AWS, you can use EMR, a managed Spark platform. To configure Feast to use it, set at least the following options in Feast config: + +| Feast Setting | Value | +| :--- | :--- | +| `SPARK_LAUNCHER` | `"emr"` | +| `SPARK_STAGING_LOCATION` | S3 URL to use as a staging location, must be readable and writable by Feast. Ex.: `s3://some-bucket/some-prefix` | + +See [Feast documentation](https://api.docs.feast.dev/python/#module-feast.constants) for more configuration options for EMR. + diff --git a/docs/reference/limitations.md b/docs/reference/limitations.md index d0d3ec5a773..efde485a805 100644 --- a/docs/reference/limitations.md +++ b/docs/reference/limitations.md @@ -43,5 +43,10 @@ | Limitation | Motivation | | :--- | :--- | | Once data has been ingested into Feast, there is currently no way to delete the data without manually going to the database and deleting it. However, during retrieval only the latest rows will be returned for a specific key \(`event_timestamp`, `entity`\) based on its `created_timestamp`. | This functionality simply doesn't exist yet as a Feast API | -| During the ingestion of data into BigQuery, `event_timestamp` is rounded down to seconds. E.g., `2020-08-21T08:40:19.906 -> 2020-08-21T08:40:19.000` | This ensures that floating point rounding errors do not occur during the retrieval of feature data, since this step requires time based joins | + +### Storage + +| Limitation | Motivation | +| :--- | :--- | +| Feast does not support offline storage in Feast 0.8 | As part of our re-architecture of Feast, we moved from GCP to cloud-agnostic deployments. Developing offline storage support that is available in all cloud environments is a pending action. | diff --git a/docs/reference/metrics-reference/README.md b/docs/reference/metrics-reference.md similarity index 92% rename from docs/reference/metrics-reference/README.md rename to docs/reference/metrics-reference.md index 4ccd6cb9ce0..34c97c7be60 100644 --- a/docs/reference/metrics-reference/README.md +++ b/docs/reference/metrics-reference.md @@ -1,12 +1,16 @@ # Metrics Reference +{% hint style="warning" %} +This page applies to Feast 0.7. The content may be out of date for Feast 0.8+ +{% endhint %} + Reference of the metrics that each Feast component exports: -* [Feast Core](./#feast-core) -* [Feast Serving](./#feast-serving) -* [Feast Ingestion Job](./#feast-ingestion-job) +* [Feast Core](metrics-reference.md#feast-core) +* [Feast Serving](metrics-reference.md#feast-serving) +* [Feast Ingestion Job](metrics-reference.md#feast-ingestion-job) -For how to configure Feast to export Metrics, see the [Metrics user guide.](../../advanced/metrics.md) +For how to configure Feast to export Metrics, see the [Metrics user guide.](../advanced/metrics.md) ## Feast Core @@ -44,8 +48,8 @@ Feast Serving exports the following metrics: | :--- | :--- | :--- | | `feast_serving_request_latency_seconds` | Feast Serving's latency in serving Requests in Seconds. | `method` | | `feast_serving_request_feature_count` | No. of requests retrieving a Feature from Feast Serving. | `project`, `feature_name` | -| `feast_serving_not_found_feature_count` | No. of requests retrieving a Feature has resulted in a [`NOT_FOUND` field status.](../../user-guide/feature-retrieval.md#online-field-statuses) | `project`, `feature_name` | -| `feast_serving_stale_feature_count` | No. of requests retrieving a Feature resulted in a [`OUTSIDE_MAX_AGE` field status.](../../user-guide/feature-retrieval.md#online-field-statuses) | `project`, `feature_name` | +| `feast_serving_not_found_feature_count` | No. of requests retrieving a Feature has resulted in a [`NOT_FOUND` field status.](../user-guide/getting-training-features.md#online-field-statuses) | `project`, `feature_name` | +| `feast_serving_stale_feature_count` | No. of requests retrieving a Feature resulted in a [`OUTSIDE_MAX_AGE` field status.](../user-guide/getting-training-features.md#online-field-statuses) | `project`, `feature_name` | | `feast_serving_grpc_request_count` | Total gRPC requests served. | `method` | **Metric Tags** @@ -61,11 +65,7 @@ Exported Feast Serving metrics may be filtered by the following tags/keys ## Feast Ingestion Job -Feast Ingestion computes both metrics an statistics on [data ingestion.](../../user-guide/data-ingestion.md) Make sure you familar with data ingestion concepts before proceeding. - -{% hint style="info" %} -For documentation on Feature value statistics computed by the Ingestion Job see [Statistics]() -{% endhint %} +Feast Ingestion computes both metrics an statistics on [data ingestion.](../user-guide/define-and-ingest-features.md) Make sure you familar with data ingestion concepts before proceeding. **Metrics Namespace** diff --git a/docs/roadmap.md b/docs/roadmap.md index 5ec26f85a1c..4f1a59c751c 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -1,32 +1,44 @@ # Roadmap -### Feast 0.8 +## Feast 0.9 + +[Discussion](https://github.com/feast-dev/feast/issues/1131) + +### New Functionality + +* Feast Job Service +* Delta offline store support. Optional for users +* Push based ingestion into offline store +* On-prem support \(Open source storage and launcher\) +* Azure support + +## Feast 0.8 [Discussion](https://github.com/feast-dev/feast/issues/1018) [Feast 0.8 RFC](https://docs.google.com/document/d/1snRxVb8ipWZjCiLlfkR4Oc28p7Fkv_UXjvxBFWjRBj4/edit#heading=h.yvkhw2cuvx5) -#### **New Functionality** +### **New Functionality** 1. Add support for AWS \(data sources and deployment\) 2. Add support for local deployment 3. Add support for Spark based ingestion 4. Add support for Spark based historical retrieval -#### **Technical debt, refactoring, or housekeeping** +### **Technical debt, refactoring, or housekeeping** 1. Move job management functionality to SDK 2. Remove Apache Beam based ingestion 3. Allow direct ingestion from batch sources that does not pass through stream 4. Remove Feast Historical Serving abstraction to allow direct access from Feast SDK to data sources for retrieval -### Feast 0.7 +## Feast 0.7 [Discussion](https://github.com/feast-dev/feast/issues/834) [GitHub Milestone](https://github.com/feast-dev/feast/milestone/4) -#### **New Functionality** +### **New Functionality** 1. Label based Ingestion Job selector for Job Controller [\#903](https://github.com/feast-dev/feast/pull/903) 2. Authentication Support for Java & Go SDKs [\#971](https://github.com/feast-dev/feast/pull/971) @@ -35,19 +47,19 @@ 5. Request Response Logging support via Fluentd [\#961](https://github.com/feast-dev/feast/pull/961) 6. Feast Core Rest Endpoints [\#878](https://github.com/feast-dev/feast/pull/878) -#### **Technical debt, refactoring, or housekeeping** +### **Technical debt, refactoring, or housekeeping** 1. Improved integration testing framework [\#886](https://github.com/feast-dev/feast/pull/886) 2. Rectify all flaky batch tests [\#953](https://github.com/feast-dev/feast/pull/953), [\#982](https://github.com/feast-dev/feast/pull/982) 3. Decouple job management from Feast Core [\#951](https://github.com/feast-dev/feast/pull/951) -### Feast 0.6 +## Feast 0.6 [Discussion](https://github.com/feast-dev/feast/issues/767) [GitHub Milestone](https://github.com/feast-dev/feast/milestone/3) -#### New functionality +### New functionality 1. Batch statistics and validation [\#612](https://github.com/feast-dev/feast/pull/612) 2. Authentication and authorization [\#554](https://github.com/feast-dev/feast/pull/554) @@ -55,29 +67,29 @@ 4. Improved searching and filtering of features and entities 5. Python support for labels [\#663](https://github.com/feast-dev/feast/issues/663) -#### Technical debt, refactoring, or housekeeping +### Technical debt, refactoring, or housekeeping 1. Improved job life cycle management [\#761](https://github.com/feast-dev/feast/issues/761) 2. Compute and write metrics for rows prior to store writes [\#763](https://github.com/feast-dev/feast/pull/763) -### Feast 0.5 +## Feast 0.5 -[Discussion](https://github.com/gojek/feast/issues/527) +[Discussion](https://github.com/feast-dev/feast/issues/527) -#### New functionality +### New functionality 1. Streaming statistics and validation \(M1 from [Feature Validation RFC](https://docs.google.com/document/d/1TPmd7r4mniL9Y-V_glZaWNo5LMXLshEAUpYsohojZ-8/edit)\) -2. Support for Redis Clusters \([\#478](https://github.com/gojek/feast/issues/478), [\#502](https://github.com/gojek/feast/issues/502)\) -3. Add feature and feature set labels, i.e. key/value registry metadata \([\#463](https://github.com/gojek/feast/issues/463)\) -4. Job management API \([\#302](https://github.com/gojek/feast/issues/302)\) - -#### Technical debt, refactoring, or housekeeping - -1. Clean up and document all configuration options \([\#525](https://github.com/gojek/feast/issues/525)\) -2. Externalize storage interfaces \([\#402](https://github.com/gojek/feast/issues/402)\) -3. Reduce memory usage in Redis \([\#515](https://github.com/gojek/feast/issues/515)\) -4. Support for handling out of order ingestion \([\#273](https://github.com/gojek/feast/issues/273)\) -5. Remove feature versions and enable automatic data migration \([\#386](https://github.com/gojek/feast/issues/386)\) \([\#462](https://github.com/gojek/feast/issues/462)\) -6. Tracking of batch ingestion by with dataset\_id/job\_id \([\#461](https://github.com/gojek/feast/issues/461)\) -7. Write Beam metrics after ingestion to store \(not prior\) \([\#489](https://github.com/gojek/feast/issues/489)\) +2. Support for Redis Clusters \([\#478](https://github.com/feast-dev/feast/issues/478), [\#502](https://github.com/feast-dev/feast/issues/502)\) +3. Add feature and feature set labels, i.e. key/value registry metadata \([\#463](https://github.com/feast-dev/feast/issues/463)\) +4. Job management API \([\#302](https://github.com/feast-dev/feast/issues/302)\) + +### Technical debt, refactoring, or housekeeping + +1. Clean up and document all configuration options \([\#525](https://github.com/feast-dev/feast/issues/525)\) +2. Externalize storage interfaces \([\#402](https://github.com/feast-dev/feast/issues/402)\) +3. Reduce memory usage in Redis \([\#515](https://github.com/feast-dev/feast/issues/515)\) +4. Support for handling out of order ingestion \([\#273](https://github.com/feast-dev/feast/issues/273)\) +5. Remove feature versions and enable automatic data migration \([\#386](https://github.com/feast-dev/feast/issues/386)\) \([\#462](https://github.com/feast-dev/feast/issues/462)\) +6. Tracking of batch ingestion by with dataset\_id/job\_id \([\#461](https://github.com/feast-dev/feast/issues/461)\) +7. Write Beam metrics after ingestion to store \(not prior\) \([\#489](https://github.com/feast-dev/feast/issues/489)\) diff --git a/docs/user-guide/data-ingestion.md b/docs/user-guide/data-ingestion.md deleted file mode 100644 index e29953d173e..00000000000 --- a/docs/user-guide/data-ingestion.md +++ /dev/null @@ -1,42 +0,0 @@ -# Getting data into Feast - -In order to retrieve features for both training and serving, Feast requires data being ingested into the offline and online stores. - -{% hint style="warning" %} -Offline storage support will not be available until v0.9. Only Online storage support exists currently. -{% endhint %} - -Users are expected to already have either a batch or stream source with data materialized in it, ready to be ingested into Feast. Upon providing their external data sources in feature table specifications and registering them, users can now ingest data into Feast using Spark jobs. - -The following depicts an example ingestion flow from the specified data source to online store. - -### Batch Source to Online Store - -```python -from feast import Client -from datetime import datetime, timedelta - -client = Client(core_url="localhost:6565") -driver_ft = client.get_feature_table("driver_trips") - -# Initialize date ranges -today = datetime.now() -yesterday = today - timedelta(1) - -client.start_offline_to_online_ingestion( - driver_ft, yesterday, today -) -``` - -### Stream Source to Online Store - -```python -from feast import Client -from datetime import datetime, timedelta - -client = Client(core_url="localhost:6565") -driver_ft = client.get_feature_table("driver_trips") - -client.start_stream_to_online_ingestion(driver_ft) -``` - diff --git a/docs/user-guide/define-and-ingest-features.md b/docs/user-guide/define-and-ingest-features.md new file mode 100644 index 00000000000..d55fcb1d857 --- /dev/null +++ b/docs/user-guide/define-and-ingest-features.md @@ -0,0 +1,56 @@ +# Define and ingest features + +In order to retrieve features for both training and serving, Feast requires data being ingested into its offline and online stores. + +{% hint style="warning" %} +Feast 0.8 does not have an offline store. Only Online storage support exists currently. Feast 0.9 will have offline storage support. In Feast 0.8, historical data is retrieved directly from batch sources. +{% endhint %} + +Users are expected to already have either a batch or stream source with data stored in it, ready to be ingested into Feast. Once a feature table \(with the corresponding sources\) has been registered with Feast, it is possible to load data from this source into stores. + +The following depicts an example ingestion flow from a data source to the online store. + +### Batch Source to Online Store + +```python +from feast import Client +from datetime import datetime, timedelta + +client = Client(core_url="localhost:6565") +driver_ft = client.get_feature_table("driver_trips") + +# Initialize date ranges +today = datetime.now() +yesterday = today - timedelta(1) + +# Launches a short-lived job that ingests data over the provided date range. +client.start_offline_to_online_ingestion( + driver_ft, yesterday, today +) +``` + +### Stream Source to Online Store + +```python +from feast import Client +from datetime import datetime, timedelta + +client = Client(core_url="localhost:6565") +driver_ft = client.get_feature_table("driver_trips") + +# Launches a long running streaming ingestion job +client.start_stream_to_online_ingestion(driver_ft) +``` + +### Batch Source to Offline Store + +{% hint style="danger" %} +Not supported in Feast 0.8 +{% endhint %} + +### Stream Source to Offline Store + +{% hint style="danger" %} +Not supported in Feast 0.8 +{% endhint %} + diff --git a/docs/user-guide/feature-retrieval.md b/docs/user-guide/feature-retrieval.md deleted file mode 100644 index 3a6d072a410..00000000000 --- a/docs/user-guide/feature-retrieval.md +++ /dev/null @@ -1,61 +0,0 @@ -# Getting training features - -Feast provides a historical retrieval interface for exporting feature data to train machine learning models. Essentially, users are able to retrieve features from any feature tables and join them together in a single response dataset. The only requirement is that the user provides the correct entities and timestamps in order to look up the features. - -Historical feature retrieval can be done through the [Feast SDK](https://api.docs.feast.dev/python). - -{% hint style="warning" %} -Historical Retrieval currently pulls from batch sources for Feast v0.8, and offline storage support will not be available until v0.9. -{% endhint %} - -{% hint style="info" %} -By default, Feast infers that the features specified belong to the `default` project. To retrieve from another project, specify the `project` parameter when retrieving features. -{% endhint %} - -## **Point-in-time-correct Join** - -Feast does a point in time correct query from a single feature table. For each entity key and event timestamp combination that is provided by `entity_source`, Feast determines the values of all the features in the `feature_refs` list at that respective point in time and then joins features values to that specific entity value and event timestamp, and repeats this process for all timestamps. - -This is called a point in time correct join. - -Below is an example of how a point-in-time-correct join works. We have two DataFrames. The first is the `entity dataframe` that contains timestamps, entities, and labels. The user would like to have driver features joined onto this `entity dataframe` from the `driver dataframe` to produce a `joined dataframe` upon materializing the view that contains both labels and features. They would then like to train their model on this output - -![](../.gitbook/assets/point_in_time_join%20%281%29.png) - -Typically the `input 1` DataFrame would be provided by the user through `entity_source`, and the `input 2` DataFrame would already be ingested into Feast. To join these two, the user would call Feast as follows: - -```python -# Feature references with target feature -feature_refs = [ - "driver_trips:average_daily_rides", - "driver_trips:maximum_daily_rides", - "driver_trips:rating", - "trip_completed", -] - -# Define entity source -entity_source = FileSource( - "event_timestamp", - ParquetFormat(), - "gs://some-bucket/customer" -) - -# Retrieve historical dataset from Feast. -historical_feature_retrieval_job = client.get_historical_features( - feature_refs=feature_refs, - entity_rows=entity_source -) - -# Retrieve the output uri to materialize the dataset object into a Pandas DataFrame etc. -# Eg. gs://some-bucket/output/, s3://*, file://* -output_file_uri = historical_feature_retrieval_job.get_output_file_uri() -``` - -Feast is able to intelligently join feature data with different timestamps to a single basis table in a point-in-time-correct way. This allows users to join daily batch data with high-frequency event data transparently. They simply need to provide the feature references. - -{% hint style="info" %} -Feast can retrieve features from any amount of feature tables, as long as they occur on the same entities. -{% endhint %} - -Point-in-time-correct joins also prevents the occurrence of feature leakage by trying to accurate the state of the world at a single point in time, instead of just joining features based on the nearest timestamps. - diff --git a/docs/user-guide/getting-online-features.md b/docs/user-guide/getting-online-features.md index 722bb2ade4c..c16dc08a013 100644 --- a/docs/user-guide/getting-online-features.md +++ b/docs/user-guide/getting-online-features.md @@ -1,10 +1,10 @@ # Getting online features -Feast provides an online retrieval interface for serving. Data ingested into the online store comes from both batch and stream sources. +Feast provides an API through which online feature values can be retrieved. This allows teams to look up feature values at low latency in production during model serving, in order to make online predictions. -When data is ingested from a batch source, users can retrieve the same features used in training models from the low latency online store to be used in production. When data is ingested from a stream source, features that are retrieved by users are of the latest values, which are not yet used in training models. - -Online feature retrieval works in much the same way as batch retrieval, with one important distinction: Online stores only maintain the current state of features, i.e latest feature values. No historical data is served. +{% hint style="info" %} +Online stores only maintain the current state of features, i.e latest feature values. No historical data is stored or served. +{% endhint %} ```python from feast import Client @@ -36,35 +36,19 @@ response_dict = response.to_dict() print(response_dict) ``` -{% hint style="info" %} -When no project is specified when retrieving features with get\_online\_feature\(\), Feast infers that the features specified belong to the default project. To retrieve from another project, specify the project parameter when retrieving features. -{% endhint %} +The online store must be populated through [ingestion jobs](define-and-ingest-features.md#batch-source-to-online-store) prior to being used for online serving. -Feast Serving provides a [gRPC API](https://api.docs.feast.dev/grpc/feast.serving.pb.html) that is backed by [Redis](https://redis.io/). We also provide support for [Python](https://api.docs.feast.dev/python/), [Go](https://godoc.org/github.com/gojek/feast/sdk/go), and [Java](https://javadoc.io/doc/dev.feast) clients. +Feast Serving provides a [gRPC API](https://api.docs.feast.dev/grpc/feast.serving.pb.html) that is backed by [Redis](https://redis.io/). We have native clients in [Python](https://api.docs.feast.dev/python/), [Go](https://godoc.org/github.com/gojek/feast/sdk/go), and [Java](https://javadoc.io/doc/dev.feast). ### Online Field Statuses -Online Serving also returns Online Field Statuses when retrieving features. These status values gives useful insight into situations where Online Serving returns unset values. It also allows better of handling of the different possible cases represented by each status:for feature in features: - -```python -response_dict = response.to_dict() - -for feature_ref in feature_refs: - # field status can be obtained from the response's field values - status = response_dict["field_values"]["statuses"][feature_ref] - - if status == GetOnlineFeaturesResponse.FieldStatus.NOT_FOUND: - # handle case where feature value has not been ingested - elif status == GetOnlineFeaturesResponse.FieldStatus.PRESENT: - # feature value is present and can be used - value = response_dict["field_values"]["statuses"][feature_ref] -``` +Feast also returns status codes when retrieving features from the Feast Serving API. These status code give useful insight into the quality of data being served. | Status | Meaning | | :--- | :--- | -| NOT\_FOUND | Unset values returned as the feature value was not found in the online store. This might mean that no feature value was ingested for this feature. | -| NULL\_VALUE | Unset values returned as the ingested feature value was also unset. | -| OUTSIDE\_MAX\_AGE | Unset values returned as the age of the feature value \(time since the value was ingested\) has exceeded the Feature Set's max age, which the feature was defined in. | -| PRESENT | Set values are returned for the requested feature. | -| UNKNOWN | Status signifies the field status is unset for the requested feature. Might mean that the Feast version does not support Field Statuses. | +| NOT\_FOUND | The feature value was not found in the online store. This might mean that no feature value was ingested for this feature. | +| NULL\_VALUE | A entity key was successfully found but no feature values had been set. This status code should not occur during normal operation. | +| OUTSIDE\_MAX\_AGE | The age of the feature row in the online store \(in terms of its event timestamp\) has exceeded the maximum age defined within the feature table. | +| PRESENT | The feature values have been found and are within the maximum age. | +| UNKNOWN | Indicates a system failure. | diff --git a/docs/user-guide/getting-training-features.md b/docs/user-guide/getting-training-features.md new file mode 100644 index 00000000000..9f33fd79fc7 --- /dev/null +++ b/docs/user-guide/getting-training-features.md @@ -0,0 +1,72 @@ +# Getting training features + +Feast provides a historical retrieval interface for exporting feature data in order to train machine learning models. Essentially, users are able to enrich their data with features from any feature tables. + +### Retrieving historical features + +Below is an example of the process required to produce a training dataset: + +```python +# Feature references with target feature +feature_refs = [ + "driver_trips:average_daily_rides", + "driver_trips:maximum_daily_rides", + "driver_trips:rating", + "driver_trips:rating:trip_completed", +] + +# Define entity source +entity_source = FileSource( + "event_timestamp", + ParquetFormat(), + "gs://some-bucket/customer" +) + +# Retrieve historical dataset from Feast. +historical_feature_retrieval_job = client.get_historical_features( + feature_refs=feature_refs, + entity_rows=entity_source +) + +output_file_uri = historical_feature_retrieval_job.get_output_file_uri() +``` + +#### 1. Define feature references + +[Feature references](../concepts/glossary.md#feature-references) define the specific features that will be retrieved from Feast. These features can come from multiple feature tables. The only requirement is that the feature tables that make up the feature references have the same entity \(or composite entity\). + +**2. Define an entity dataframe** + +Feast needs to join feature values onto specific entities at specific points in time. Thus, it is necessary to provide an [entity dataframe](../concepts/glossary.md#entity-dataframe) as part of the `get_historical_features` method. In the example above we are defining an entity source. This source is an external file that provides Feast with the entity dataframe. + +**3. Launch historical retrieval job** + +Once the feature references and an entity source are defined, it is possible to call `get_historical_features()`. This method launches a job that extracts features from the sources defined in the provided feature tables, joins them onto the provided entity source, and returns a reference to the training dataset that is produced. + +Please see the [Feast SDK](https://api.docs.feast.dev/python) for more details. + +### Point-in-time Joins + +Feast always joins features onto entity data in a point-in-time correct way. The process can be described through an example. + +In the example below there are two tables \(or dataframes\): + +* The dataframe on the left is the [entity dataframe](../concepts/glossary.md#entity-dataframe) that contains timestamps, entities, and the target variable \(trip\_completed\). This dataframe is provided to Feast through an entity source. +* The dataframe on the right contains driver features. This dataframe is represented in Feast through a feature table and its accompanying data source\(s\). + +The user would like to have the driver features joined onto the entity dataframe to produce a training dataset that contains both the target \(trip\_completed\) and features \(average\_daily\_rides, maximum\_daily\_rides, rating\). This dataset will then be used to train their model. + +![](../.gitbook/assets/point_in_time_join%20%281%29%20%282%29%20%281%29.png) + +Feast is able to intelligently join feature data with different timestamps to a single entity dataframe. It does this through a point-in-time join as follows: + +1. Feast loads the entity dataframe and all feature tables \(driver dataframe\) into the same location. This can either be a database or in memory. +2. For each [entity row](../concepts/glossary.md#entity-rows) in the [entity dataframe](getting-online-features.md), Feast tries to find feature values in each feature table to join to it. Feast extracts the timestamp and entity key of each row in the entity dataframe and scans backward through the feature table until it finds a matching entity key. +3. If the event timestamp of the matching entity key within the driver feature table is within the maximum age configured for the feature table, then the features at that entity key are joined onto the entity dataframe. If the event timestamp is outside of the maximum age, then only null values are returned. +4. If multiple entity keys are found with the same event timestamp, then they are deduplicated by the created timestamp, with newer values taking precedence. +5. Feast repeats this joining process for all feature tables and returns the resulting dataset. + +{% hint style="info" %} +Point-in-time correct joins attempts to prevent the occurrence of feature leakage by trying to recreate the state of the world at a single point in time, instead of joining features based on exact timestamps only. +{% endhint %} + diff --git a/docs/user-guide/overview.md b/docs/user-guide/overview.md new file mode 100644 index 00000000000..2d6eb9981bb --- /dev/null +++ b/docs/user-guide/overview.md @@ -0,0 +1,32 @@ +# Overview + +### Using Feast + +Feast development happens through three key workflows: + +1. [Define and load feature data into Feast](define-and-ingest-features.md) +2. [Retrieve historical features for training models](getting-training-features.md) +3. [Retrieve online features for serving models](getting-online-features.md) + +### Defining feature tables and ingesting data into Feast + +Feature creators model the data within their organization into Feast through the definition of [feature tables](../concepts/feature-tables.md) that contain [data sources](../concepts/sources.md). Feature tables are both a schema and a means of identifying data sources for features, and allow Feast to know how to interpret your data, and where to find it. + +After registering a feature table with Feast, users can trigger an ingestion from their data source into Feast. This loads feature values from an upstream data source into Feast stores through ingestion jobs. + +Visit [feature tables](../concepts/feature-tables.md#overview) to learn more about them. + +{% page-ref page="define-and-ingest-features.md" %} + +### Retrieving historical features for training + +In order to generate a training dataset it is necessary to provide both an [entity dataframe ](../concepts/glossary.md#entity-dataframe)and feature references through the[ Feast SDK](https://api.docs.feast.dev/python/) to retrieve historical features. For historical serving, Feast requires that you provide the entities and timestamps for the corresponding feature data. Feast produces a point-in-time correct dataset using the requested features. These features can be requested from an unlimited number of feature sets. + +{% page-ref page="getting-training-features.md" %} + +### Retrieving online features for online serving + +Online retrieval uses feature references through the [Feast Online Serving API](https://api.docs.feast.dev/grpc/feast.serving.pb.html) to retrieve online features. Online serving allows for very low latency requests to feature data at very high throughput. + +{% page-ref page="getting-online-features.md" %} + diff --git a/examples/minimal/minimal_ride_hailing.ipynb b/examples/minimal/minimal_ride_hailing.ipynb index fd6b1d963ef..bc170fa1f8d 100644 --- a/examples/minimal/minimal_ride_hailing.ipynb +++ b/examples/minimal/minimal_ride_hailing.ipynb @@ -328,9 +328,9 @@ "metadata": {}, "outputs": [], "source": [ - "client.apply_entity(driver_id)\n", - "client.apply_feature_table(driver_statistics)\n", - "client.apply_feature_table(driver_trips)" + "client.apply(driver_id)\n", + "client.apply(driver_statistics)\n", + "client.apply(driver_trips)" ] }, { @@ -571,6 +571,15 @@ " files = [\"s3://\" + path for path in fs.glob(uri + '/part-*')]\n", " ds = ParquetDataset(files, filesystem=fs)\n", " return ds.read().to_pandas()\n", + " elif parsed_uri.scheme == 'wasbs':\n", + " import adlfs\n", + " fs = adlfs.AzureBlobFileSystem(\n", + " account_name=os.getenv('FEAST_AZURE_BLOB_ACCOUNT_NAME'), account_key=os.getenv('FEAST_AZURE_BLOB_ACCOUNT_ACCESS_KEY')\n", + " )\n", + " uripath = parsed_uri.username + parsed_uri.path\n", + " files = fs.glob(uripath + '/part-*')\n", + " ds = ParquetDataset(files, filesystem=fs)\n", + " return ds.read().to_pandas()\n", " else:\n", " raise ValueError(f\"Unsupported URL scheme {uri}\")" ] @@ -1228,7 +1237,7 @@ " topic=\"driver_trips\",\n", " message_format=AvroFormat(avro_schema_json)\n", ")\n", - "client.apply_feature_table(driver_trips)" + "client.apply(driver_trips)" ] }, { @@ -1275,6 +1284,12 @@ "metadata": {}, "outputs": [], "source": [ + "# Note: depending on the Kafka configuration you may need to create the Kafka topic first, like below:\n", + "#from confluent_kafka.admin import AdminClient, NewTopic\n", + "#admin = AdminClient({'bootstrap.servers': KAFKA_BROKER})\n", + "#new_topic = NewTopic('driver_trips', num_partitions=1, replication_factor=3)\n", + "#admin.create_topics(new_topic)\n", + "\n", "for record in trips_df.drop(columns=['created']).to_dict('record'):\n", " record[\"datetime\"] = (\n", " record[\"datetime\"].to_pydatetime().replace(tzinfo=pytz.utc)\n", @@ -1490,4 +1505,4 @@ }, "nbformat": 4, "nbformat_minor": 4 -} \ No newline at end of file +} diff --git a/go.mod b/go.mod index de70423b33e..6e71e5b637e 100644 --- a/go.mod +++ b/go.mod @@ -23,9 +23,8 @@ require ( github.com/woop/protoc-gen-doc v1.3.0 // indirect go.opencensus.io v0.22.3 // indirect golang.org/x/lint v0.0.0-20200302205851-738671d3881b // indirect - golang.org/x/net v0.0.0-20200822124328-c89045814202 - golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9 // indirect - golang.org/x/tools v0.0.0-20201017001424-6003fad69a88 // indirect + golang.org/x/net v0.0.0-20201021035429-f5854403a974 + golang.org/x/tools v0.0.0-20201124005743-911501bfb504 // indirect google.golang.org/grpc v1.29.1 google.golang.org/protobuf v1.25.0 // indirect gopkg.in/russross/blackfriday.v2 v2.0.0 // indirect diff --git a/go.sum b/go.sum index b9ec81936fc..bd68e3f8638 100644 --- a/go.sum +++ b/go.sum @@ -403,6 +403,7 @@ golang.org/x/net v0.0.0-20200320220750-118fecf932d8/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20200513185701-a91f0712d120 h1:EZ3cVSzKOlJxAd8e8YAJ7no8nNypTxexh/YE/xW3ZEY= golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190402181905-9f3314589c9a/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -414,6 +415,7 @@ golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20170830134202-bb24a47a89ea/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -436,6 +438,7 @@ golang.org/x/sys v0.0.0-20200321134203-328b4cd54aae/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9 h1:YTzHMGlqJu67/uEo1lBv0n3wBXhXNeUbB1XfN2vmTm0= golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.0.0-20160726164857-2910a502d2bf/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -443,6 +446,7 @@ golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3 golang.org/x/text v0.3.1-0.20181227161524-e6919f6577db/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/time v0.0.0-20161028155119-f51c12702a4d/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -500,6 +504,8 @@ golang.org/x/tools v0.0.0-20201015182029-a5d9e455e9c4 h1:rQWkJiVIyJ3PgiSHL+RXc8x golang.org/x/tools v0.0.0-20201015182029-a5d9e455e9c4/go.mod h1:z6u4i615ZeAfBE4XtMziQW1fSVJXACjjbWkB/mvPzlU= golang.org/x/tools v0.0.0-20201017001424-6003fad69a88 h1:ZB1XYzdDo7c/O48jzjMkvIjnC120Z9/CwgDWhePjQdQ= golang.org/x/tools v0.0.0-20201017001424-6003fad69a88/go.mod h1:z6u4i615ZeAfBE4XtMziQW1fSVJXACjjbWkB/mvPzlU= +golang.org/x/tools v0.0.0-20201124005743-911501bfb504 h1:jOKV2ysikH1GANB7t2LotmhyvkkPvl7HQoEXkV6slJA= +golang.org/x/tools v0.0.0-20201124005743-911501bfb504/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= diff --git a/infra/charts/feast/Chart.yaml b/infra/charts/feast/Chart.yaml index 323f4f3389d..354847f1a81 100644 --- a/infra/charts/feast/Chart.yaml +++ b/infra/charts/feast/Chart.yaml @@ -1,4 +1,4 @@ apiVersion: v1 description: Feature store for machine learning. name: feast -version: 0.8.0 +version: 0.9.0 diff --git a/infra/charts/feast/README.md b/infra/charts/feast/README.md index 5260efc4823..2b64b49a97d 100644 --- a/infra/charts/feast/README.md +++ b/infra/charts/feast/README.md @@ -1,7 +1,7 @@ feast ===== -Feature store for machine learning. Current chart version is `0.8.0` +Feature store for machine learning. Current chart version is `0.9.0` ## Installation @@ -11,9 +11,9 @@ https://docs.feast.dev/v/master/getting-started/deploying-feast/kubernetes | Repository | Name | Version | |------------|------|---------| -| | feast-core | 0.8.0 | -| | feast-jupyter | 0.8.0 | -| | feast-serving | 0.8.0 | +| | feast-core | 0.9.0 | +| | feast-jupyter | 0.9.0 | +| | feast-serving | 0.9.0 | | | prometheus-statsd-exporter | 0.1.2 | | https://charts.bitnami.com/bitnami/ | kafka | 11.8.8 | | https://kubernetes-charts.storage.googleapis.com/ | grafana | 5.0.5 | diff --git a/infra/charts/feast/charts/feast-core/Chart.yaml b/infra/charts/feast/charts/feast-core/Chart.yaml index 0857cfe4a5d..e8ce2818f97 100644 --- a/infra/charts/feast/charts/feast-core/Chart.yaml +++ b/infra/charts/feast/charts/feast-core/Chart.yaml @@ -1,4 +1,4 @@ apiVersion: v1 description: Feast Core registers feature specifications. name: feast-core -version: 0.8.0 +version: 0.9.0 diff --git a/infra/charts/feast/charts/feast-core/README.md b/infra/charts/feast/charts/feast-core/README.md index 45607047a3c..01d7e76678a 100644 --- a/infra/charts/feast/charts/feast-core/README.md +++ b/infra/charts/feast/charts/feast-core/README.md @@ -2,7 +2,7 @@ feast-core ========== Feast Core registers feature specifications. -Current chart version is `0.8.0` +Current chart version is `0.9.0` @@ -18,7 +18,7 @@ Current chart version is `0.8.0` | "application.yaml".enabled | bool | `true` | Flag to include the default [configuration](https://github.com/feast-dev/feast/blob/master/core/src/main/resources/application.yml). Please set `application-override.yaml` to override this configuration. | | envOverrides | object | `{}` | Extra environment variables to set | | image.pullPolicy | string | `"IfNotPresent"` | Image pull policy | -| image.repository | string | `"gcr.io/kf-feast/feast-core"` | Docker image repository | +| image.repository | string | `"feastdev/feast-core"` | Docker image repository | | image.tag | string | `"develop"` | Image tag | | ingress.grpc.annotations | object | `{}` | Extra annotations for the ingress | | ingress.grpc.auth.enabled | bool | `false` | Flag to enable auth | diff --git a/infra/charts/feast/charts/feast-core/values.yaml b/infra/charts/feast/charts/feast-core/values.yaml index 423373e61cd..6f5be009d9d 100644 --- a/infra/charts/feast/charts/feast-core/values.yaml +++ b/infra/charts/feast/charts/feast-core/values.yaml @@ -3,9 +3,9 @@ replicaCount: 1 image: # image.repository -- Docker image repository - repository: gcr.io/kf-feast/feast-core + repository: feastdev/feast-core # image.tag -- Image tag - tag: develop + tag: 0.9.0 # image.pullPolicy -- Image pull policy pullPolicy: IfNotPresent diff --git a/infra/charts/feast/charts/feast-jobservice/Chart.yaml b/infra/charts/feast/charts/feast-jobservice/Chart.yaml index d4183b197ab..5f8943fbada 100644 --- a/infra/charts/feast/charts/feast-jobservice/Chart.yaml +++ b/infra/charts/feast/charts/feast-jobservice/Chart.yaml @@ -1,4 +1,4 @@ apiVersion: v1 description: Feast Job Service manage ingestion jobs. name: feast-jobservice -version: 0.8.0 +version: 0.9.0 diff --git a/infra/charts/feast/charts/feast-jobservice/README.md b/infra/charts/feast/charts/feast-jobservice/README.md index 859889978ac..2fe24dc6a70 100644 --- a/infra/charts/feast/charts/feast-jobservice/README.md +++ b/infra/charts/feast/charts/feast-jobservice/README.md @@ -2,7 +2,7 @@ feast-jobservice ================ Feast Job Service manage ingestion jobs. -Current chart version is `0.8.0` +Current chart version is `0.9.0` @@ -18,7 +18,7 @@ Current chart version is `0.8.0` | gcpServiceAccount.existingSecret.key | string | `"credentials.json"` | Key in the secret data (file name of the service account) | | gcpServiceAccount.existingSecret.name | string | `"feast-gcp-service-account"` | Name of the existing secret containing the service account | | image.pullPolicy | string | `"IfNotPresent"` | Image pull policy | -| image.repository | string | `"gcr.io/kf-feast/feast-jobservice"` | Docker image repository | +| image.repository | string | `"feastdev/feast-jobservice"` | Docker image repository | | image.tag | string | `"develop"` | Image tag | | ingress.grpc.annotations | object | `{}` | Extra annotations for the ingress | | ingress.grpc.auth.enabled | bool | `false` | Flag to enable auth | diff --git a/infra/charts/feast/charts/feast-jobservice/templates/configmap.yaml b/infra/charts/feast/charts/feast-jobservice/templates/configmap.yaml new file mode 100644 index 00000000000..356358fd3d7 --- /dev/null +++ b/infra/charts/feast/charts/feast-jobservice/templates/configmap.yaml @@ -0,0 +1,16 @@ +{{- if .Values.sparkOperator.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ template "feast-jobservice.fullname" . }}-spark-template + namespace: {{ .Release.Namespace }} + labels: + app: {{ template "feast-jobservice.name" . }} + component: jobservice + chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +data: + jobTemplate.yaml: | +{{- toYaml .Values.sparkOperator.jobTemplate | nindent 4 }} +{{- end }} \ No newline at end of file diff --git a/infra/charts/feast/charts/feast-jobservice/templates/deployment.yaml b/infra/charts/feast/charts/feast-jobservice/templates/deployment.yaml index 00667ccbb0f..506d8a5cfd8 100644 --- a/infra/charts/feast/charts/feast-jobservice/templates/deployment.yaml +++ b/infra/charts/feast/charts/feast-jobservice/templates/deployment.yaml @@ -37,13 +37,18 @@ spec: {{- toYaml . | nindent 8 }} {{- end }} - {{- if .Values.secrets }} + {{- if or .Values.secrets .Values.sparkOperator.enabled }} volumes: + {{- end }} {{- range $secret := .Values.secrets }} - name: {{ $secret }} secret: secretName: {{ $secret }} {{- end }} + {{- if .Values.sparkOperator.enabled }} + - name: {{ template "feast-jobservice.fullname" . }}-spark-template + configMap: + name: {{ template "feast-jobservice.fullname" . }}-spark-template {{- end }} containers: @@ -51,16 +56,24 @@ spec: image: {{ .Values.image.repository }}:{{ .Values.image.tag }} imagePullPolicy: {{ .Values.image.pullPolicy }} - {{- if .Values.secrets }} + {{- if or .Values.secrets .Values.sparkOperator.enabled }} volumeMounts: + {{- end }} {{- range $secret := .Values.secrets }} - name: {{ $secret }} mountPath: "/etc/secrets/{{ $secret }}" readOnly: true {{- end }} + {{- if .Values.sparkOperator.enabled }} + - name: {{ template "feast-jobservice.fullname" . }}-spark-template + mountPath: "/etc/configs" {{- end }} env: + {{- if .Values.sparkOperator.enabled }} + - name: FEAST_SPARK_K8S_JOB_TEMPLATE_PATH + value: /etc/configs/jobTemplate.yaml + {{- end }} {{- range $key, $value := .Values.envOverrides }} - name: {{ printf "%s" $key | replace "." "_" | upper | quote }} {{- if eq (kindOf $value) "map" }} diff --git a/infra/charts/feast/charts/feast-jobservice/values.yaml b/infra/charts/feast/charts/feast-jobservice/values.yaml index 03b8296a265..d2ebdc2795d 100644 --- a/infra/charts/feast/charts/feast-jobservice/values.yaml +++ b/infra/charts/feast/charts/feast-jobservice/values.yaml @@ -3,9 +3,9 @@ replicaCount: 1 image: # image.repository -- Docker image repository - repository: gcr.io/kf-feast/feast-jobservice + repository: feastdev/feast-jobservice # image.tag -- Image tag - tag: develop + tag: 0.9.0 # image.pullPolicy -- Image pull policy pullPolicy: IfNotPresent @@ -21,6 +21,12 @@ gcpServiceAccount: # gcpProjectId -- Project ID to use when using Google Cloud services such as BigQuery, Cloud Storage and Dataflow gcpProjectId: "" +sparkOperator: + # sparkOperator.enabled -- Flag to create and mount custom job template on the jobservice deployment as configmap + enabled: false + # sparkOperator.jobTemplate -- Content of the job template, in yaml format + jobTemplate: {} + prometheus: # prometheus.enabled -- Flag to enable scraping of metrics enabled: true @@ -124,3 +130,6 @@ envOverrides: {} # podLabels -- Labels to be added to Feast Job Service pods podLabels: {} + +# secrets -- Arbitrary secrets to mount on the job service pod, on /etc/secrets/ +secrets: [] diff --git a/infra/charts/feast/charts/feast-jupyter/Chart.yaml b/infra/charts/feast/charts/feast-jupyter/Chart.yaml index 3f8131fa427..3df73756ae8 100644 --- a/infra/charts/feast/charts/feast-jupyter/Chart.yaml +++ b/infra/charts/feast/charts/feast-jupyter/Chart.yaml @@ -1,4 +1,4 @@ apiVersion: v1 description: Feast Jupyter provides a Jupyter server with pre-installed Feast SDK name: feast-jupyter -version: 0.8.0 +version: 0.9.0 diff --git a/infra/charts/feast/charts/feast-jupyter/README.md b/infra/charts/feast/charts/feast-jupyter/README.md index 5bb26b4357a..548bf17c39f 100644 --- a/infra/charts/feast/charts/feast-jupyter/README.md +++ b/infra/charts/feast/charts/feast-jupyter/README.md @@ -2,7 +2,7 @@ feast-jupyter ============= Feast Jupyter provides a Jupyter server with pre-installed Feast SDK -Current chart version is `0.8.0` +Current chart version is `0.9.0` @@ -17,6 +17,6 @@ Current chart version is `0.8.0` | gcpServiceAccount.existingSecret.key | string | `"credentials.json"` | Key in the secret data (file name of the service account) | | gcpServiceAccount.existingSecret.name | string | `"feast-gcp-service-account"` | Name of the existing secret containing the service account | | image.pullPolicy | string | `"Always"` | Image pull policy | -| image.repository | string | `"gcr.io/kf-feast/feast-jupyter"` | Docker image repository | +| image.repository | string | `"feastdev/feast-jupyter"` | Docker image repository | | image.tag | string | `"develop"` | Image tag | | replicaCount | int | `1` | Number of pods that will be created | diff --git a/infra/charts/feast/charts/feast-jupyter/values.yaml b/infra/charts/feast/charts/feast-jupyter/values.yaml index 05078466a5e..b582edacfa9 100644 --- a/infra/charts/feast/charts/feast-jupyter/values.yaml +++ b/infra/charts/feast/charts/feast-jupyter/values.yaml @@ -3,9 +3,9 @@ replicaCount: 1 image: # image.repository -- Docker image repository - repository: gcr.io/kf-feast/feast-jupyter + repository: feastdev/feast-jupyter # image.tag -- Image tag - tag: develop + tag: 0.9.0 # image.pullPolicy -- Image pull policy pullPolicy: Always diff --git a/infra/charts/feast/charts/feast-serving/Chart.yaml b/infra/charts/feast/charts/feast-serving/Chart.yaml index fb007dd04dc..d753d4b3609 100644 --- a/infra/charts/feast/charts/feast-serving/Chart.yaml +++ b/infra/charts/feast/charts/feast-serving/Chart.yaml @@ -1,4 +1,4 @@ apiVersion: v1 description: Feast Serving serves low-latency latest features and historical batch features. name: feast-serving -version: 0.8.0 +version: 0.9.0 diff --git a/infra/charts/feast/charts/feast-serving/README.md b/infra/charts/feast/charts/feast-serving/README.md index acfdc0040f9..932b778c0ae 100644 --- a/infra/charts/feast/charts/feast-serving/README.md +++ b/infra/charts/feast/charts/feast-serving/README.md @@ -2,7 +2,7 @@ feast-serving ============= Feast Serving serves low-latency latest features and historical batch features. -Current chart version is `0.8.0` +Current chart version is `0.9.0` @@ -22,7 +22,7 @@ Current chart version is `0.8.0` | gcpServiceAccount.existingSecret.key | string | `"credentials.json"` | Key in the secret data (file name of the service account) | | gcpServiceAccount.existingSecret.name | string | `"feast-gcp-service-account"` | Name of the existing secret containing the service account | | image.pullPolicy | string | `"IfNotPresent"` | Image pull policy | -| image.repository | string | `"gcr.io/kf-feast/feast-serving"` | Docker image repository | +| image.repository | string | `"feastdev/feast-serving"` | Docker image repository | | image.tag | string | `"develop"` | Image tag | | ingress.grpc.annotations | object | `{}` | Extra annotations for the ingress | | ingress.grpc.auth.enabled | bool | `false` | Flag to enable auth | diff --git a/infra/charts/feast/charts/feast-serving/templates/deployment.yaml b/infra/charts/feast/charts/feast-serving/templates/deployment.yaml index 3df799df1d7..1d6e3aadc4c 100644 --- a/infra/charts/feast/charts/feast-serving/templates/deployment.yaml +++ b/infra/charts/feast/charts/feast-serving/templates/deployment.yaml @@ -117,7 +117,11 @@ spec: {{- if .Values.livenessProbe.enabled }} livenessProbe: exec: - command: ["grpc-health-probe", "-addr=:{{ .Values.service.grpc.targetPort }}"] + command: + - "grpc-health-probe" + - "-addr=:{{ .Values.service.grpc.targetPort }}" + - "-connect-timeout={{ .Values.livenessProbe.timeoutSeconds }}s" + - "-rpc-timeout={{ .Values.livenessProbe.timeoutSeconds }}s" initialDelaySeconds: {{ .Values.livenessProbe.initialDelaySeconds }} periodSeconds: {{ .Values.livenessProbe.periodSeconds }} successThreshold: {{ .Values.livenessProbe.successThreshold }} @@ -128,7 +132,11 @@ spec: {{- if .Values.readinessProbe.enabled }} readinessProbe: exec: - command: ["grpc-health-probe", "-addr=:{{ .Values.service.grpc.targetPort }}"] + command: + - "grpc-health-probe" + - "-addr=:{{ .Values.service.grpc.targetPort }}" + - "-connect-timeout={{ .Values.readinessProbe.timeoutSeconds }}s" + - "-rpc-timeout={{ .Values.readinessProbe.timeoutSeconds }}s" initialDelaySeconds: {{ .Values.readinessProbe.initialDelaySeconds }} periodSeconds: {{ .Values.readinessProbe.periodSeconds }} successThreshold: {{ .Values.readinessProbe.successThreshold }} diff --git a/infra/charts/feast/charts/feast-serving/values.yaml b/infra/charts/feast/charts/feast-serving/values.yaml index 06dbb85cb97..a0eb8ef2086 100644 --- a/infra/charts/feast/charts/feast-serving/values.yaml +++ b/infra/charts/feast/charts/feast-serving/values.yaml @@ -3,9 +3,9 @@ replicaCount: 1 image: # image.repository -- Docker image repository - repository: gcr.io/kf-feast/feast-serving + repository: feastdev/feast-serving # image.tag -- Image tag - tag: develop + tag: 0.9.0 # image.pullPolicy -- Image pull policy pullPolicy: IfNotPresent diff --git a/infra/charts/feast/charts/prometheus-statsd-exporter/README.md b/infra/charts/feast/charts/prometheus-statsd-exporter/README.md index 8a6739f393b..61f8ffe64e3 100644 --- a/infra/charts/feast/charts/prometheus-statsd-exporter/README.md +++ b/infra/charts/feast/charts/prometheus-statsd-exporter/README.md @@ -38,7 +38,7 @@ $ helm delete my-release |`extraArgs` | key:value list of extra arguments to give the binary | `{}` | |`image.pullPolicy` | Image pull policy | `IfNotPresent` | |`image.repository` | Image repository | `prom/statsd-exporter` | -|`image.tag` | Image tag | `v0.8.0` | +|`image.tag` | Image tag | `v0.9.0` | |`ingress.enabled` | enable ingress | `false` | |`ingress.path` | ingress base path | `/` | |`ingress.host` | Ingress accepted hostnames | `nil` | diff --git a/infra/charts/feast/requirements.lock b/infra/charts/feast/requirements.lock index 2936f5007d4..8a87c90b5c3 100644 --- a/infra/charts/feast/requirements.lock +++ b/infra/charts/feast/requirements.lock @@ -1,13 +1,13 @@ dependencies: - name: feast-core repository: "" - version: 0.8.0 + version: 0.9.0 - name: feast-serving repository: "" - version: 0.8.0 + version: 0.9.0 - name: feast-jupyter repository: "" - version: 0.8.0 + version: 0.9.0 - name: postgresql repository: https://kubernetes-charts.storage.googleapis.com/ version: 8.6.1 diff --git a/infra/charts/feast/requirements.yaml b/infra/charts/feast/requirements.yaml index dd47411da23..9987c968145 100644 --- a/infra/charts/feast/requirements.yaml +++ b/infra/charts/feast/requirements.yaml @@ -1,17 +1,17 @@ dependencies: - name: feast-core - version: 0.8.0 + version: 0.9.0 condition: feast-core.enabled - name: feast-serving alias: feast-online-serving - version: 0.8.0 + version: 0.9.0 condition: feast-online-serving.enabled - name: feast-jupyter - version: 0.8.0 + version: 0.9.0 condition: feast-jupyter.enabled - name: postgresql version: 8.6.1 - repository: https://kubernetes-charts.storage.googleapis.com/ + repository: https://charts.helm.sh/stable condition: postgresql.enabled - name: kafka version: 11.8.8 @@ -19,16 +19,16 @@ dependencies: condition: kafka.enabled - name: redis version: 10.5.6 - repository: https://kubernetes-charts.storage.googleapis.com/ + repository: https://charts.helm.sh/stable condition: redis.enabled - name: prometheus-statsd-exporter version: 0.1.2 condition: prometheus-statsd-exporter.enabled - name: prometheus version: 11.0.2 - repository: https://kubernetes-charts.storage.googleapis.com/ + repository: https://charts.helm.sh/stable condition: prometheus.enabled - name: grafana version: 5.0.5 - repository: https://kubernetes-charts.storage.googleapis.com/ + repository: https://charts.helm.sh/stable condition: grafana.enabled diff --git a/infra/docker-compose/.env.sample b/infra/docker-compose/.env.sample index ea98b441d6e..8c48fbc976b 100644 --- a/infra/docker-compose/.env.sample +++ b/infra/docker-compose/.env.sample @@ -2,4 +2,5 @@ COMPOSE_PROJECT_NAME=feast FEAST_VERSION=develop FEAST_CORE_CONFIG=./core/core.yml FEAST_ONLINE_SERVING_CONFIG=./serving/online-serving.yml -GCP_SERVICE_ACCOUNT=./gcp-service-accounts/placeholder.json \ No newline at end of file +GCP_SERVICE_ACCOUNT=./gcp-service-accounts/placeholder.json +INGESTION_JAR_PATH=https://storage.googleapis.com/feast-jobs/spark/ingestion/feast-ingestion-spark-develop.jar \ No newline at end of file diff --git a/infra/docker-compose/docker-compose.yml b/infra/docker-compose/docker-compose.yml index 90c94e0055c..98131d6ccf0 100644 --- a/infra/docker-compose/docker-compose.yml +++ b/infra/docker-compose/docker-compose.yml @@ -36,6 +36,10 @@ services: FEAST_HISTORICAL_FEATURE_OUTPUT_LOCATION: file:///shared/historical_feature_output FEAST_HISTORICAL_FEATURE_OUTPUT_FORMAT: parquet FEAST_REDIS_HOST: redis + FEAST_SPARK_INGESTION_JAR: ${INGESTION_JAR_PATH} + FEAST_STATSD_ENABLED: "true" + FEAST_STATSD_HOST: prometheus_statsd + FEAST_STATSD_PORT: 9125 jupyter: image: gcr.io/kf-feast/feast-jupyter:${FEAST_VERSION} @@ -105,4 +109,10 @@ services: redis: image: redis:5-alpine ports: - - "6379:6379" \ No newline at end of file + - "6379:6379" + + prometheus_statsd: + image: prom/statsd-exporter:v0.12.1 + ports: + - "9125:9125" + - "9102:9102" \ No newline at end of file diff --git a/infra/docker/ci/Dockerfile b/infra/docker/ci/Dockerfile index c6ef17aabd6..b4f4504b5ed 100644 --- a/infra/docker/ci/Dockerfile +++ b/infra/docker/ci/Dockerfile @@ -1,14 +1,35 @@ -FROM maven:3.6-jdk-11 +FROM ubuntu:18.04 -# Install Google Cloud SDK -RUN echo "deb [signed-by=/usr/share/keyrings/cloud.google.gpg] http://packages.cloud.google.com/apt cloud-sdk main" \ - | tee -a /etc/apt/sources.list.d/google-cloud-sdk.list && \ - curl https://packages.cloud.google.com/apt/doc/apt-key.gpg \ - | apt-key --keyring /usr/share/keyrings/cloud.google.gpg \ - add - && apt-get update -y && apt-get install google-cloud-sdk -y +ARG REVISION +ENV DEBIAN_FRONTEND=noninteractive + +RUN apt-get update && apt-get install -y curl unzip locales software-properties-common && \ + apt-add-repository ppa:git-core/ppa && \ + apt update && apt install -y git + +# Install Java (by default openjdk-11) +RUN apt-get install -y default-jdk + +RUN locale-gen en_US.UTF-8 && update-locale LANG=en_US.utf8 +ENV LANG='en_US.UTF-8' LANGUAGE='en_US:en' LC_ALL='en_US.UTF-8' + +# Install maven +ARG MAVEN_VERSION=3.6.3 +ARG SHA=c35a1803a6e70a126e80b2b3ae33eed961f83ed74d18fcd16909b2d44d7dada3203f1ffe726c17ef8dcca2dcaa9fca676987befeadc9b9f759967a8cb77181c0 +ARG BASE_URL=https://apache.osuosl.org/maven/maven-3/${MAVEN_VERSION}/binaries + +RUN mkdir -p /usr/share/maven /usr/share/maven/ref \ + && curl -fsSL -o /tmp/apache-maven.tar.gz ${BASE_URL}/apache-maven-${MAVEN_VERSION}-bin.tar.gz \ + && echo "${SHA} /tmp/apache-maven.tar.gz" | sha512sum -c - \ + && tar -xzf /tmp/apache-maven.tar.gz -C /usr/share/maven --strip-components=1 \ + && rm -f /tmp/apache-maven.tar.gz \ + && ln -s /usr/share/maven/bin/mvn /usr/bin/mvn + +ENV MAVEN_HOME /usr/share/maven +ENV MAVEN_CONFIG "/root/.m2" # Install Make and Python -ENV PYTHON_VERSION 3.7 +ENV PYTHON_VERSION 3.6 RUN apt-get install -y build-essential curl python${PYTHON_VERSION} \ python${PYTHON_VERSION}-dev python${PYTHON_VERSION}-distutils && \ @@ -18,6 +39,13 @@ RUN apt-get install -y build-essential curl python${PYTHON_VERSION} \ python get-pip.py --force-reinstall && \ rm get-pip.py +# Install Google Cloud SDK +RUN echo "deb [signed-by=/usr/share/keyrings/cloud.google.gpg] http://packages.cloud.google.com/apt cloud-sdk main" \ + | tee -a /etc/apt/sources.list.d/google-cloud-sdk.list && \ + curl https://packages.cloud.google.com/apt/doc/apt-key.gpg \ + | apt-key --keyring /usr/share/keyrings/cloud.google.gpg \ + add - && apt-get update -y && apt-get install google-cloud-sdk -y + # Instal boto3 RUN pip install boto3==1.16.10 @@ -43,4 +71,18 @@ RUN PROTOC_ZIP=protoc-${PROTOC_VERSION}-linux-x86_64.zip && \ go get gopkg.in/russross/blackfriday.v2 && \ git clone https://github.com/istio/tools/ && \ cd tools/cmd/protoc-gen-docs && \ - go build && mkdir -p $HOME/bin && cp protoc-gen-docs $HOME/bin \ No newline at end of file + go build && mkdir -p $HOME/bin && cp protoc-gen-docs $HOME/bin + +# Install AZ CLI +RUN curl -sL https://aka.ms/InstallAzureCLIDeb | bash + +# Install kubectl +RUN apt-get install -y kubectl=1.20.2-00 + +# Install helm +RUN curl -fsSL -o get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 && \ + chmod 700 get_helm.sh && \ + ./get_helm.sh --version v3.4.2 + +# Install jq +RUN apt-get install -y jq diff --git a/infra/docker/core/Dockerfile b/infra/docker/core/Dockerfile index bf2e17cf076..c462673780b 100644 --- a/infra/docker/core/Dockerfile +++ b/infra/docker/core/Dockerfile @@ -9,13 +9,11 @@ WORKDIR /build COPY pom.xml . COPY datatypes/java/pom.xml datatypes/java/pom.xml COPY common/pom.xml common/pom.xml -COPY ingestion/pom.xml ingestion/pom.xml COPY core/pom.xml core/pom.xml COPY serving/pom.xml serving/pom.xml COPY storage/api/pom.xml storage/api/pom.xml COPY storage/connectors/pom.xml storage/connectors/pom.xml COPY storage/connectors/redis/pom.xml storage/connectors/redis/pom.xml -COPY storage/connectors/bigquery/pom.xml storage/connectors/bigquery/pom.xml COPY sdk/java/pom.xml sdk/java/pom.xml COPY docs/coverage/java/pom.xml docs/coverage/java/pom.xml COPY protos/ protos/ @@ -23,14 +21,14 @@ COPY protos/ protos/ # Setting Maven repository .m2 directory relative to /build folder gives the # user to optionally use cached repository when building the image by copying # the existing .m2 directory to $FEAST_REPO_ROOT/.m2 -ENV MAVEN_OPTS="-Dmaven.repo.local=/build/.m2/repository -DdependencyLocationsEnabled=false" +ENV MAVEN_OPTS="-Dmaven.repo.local=/build/.m2/repository -DdependencyLocationsEnabled=false -Dmaven.wagon.httpconnectionManager.ttlSeconds=25 -Dmaven.wagon.http.retryHandler.count=3" COPY pom.xml .m2/* .m2/ RUN mvn dependency:go-offline -DexcludeGroupIds:dev.feast 2>/dev/null || true COPY . . -ARG REVISION=dev -RUN mvn --also-make --projects core -Drevision=$REVISION \ +ARG VERSION=dev +RUN mvn --also-make --projects core -Drevision=$VERSION \ -DskipUTs=true --batch-mode clean package # @@ -46,9 +44,9 @@ RUN wget -q https://github.com/grpc-ecosystem/grpc-health-probe/releases/downloa # ============================================================ FROM openjdk:11-jre as production -ARG REVISION=dev +ARG VERSION=dev -COPY --from=builder /build/core/target/feast-core-$REVISION-exec.jar /opt/feast/feast-core.jar +COPY --from=builder /build/core/target/feast-core-$VERSION-exec.jar /opt/feast/feast-core.jar COPY --from=builder /usr/bin/grpc-health-probe /usr/bin/grpc-health-probe CMD ["java",\ diff --git a/infra/docker/jobcontroller/Dockerfile b/infra/docker/jobcontroller/Dockerfile deleted file mode 100644 index b4115c3930a..00000000000 --- a/infra/docker/jobcontroller/Dockerfile +++ /dev/null @@ -1,71 +0,0 @@ -# ============================================================ -# Build stage 1: Builder -# ============================================================ - -FROM maven:3.6-jdk-11 as builder - -WORKDIR /build - -COPY pom.xml . -COPY datatypes/java/pom.xml datatypes/java/pom.xml -COPY common/pom.xml common/pom.xml -COPY ingestion/pom.xml ingestion/pom.xml -COPY core/pom.xml core/pom.xml -COPY serving/pom.xml serving/pom.xml -COPY storage/api/pom.xml storage/api/pom.xml -COPY storage/connectors/pom.xml storage/connectors/pom.xml -COPY storage/connectors/redis/pom.xml storage/connectors/redis/pom.xml -COPY storage/connectors/bigquery/pom.xml storage/connectors/bigquery/pom.xml -COPY sdk/java/pom.xml sdk/java/pom.xml -COPY job-controller/pom.xml job-controller/pom.xml -COPY docs/coverage/java/pom.xml docs/coverage/java/pom.xml -COPY protos/ protos/ - -# Setting Maven repository .m2 directory relative to /build folder gives the -# user to optionally use cached repository when building the image by copying -# the existing .m2 directory to $FEAST_REPO_ROOT/.m2 -ENV MAVEN_OPTS="-Dmaven.repo.local=/build/.m2/repository -DdependencyLocationsEnabled=false" -COPY pom.xml .m2/* .m2/ -RUN mvn dependency:go-offline -DexcludeGroupIds:dev.feast 2>/dev/null || true - -COPY . . - -ARG REVISION=dev -RUN mvn --also-make --projects job-controller,ingestion -Drevision=$REVISION \ - -DskipUTs=true --batch-mode clean package -# -# Unpack the jar and copy the files into production Docker image -# for faster startup time when starting Dataflow jobs from Feast Job Controller. -# This is because we need to stage the classes and dependencies when using Dataflow. -# The final size of the production image will be bigger but it seems -# a good tradeoff between speed and size. -# -# https://github.com/feast-dev/feast/pull/291 -RUN apt-get -qq update && apt-get -y install unar && \ - unar /build/job-controller/target/feast-job-controller-$REVISION-exec.jar -o /build/job-controller/target/ - -# -# Download grpc_health_probe to run health check for Feast Serving -# https://kubernetes.io/blog/2018/10/01/health-checking-grpc-servers-on-kubernetes/ -# -RUN wget -q https://github.com/grpc-ecosystem/grpc-health-probe/releases/download/v0.3.1/grpc_health_probe-linux-amd64 \ - -O /usr/bin/grpc-health-probe && \ - chmod +x /usr/bin/grpc-health-probe - -# ============================================================ -# Build stage 2: Production -# ============================================================ - -FROM openjdk:11-jre as production -ARG REVISION=dev - -COPY --from=builder /build/job-controller/target/feast-job-controller-$REVISION-exec.jar /opt/feast/feast-job-controller.jar -# Required for staging jar dependencies when submitting Dataflow jobs. -COPY --from=builder /build/job-controller/target/feast-job-controller-$REVISION-exec /opt/feast/feast-job-controller -COPY --from=builder /usr/bin/grpc-health-probe /usr/bin/grpc-health-probe - -CMD ["java",\ - "-Xms2048m",\ - "-Xmx2048m",\ - "-jar",\ - "/opt/feast/feast-job-controller.jar"] diff --git a/infra/docker/jobcontroller/Dockerfile.dev b/infra/docker/jobcontroller/Dockerfile.dev deleted file mode 100644 index da238dd2235..00000000000 --- a/infra/docker/jobcontroller/Dockerfile.dev +++ /dev/null @@ -1,8 +0,0 @@ -FROM openjdk:11-jre -ARG REVISION=dev -ADD $PWD/core/target/feast-job-controller-$REVISION-exec.jar /opt/feast/feast-job-controller.jar -CMD ["java",\ - "-Xms2048m",\ - "-Xmx2048m",\ - "-jar",\ - "/opt/feast/feast-job-controller.jar"] diff --git a/infra/docker/jobservice/Dockerfile b/infra/docker/jobservice/Dockerfile index f3f8127ec75..bad0983f59e 100644 --- a/infra/docker/jobservice/Dockerfile +++ b/infra/docker/jobservice/Dockerfile @@ -1,4 +1,4 @@ -FROM jupyter/pyspark-notebook:ae5f7e104dd5 +FROM jupyter/pyspark-notebook:399cbb986c6b USER root WORKDIR /feast @@ -14,7 +14,7 @@ RUN apt-get update && apt-get -y install make git wget RUN make compile-protos-python # Install Feast SDK -COPY .git .git +RUN git init . COPY README.md README.md RUN pip install -U -e sdk/python RUN pip install "s3fs" "boto3" "urllib3>=1.25.4" diff --git a/infra/docker/jupyter/Dockerfile b/infra/docker/jupyter/Dockerfile index 69aa3622caf..5126aaffecc 100644 --- a/infra/docker/jupyter/Dockerfile +++ b/infra/docker/jupyter/Dockerfile @@ -1,4 +1,4 @@ -FROM jupyter/pyspark-notebook:ae5f7e104dd5 +FROM jupyter/pyspark-notebook:399cbb986c6b USER root WORKDIR /feast @@ -15,7 +15,7 @@ RUN make compile-protos-python RUN pip install -r sdk/python/requirements-ci.txt # Install Feast SDK -COPY .git .git +RUN git init . COPY README.md README.md RUN pip install -e sdk/python -U RUN pip install "s3fs" "boto3" "urllib3>=1.25.4" diff --git a/infra/docker/serving/Dockerfile b/infra/docker/serving/Dockerfile index 960e2848906..395b344c7c1 100644 --- a/infra/docker/serving/Dockerfile +++ b/infra/docker/serving/Dockerfile @@ -9,13 +9,11 @@ WORKDIR /build COPY pom.xml . COPY datatypes/java/pom.xml datatypes/java/pom.xml COPY common/pom.xml common/pom.xml -COPY ingestion/pom.xml ingestion/pom.xml COPY core/pom.xml core/pom.xml COPY serving/pom.xml serving/pom.xml COPY storage/api/pom.xml storage/api/pom.xml COPY storage/connectors/pom.xml storage/connectors/pom.xml COPY storage/connectors/redis/pom.xml storage/connectors/redis/pom.xml -COPY storage/connectors/bigquery/pom.xml storage/connectors/bigquery/pom.xml COPY sdk/java/pom.xml sdk/java/pom.xml COPY docs/coverage/java/pom.xml docs/coverage/java/pom.xml COPY protos/ protos/ @@ -23,14 +21,14 @@ COPY protos/ protos/ # Setting Maven repository .m2 directory relative to /build folder gives the # user to optionally use cached repository when building the image by copying # the existing .m2 directory to $FEAST_REPO_ROOT/.m2 -ENV MAVEN_OPTS="-Dmaven.repo.local=/build/.m2/repository -DdependencyLocationsEnabled=false" +ENV MAVEN_OPTS="-Dmaven.repo.local=/build/.m2/repository -DdependencyLocationsEnabled=false -Dmaven.wagon.httpconnectionManager.ttlSeconds=25 -Dmaven.wagon.http.retryHandler.count=3" COPY pom.xml .m2/* .m2/ RUN mvn dependency:go-offline -DexcludeGroupIds:dev.feast 2>/dev/null || true COPY . . -ARG REVISION=dev -RUN mvn --also-make --projects serving -Drevision=$REVISION \ +ARG VERSION=dev +RUN mvn --also-make --projects serving -Drevision=$VERSION \ -DskipUTs=true --batch-mode clean package # # Download grpc_health_probe to run health check for Feast Serving @@ -44,12 +42,12 @@ RUN wget -q https://github.com/grpc-ecosystem/grpc-health-probe/releases/downloa # Build stage 2: Production # ============================================================ -FROM openjdk:11-jre-slim as production -ARG REVISION=dev -COPY --from=builder /build/serving/target/feast-serving-$REVISION-exec.jar /opt/feast/feast-serving.jar +FROM amazoncorretto:11 as production +ARG VERSION=dev +COPY --from=builder /build/serving/target/feast-serving-$VERSION-exec.jar /opt/feast/feast-serving.jar COPY --from=builder /usr/bin/grpc-health-probe /usr/bin/grpc-health-probe CMD ["java",\ - "-Xms1024m",\ - "-Xmx1024m",\ + "-Xms1g",\ + "-Xmx4g",\ "-jar",\ "/opt/feast/feast-serving.jar"] diff --git a/infra/docker/tests/Dockerfile b/infra/docker/tests/Dockerfile new file mode 100644 index 00000000000..1f127b9bed0 --- /dev/null +++ b/infra/docker/tests/Dockerfile @@ -0,0 +1,24 @@ +ARG BASE_IMAGE=gcr.io/kf-feast/feast-ci:latest + +FROM ${BASE_IMAGE} + +RUN mkdir -p /src/sdk /src/spark/ingestion + +COPY sdk/python /src/sdk/python + +COPY README.md /src/README.md + +WORKDIR /src + +RUN pip install -r sdk/python/requirements-ci.txt + +RUN git init . +RUN pip install -e sdk/python -U +RUN pip install "s3fs" "boto3" "urllib3>=1.25.4" + +COPY tests /src/tests + +RUN pip install -r tests/requirements.txt + +COPY infra/scripts /src/infra/scripts +COPY spark/ingestion/target /src/spark/ingestion/target diff --git a/infra/scripts/azure-runner.sh b/infra/scripts/azure-runner.sh new file mode 100755 index 00000000000..dde64e9e521 --- /dev/null +++ b/infra/scripts/azure-runner.sh @@ -0,0 +1,65 @@ +#!/usr/bin/env bash + +set -euo pipefail + +STEP_BREADCRUMB='~~~~~~~~' +SECONDS=0 +TIMEFORMAT="${STEP_BREADCRUMB} took %R seconds" + +GIT_TAG=$PULL_PULL_SHA +GIT_REMOTE_URL=https://github.com/feast-dev/feast.git + +echo "########## Starting e2e tests for ${GIT_REMOTE_URL} ${GIT_TAG} ###########" + +# Note requires running in root feast directory +source infra/scripts/k8s-common-functions.sh + +# Workaround for COPY command in core docker image that pulls local maven repo into the image +# itself. +mkdir .m2 2>/dev/null || true + +# Log into k8s. +echo "${STEP_BREADCRUMB} Updating kubeconfig" +az login --service-principal -u "$AZ_SERVICE_PRINCIPAL_ID" -p "$AZ_SERVICE_PRINCIPAL_PASS" --tenant "$AZ_SERVICE_PRINCIPAL_TENANT_ID" >/dev/null +az aks get-credentials --resource-group "$RESOURCE_GROUP" --name "$AKS_CLUSTER_NAME" + +# Sanity check that kubectl is working. +echo "${STEP_BREADCRUMB} k8s sanity check" +kubectl get pods + +# e2e test - runs in sparkop namespace for consistency with AWS sparkop test. +NAMESPACE=sparkop +RELEASE=sparkop + +# Delete old helm release and PVCs +k8s_cleanup "$RELEASE" "$NAMESPACE" + +wait_for_images "${DOCKER_REPOSITORY}" "${GIT_TAG}" + +# Helm install everything in a namespace +helm_install "$RELEASE" "${DOCKER_REPOSITORY}" "${GIT_TAG}" "$NAMESPACE" \ + --set "feast-jobservice.envOverrides.FEAST_AZURE_BLOB_ACCOUNT_NAME=${AZURE_BLOB_ACCOUNT_NAME}" \ + --set "feast-jobservice.envOverrides.FEAST_AZURE_BLOB_ACCOUNT_ACCESS_KEY=${AZURE_BLOB_ACCOUNT_ACCESS_KEY}" + +# Delete old test running pod if it exists +kubectl delete pod -n "$NAMESPACE" ci-test-runner 2>/dev/null || true + +# Delete all sparkapplication resources that may be left over from the previous test runs. +kubectl delete sparkapplication --all -n "$NAMESPACE" || true + +# Make sure the test pod has permissions to create sparkapplication resources +setup_sparkop_role + +# Run the test suite as a one-off pod. +echo "${STEP_BREADCRUMB} Running the test suite" +time kubectl run -n "$NAMESPACE" -i ci-test-runner \ + --pod-running-timeout=5m \ + --restart=Never \ + --image="${DOCKER_REPOSITORY}/feast-ci:${GIT_TAG}" \ + --env="STAGING_PATH=${STAGING_PATH}" \ + --env="FEAST_AZURE_BLOB_ACCOUNT_NAME=${AZURE_BLOB_ACCOUNT_NAME}" \ + --env="FEAST_AZURE_BLOB_ACCOUNT_ACCESS_KEY=${AZURE_BLOB_ACCOUNT_ACCESS_KEY}" \ + -- \ + bash -c "mkdir src && cd src && git clone ${GIT_REMOTE_URL} && cd feast && git config remote.origin.fetch '+refs/pull/*:refs/remotes/origin/pull/*' && git fetch -q && git checkout ${GIT_TAG} && ./infra/scripts/setup-e2e-env-sparkop.sh && ./infra/scripts/test-end-to-end-sparkop.sh" + +echo "########## e2e tests took $SECONDS seconds ###########" diff --git a/infra/scripts/build-ingestion-py-dependencies.sh b/infra/scripts/build-ingestion-py-dependencies.sh new file mode 100755 index 00000000000..c57a0979a20 --- /dev/null +++ b/infra/scripts/build-ingestion-py-dependencies.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash +set -euo pipefail +PLATFORM=$1 +DESTINATION=$2 +PACKAGES=${PACKAGES:-"great-expectations==0.13.2 pyarrow==2.0.0 datadog==0.39.0"} + +tmp_dir=$(mktemp -d) + +pip3 install -t ${tmp_dir}/libs $PACKAGES + +cd $tmp_dir +tar -czf pylibs-ge-$PLATFORM.tar.gz libs/ +if [[ $DESTINATION == gs* ]]; then + gsutil cp pylibs-ge-$PLATFORM.tar.gz $DESTINATION +else + mv pylibs-ge-$PLATFORM.tar.gz $DESTINATION +fi diff --git a/infra/scripts/codebuild-entrypoint.sh b/infra/scripts/codebuild-entrypoint.sh new file mode 100755 index 00000000000..bf7792da349 --- /dev/null +++ b/infra/scripts/codebuild-entrypoint.sh @@ -0,0 +1,157 @@ +#!/bin/bash + +set -euo pipefail + +STEP_BREADCRUMB='~~~~~~~~' +SECONDS=0 +TIMEFORMAT="${STEP_BREADCRUMB} took %R seconds" + +function maybe_build_push_docker { + # Build and push docker image, tagged with SHA tag, if it doesn't exist already. + NAME=$1 + TARGET=$NAME-docker + SUFFIX=feast-$NAME + + if ! aws ecr describe-images --repository-name "feast-ci/feast/$SUFFIX" "--image-ids=imageTag=${GIT_TAG}" >/dev/null ; then + make "build-$TARGET" "push-$TARGET" REGISTRY="${DOCKER_REPOSITORY}" VERSION="${GIT_TAG}" + else + echo "Image ${DOCKER_REPOSITORY}/$SUFFIX:$GIT_TAG already exists, skipping docker build" + fi +} + +source infra/scripts/k8s-common-functions.sh + +GIT_TAG=${CODEBUILD_RESOLVED_SOURCE_VERSION} + +echo "########## Starting stage $STAGE for ${CODEBUILD_SOURCE_REPO_URL} ${GIT_TAG} ###########" + +# This seems to make builds a bit faster. +export DOCKER_BUILDKIT=1 + +# Workaround for COPY command in core docker image that pulls local maven repo into the image +# itself. +mkdir .m2 2>/dev/null || true +mkdir deps/feast/.m2 2>/dev/null || true + +# Log into k8s. +echo "${STEP_BREADCRUMB} Updating kubeconfig" +aws eks update-kubeconfig --name "$EKS_CLUSTER_NAME" + +# chmod kubeconfig so it doesn't complain all the time +chmod 755 ~/.kube/config + +# Sanity check that kubectl is working. +echo "${STEP_BREADCRUMB} k8s sanity check" +kubectl get pods + +case $STAGE in + core-docker) + maybe_build_push_docker core + ;; + serving-docker) + maybe_build_push_docker serving + ;; + jupyter-docker) + maybe_build_push_docker jupyter + ;; + jobservice-docker) + maybe_build_push_docker jobservice + ;; + ci-docker) + maybe_build_push_docker ci + ;; + e2e-test-emr) + # EMR test - runs in default namespace. + + # Copy cluster config template generated for us by terraform. + aws s3 cp "${EMR_TEMPLATE_YML}" emr_cluster.yaml + + # Delete old helm release and PVCs + k8s_cleanup cicd default + + # Create cluster OR get existing EMR cluster id. In the latter case, clean up any steps + # already running there from previous test runs. + echo "${STEP_BREADCRUMB} Creating EMR cluster, this can take up 10 minutes." + CLUSTER_ID=$(time emr_cluster.py --template emr_cluster.yaml ensure --cleanup) + + # Get (any) node IP. EMR will use this to connect to Kafka and Redis. We make them + # available to the EMR job by exposing them as NodePort services. + NODE_IP=$(kubectl get nodes -o custom-columns=Name:.metadata.name | tail -n1) + + # Helm install everything. + # + # This may occasionally run into "provided port is already allocated" error due to + # https://github.com/kubernetes/kubernetes/issues/85894 + helm_install cicd "$DOCKER_REPOSITORY" "$GIT_TAG" default \ + --set "redis.master.service.type=NodePort" \ + --set "redis.master.service.nodePort=32379" \ + --set "kafka.externalAccess.service.type=NodePort" \ + --set "kafka.externalAccess.enabled=true" \ + --set "kafka.externalAccess.service.nodePorts[0]=30092" \ + --set "kafka.externalAccess.service.domain=${NODE_IP}" \ + --set "kafka.service.externalPort=30094" + + # Run the test suite as a one-off pod. We could also run it here, in the codebuild container + # itself, but that'd require more networking setup to make feast services available + # outside k8s cluster. + kubectl delete pod ci-test-runner 2>/dev/null || true + + echo "${STEP_BREADCRUMB} Running the test suite" + time kubectl run --rm -i ci-test-runner \ + --restart=Never \ + --image="${DOCKER_REPOSITORY}/feast-ci:${GIT_TAG}" \ + --env="CLUSTER_ID=$CLUSTER_ID" \ + --env="STAGING_PATH=$STAGING_PATH" \ + --env="NODE_IP=$NODE_IP" \ + -- \ + bash -c "mkdir src && cd src && git clone $CODEBUILD_SOURCE_REPO_URL && cd feast* && git config remote.origin.fetch '+refs/pull/*:refs/remotes/origin/pull/*' && git fetch -q && git checkout $CODEBUILD_RESOLVED_SOURCE_VERSION && ./infra/scripts/setup-e2e-env-aws.sh && ./infra/scripts/test-end-to-end-aws.sh" + + ;; + e2e-test-sparkop) + # spark k8s test - runs in sparkop namespace (so it doesn't interfere with a concurrently + # running EMR test). + NAMESPACE=sparkop + RELEASE=sparkop + + # Clean up old release + k8s_cleanup "$RELEASE" "$NAMESPACE" + + # Helm install everything in a namespace + helm_install "$RELEASE" "${DOCKER_REPOSITORY}" "${GIT_TAG}" "$NAMESPACE" + + # Delete old test running pod if it exists + kubectl delete pod -n "$NAMESPACE" ci-test-runner 2>/dev/null || true + + # Delete all sparkapplication resources that may be left over from the previous test runs. + kubectl delete sparkapplication --all -n "$NAMESPACE" || true + + # Make sure the test pod has permissions to create sparkapplication resources + setup_sparkop_role + + # Run the test suite as a one-off pod. + echo "${STEP_BREADCRUMB} Running the test suite" + if ! time kubectl run --rm -n "$NAMESPACE" -i ci-test-runner \ + --restart=Never \ + --image="${DOCKER_REPOSITORY}/feast-ci:${GIT_TAG}" \ + --env="STAGING_PATH=$STAGING_PATH" \ + -- \ + bash -c "mkdir src && cd src && git clone $CODEBUILD_SOURCE_REPO_URL && cd feast* && git config remote.origin.fetch '+refs/pull/*:refs/remotes/origin/pull/*' && git fetch -q && git checkout $CODEBUILD_RESOLVED_SOURCE_VERSION && ./infra/scripts/setup-e2e-env-sparkop.sh && ./infra/scripts/test-end-to-end-sparkop.sh" ; then + + readarray -t CRASHED_PODS < <(kubectl get pods --no-headers=true --namespace sparkop | grep Error | awk '{ print $1 }') + + for POD in "${CRASHED_PODS[@]}"; do + echo "Logs from crashed pod $POD:" + kubectl logs --namespace sparkop "$POD" + done + fi + + ;; + cleanup) + emr_cluster.py --template emr_cluster.yaml destroy + ;; + *) + echo "Unknown stage $STAGE" + ;; +esac + +echo "########## Stage $STAGE took $SECONDS seconds ###########" diff --git a/infra/scripts/codebuild_runner.py b/infra/scripts/codebuild_runner.py index 7b5ee0ee3e0..382ce9494c3 100755 --- a/infra/scripts/codebuild_runner.py +++ b/infra/scripts/codebuild_runner.py @@ -11,6 +11,7 @@ import sys import argparse import boto3 +from botocore.config import Config class LogTailer: @@ -125,7 +126,14 @@ def source_version_from_prow_job_spec(job_spec: Dict[str, Any]) -> str: async def run_build(project_name: str, source_version: str, source_location: str): print(f"Building {project_name} at {source_version}", file=sys.stderr) - logs_client = boto3.client("logs", region_name="us-west-2") + + config = Config( + retries = { + 'max_attempts': 10, + } + ) + + logs_client = boto3.client("logs", region_name="us-west-2", config=config) codebuild_client = boto3.client("codebuild", region_name="us-west-2") print("Submitting the build..", file=sys.stderr) @@ -162,7 +170,7 @@ async def run_build(project_name: str, source_version: str, source_location: str log_group=build["logs"]["groupName"], ) - waiter_task = asyncio.create_task( + waiter_task = asyncio.get_event_loop().create_task( _wait_build_state( codebuild_client, build_id, @@ -188,4 +196,5 @@ async def run_build(project_name: str, source_version: str, source_location: str if __name__ == "__main__": - asyncio.run(main()) + loop = asyncio.get_event_loop() + loop.run_until_complete(main()) diff --git a/infra/scripts/install-helm.sh b/infra/scripts/install-helm.sh index 3a6221a2e03..3686f9dfdb1 100755 --- a/infra/scripts/install-helm.sh +++ b/infra/scripts/install-helm.sh @@ -2,8 +2,8 @@ set -e readonly HELM_URL=https://storage.googleapis.com/kubernetes-helm readonly HELM_TARBALL="helm-${HELM_VERSION}-linux-amd64.tar.gz" -readonly STABLE_REPO_URL=https://kubernetes-charts.storage.googleapis.com/ -readonly INCUBATOR_REPO_URL=https://kubernetes-charts-incubator.storage.googleapis.com/ +readonly STABLE_REPO_URL=https://charts.helm.sh/stable +readonly INCUBATOR_REPO_URL=https://charts.helm.sh/incubator curl -s "https://get.helm.sh/helm-${HELM_VERSION}-linux-amd64.tar.gz" | tar -C /tmp -xz sudo mv /tmp/linux-amd64/helm /usr/bin/helm helm init --client-only diff --git a/infra/scripts/k8s-common-functions.sh b/infra/scripts/k8s-common-functions.sh new file mode 100644 index 00000000000..b6161fcfd46 --- /dev/null +++ b/infra/scripts/k8s-common-functions.sh @@ -0,0 +1,119 @@ +#!/bin/bash + +set -euo pipefail + +function wait_for_images { + local DOCKER_REPOSITORY=$1 + local GIT_TAG=$2 + # Wait for images to be available in the docker repository; ci is the last image built + timeout 15m bash -c "while ! gcloud container images list-tags ${DOCKER_REPOSITORY}/feast-ci --format=json | jq -e \".[] | select(.tags[] | contains (\\\"${GIT_TAG}\\\"))\" > /dev/null; do sleep 10s; done" +} + +function k8s_cleanup { + local RELEASE=$1 + local NAMESPACE=$2 + + # Create namespace if it doesn't exist. + kubectl create namespace "$NAMESPACE" || true + + # Uninstall previous feast release if there is any. + helm uninstall "$RELEASE" -n "$NAMESPACE" || true + + # `helm uninstall` doesn't remove PVCs, delete them manually. + time kubectl delete pvc --all -n "$NAMESPACE" || true + + kubectl get service -n "$NAMESPACE" + + # Set a new postgres password. Note that the postgres instance is not available outside + # the k8s cluster anyway so it doesn't have to be super secure. + echo "${STEP_BREADCRUMB:-} Setting PG password" + + # use either shasum or md5sum, whichever exists + SUM=$(which md5sum shasum | grep -v "not found" | tail -n1 || true ) + + PG_PASSWORD=$(head -c 59 /dev/urandom | $SUM | head -c 16) + kubectl delete secret feast-postgresql -n "$NAMESPACE" || true + kubectl create secret generic feast-postgresql --from-literal=postgresql-password="$PG_PASSWORD" -n "$NAMESPACE" +} + +function helm_install { + # helm install Feast into k8s cluster and display a nice error if it fails. + # Usage: helm_install $RELEASE $DOCKER_REPOSITORY $GIT_TAG ... + # Args: + # $RELEASE is helm release name + # $DOCKER_REPOSITORY is the docker repo containing feast images tagged with $GIT_TAG + # ... you can pass additional args to this function that are passed on to helm install + # $NAMESPACE is the namespace name + + local RELEASE=$1 + local DOCKER_REPOSITORY=$2 + local GIT_TAG=$3 + local NAMESPACE=$4 + + shift 4 + + # We skip statsd exporter and other metrics stuff since we're not using it anyway, and it + # has some issues with unbound PVCs (that cause kubectl delete pvc to hang). + echo "${STEP_BREADCRUMB:-} Helm installing feast" + + if ! time helm install --wait "$RELEASE" "${HELM_CHART_LOCATION:-./infra/charts/feast}" \ + --timeout 15m \ + --set "feast-jupyter.image.repository=${DOCKER_REPOSITORY}/feast-jupyter" \ + --set "feast-jupyter.image.tag=${GIT_TAG}" \ + --set "feast-online-serving.image.repository=${DOCKER_REPOSITORY}/feast-serving" \ + --set "feast-online-serving.image.tag=${GIT_TAG}" \ + --set "feast-jobservice.image.repository=${DOCKER_REPOSITORY}/feast-jobservice" \ + --set "feast-jobservice.image.tag=${GIT_TAG}" \ + --set "feast-core.image.repository=${DOCKER_REPOSITORY}/feast-core" \ + --set "feast-core.image.tag=${GIT_TAG}" \ + --set "prometheus-statsd-exporter.enabled=false" \ + --set "prometheus.enabled=false" \ + --set "grafana.enabled=false" \ + --set "feast-jobservice.enabled=false" \ + --namespace "$NAMESPACE" \ + "$@" ; then + + echo "Error during helm install. " + kubectl -n "$NAMESPACE" get pods + + readarray -t CRASHED_PODS < <(kubectl -n "$NAMESPACE" get pods --no-headers=true | grep "$RELEASE" | awk '{if ($2 == "0/1") { print $1 } }') + echo "Crashed pods: ${CRASHED_PODS[*]}" + + for POD in "${CRASHED_PODS[@]}"; do + echo "Logs from pod error $POD:" + kubectl -n "$NAMESPACE" logs "$POD" --previous + done + + exit 1 + fi +} + +function setup_sparkop_role { + # Set up permissions for the default user in sparkop namespace so that Feast SDK can manage + # sparkapplication resources from the test runner pod. + + cat </dev/null || true + +# Create the job +kubectl apply -n ${NAMESPACE} -f "$JOB_SPEC" + +# Wait for job to have a pod. +for i in {1..10} +do + POD=$(kubectl get pods -n ${NAMESPACE} --selector=job-name=$JOB_NAME --output=jsonpath='{.items[0].metadata.name}') + if [ ! -z "$POD" ]; then + break + else + sleep 1 + fi +done + +echo "Waiting for pod to be ready:" +kubectl wait -n ${NAMESPACE} --for=condition=ContainersReady "pod/$POD" --timeout=60s || true + +echo "Job output:" +kubectl logs -n ${NAMESPACE} -f "job/$JOB_NAME" + +# Can't wait for both conditions at once, so wait for complete first then wait for failure +kubectl wait -n ${NAMESPACE} --for=condition=complete "job/$JOB_NAME" --timeout=60s && exit 0 +kubectl wait -n ${NAMESPACE} --for=condition=failure "job/$JOB_NAME" --timeout=60s && exit 1 diff --git a/infra/scripts/setup-e2e-env-aws.sh b/infra/scripts/setup-e2e-env-aws.sh index 6521f941e25..dbc2859daef 100755 --- a/infra/scripts/setup-e2e-env-aws.sh +++ b/infra/scripts/setup-e2e-env-aws.sh @@ -2,13 +2,14 @@ make compile-protos-python -python -m pip install --upgrade pip setuptools wheel +python -m pip install --upgrade pip==20.2 setuptools wheel python -m pip install -qr sdk/python/requirements-dev.txt python -m pip install -qr tests/requirements.txt # Using mvn -q to make it less verbose. This step happens after docker containers were -# succesfully built so it should be unlikely to fail. +# succesfully built so it should be unlikely to fail, therefore we likely won't need detailed logs. echo "########## Building ingestion jar" TIMEFORMAT='########## took %R seconds' -time mvn -q --no-transfer-progress -Dmaven.javadoc.skip=true -Dgpg.skip -DskipUTs=true clean package + +time make build-java-no-tests REVISION=develop MAVEN_EXTRA_OPTS="-q --no-transfer-progress" diff --git a/infra/scripts/setup-e2e-env-sparkop.sh b/infra/scripts/setup-e2e-env-sparkop.sh new file mode 100755 index 00000000000..dbc2859daef --- /dev/null +++ b/infra/scripts/setup-e2e-env-sparkop.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +make compile-protos-python + +python -m pip install --upgrade pip==20.2 setuptools wheel + +python -m pip install -qr sdk/python/requirements-dev.txt +python -m pip install -qr tests/requirements.txt + +# Using mvn -q to make it less verbose. This step happens after docker containers were +# succesfully built so it should be unlikely to fail, therefore we likely won't need detailed logs. +echo "########## Building ingestion jar" +TIMEFORMAT='########## took %R seconds' + +time make build-java-no-tests REVISION=develop MAVEN_EXTRA_OPTS="-q --no-transfer-progress" diff --git a/infra/scripts/setup-e2e-local.sh b/infra/scripts/setup-e2e-local.sh new file mode 100644 index 00000000000..3432673d6dc --- /dev/null +++ b/infra/scripts/setup-e2e-local.sh @@ -0,0 +1,26 @@ +#!/bin/bash +set -euo pipefail + +STEP_BREADCRUMB='~~~~~~~~' + +pushd "$(dirname $0)" +source k8s-common-functions.sh + +# spark k8s test - runs in sparkop namespace (so it doesn't interfere with a concurrently +# running EMR test). +NAMESPACE=sparkop +RELEASE=sparkop + +# Clean up old release +k8s_cleanup "$RELEASE" "$NAMESPACE" + +# Helm install everything in a namespace +helm_install "$RELEASE" "${DOCKER_REPOSITORY}" "${GIT_TAG}" "$NAMESPACE" --create-namespace + +# Delete all sparkapplication resources that may be left over from the previous test runs. +kubectl delete sparkapplication --all -n "$NAMESPACE" || true + +# Make sure the test pod has permissions to create sparkapplication resources +setup_sparkop_role + +echo "DONE" \ No newline at end of file diff --git a/infra/scripts/sync-helm-charts.sh b/infra/scripts/sync-helm-charts.sh index 8c242aeae69..acb4effe66e 100755 --- a/infra/scripts/sync-helm-charts.sh +++ b/infra/scripts/sync-helm-charts.sh @@ -30,6 +30,8 @@ fi exit_code=0 +helm repo add bitnami https://charts.bitnami.com/bitnami + for dir in "$repo_dir"/*; do if helm dep update "$dir" && helm dep build "$dir"; then helm package --destination "$sync_dir" "$dir" diff --git a/infra/scripts/test-docker-compose.sh b/infra/scripts/test-docker-compose.sh index 6ac950aec04..e2454f9fe85 100755 --- a/infra/scripts/test-docker-compose.sh +++ b/infra/scripts/test-docker-compose.sh @@ -13,6 +13,7 @@ clean_up () { ARG=$? # Shut down docker-compose images + docker-compose down exit $ARG @@ -69,4 +70,4 @@ docker exec \ -e DISABLE_FEAST_SERVICE_FIXTURES=true \ --user root \ feast_jupyter_1 bash \ - -c 'cd /feast/tests && python -m pip install -r requirements.txt && pytest e2e/ --ingestion-jar https://storage.googleapis.com/feast-jobs/spark/ingestion/feast-ingestion-spark-${FEAST_VERSION}.jar --redis-url redis:6379 --core-url core:6565 --serving-url online_serving:6566 --job-service-url jobservice:6568 --staging-path file:///shared/staging/ --kafka-brokers kafka:9092' + -c 'cd /feast/tests && python -m pip install -r requirements.txt && pytest e2e/ --ingestion-jar https://storage.googleapis.com/feast-jobs/spark/ingestion/feast-ingestion-spark-${FEAST_VERSION}.jar --redis-url redis:6379 --core-url core:6565 --serving-url online_serving:6566 --job-service-url jobservice:6568 --staging-path file:///shared/staging/ --kafka-brokers kafka:9092 --statsd-url prometheus_statsd:9125 --prometheus-url prometheus_statsd:9102 --feast-version develop' diff --git a/infra/scripts/test-end-to-end-aws.sh b/infra/scripts/test-end-to-end-aws.sh index 981118fd2e5..807f439f6d6 100755 --- a/infra/scripts/test-end-to-end-aws.sh +++ b/infra/scripts/test-end-to-end-aws.sh @@ -8,6 +8,7 @@ export DISABLE_FEAST_SERVICE_FIXTURES=1 export DISABLE_SERVICE_FIXTURES=1 PYTHONPATH=sdk/python pytest tests/e2e/ \ + --feast-version develop \ --core-url cicd-feast-core:6565 \ --serving-url cicd-feast-online-serving:6566 \ --env aws \ diff --git a/infra/scripts/test-end-to-end-gcp.sh b/infra/scripts/test-end-to-end-gcp.sh index 3dab0513f6c..cc54d9c8f88 100755 --- a/infra/scripts/test-end-to-end-gcp.sh +++ b/infra/scripts/test-end-to-end-gcp.sh @@ -1,7 +1,10 @@ #!/usr/bin/env bash export DISABLE_SERVICE_FIXTURES=1 +export MAVEN_OPTS="-Dmaven.repo.local=/tmp/.m2/repository -DdependencyLocationsEnabled=false -Dmaven.wagon.httpconnectionManager.ttlSeconds=25 -Dmaven.wagon.http.retryHandler.count=3 -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false" +export MAVEN_CACHE="gs://feast-templocation-kf-feast/.m2.2020-11-17.tar" +infra/scripts/download-maven-cache.sh --archive-uri ${MAVEN_CACHE} --output-dir /tmp apt-get update && apt-get install -y redis-server postgresql libpq-dev make build-java-no-tests REVISION=develop diff --git a/infra/scripts/test-end-to-end-local.sh b/infra/scripts/test-end-to-end-local.sh new file mode 100755 index 00000000000..ec6fd0bff34 --- /dev/null +++ b/infra/scripts/test-end-to-end-local.sh @@ -0,0 +1,69 @@ +#!/usr/bin/env bash + +set -euo pipefail + +export DISABLE_FEAST_SERVICE_FIXTURES=1 +export DISABLE_SERVICE_FIXTURES=1 + +export FEAST_SPARK_K8S_NAMESPACE=sparkop +export FEAST_S3_ENDPOINT_URL=http://minio.minio.svc.cluster.local:9000 + +# Used by tests +export AWS_S3_ENDPOINT_URL=http://minio.minio.svc.cluster.local:9000 + +cat << SPARK_CONF_END >/tmp/spark_conf.yml +apiVersion: "sparkoperator.k8s.io/v1beta2" +kind: SparkApplication +metadata: + namespace: default +spec: + type: Scala + mode: cluster + image: "gcr.io/kf-feast/spark-py:v3.0.1" + imagePullPolicy: Always + sparkVersion: "3.0.1" + timeToLiveSeconds: 3600 + pythonVersion: "3" + sparkConf: + "spark.hadoop.fs.s3a.endpoint": http://minio.minio.svc.cluster.local:9000 + "spark.hadoop.fs.s3a.path.style.access": "true" + "spark.hadoop.fs.s3a.access.key": ${AWS_ACCESS_KEY_ID} + "spark.hadoop.fs.s3a.secret.key": ${AWS_SECRET_ACCESS_KEY} + restartPolicy: + type: Never + volumes: + - name: "test-volume" + hostPath: + path: "/tmp" + type: Directory + driver: + cores: 1 + coreLimit: "1200m" + memory: "512m" + labels: + version: 3.0.1 + serviceAccount: spark + volumeMounts: + - name: "test-volume" + mountPath: "/tmp" + executor: + cores: 1 + instances: 1 + memory: "512m" + labels: + version: 3.0.1 + volumeMounts: + - name: "test-volume" + mountPath: "/tmp" +SPARK_CONF_END +export FEAST_SPARK_K8S_JOB_TEMPLATE_PATH=/tmp/spark_conf.yml + +PYTHONPATH=sdk/python pytest tests/e2e/ \ + --feast-version develop \ + --core-url sparkop-feast-core:6565 \ + --serving-url sparkop-feast-online-serving:6566 \ + --env k8s \ + --staging-path s3a://feast-staging \ + --redis-url sparkop-redis-master.sparkop.svc.cluster.local:6379 \ + --kafka-brokers sparkop-kafka.sparkop.svc.cluster.local:9092 \ + -m "not bq" \ No newline at end of file diff --git a/infra/scripts/test-end-to-end-sparkop.sh b/infra/scripts/test-end-to-end-sparkop.sh new file mode 100755 index 00000000000..035acc39f55 --- /dev/null +++ b/infra/scripts/test-end-to-end-sparkop.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash + +set -euo pipefail + +pip install "s3fs" "boto3" "urllib3>=1.25.4" + +export DISABLE_FEAST_SERVICE_FIXTURES=1 +export DISABLE_SERVICE_FIXTURES=1 + +export FEAST_SPARK_K8S_NAMESPACE=sparkop + +PYTHONPATH=sdk/python pytest tests/e2e/ \ + --feast-version develop \ + --core-url sparkop-feast-core:6565 \ + --serving-url sparkop-feast-online-serving:6566 \ + --env k8s \ + --staging-path $STAGING_PATH \ + --redis-url sparkop-redis-master.sparkop.svc.cluster.local:6379 \ + --kafka-brokers sparkop-kafka.sparkop.svc.cluster.local:9092 \ + -m "not bq" \ No newline at end of file diff --git a/infra/scripts/test-end-to-end.sh b/infra/scripts/test-end-to-end.sh index 60f9c33a140..c4853dcbcf2 100755 --- a/infra/scripts/test-end-to-end.sh +++ b/infra/scripts/test-end-to-end.sh @@ -1,5 +1,9 @@ #!/usr/bin/env bash +export MAVEN_OPTS="-Dmaven.repo.local=/tmp/.m2/repository -DdependencyLocationsEnabled=false -Dmaven.wagon.httpconnectionManager.ttlSeconds=25 -Dmaven.wagon.http.retryHandler.count=3 -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false" +export MAVEN_CACHE="gs://feast-templocation-kf-feast/.m2.2020-11-17.tar" + +infra/scripts/download-maven-cache.sh --archive-uri ${MAVEN_CACHE} --output-dir /tmp apt-get update && apt-get install -y redis-server postgresql libpq-dev make build-java-no-tests REVISION=develop diff --git a/infra/scripts/test_job.yaml b/infra/scripts/test_job.yaml new file mode 100644 index 00000000000..4995b7d4f06 --- /dev/null +++ b/infra/scripts/test_job.yaml @@ -0,0 +1,35 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: test-runner + namespace: sparkop +spec: + backoffLimit: 1 + template: + spec: + containers: + - name: ubuntu + image: feast:local + command: ["bash", "-c", "./infra/scripts/test-end-to-end-local.sh"] + imagePullPolicy: Never + args: + - bash + stdin: true + stdinOnce: true + tty: true + env: + - name: AWS_ACCESS_KEY_ID + valueFrom: + secretKeyRef: + name: minio + key: accesskey + - name: AWS_SECRET_ACCESS_KEY + valueFrom: + secretKeyRef: + name: minio + key: secretkey + - name: AWS_DEFAULT_REGION + value: us-east-1 + - name: AWS_S3_SIGNATURE_VERSION + value: s3v4 + restartPolicy: Never diff --git a/infra/scripts/validate-helm-chart-docker-image.sh b/infra/scripts/validate-helm-chart-docker-image.sh new file mode 100755 index 00000000000..df00b30a208 --- /dev/null +++ b/infra/scripts/validate-helm-chart-docker-image.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash + +# Get project root +PROJECT_ROOT_DIR=$(git rev-parse --show-toplevel) + +# Should have no "develop" tags +grep -R "tag: develop" "$PROJECT_ROOT_DIR"/infra/charts || true +COUNT=$(grep -R "tag: develop" "$PROJECT_ROOT_DIR"/infra/charts | wc -l) + +if [ "$COUNT" -gt 0 ]; then + echo 'Found more than one instance of "develop" in an image tag. Please replace with correct release version.'; + exit 1 +else + echo 'No "develop" tags found, continuing'; +fi + +# Should have no "gcr" images +grep -R "gcr.io" "$PROJECT_ROOT_DIR"/infra/charts || true +COUNT=$(grep -R "gcr.io" "$PROJECT_ROOT_DIR"/infra/charts | wc -l) + +if [ "$COUNT" -gt 0 ]; then + echo 'Found more than one instance of "gcr.io" in charts. Please replace with https://hub.docker.com/r/feastdev feast image.'; + exit 1 +else + echo 'No "gcr.io" instances found, continuing'; +fi + +# Should have no "SNAPSHOT" versions +grep -R "SNAPSHOT" "$PROJECT_ROOT_DIR"/infra/charts || true +COUNT=$(grep -R "SNAPSHOT" "$PROJECT_ROOT_DIR"/infra/charts | wc -l) + +if [ "$COUNT" -gt 0 ]; then + echo 'Found more than one instance of "SNAPSHOT" in charts. Please ensure that no SNAPSHOT charts are published.'; + exit 1 +else + echo 'No "SNAPSHOT" instances found, continuing'; +fi \ No newline at end of file diff --git a/infra/scripts/validate-version-consistency.sh b/infra/scripts/validate-version-consistency.sh index a021e119a3b..30e294ee416 100755 --- a/infra/scripts/validate-version-consistency.sh +++ b/infra/scripts/validate-version-consistency.sh @@ -96,9 +96,6 @@ declare -a files_to_validate_version=( "infra/docker-compose/.env.sample,1,${FEAST_RELEASE_VERSION}" "datatypes/java/README.md,1,${FEAST_MASTER_VERSION}" "docs/contributing/development-guide.md,4,${FEAST_MASTER_VERSION}" - "docs/administration/audit-logging.md,1,${FEAST_STABLE_VERSION}" - "docs/getting-started/deploying-feast/docker-compose.md,1,${FEAST_STABLE_VERSION}" - "README.md,1,${FEAST_STABLE_VERSION}" "CHANGELOG.md,2,${FEAST_STABLE_VERSION}" ) diff --git a/infra/terraform/aws/rds.tf b/infra/terraform/aws/rds.tf index 71f338e0fed..52a8e147f36 100644 --- a/infra/terraform/aws/rds.tf +++ b/infra/terraform/aws/rds.tf @@ -5,7 +5,7 @@ resource "random_password" "db_password" { } module "rds_cluster" { - source = "git::https://github.com/cloudposse/terraform-aws-rds-cluster.git?ref=tags/0.35.0" + source = "git::https://github.com/cloudposse/terraform-aws-rds-cluster.git?ref=tags/0.36.0" name = "${var.name_prefix}-db" engine = "aurora-postgresql" engine_mode = "serverless" diff --git a/infra/terraform/azure/README.md b/infra/terraform/azure/README.md new file mode 100644 index 00000000000..b22c870d12a --- /dev/null +++ b/infra/terraform/azure/README.md @@ -0,0 +1,36 @@ +# Terraform config for Feast on Azure + +This serves as a guide on how to deploy Feast on Azure. At the end of this guide, we will have provisioned: +1. AKS cluster +2. Feast services running on AKS +3. Azure Cache (Redis) as online store +4. Spark operator on AKS +5. Kafka running on HDInsight. + +# Steps + +1. Create a tfvars file, e.g. `my.tfvars`. A sample configuration is as below: + +``` +name_prefix = "feast09" +resource_group = "Feast" # pre-exisiting resource group +``` + +3. Configure tf state backend, e.g.: +``` +terraform { + backend "azurerm" { + storage_account_name = "" + container_name = "" + key = "" + } +} +``` + +3. Use `terraform apply -var-file="my.tfvars"` to deploy. + +Note: to get the list of Kafka brokers needed for streaming ingestion, use + +`curl -sS -u : -G https://.azurehdinsight.net/api/v1/clusters//services/KAFKA/components/KAFKA_BROKER | jq -r '["\(.host_components[].HostRoles.host_name):9092"] | join(",")'` + +where the Kafka gateway username is -kafka-gateway, the Kafka cluster name is -kafka, and the Kafka gateway password is a kubectl secret under the name feast-kafka-gateway. diff --git a/infra/terraform/azure/aks.tf b/infra/terraform/azure/aks.tf new file mode 100644 index 00000000000..c0899d49c99 --- /dev/null +++ b/infra/terraform/azure/aks.tf @@ -0,0 +1,15 @@ +resource "azurerm_kubernetes_cluster" "main" { + name = "${var.name_prefix}-aks" + location = data.azurerm_resource_group.main.location + resource_group_name = data.azurerm_resource_group.main.name + dns_prefix = var.name_prefix + default_node_pool { + name = var.name_prefix + vm_size = var.aks_machine_type + node_count = var.aks_node_count + vnet_subnet_id = azurerm_subnet.main.id + } + identity { + type = "SystemAssigned" + } +} diff --git a/infra/terraform/azure/helm.tf b/infra/terraform/azure/helm.tf new file mode 100644 index 00000000000..8c28762a438 --- /dev/null +++ b/infra/terraform/azure/helm.tf @@ -0,0 +1,101 @@ +locals { + feast_postgres_secret_name = "${var.name_prefix}-postgres-secret" + feast_helm_values = { + redis = { + enabled = false + } + + grafana = { + enabled = false + } + + kafka = { + enabled = false + } + + postgresql = { + existingSecret = local.feast_postgres_secret_name + } + + feast-core = { + postgresql = { + existingSecret = local.feast_postgres_secret_name + } + } + + feast-online-serving = { + enabled = true + "application-override.yaml" = { + feast = { + core-host = "${var.name_prefix}-feast-core" + core-grpc-port = 6565 + active_store = "online_store" + stores = [ + { + name = "online_store" + type = "REDIS" + config = { + host = azurerm_redis_cache.main.hostname + port = azurerm_redis_cache.main.ssl_port + ssl = true + } + } + ] + } + } + } + + feast-jupyter = { + enabled = true + envOverrides = { + feast_redis_host = azurerm_redis_cache.main.hostname, + feast_redis_port = azurerm_redis_cache.main.ssl_port, + feast_redis_ssl = true + feast_spark_launcher = "k8s" + feast_spark_staging_location = "wasbs://${azurerm_storage_container.staging.name}@${azurerm_storage_account.main.name}.blob.core.windows.net/artifacts/" + feast_historical_feature_output_location : "wasbs://${azurerm_storage_container.staging.name}@${azurerm_storage_account.main.name}.blob.core.windows.net/out/" + feast_historical_feature_output_format : "parquet" + demo_data_location : "wasbs://${azurerm_storage_container.staging.name}@${azurerm_storage_account.main.name}.blob.core.windows.net/test-data/" + feast_azure_blob_account_name = azurerm_storage_account.main.name + feast_azure_blob_account_access_key = azurerm_storage_account.main.primary_access_key + } + } + } +} + +resource "random_password" "feast-postgres-password" { + length = 16 + special = false +} + +resource "kubernetes_secret" "feast-postgres-secret" { + metadata { + name = local.feast_postgres_secret_name + } + data = { + postgresql-password = random_password.feast-postgres-password.result + } +} + +resource "helm_release" "feast" { + depends_on = [kubernetes_secret.feast-postgres-secret] + + name = var.name_prefix + namespace = var.aks_namespace + chart = "../../charts/feast" + + values = [ + yamlencode(local.feast_helm_values) + ] +} + +resource "helm_release" "sparkop" { + name = "sparkop" + namespace = "default" + repository = "https://googlecloudplatform.github.io/spark-on-k8s-operator" + chart = "spark-operator" + set { + name = "serviceAccounts.spark.name" + value = "spark" + } +} diff --git a/infra/terraform/azure/kafka.tf b/infra/terraform/azure/kafka.tf new file mode 100644 index 00000000000..a7403ff709b --- /dev/null +++ b/infra/terraform/azure/kafka.tf @@ -0,0 +1,75 @@ +resource "azurerm_hdinsight_kafka_cluster" "main" { + name = "${var.name_prefix}-kafka" + location = data.azurerm_resource_group.main.location + resource_group_name = data.azurerm_resource_group.main.name + cluster_version = "4.0" + tier = "Standard" + + component_version { + kafka = "2.1" + } + + gateway { + enabled = true + username = "${var.name_prefix}-kafka-gateway" + password = random_password.feast-kafka-gateway-password.result + } + + storage_account { + is_default = true + storage_account_key = azurerm_storage_account.main.primary_access_key + storage_container_id = azurerm_storage_container.kafka.id + } + + roles { + head_node { + vm_size = var.kafka_head_vm_size + username = "${var.name_prefix}-kafka-user" + password = random_password.feast-kafka-role-password.result + subnet_id = azurerm_subnet.kafka.id + virtual_network_id = azurerm_virtual_network.main.id + } + worker_node { + vm_size = var.kafka_worker_vm_size + username = "${var.name_prefix}-kafka-user" + password = random_password.feast-kafka-role-password.result + number_of_disks_per_node = var.kafka_worker_disks_per_node + target_instance_count = var.kafka_worker_target_instance_count + subnet_id = azurerm_subnet.kafka.id + virtual_network_id = azurerm_virtual_network.main.id + } + zookeeper_node { + vm_size = var.kafka_zookeeper_vm_size + username = "${var.name_prefix}-kafka-user" + password = random_password.feast-kafka-role-password.result + subnet_id = azurerm_subnet.kafka.id + virtual_network_id = azurerm_virtual_network.main.id + } + } +} + +resource "random_password" "feast-kafka-role-password" { + length = 16 + special = false + min_upper = 1 + min_lower = 1 + min_numeric = 1 +} + +resource "random_password" "feast-kafka-gateway-password" { + length = 16 + special = true + min_upper = 1 + min_lower = 1 + min_special = 1 + min_numeric = 1 +} + +resource "kubernetes_secret" "feast-kafka-gateway-secret" { + metadata { + name = "feast-kafka-gateway" + } + data = { + kafka-gateway-password = random_password.feast-kafka-gateway-password.result + } +} diff --git a/infra/terraform/azure/provider.tf b/infra/terraform/azure/provider.tf new file mode 100644 index 00000000000..916c10143fc --- /dev/null +++ b/infra/terraform/azure/provider.tf @@ -0,0 +1,28 @@ +provider "azurerm" { + version = "=2.40.0" + features {} +} + +provider "helm" { + version = "~> 1.3.2" + kubernetes { + host = azurerm_kubernetes_cluster.main.kube_config.0.host + username = azurerm_kubernetes_cluster.main.kube_config.0.username + password = azurerm_kubernetes_cluster.main.kube_config.0.password + client_certificate = base64decode(azurerm_kubernetes_cluster.main.kube_config.0.client_certificate) + client_key = base64decode(azurerm_kubernetes_cluster.main.kube_config.0.client_key) + cluster_ca_certificate = base64decode(azurerm_kubernetes_cluster.main.kube_config.0.cluster_ca_certificate) + load_config_file = false + } +} + +provider "kubernetes" { + version = "~> 1.13.3" + host = azurerm_kubernetes_cluster.main.kube_config.0.host + username = azurerm_kubernetes_cluster.main.kube_config.0.username + password = azurerm_kubernetes_cluster.main.kube_config.0.password + client_certificate = base64decode(azurerm_kubernetes_cluster.main.kube_config.0.client_certificate) + client_key = base64decode(azurerm_kubernetes_cluster.main.kube_config.0.client_key) + cluster_ca_certificate = base64decode(azurerm_kubernetes_cluster.main.kube_config.0.cluster_ca_certificate) + load_config_file = false +} diff --git a/infra/terraform/azure/redis.tf b/infra/terraform/azure/redis.tf new file mode 100644 index 00000000000..c6e85a4a0b8 --- /dev/null +++ b/infra/terraform/azure/redis.tf @@ -0,0 +1,12 @@ +resource "azurerm_redis_cache" "main" { + name = "${var.name_prefix}-redis" + location = data.azurerm_resource_group.main.location + resource_group_name = data.azurerm_resource_group.main.name + capacity = var.redis_capacity + family = "P" + sku_name = "Premium" + redis_configuration { + enable_authentication = false + } + subnet_id = azurerm_subnet.redis.id +} diff --git a/infra/terraform/azure/sparkop.tf b/infra/terraform/azure/sparkop.tf new file mode 100644 index 00000000000..e4aa8d7acab --- /dev/null +++ b/infra/terraform/azure/sparkop.tf @@ -0,0 +1,27 @@ +resource "kubernetes_role" "sparkop-user" { + metadata { + name = "use-spark-operator" + namespace = var.aks_namespace + } + rule { + api_groups = ["sparkoperator.k8s.io"] + resources = ["sparkapplications"] + verbs = ["create", "delete", "deletecollection", "get", "list", "update", "watch", "patch"] + } +} + +resource "kubernetes_role_binding" "sparkop-user" { + metadata { + name = "use-spark-operator" + namespace = var.aks_namespace + } + role_ref { + api_group = "rbac.authorization.k8s.io" + kind = "Role" + name = kubernetes_role.sparkop-user.metadata[0].name + } + subject { + kind = "ServiceAccount" + name = "default" + } +} diff --git a/infra/terraform/azure/storage.tf b/infra/terraform/azure/storage.tf new file mode 100644 index 00000000000..08db2386a49 --- /dev/null +++ b/infra/terraform/azure/storage.tf @@ -0,0 +1,21 @@ +resource "azurerm_storage_account" "main" { + name = "${var.name_prefix}storage" + resource_group_name = data.azurerm_resource_group.main.name + location = data.azurerm_resource_group.main.location + account_kind = "StorageV2" + account_tier = "Standard" + account_replication_type = var.storage_account_replication_type + allow_blob_public_access = true +} + +resource "azurerm_storage_container" "staging" { + name = "staging" + storage_account_name = azurerm_storage_account.main.name + container_access_type = "blob" +} + +resource "azurerm_storage_container" "kafka" { + name = "kafkastorage" + storage_account_name = azurerm_storage_account.main.name + container_access_type = "blob" +} diff --git a/infra/terraform/azure/variables.tf b/infra/terraform/azure/variables.tf new file mode 100644 index 00000000000..be4e7f2c19d --- /dev/null +++ b/infra/terraform/azure/variables.tf @@ -0,0 +1,57 @@ +variable "resource_group" { + type = string +} + +variable "name_prefix" { + type = string +} + +variable "aks_machine_type" { + type = string + default = "Standard_DS2_v2" +} + +variable "aks_node_count" { + type = number + default = 2 +} + +variable "redis_capacity" { + type = number + default = 2 +} + +variable "storage_account_replication_type" { + type = string + default = "LRS" +} + +variable "aks_namespace" { + type = string + default = "default" +} + +variable "kafka_head_vm_size" { + type = string + default = "Standard_DS3_v2" +} + +variable "kafka_worker_vm_size" { + type = string + default = "A5" +} + +variable "kafka_zookeeper_vm_size" { + type = string + default = "Standard_DS3_v2" +} + +variable "kafka_worker_disks_per_node" { + type = number + default = 3 +} + +variable "kafka_worker_target_instance_count" { + type = number + default = 3 +} diff --git a/infra/terraform/azure/vnet.tf b/infra/terraform/azure/vnet.tf new file mode 100644 index 00000000000..db790991e01 --- /dev/null +++ b/infra/terraform/azure/vnet.tf @@ -0,0 +1,31 @@ +data "azurerm_resource_group" "main" { + name = var.resource_group +} + +resource "azurerm_virtual_network" "main" { + name = "${var.name_prefix}-vnet" + location = data.azurerm_resource_group.main.location + resource_group_name = data.azurerm_resource_group.main.name + address_space = ["10.1.0.0/16"] +} + +resource "azurerm_subnet" "main" { + name = "${var.name_prefix}-aks-subnet" + resource_group_name = data.azurerm_resource_group.main.name + virtual_network_name = azurerm_virtual_network.main.name + address_prefixes = ["10.1.0.0/24"] +} + +resource "azurerm_subnet" "redis" { + name = "${var.name_prefix}-redis-subnet" + resource_group_name = data.azurerm_resource_group.main.name + virtual_network_name = azurerm_virtual_network.main.name + address_prefixes = ["10.1.128.0/24"] +} + +resource "azurerm_subnet" "kafka" { + name = "${var.name_prefix}-kafka-subnet" + resource_group_name = data.azurerm_resource_group.main.name + virtual_network_name = azurerm_virtual_network.main.name + address_prefixes = ["10.1.64.0/24"] +} diff --git a/infra/terraform/gcp/README.md b/infra/terraform/gcp/README.md new file mode 100644 index 00000000000..b9a738049de --- /dev/null +++ b/infra/terraform/gcp/README.md @@ -0,0 +1,35 @@ +# Terraform config for feast on GCP + +This serves as a guide on how to deploy Feast on GCP. At the end of this guide, we will have provisioned: +1. GKE cluster +2. Feast services running on GKE +3. Google Memorystore (Redis) as online store +4. Dataproc cluster +4. Kafka running on GKE, exposed to the dataproc cluster via internal load balancer. + +# Steps + +1. Create a tfvars file, e.g. `my.tfvars`. A sample configuration is as below: + +``` +gcp_project_name = "kf-feast" +name_prefix = "feast-0-8" +region = "asia-east1" +gke_machine_type = "n1-standard-2" +network = "default" +subnetwork = "default" +dataproc_staging_bucket = "kf-feast-dataproc-staging-test" +``` + +3. Configure tf state backend, e.g.: +``` +terraform { + backend "gcs" { + bucket = "" + prefix = "terraform/feast" + } +} +``` + +3. Use `terraform apply -var-file="my.tfvars"` to deploy. + diff --git a/infra/terraform/gcp/dataproc.tf b/infra/terraform/gcp/dataproc.tf new file mode 100644 index 00000000000..b9cc26af58d --- /dev/null +++ b/infra/terraform/gcp/dataproc.tf @@ -0,0 +1,68 @@ +resource "google_storage_bucket" "dataproc_staging_bucket" { + name = var.dataproc_staging_bucket + project = var.gcp_project_name + location = var.region + force_destroy = true +} + +resource "google_dataproc_autoscaling_policy" "feast_dataproc_cluster_asp" { + policy_id = var.name_prefix + location = var.region + project = var.gcp_project_name + + worker_config { + min_instances = var.min_dataproc_worker_count + max_instances = var.max_dataproc_worker_count + } + + basic_algorithm { + yarn_config { + graceful_decommission_timeout = "3600s" + scale_down_factor = 0.5 + scale_up_factor = 0.5 + } + } +} + +resource "google_dataproc_cluster" "feast_dataproc_cluster" { + project = var.gcp_project_name + name = var.name_prefix + region = var.region + + cluster_config { + staging_bucket = google_storage_bucket.dataproc_staging_bucket.name + + master_config { + num_instances = 1 + machine_type = var.dataproc_master_instance_type + disk_config { + boot_disk_type = var.dataproc_master_disk_type + boot_disk_size_gb = var.dataproc_master_disk_size + } + } + + worker_config { + num_instances = var.min_dataproc_worker_count + machine_type = var.dataproc_worker_instance_type + disk_config { + boot_disk_type = var.dataproc_worker_disk_type + boot_disk_size_gb = var.dataproc_worker_disk_size + } + } + + gce_cluster_config { + subnetwork = var.subnetwork + service_account = google_service_account.feast_sa.email + + internal_ip_only = true + } + + software_config { + image_version = var.dataproc_image_version + } + + autoscaling_config { + policy_uri = google_dataproc_autoscaling_policy.feast_dataproc_cluster_asp.name + } + } +} diff --git a/infra/terraform/gcp/feast.tf b/infra/terraform/gcp/feast.tf new file mode 100644 index 00000000000..e528cd15eef --- /dev/null +++ b/infra/terraform/gcp/feast.tf @@ -0,0 +1,121 @@ +locals { + feast_postgres_secret_name = "${var.name_prefix}-postgres-secret" + feast_helm_values = { + redis = { + enabled = false + } + + kafka = { + enabled = true + externalAccess = { + enabled = true + service = { + types = "LoadBalancer" + port = 9094 + loadBalancerIPs = [google_compute_address.kafka_broker.address] + loadBalancerSourceRanges = ["10.0.0.0/8"] + annotations = { + "cloud.google.com/load-balancer-type" = "Internal" + } + } + } + } + + grafana = { + enabled = false + } + + + postgresql = { + existingSecret = local.feast_postgres_secret_name + } + + feast-core = { + postgresql = { + existingSecret = local.feast_postgres_secret_name + } + } + + feast-online-serving = { + enabled = true + "application-override.yaml" = { + feast = { + core-host = "${var.name_prefix}-feast-core" + core-grpc-port = 6565 + active_store = "online_store" + stores = [ + { + name = "online_store" + type = "REDIS" + config = { + host = google_redis_instance.online_store.host + port = 6379 + subscriptions = [ + { + name = "*" + project = "*" + } + ] + } + } + ] + } + } + } + + feast-jupyter = { + enabled = true + envOverrides = { + feast_redis_host = google_redis_instance.online_store.host, + feast_redis_port = 6379, + feast_spark_launcher = "dataproc" + feast_dataproc_cluster_name = google_dataproc_cluster.feast_dataproc_cluster.name + feast_dataproc_project = var.gcp_project_name + feast_dataproc_region = var.region + feast_spark_staging_location = "gs://${var.dataproc_staging_bucket}/artifacts/" + feast_historical_feature_output_location : "gs://${var.dataproc_staging_bucket}/out/" + feast_historical_feature_output_format : "parquet" + demo_kafka_brokers : "${google_compute_address.kafka_broker.address}:9094" + demo_data_location : "gs://${var.dataproc_staging_bucket}/test-data/" + } + gcpServiceAccount = { + enabled = true + name = var.feast_sa_secret_name + key = "credentials.json" + } + } + } +} + +resource "random_password" "feast-postgres-password" { + length = 16 + special = false +} + +resource "kubernetes_secret" "feast-postgres-secret" { + metadata { + name = local.feast_postgres_secret_name + } + data = { + postgresql-password = random_password.feast-postgres-password.result + } +} + +resource "google_compute_address" "kafka_broker" { + project = var.gcp_project_name + region = var.region + subnetwork = var.subnetwork + name = "${var.name_prefix}-kafka" + address_type = "INTERNAL" +} + +resource "helm_release" "feast" { + depends_on = [kubernetes_secret.feast-postgres-secret, kubernetes_secret.feast_sa_secret] + + name = var.name_prefix + chart = "../../charts/feast" + + values = [ + yamlencode(local.feast_helm_values) + ] +} diff --git a/infra/terraform/gcp/gke.tf b/infra/terraform/gcp/gke.tf new file mode 100644 index 00000000000..5dc4a02d077 --- /dev/null +++ b/infra/terraform/gcp/gke.tf @@ -0,0 +1,19 @@ +resource "google_container_cluster" "feast_gke_cluster" { + name = "${var.name_prefix}-cluster" + location = var.region + network = var.network + subnetwork = var.subnetwork + + initial_node_count = var.gke_node_count + node_config { + machine_type = var.gke_machine_type + } + + ip_allocation_policy { + } +} + +data "google_container_cluster" "feast_gke_cluster" { + location = var.region + name = google_container_cluster.feast_gke_cluster.name +} diff --git a/infra/terraform/gcp/iam.tf b/infra/terraform/gcp/iam.tf new file mode 100644 index 00000000000..fb0dd38ef9d --- /dev/null +++ b/infra/terraform/gcp/iam.tf @@ -0,0 +1,37 @@ +resource "google_service_account" "feast_sa" { + account_id = var.name_prefix + display_name = var.name_prefix + project = var.gcp_project_name +} + +resource "google_service_account_key" "feast_sa" { + service_account_id = google_service_account.feast_sa.name +} + +resource "google_project_iam_member" "feast_dataproc_worker" { + project = var.gcp_project_name + role = "roles/dataproc.worker" + member = "serviceAccount:${google_service_account.feast_sa.email}" +} + +resource "google_project_iam_member" "feast_dataproc_editor" { + project = var.gcp_project_name + role = "roles/dataproc.editor" + member = "serviceAccount:${google_service_account.feast_sa.email}" +} + +resource "google_project_iam_member" "feast_batch_ingestion_storage" { + project = var.gcp_project_name + role = "roles/storage.admin" + member = "serviceAccount:${google_service_account.feast_sa.email}" +} + +resource "kubernetes_secret" "feast_sa_secret" { + metadata { + name = var.feast_sa_secret_name + } + data = { + "credentials.json" = base64decode(google_service_account_key.feast_sa.private_key) + } +} + diff --git a/infra/terraform/gcp/online_store.tf b/infra/terraform/gcp/online_store.tf new file mode 100644 index 00000000000..5628eff7224 --- /dev/null +++ b/infra/terraform/gcp/online_store.tf @@ -0,0 +1,18 @@ +resource "google_redis_instance" "online_store" { + project = var.gcp_project_name + region = var.region + name = "${var.name_prefix}-online-store" + tier = var.redis_tier + memory_size_gb = var.redis_memory_size_gb + + authorized_network = data.google_compute_network.redis-network.id + + redis_version = "REDIS_5_0" + display_name = "Feast Online Store" + +} + +data "google_compute_network" "redis-network" { + project = var.gcp_project_name + name = var.network +} diff --git a/infra/terraform/gcp/provider.tf b/infra/terraform/gcp/provider.tf new file mode 100644 index 00000000000..77c77974b75 --- /dev/null +++ b/infra/terraform/gcp/provider.tf @@ -0,0 +1,27 @@ +provider "google" { + version = "~> 3.46" + project = var.gcp_project_name +} + +data "google_client_config" "gcp_client" { + provider = google +} + +provider "kubernetes" { + version = "~> 1.13.3" + + host = google_container_cluster.feast_gke_cluster.endpoint + token = data.google_client_config.gcp_client.access_token + cluster_ca_certificate = base64decode(google_container_cluster.feast_gke_cluster.master_auth.0.cluster_ca_certificate) + load_config_file = false +} + +provider "helm" { + version = "~> 1.3.2" + kubernetes { + host = google_container_cluster.feast_gke_cluster.endpoint + token = data.google_client_config.gcp_client.access_token + cluster_ca_certificate = base64decode(google_container_cluster.feast_gke_cluster.master_auth.0.cluster_ca_certificate) + load_config_file = false + } +} diff --git a/infra/terraform/gcp/variables.tf b/infra/terraform/gcp/variables.tf new file mode 100644 index 00000000000..c39046bf18a --- /dev/null +++ b/infra/terraform/gcp/variables.tf @@ -0,0 +1,103 @@ +variable "gcp_project_name" { + description = "GCP project name" +} + +variable "name_prefix" { + description = "Prefix to be used when naming the different components of Feast" +} + +variable "region" { + description = "Region for GKE and Dataproc cluster" +} + +variable "gke_machine_type" { + description = "GKE node pool machine type" + default = "n1-standard-4" +} + +variable "gke_node_count" { + description = "Number of nodes in the GKE default node pool" + default = 1 +} + +variable "gke_disk_size_gb" { + description = "Disk size for nodes in the GKE default node pool" + default = 100 +} + +variable "gke_disk_type" { + description = "Disk type for nodes in the GKE default node pool" + default = "pd-standard" +} + +variable "network" { + description = "Network for GKE and Dataproc cluster" +} + +variable "subnetwork" { + description = "Subnetwork for GKE and Dataproc cluster" +} + +variable "dataproc_staging_bucket" { + description = "GCS bucket for staging temporary files required for dataproc jobs" +} + +variable "min_dataproc_worker_count" { + description = "Minimum dataproc worker count" + default = 2 +} + +variable "max_dataproc_worker_count" { + description = "Maximum dataproc worker count" + default = 4 +} + +variable "dataproc_master_instance_type" { + description = "Machine type for dataproc cluster master" + default = "n1-standard-2" +} + +variable "dataproc_master_disk_type" { + description = "Disk type for dataproc cluster master" + default = "pd-standard" +} + +variable "dataproc_master_disk_size" { + description = "Disk size for dataproc cluster master" + default = 100 +} + +variable "dataproc_worker_instance_type" { + description = "Machine type for dataproc cluster worker" + default = "n1-standard-2" +} + +variable "dataproc_worker_disk_type" { + description = "Disk type for dataproc cluster worker" + default = "pd-standard" +} + +variable "dataproc_worker_disk_size" { + description = "Disk size for dataproc cluster worker" + default = 100 +} + +variable "dataproc_image_version" { + description = "Dataproc image version" + default = "1.5-debian10" +} + +variable "redis_tier" { + description = "GCP Redis instance tier" + default = "BASIC" +} + +variable "redis_memory_size_gb" { + description = "Redis memory size in Gb" + default = 2 +} + +variable "feast_sa_secret_name" { + description = "Kubernetes secret name for Feast GCP service account" + default = "feast-gcp-service-account" +} diff --git a/ingestion/.gitignore b/ingestion/.gitignore deleted file mode 100644 index 6747837bae0..00000000000 --- a/ingestion/.gitignore +++ /dev/null @@ -1 +0,0 @@ -example/output diff --git a/ingestion/pom.xml b/ingestion/pom.xml deleted file mode 100644 index bab8c17fb69..00000000000 --- a/ingestion/pom.xml +++ /dev/null @@ -1,237 +0,0 @@ - - - - 4.0.0 - - - dev.feast - feast-parent - ${revision} - - - Feast Ingestion - feast-ingestion - - - - - org.apache.maven.plugins - maven-shade-plugin - 3.2.1 - - - package - - shade - - - - - - feast.ingestion.ImportJobOld - - - reference.conf - - - - - - io.opencensus - io.opencensus.vendor - - - com.google.cloud.bigquery - com.google.cloud.bigquery.vendor - - - - - *:* - - META-INF/*.SF - META-INF/*.DSA - META-INF/*.RSA - - - - - - - - - - org.apache.maven.plugins - maven-enforcer-plugin - - - - enforce-bytecode-version - - enforce - - - - - 1.8 - - - - - - - - - org.jacoco - jacoco-maven-plugin - - - - - - - dev.feast - datatypes-java - ${project.version} - - - - dev.feast - feast-storage-api - ${project.version} - - - - dev.feast - feast-storage-connector-redis - ${project.version} - - - - dev.feast - feast-storage-connector-bigquery - ${project.version} - - - - dev.feast - feast-common - ${project.version} - - - - dev.feast - feast-common-test - ${project.version} - test - - - - com.google.auto.value - auto-value-annotations - 1.6.6 - - - - com.google.cloud - google-cloud-bigquery - - - - com.google.protobuf - protobuf-java - - - com.google.protobuf - protobuf-java-util - - - - org.apache.kafka - kafka-clients - - - - joda-time - joda-time - - - - org.apache.beam - beam-runners-google-cloud-dataflow-java - ${org.apache.beam.version} - - - - org.apache.beam - beam-runners-direct-java - ${org.apache.beam.version} - - - - org.apache.beam - beam-sdks-java-io-kafka - ${org.apache.beam.version} - - - - io.lettuce - lettuce-core - - - - org.slf4j - slf4j-api - - - - org.slf4j - slf4j-simple - 1.7.30 - test - - - - com.google.guava - guava - - - com.datadoghq - java-dogstatsd-client - 2.8.1 - - - - - org.apache.commons - commons-math3 - 3.6.1 - - - - diff --git a/ingestion/src/main/java/feast/ingestion/ImportJob.java b/ingestion/src/main/java/feast/ingestion/ImportJob.java deleted file mode 100644 index e51d05608aa..00000000000 --- a/ingestion/src/main/java/feast/ingestion/ImportJob.java +++ /dev/null @@ -1,225 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2019 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.ingestion; - -import static feast.ingestion.utils.StoreUtil.getFeatureSink; - -import com.google.protobuf.InvalidProtocolBufferException; -import feast.common.models.FeatureSetReference; -import feast.ingestion.options.ImportOptions; -import feast.ingestion.transform.FeatureRowToStoreAllocator; -import feast.ingestion.transform.ProcessAndValidateFeatureRows; -import feast.ingestion.transform.ReadFromSource; -import feast.ingestion.transform.metrics.WriteFailureMetricsTransform; -import feast.ingestion.transform.metrics.WriteInflightMetricsTransform; -import feast.ingestion.transform.metrics.WriteSuccessMetricsTransform; -import feast.ingestion.transform.specs.ReadFeatureSetSpecs; -import feast.ingestion.transform.specs.WriteFeatureSetSpecAck; -import feast.ingestion.utils.SpecUtil; -import feast.proto.core.FeatureSetProto.FeatureSetSpec; -import feast.proto.core.IngestionJobProto.SpecsStreamingUpdateConfig; -import feast.proto.core.SourceProto.Source; -import feast.proto.core.StoreProto.Store; -import feast.proto.types.FeatureRowProto.FeatureRow; -import feast.storage.api.writer.DeadletterSink; -import feast.storage.api.writer.FailedElement; -import feast.storage.api.writer.FeatureSink; -import feast.storage.api.writer.WriteResult; -import feast.storage.connectors.bigquery.writer.BigQueryDeadletterSink; -import java.io.IOException; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; -import org.apache.beam.sdk.Pipeline; -import org.apache.beam.sdk.PipelineResult; -import org.apache.beam.sdk.extensions.protobuf.ProtoCoder; -import org.apache.beam.sdk.options.PipelineOptionsFactory; -import org.apache.beam.sdk.options.PipelineOptionsValidator; -import org.apache.beam.sdk.transforms.*; -import org.apache.beam.sdk.values.*; -import org.apache.commons.lang3.tuple.Pair; -import org.slf4j.Logger; - -public class ImportJob { - - // Tag for main output containing Feature Row that has been successfully processed. - private static final TupleTag FEATURE_ROW_OUT = new TupleTag() {}; - - // Tag for deadletter output containing elements and error messages from invalid input/transform. - private static final TupleTag DEADLETTER_OUT = new TupleTag() {}; - private static final Logger log = org.slf4j.LoggerFactory.getLogger(ImportJob.class); - - /** - * @param args arguments to be passed to Beam pipeline - * @throws InvalidProtocolBufferException if options passed to the pipeline are invalid - */ - public static void main(String[] args) throws IOException { - ImportOptions options = - PipelineOptionsFactory.fromArgs(args).withValidation().create().as(ImportOptions.class); - runPipeline(options); - } - - @SuppressWarnings("UnusedReturnValue") - public static PipelineResult runPipeline(ImportOptions options) throws IOException { - /* - * Steps: - * 1. Read FeatureSetSpec messages from kafka - * 2. Read messages from Feast Source as FeatureRow - * 3. Validate the feature rows to ensure the schema matches what is registered to the system - * 4. Distribute rows across stores by subscription - * 5. Write FeatureRow to the corresponding Store - * 6. Write elements that failed to be processed to a dead letter queue. - * 7. Write metrics to a metrics sink - * 8. Send ack on receiving FeatureSetSpec - */ - - PipelineOptionsValidator.validate(ImportOptions.class, options); - Pipeline pipeline = Pipeline.create(options); - - log.info("Starting import job with settings: \n{}", options.toString()); - - List stores = SpecUtil.parseStoreJsonList(options.getStoresJson()); - Source source = SpecUtil.parseSourceJson(options.getSourceJson()); - SpecsStreamingUpdateConfig specsStreamingUpdateConfig = - SpecUtil.parseSpecsStreamingUpdateConfig(options.getSpecsStreamingUpdateConfigJson()); - - // Step 1. Read FeatureSetSpecs from Spec source - PCollection> featureSetSpecs = - pipeline.apply( - "ReadFeatureSetSpecs", - ReadFeatureSetSpecs.newBuilder() - .setSource(source) - .setStores(stores) - .setSpecsStreamingUpdateConfig(specsStreamingUpdateConfig) - .build()); - - PCollectionView>> globalSpecView = - featureSetSpecs - .apply(MapElements.via(new ReferenceToString())) - .apply("GlobalSpecView", View.asMultimap()); - - // Step 2. Read messages from Feast Source as FeatureRow. - PCollectionTuple convertedFeatureRows = - pipeline.apply( - "ReadFeatureRowFromSource", - ReadFromSource.newBuilder() - .setSource(source) - .setSuccessTag(FEATURE_ROW_OUT) - .setFailureTag(DEADLETTER_OUT) - .setKafkaConsumerProperties( - options.getKafkaConsumerProperties() == null - ? new HashMap<>() - : options.getKafkaConsumerProperties()) - .build()); - - // Step 3. Process and validate incoming FeatureRows - PCollectionTuple validatedRows = - convertedFeatureRows - .get(FEATURE_ROW_OUT) - .apply( - ProcessAndValidateFeatureRows.newBuilder() - .setDefaultProject(options.getDefaultFeastProject()) - .setFeatureSetSpecs(globalSpecView) - .setSuccessTag(FEATURE_ROW_OUT) - .setFailureTag(DEADLETTER_OUT) - .build()); - - Map> storeTags = - stores.stream() - .map(s -> Pair.of(s, new TupleTag())) - .collect(Collectors.toMap(Pair::getLeft, Pair::getRight)); - - // Step 4. Allocate validated rows to stores by store subscription - PCollectionTuple storeAllocatedRows = - validatedRows - .get(FEATURE_ROW_OUT) - .apply( - FeatureRowToStoreAllocator.newBuilder() - .setStores(stores) - .setStoreTags(storeTags) - .build()); - - PCollectionList sinkReadiness = PCollectionList.empty(pipeline); - - for (Store store : stores) { - FeatureSink featureSink = getFeatureSink(store); - - sinkReadiness = sinkReadiness.and(featureSink.prepareWrite(featureSetSpecs)); - PCollection rowsForStore = - storeAllocatedRows.get(storeTags.get(store)).setCoder(ProtoCoder.of(FeatureRow.class)); - - // Step 5. Write metrics of successfully validated rows - rowsForStore.apply( - "WriteInflightMetrics", WriteInflightMetricsTransform.create(store.getName())); - - // Step 6. Write FeatureRow to the corresponding Store. - WriteResult writeFeatureRows = - rowsForStore.apply("WriteFeatureRowToStore", featureSink.writer()); - - // Step 7. Write FailedElements to a dead letter table in BigQuery. - if (options.getDeadLetterTableSpec() != null) { - // TODO: make deadletter destination type configurable - DeadletterSink deadletterSink = - new BigQueryDeadletterSink(options.getDeadLetterTableSpec()); - - writeFeatureRows - .getFailedInserts() - .apply("WriteFailedElements_WriteFeatureRowToStore", deadletterSink.write()); - } - - // Step 8. Write metrics to a metrics sink. - writeFeatureRows - .getSuccessfulInserts() - .apply("WriteSuccessMetrics", WriteSuccessMetricsTransform.create(store.getName())); - - writeFeatureRows - .getFailedInserts() - .apply("WriteFailureMetrics", WriteFailureMetricsTransform.create(store.getName())); - } - - if (options.getDeadLetterTableSpec() != null) { - DeadletterSink deadletterSink = new BigQueryDeadletterSink(options.getDeadLetterTableSpec()); - - convertedFeatureRows - .get(DEADLETTER_OUT) - .apply("WriteFailedElements_ReadFromSource", deadletterSink.write()); - - validatedRows - .get(DEADLETTER_OUT) - .apply("WriteFailedElements_ValidateRows", deadletterSink.write()); - } - - sinkReadiness - .apply(Flatten.pCollections()) - .apply( - "WriteAck", - WriteFeatureSetSpecAck.newBuilder() - .setSinksCount(stores.size()) - .setSpecsStreamingUpdateConfig(specsStreamingUpdateConfig) - .build()); - - return pipeline.run(); - } - - private static class ReferenceToString - extends SimpleFunction, KV> { - public KV apply(KV input) { - return KV.of(input.getKey().getReference(), input.getValue()); - } - } -} diff --git a/ingestion/src/main/java/feast/ingestion/coders/FailsafeFeatureRowCoder.java b/ingestion/src/main/java/feast/ingestion/coders/FailsafeFeatureRowCoder.java deleted file mode 100644 index 6985941ad33..00000000000 --- a/ingestion/src/main/java/feast/ingestion/coders/FailsafeFeatureRowCoder.java +++ /dev/null @@ -1,109 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2019 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.ingestion.coders; - -import feast.ingestion.values.FailsafeFeatureRow; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.util.Arrays; -import java.util.List; -import org.apache.beam.sdk.coders.Coder; -import org.apache.beam.sdk.coders.CoderException; -import org.apache.beam.sdk.coders.CustomCoder; -import org.apache.beam.sdk.coders.NullableCoder; -import org.apache.beam.sdk.coders.StringUtf8Coder; -import org.apache.beam.sdk.values.TypeDescriptor; -import org.apache.beam.sdk.values.TypeParameter; - -/** - * Adapted from: - * https://github.com/GoogleCloudPlatform/DataflowTemplates/blob/master/src/main/java/com/google/cloud/teleport/coders/FailsafeElementCoder.java - * - *

The {@link FailsafeFeatureRowCoder} encodes and decodes {@link FailsafeFeatureRow} objects. - * - *

This coder is necessary until Avro supports parameterized types (AVRO-1571) without requiring to - * explicitly specifying the schema for the type. - * - * @param The type of the original payload to be encoded. - * @param The type of the current payload to be encoded. - */ -public class FailsafeFeatureRowCoder - extends CustomCoder> { - - private static final NullableCoder STRING_CODER = NullableCoder.of(StringUtf8Coder.of()); - private final Coder originalPayloadCoder; - private final Coder currentPayloadCoder; - - private FailsafeFeatureRowCoder( - Coder originalPayloadCoder, Coder currentPayloadCoder) { - this.originalPayloadCoder = originalPayloadCoder; - this.currentPayloadCoder = currentPayloadCoder; - } - - public Coder getOriginalPayloadCoder() { - return originalPayloadCoder; - } - - public Coder getCurrentPayloadCoder() { - return currentPayloadCoder; - } - - public static FailsafeFeatureRowCoder of( - Coder originalPayloadCoder, Coder currentPayloadCoder) { - return new FailsafeFeatureRowCoder<>(originalPayloadCoder, currentPayloadCoder); - } - - @Override - public void encode(FailsafeFeatureRow value, OutputStream outStream) - throws IOException { - if (value == null) { - throw new CoderException("The FailsafeFeatureRowCoder cannot encode a null object!"); - } - - originalPayloadCoder.encode(value.getOriginalPayload(), outStream); - currentPayloadCoder.encode(value.getPayload(), outStream); - STRING_CODER.encode(value.getErrorMessage(), outStream); - STRING_CODER.encode(value.getStacktrace(), outStream); - } - - @Override - public FailsafeFeatureRow decode(InputStream inStream) throws IOException { - - OriginalT originalPayload = originalPayloadCoder.decode(inStream); - CurrentT currentPayload = currentPayloadCoder.decode(inStream); - String errorMessage = STRING_CODER.decode(inStream); - String stacktrace = STRING_CODER.decode(inStream); - - return FailsafeFeatureRow.of(originalPayload, currentPayload) - .setErrorMessage(errorMessage) - .setStacktrace(stacktrace); - } - - @Override - public List> getCoderArguments() { - return Arrays.asList(originalPayloadCoder, currentPayloadCoder); - } - - @Override - public TypeDescriptor> getEncodedTypeDescriptor() { - return new TypeDescriptor>() {}.where( - new TypeParameter() {}, originalPayloadCoder.getEncodedTypeDescriptor()) - .where(new TypeParameter() {}, currentPayloadCoder.getEncodedTypeDescriptor()); - } -} diff --git a/ingestion/src/main/java/feast/ingestion/options/BZip2Compressor.java b/ingestion/src/main/java/feast/ingestion/options/BZip2Compressor.java deleted file mode 100644 index b7e4e6ee0af..00000000000 --- a/ingestion/src/main/java/feast/ingestion/options/BZip2Compressor.java +++ /dev/null @@ -1,47 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.ingestion.options; - -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import org.apache.commons.compress.compressors.bzip2.BZip2CompressorOutputStream; - -public class BZip2Compressor implements OptionCompressor { - - private final OptionByteConverter byteConverter; - - public BZip2Compressor(OptionByteConverter byteConverter) { - this.byteConverter = byteConverter; - } - /** - * Compress pipeline option using BZip2 - * - * @param option Pipeline option value - * @return BZip2 compressed option value - * @throws IOException - */ - @Override - public byte[] compress(T option) throws IOException { - ByteArrayOutputStream compressedStream = new ByteArrayOutputStream(); - try (BZip2CompressorOutputStream bzip2Output = - new BZip2CompressorOutputStream(compressedStream)) { - bzip2Output.write(byteConverter.toByte(option)); - } - - return compressedStream.toByteArray(); - } -} diff --git a/ingestion/src/main/java/feast/ingestion/options/BZip2Decompressor.java b/ingestion/src/main/java/feast/ingestion/options/BZip2Decompressor.java deleted file mode 100644 index ce49c1be6e6..00000000000 --- a/ingestion/src/main/java/feast/ingestion/options/BZip2Decompressor.java +++ /dev/null @@ -1,38 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.ingestion.options; - -import java.io.ByteArrayInputStream; -import java.io.IOException; -import org.apache.commons.compress.compressors.bzip2.BZip2CompressorInputStream; - -public class BZip2Decompressor implements OptionDecompressor { - - private final InputStreamConverter inputStreamConverter; - - public BZip2Decompressor(InputStreamConverter inputStreamConverter) { - this.inputStreamConverter = inputStreamConverter; - } - - @Override - public T decompress(byte[] compressed) throws IOException { - try (ByteArrayInputStream inputStream = new ByteArrayInputStream(compressed); - BZip2CompressorInputStream bzip2Input = new BZip2CompressorInputStream(inputStream)) { - return inputStreamConverter.readStream(bzip2Input); - } - } -} diff --git a/ingestion/src/main/java/feast/ingestion/options/ImportOptions.java b/ingestion/src/main/java/feast/ingestion/options/ImportOptions.java deleted file mode 100644 index 7416ee01f78..00000000000 --- a/ingestion/src/main/java/feast/ingestion/options/ImportOptions.java +++ /dev/null @@ -1,122 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2019 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.ingestion.options; - -import java.util.List; -import java.util.Map; -import org.apache.beam.runners.dataflow.options.DataflowPipelineOptions; -import org.apache.beam.runners.direct.DirectOptions; -import org.apache.beam.sdk.options.Default; -import org.apache.beam.sdk.options.Description; -import org.apache.beam.sdk.options.PipelineOptions; -import org.apache.beam.sdk.options.Validation.Required; - -/** Options passed to Beam to influence the job's execution environment */ -public interface ImportOptions extends PipelineOptions, DataflowPipelineOptions, DirectOptions { - - @Required - @Description( - "Default feast project to apply to incoming rows that do not specify project in its feature set reference.") - String getDefaultFeastProject(); - - void setDefaultFeastProject(String defaultProject); - - @Required - @Description( - "JSON string representation of the SpecsPipe configuration." - + "Job will use this to know where read new FeatureSetSpec from (kafka broker & topic)" - + "and where send acknowledgment on successful update of job's state to." - + "SpecsStreamingUpdateConfig follows the format in feast.core.IngestionJob.SpecsStreamingUpdateConfig proto." - + "The conversion of Proto message to JSON should follow this mapping:" - + "https://developers.google.com/protocol-buffers/docs/proto3#json" - + "Please minify and remove all insignificant whitespace such as newline in the JSON string" - + "to prevent error when parsing the options") - String getSpecsStreamingUpdateConfigJson(); - - void setSpecsStreamingUpdateConfigJson(String json); - - @Required - @Description( - "JSON string representation of the Source that will be used to read FeatureRows from." - + "Source follows the format in featst.core.Source proto. Currently only kafka source is supported" - + "The conversion of Proto message to JSON should follow this mapping:" - + "https://developers.google.com/protocol-buffers/docs/proto3#json" - + "Please minify and remove all insignificant whitespace such as newline in the JSON string" - + "to prevent error when parsing the options") - String getSourceJson(); - - void setSourceJson(String json); - - @Required - @Description( - "JSON string representation of the Store that import job will write FeatureRow to." - + "Store follows the format in feast.core.Store proto." - + "Multiple Store can be passed by specifying '--store={...}' multiple times" - + "The conversion of Proto message to JSON should follow this mapping:" - + "https://developers.google.com/protocol-buffers/docs/proto3#json" - + "Please minify and remove all insignificant whitespace such as newline in the JSON string" - + "to prevent error when parsing the options") - List getStoresJson(); - - void setStoresJson(List storeJson); - - @Description("Properties Map for Kafka Consumer used to pull FeatureRows") - Map getKafkaConsumerProperties(); - - void setKafkaConsumerProperties(Map kafkaConsumerProperties); - - @Description( - "(Optional) Deadletter elements will be written to this BigQuery table." - + "Table spec must follow this format PROJECT_ID:DATASET_ID.PROJECT_ID" - + "The table will be created if not exists.") - String getDeadLetterTableSpec(); - - /** - * @param deadLetterTableSpec (Optional) BigQuery table for storing elements that failed to be - * processed. Table spec must follow this format PROJECT_ID:DATASET_ID.PROJECT_ID - */ - void setDeadLetterTableSpec(String deadLetterTableSpec); - - @Description("MetricsAccumulator exporter type to instantiate. Supported type: statsd") - @Default.String("none") - String getMetricsExporterType(); - - void setMetricsExporterType(String metricsExporterType); - - @Description("Host to write the metrics to. Required if the metrics exporter is set to StatsD.") - @Default.String("localhost") - String getStatsdHost(); - - void setStatsdHost(String StatsdHost); - - @Description( - "Port on StatsD server to write metrics to. Required if the metrics exporter is set to StatsD.") - @Default.Integer(8125) - int getStatsdPort(); - - void setStatsdPort(int StatsdPort); - - @Description( - "Fixed window size in seconds (default 60) to apply before aggregating the numerical value of " - + "features and exporting the aggregated values as metrics. Refer to " - + "feast/ingestion/transform/metrics/WriteFeatureValueMetricsDoFn.java" - + "for the metric nameas and types used.") - @Default.Integer(60) - int getWindowSizeInSecForFeatureValueMetric(); - - void setWindowSizeInSecForFeatureValueMetric(int seconds); -} diff --git a/ingestion/src/main/java/feast/ingestion/options/OptionByteConverter.java b/ingestion/src/main/java/feast/ingestion/options/OptionByteConverter.java deleted file mode 100644 index ff5a41a627d..00000000000 --- a/ingestion/src/main/java/feast/ingestion/options/OptionByteConverter.java +++ /dev/null @@ -1,30 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.ingestion.options; - -import java.io.IOException; - -public interface OptionByteConverter { - - /** - * Used in conjunction with {@link OptionCompressor} to compress the pipeline option - * - * @param option Pipeline option value - * @return byte representation of the pipeline option value, without compression. - */ - byte[] toByte(T option) throws IOException; -} diff --git a/ingestion/src/main/java/feast/ingestion/options/OptionDecompressor.java b/ingestion/src/main/java/feast/ingestion/options/OptionDecompressor.java deleted file mode 100644 index affeafdaa0b..00000000000 --- a/ingestion/src/main/java/feast/ingestion/options/OptionDecompressor.java +++ /dev/null @@ -1,30 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.ingestion.options; - -import java.io.IOException; - -public interface OptionDecompressor { - - /** - * Decompress pipeline option from byte array. - * - * @param compressed Compressed pipeline option value - * @return Decompressed pipeline option - */ - T decompress(byte[] compressed) throws IOException; -} diff --git a/ingestion/src/main/java/feast/ingestion/options/StringListStreamConverter.java b/ingestion/src/main/java/feast/ingestion/options/StringListStreamConverter.java deleted file mode 100644 index d7277f3c7d6..00000000000 --- a/ingestion/src/main/java/feast/ingestion/options/StringListStreamConverter.java +++ /dev/null @@ -1,41 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.ingestion.options; - -import java.io.BufferedReader; -import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.util.List; -import java.util.stream.Collectors; - -public class StringListStreamConverter implements InputStreamConverter> { - - /** - * Convert Input byte stream to newline separated strings - * - * @param inputStream Input byte stream - * @return List of string - */ - @Override - public List readStream(InputStream inputStream) throws IOException { - BufferedReader reader = new BufferedReader(new InputStreamReader(inputStream)); - List stringList = reader.lines().collect(Collectors.toList()); - reader.close(); - return stringList; - } -} diff --git a/ingestion/src/main/java/feast/ingestion/transform/FeatureRowToStoreAllocator.java b/ingestion/src/main/java/feast/ingestion/transform/FeatureRowToStoreAllocator.java deleted file mode 100644 index aa101203e08..00000000000 --- a/ingestion/src/main/java/feast/ingestion/transform/FeatureRowToStoreAllocator.java +++ /dev/null @@ -1,93 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.ingestion.transform; - -import static feast.ingestion.utils.SpecUtil.parseFeatureSetReference; - -import com.google.auto.value.AutoValue; -import feast.common.models.Store; -import feast.proto.core.StoreProto; -import feast.proto.types.FeatureRowProto; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; -import org.apache.beam.sdk.transforms.DoFn; -import org.apache.beam.sdk.transforms.PTransform; -import org.apache.beam.sdk.transforms.ParDo; -import org.apache.beam.sdk.values.PCollection; -import org.apache.beam.sdk.values.PCollectionTuple; -import org.apache.beam.sdk.values.TupleTag; -import org.apache.beam.sdk.values.TupleTagList; -import org.apache.commons.lang3.tuple.Pair; - -/** - * For each incoming {@link FeatureRowProto.FeatureRow} allocator choose only stores that - * subscripted to its project and featureSet names. - * - *

Return PCollectionTuple with one {@link TupleTag} per {@link StoreProto.Store}. Tags must be - * generated in advance. - */ -@AutoValue -public abstract class FeatureRowToStoreAllocator - extends PTransform, PCollectionTuple> { - public abstract List getStores(); - - public abstract Map> getStoreTags(); - - public static Builder newBuilder() { - return new AutoValue_FeatureRowToStoreAllocator.Builder(); - } - - @AutoValue.Builder - public abstract static class Builder { - public abstract Builder setStores(List stores); - - public abstract Builder setStoreTags( - Map> tags); - - public abstract FeatureRowToStoreAllocator build(); - } - - @Override - public PCollectionTuple expand(PCollection input) { - return input.apply( - "AssignRowToStore", - ParDo.of( - new DoFn() { - @ProcessElement - public void process(ProcessContext c, @Element FeatureRowProto.FeatureRow row) { - Pair projectAndSetNames = - parseFeatureSetReference(row.getFeatureSet()); - getStores().stream() - .filter( - s -> - Store.isSubscribedToFeatureSet( - s.getSubscriptionsList(), - projectAndSetNames.getLeft(), - projectAndSetNames.getRight())) - .forEach(s -> c.output(getStoreTags().get(s), row)); - } - }) - .withOutputTags( - getStoreTags().get(getStores().get(0)), - TupleTagList.of( - getStores().stream() - .skip(1) - .map(getStoreTags()::get) - .collect(Collectors.toList())))); - } -} diff --git a/ingestion/src/main/java/feast/ingestion/transform/ProcessAndValidateFeatureRows.java b/ingestion/src/main/java/feast/ingestion/transform/ProcessAndValidateFeatureRows.java deleted file mode 100644 index 729bff69a7f..00000000000 --- a/ingestion/src/main/java/feast/ingestion/transform/ProcessAndValidateFeatureRows.java +++ /dev/null @@ -1,77 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2019 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.ingestion.transform; - -import com.google.auto.value.AutoValue; -import feast.ingestion.transform.fn.ProcessFeatureRowDoFn; -import feast.ingestion.transform.fn.ValidateFeatureRowDoFn; -import feast.proto.core.FeatureSetProto; -import feast.proto.types.FeatureRowProto.FeatureRow; -import feast.storage.api.writer.FailedElement; -import java.util.Map; -import org.apache.beam.sdk.transforms.PTransform; -import org.apache.beam.sdk.transforms.ParDo; -import org.apache.beam.sdk.values.*; - -@AutoValue -public abstract class ProcessAndValidateFeatureRows - extends PTransform, PCollectionTuple> { - - public abstract PCollectionView>> - getFeatureSetSpecs(); - - public abstract String getDefaultProject(); - - public abstract TupleTag getSuccessTag(); - - public abstract TupleTag getFailureTag(); - - public static Builder newBuilder() { - return new AutoValue_ProcessAndValidateFeatureRows.Builder(); - } - - @AutoValue.Builder - public abstract static class Builder { - - public abstract Builder setFeatureSetSpecs( - PCollectionView>> featureSets); - - public abstract Builder setDefaultProject(String defaultProject); - - public abstract Builder setSuccessTag(TupleTag successTag); - - public abstract Builder setFailureTag(TupleTag failureTag); - - public abstract ProcessAndValidateFeatureRows build(); - } - - @Override - public PCollectionTuple expand(PCollection input) { - return input - .apply("ProcessFeatureRows", ParDo.of(new ProcessFeatureRowDoFn(getDefaultProject()))) - .apply( - "ValidateFeatureRows", - ParDo.of( - ValidateFeatureRowDoFn.newBuilder() - .setFeatureSets(getFeatureSetSpecs()) - .setSuccessTag(getSuccessTag()) - .setFailureTag(getFailureTag()) - .build()) - .withSideInputs(getFeatureSetSpecs()) - .withOutputTags(getSuccessTag(), TupleTagList.of(getFailureTag()))); - } -} diff --git a/ingestion/src/main/java/feast/ingestion/transform/ReadFromSource.java b/ingestion/src/main/java/feast/ingestion/transform/ReadFromSource.java deleted file mode 100644 index 766e2adaf1b..00000000000 --- a/ingestion/src/main/java/feast/ingestion/transform/ReadFromSource.java +++ /dev/null @@ -1,115 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2019 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.ingestion.transform; - -import com.google.auto.value.AutoValue; -import com.google.common.base.Preconditions; -import feast.ingestion.transform.fn.KafkaRecordToFeatureRowDoFn; -import feast.proto.core.SourceProto.Source; -import feast.proto.core.SourceProto.SourceType; -import feast.proto.types.FeatureRowProto.FeatureRow; -import feast.storage.api.writer.FailedElement; -import java.util.Arrays; -import java.util.HashMap; -import java.util.Map; -import java.util.stream.Collectors; -import org.apache.beam.sdk.io.kafka.KafkaIO; -import org.apache.beam.sdk.transforms.PTransform; -import org.apache.beam.sdk.transforms.ParDo; -import org.apache.beam.sdk.values.PBegin; -import org.apache.beam.sdk.values.PCollectionTuple; -import org.apache.beam.sdk.values.TupleTag; -import org.apache.beam.sdk.values.TupleTagList; -import org.apache.kafka.clients.consumer.ConsumerConfig; - -@AutoValue -public abstract class ReadFromSource extends PTransform { - - public abstract Source getSource(); - - public abstract TupleTag getSuccessTag(); - - public abstract TupleTag getFailureTag(); - - public abstract Map getKafkaConsumerProperties(); - - public static Builder newBuilder() { - return new AutoValue_ReadFromSource.Builder(); - } - - @AutoValue.Builder - public abstract static class Builder { - - public abstract Builder setSource(Source source); - - public abstract Builder setSuccessTag(TupleTag successTag); - - public abstract Builder setFailureTag(TupleTag failureTag); - - public abstract Builder setKafkaConsumerProperties(Map kafkaConsumerProperties); - - abstract ReadFromSource autobuild(); - - public ReadFromSource build() { - ReadFromSource read = autobuild(); - Source source = read.getSource(); - Preconditions.checkState( - source.getType().equals(SourceType.KAFKA), - "Source type must be KAFKA. Please raise an issue in https://github.com/feast-dev/feast/issues to request additional source types."); - Preconditions.checkState( - !source.getKafkaSourceConfig().getBootstrapServers().isEmpty(), - "bootstrap_servers cannot be empty."); - Preconditions.checkState( - !source.getKafkaSourceConfig().getTopic().isEmpty(), "topic cannot be empty."); - return read; - } - } - - @Override - public PCollectionTuple expand(PBegin input) { - Map consumerProperties = new HashMap<>(getKafkaConsumerProperties()); - consumerProperties.put( - ConsumerConfig.GROUP_ID_CONFIG, - generateConsumerGroupId(input.getPipeline().getOptions().getJobName())); - - return input - .getPipeline() - .apply( - "ReadFromKafka", - KafkaIO.readBytes() - .withBootstrapServers(getSource().getKafkaSourceConfig().getBootstrapServers()) - .withTopic(getSource().getKafkaSourceConfig().getTopic()) - .withConsumerConfigUpdates(consumerProperties) - .withReadCommitted() - .commitOffsetsInFinalize()) - .apply( - "KafkaRecordToFeatureRow", - ParDo.of( - KafkaRecordToFeatureRowDoFn.newBuilder() - .setSuccessTag(getSuccessTag()) - .setFailureTag(getFailureTag()) - .build()) - .withOutputTags(getSuccessTag(), TupleTagList.of(getFailureTag()))); - } - - private String generateConsumerGroupId(String jobName) { - String[] split = jobName.split("-"); - String jobNameWithoutTimestamp = - Arrays.stream(split).limit(split.length - 1).collect(Collectors.joining("-")); - return "feast_import_job_" + jobNameWithoutTimestamp; - } -} diff --git a/ingestion/src/main/java/feast/ingestion/transform/WriteFailedElementToBigQuery.java b/ingestion/src/main/java/feast/ingestion/transform/WriteFailedElementToBigQuery.java deleted file mode 100644 index 0da281790c5..00000000000 --- a/ingestion/src/main/java/feast/ingestion/transform/WriteFailedElementToBigQuery.java +++ /dev/null @@ -1,89 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2019 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.ingestion.transform; - -import com.google.api.services.bigquery.model.TableRow; -import com.google.auto.value.AutoValue; -import feast.storage.api.writer.FailedElement; -import org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO; -import org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO.Write.CreateDisposition; -import org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO.Write.WriteDisposition; -import org.apache.beam.sdk.io.gcp.bigquery.WriteResult; -import org.apache.beam.sdk.transforms.DoFn; -import org.apache.beam.sdk.transforms.PTransform; -import org.apache.beam.sdk.transforms.ParDo; -import org.apache.beam.sdk.values.PCollection; - -@AutoValue -public abstract class WriteFailedElementToBigQuery - extends PTransform, WriteResult> { - public abstract String getTableSpec(); - - public abstract String getJsonSchema(); - - public static Builder newBuilder() { - return new AutoValue_WriteFailedElementToBigQuery.Builder(); - } - - @AutoValue.Builder - public abstract static class Builder { - - /** - * @param tableSpec Table spec should follow the format "PROJECT_ID:DATASET_ID.TABLE_ID". Table - * will be created if not exists. - */ - public abstract Builder setTableSpec(String tableSpec); - - /** - * @param jsonSchema JSON string describing the schema - * of the table. - */ - public abstract Builder setJsonSchema(String jsonSchema); - - public abstract WriteFailedElementToBigQuery build(); - } - - @Override - public WriteResult expand(PCollection failedElements) { - return failedElements - .apply("FailedElementToTableRow", ParDo.of(new FailedElementToTableRowFn())) - .apply( - "WriteFailedElementsToBigQuery", - BigQueryIO.writeTableRows() - .to(getTableSpec()) - .withJsonSchema(getJsonSchema()) - .withCreateDisposition(CreateDisposition.CREATE_IF_NEEDED) - .withWriteDisposition(WriteDisposition.WRITE_APPEND)); - } - - public static class FailedElementToTableRowFn extends DoFn { - @ProcessElement - public void processElement(ProcessContext context) { - final FailedElement element = context.element(); - final TableRow tableRow = - new TableRow() - .set("timestamp", element.getTimestamp().toString()) - .set("job_name", element.getJobName()) - .set("transform_name", element.getTransformName()) - .set("payload", element.getPayload()) - .set("error_message", element.getErrorMessage()) - .set("stack_trace", element.getStackTrace()); - context.output(tableRow); - } - } -} diff --git a/ingestion/src/main/java/feast/ingestion/transform/fn/KafkaRecordToFeatureRowDoFn.java b/ingestion/src/main/java/feast/ingestion/transform/fn/KafkaRecordToFeatureRowDoFn.java deleted file mode 100644 index 7654c1e2fa0..00000000000 --- a/ingestion/src/main/java/feast/ingestion/transform/fn/KafkaRecordToFeatureRowDoFn.java +++ /dev/null @@ -1,72 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2019 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.ingestion.transform.fn; - -import com.google.auto.value.AutoValue; -import com.google.protobuf.InvalidProtocolBufferException; -import feast.proto.types.FeatureRowProto.FeatureRow; -import feast.storage.api.writer.FailedElement; -import java.util.Base64; -import org.apache.beam.sdk.io.kafka.KafkaRecord; -import org.apache.beam.sdk.transforms.DoFn; -import org.apache.beam.sdk.values.TupleTag; -import org.apache.commons.lang3.exception.ExceptionUtils; - -@AutoValue -public abstract class KafkaRecordToFeatureRowDoFn - extends DoFn, FeatureRow> { - - public abstract TupleTag getSuccessTag(); - - public abstract TupleTag getFailureTag(); - - public static KafkaRecordToFeatureRowDoFn.Builder newBuilder() { - return new AutoValue_KafkaRecordToFeatureRowDoFn.Builder(); - } - - @AutoValue.Builder - public abstract static class Builder { - - public abstract Builder setSuccessTag(TupleTag successTag); - - public abstract Builder setFailureTag(TupleTag failureTag); - - public abstract KafkaRecordToFeatureRowDoFn build(); - } - - @ProcessElement - public void processElement(ProcessContext context) { - byte[] value = context.element().getKV().getValue(); - FeatureRow featureRow; - - try { - featureRow = FeatureRow.parseFrom(value); - } catch (InvalidProtocolBufferException e) { - context.output( - getFailureTag(), - FailedElement.newBuilder() - .setTransformName("KafkaRecordToFeatureRow") - .setStackTrace(ExceptionUtils.getStackTrace(e)) - .setJobName(context.getPipelineOptions().getJobName()) - .setPayload(new String(Base64.getEncoder().encode(value))) - .setErrorMessage(e.getMessage()) - .build()); - return; - } - context.output(featureRow); - } -} diff --git a/ingestion/src/main/java/feast/ingestion/transform/fn/LoggerDoFn.java b/ingestion/src/main/java/feast/ingestion/transform/fn/LoggerDoFn.java deleted file mode 100644 index 5bde9a24cb1..00000000000 --- a/ingestion/src/main/java/feast/ingestion/transform/fn/LoggerDoFn.java +++ /dev/null @@ -1,69 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2019 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.ingestion.transform.fn; - -import com.google.protobuf.Message; -import com.google.protobuf.util.JsonFormat; -import com.google.protobuf.util.JsonFormat.Printer; -import org.apache.beam.sdk.transforms.DoFn; -import org.slf4j.Logger; -import org.slf4j.event.Level; - -public class LoggerDoFn extends DoFn { - - private static final Logger log = org.slf4j.LoggerFactory.getLogger(LoggerDoFn.class); - private Level level; - private String prefix = ""; - - public LoggerDoFn(Level level) { - this.level = level; - } - - public LoggerDoFn(Level level, String prefix) { - this.level = level; - this.prefix = prefix; - } - - @ProcessElement - public void processElement(ProcessContext context) { - Printer printer = JsonFormat.printer().omittingInsignificantWhitespace(); - String message; - try { - message = prefix + printer.print(context.element()); - } catch (Exception e) { - log.error(e.getMessage(), e); - message = prefix + context.element().toString(); - } - switch (level) { - case INFO: - log.info(message); - break; - case ERROR: - log.error(message); - break; - case WARN: - log.warn(message); - break; - case DEBUG: - log.debug(message); - break; - case TRACE: - log.trace(message); - break; - } - } -} diff --git a/ingestion/src/main/java/feast/ingestion/transform/fn/ProcessFeatureRowDoFn.java b/ingestion/src/main/java/feast/ingestion/transform/fn/ProcessFeatureRowDoFn.java deleted file mode 100644 index 3680348cf0c..00000000000 --- a/ingestion/src/main/java/feast/ingestion/transform/fn/ProcessFeatureRowDoFn.java +++ /dev/null @@ -1,52 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.ingestion.transform.fn; - -import feast.proto.types.FeatureRowProto.FeatureRow; -import org.apache.beam.sdk.transforms.DoFn; - -public class ProcessFeatureRowDoFn extends DoFn { - - private String defaultProject; - - public ProcessFeatureRowDoFn(String defaultProject) { - this.defaultProject = defaultProject; - } - - @ProcessElement - public void processElement(ProcessContext context) { - FeatureRow featureRow = context.element(); - String featureSetId = stripVersion(featureRow.getFeatureSet()); - featureSetId = applyDefaultProject(featureSetId); - featureRow = featureRow.toBuilder().setFeatureSet(featureSetId).build(); - context.output(featureRow); - } - - // For backward compatibility. Will be deprecated eventually. - private String stripVersion(String featureSetId) { - String[] split = featureSetId.split(":"); - return split[0]; - } - - private String applyDefaultProject(String featureSetId) { - String[] split = featureSetId.split("/"); - if (split.length == 1) { - return defaultProject + "/" + featureSetId; - } - return featureSetId; - } -} diff --git a/ingestion/src/main/java/feast/ingestion/transform/fn/ValidateFeatureRowDoFn.java b/ingestion/src/main/java/feast/ingestion/transform/fn/ValidateFeatureRowDoFn.java deleted file mode 100644 index 7fb5626047e..00000000000 --- a/ingestion/src/main/java/feast/ingestion/transform/fn/ValidateFeatureRowDoFn.java +++ /dev/null @@ -1,128 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2019 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.ingestion.transform.fn; - -import com.google.auto.value.AutoValue; -import com.google.common.collect.Iterators; -import feast.ingestion.values.FeatureSet; -import feast.ingestion.values.Field; -import feast.proto.core.FeatureSetProto; -import feast.proto.types.FeatureRowProto.FeatureRow; -import feast.proto.types.FieldProto; -import feast.proto.types.ValueProto.Value.ValCase; -import feast.storage.api.writer.FailedElement; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import org.apache.beam.sdk.transforms.DoFn; -import org.apache.beam.sdk.values.PCollectionView; -import org.apache.beam.sdk.values.TupleTag; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -@AutoValue -public abstract class ValidateFeatureRowDoFn extends DoFn { - private static final Logger log = LoggerFactory.getLogger(ValidateFeatureRowDoFn.class); - - public abstract PCollectionView>> - getFeatureSets(); - - public abstract TupleTag getSuccessTag(); - - public abstract TupleTag getFailureTag(); - - public static Builder newBuilder() { - return new AutoValue_ValidateFeatureRowDoFn.Builder(); - } - - @AutoValue.Builder - public abstract static class Builder { - - public abstract Builder setFeatureSets( - PCollectionView>> featureSets); - - public abstract Builder setSuccessTag(TupleTag successTag); - - public abstract Builder setFailureTag(TupleTag failureTag); - - public abstract ValidateFeatureRowDoFn build(); - } - - @ProcessElement - public void processElement(ProcessContext context) { - String error = null; - FeatureRow featureRow = context.element(); - Iterable featureSetSpecs = - context.sideInput(getFeatureSets()).get(featureRow.getFeatureSet()); - if (featureSetSpecs == null) { - log.warn( - String.format( - "FeatureRow contains invalid featureSetReference %s." - + " Please check that the feature rows are being published" - + " to the correct topic on the feature stream.", - featureRow.getFeatureSet())); - return; - } - - List fields = new ArrayList<>(); - - FeatureSetProto.FeatureSetSpec latestSpec = Iterators.getLast(featureSetSpecs.iterator()); - FeatureSet featureSet = new FeatureSet(latestSpec); - - for (FieldProto.Field field : featureRow.getFieldsList()) { - Field fieldSpec = featureSet.getField(field.getName()); - if (fieldSpec == null) { - // skip - continue; - } - // If value is set in the FeatureRow, make sure the value type matches - // that defined in FeatureSetSpec - if (!field.getValue().getValCase().equals(ValCase.VAL_NOT_SET)) { - int expectedTypeFieldNumber = fieldSpec.getType().getNumber(); - int actualTypeFieldNumber = field.getValue().getValCase().getNumber(); - if (expectedTypeFieldNumber != actualTypeFieldNumber) { - error = - String.format( - "FeatureRow contains field '%s' with invalid type '%s'. Feast expects the field type to match that in FeatureSet '%s'. Please check the FeatureRow data.", - field.getName(), field.getValue().getValCase(), fieldSpec.getType()); - break; - } - } - if (!fields.contains(field)) { - fields.add(field); - } - } - - if (error != null) { - FailedElement.Builder failedElement = - FailedElement.newBuilder() - .setTransformName("ValidateFeatureRow") - .setJobName(context.getPipelineOptions().getJobName()) - .setPayload(featureRow.toString()) - .setErrorMessage(error); - if (featureSetSpecs != null) { - FeatureSetProto.FeatureSetSpec spec = Iterators.getLast(featureSetSpecs.iterator()); - failedElement = - failedElement.setProjectName(spec.getProject()).setFeatureSetName(spec.getName()); - } - context.output(getFailureTag(), failedElement.build()); - } else { - featureRow = featureRow.toBuilder().clearFields().addAllFields(fields).build(); - context.output(getSuccessTag(), featureRow); - } - } -} diff --git a/ingestion/src/main/java/feast/ingestion/transform/metrics/WriteDeadletterRowMetricsDoFn.java b/ingestion/src/main/java/feast/ingestion/transform/metrics/WriteDeadletterRowMetricsDoFn.java deleted file mode 100644 index 828fed6ceda..00000000000 --- a/ingestion/src/main/java/feast/ingestion/transform/metrics/WriteDeadletterRowMetricsDoFn.java +++ /dev/null @@ -1,84 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2019 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.ingestion.transform.metrics; - -import com.google.auto.value.AutoValue; -import com.timgroup.statsd.NonBlockingStatsDClient; -import com.timgroup.statsd.StatsDClient; -import com.timgroup.statsd.StatsDClientException; -import feast.storage.api.writer.FailedElement; -import org.apache.beam.sdk.transforms.DoFn; -import org.slf4j.Logger; - -@AutoValue -public abstract class WriteDeadletterRowMetricsDoFn extends DoFn { - - private static final Logger log = - org.slf4j.LoggerFactory.getLogger(WriteDeadletterRowMetricsDoFn.class); - - private final String INGESTION_JOB_NAME_KEY = "ingestion_job_name"; - private final String METRIC_PREFIX = "feast_ingestion"; - private final String STORE_TAG_KEY = "feast_store"; - private final String PROJECT_TAG_KEY = "feast_project_name"; - private final String FEATURE_SET_NAME_TAG_KEY = "feast_featureSet_name"; - private final String FEATURE_SET_VERSION_TAG_KEY = "feast_featureSet_version"; - - public abstract String getStoreName(); - - public abstract String getStatsdHost(); - - public abstract int getStatsdPort(); - - public StatsDClient statsd; - - public static WriteDeadletterRowMetricsDoFn.Builder newBuilder() { - return new AutoValue_WriteDeadletterRowMetricsDoFn.Builder(); - } - - @AutoValue.Builder - public abstract static class Builder { - - public abstract Builder setStoreName(String storeName); - - public abstract Builder setStatsdHost(String statsdHost); - - public abstract Builder setStatsdPort(int statsdPort); - - public abstract WriteDeadletterRowMetricsDoFn build(); - } - - @Setup - public void setup() { - statsd = new NonBlockingStatsDClient(METRIC_PREFIX, getStatsdHost(), getStatsdPort()); - } - - @ProcessElement - public void processElement(ProcessContext c) { - FailedElement ignored = c.element(); - try { - statsd.count( - "deadletter_row_count", - 1, - STORE_TAG_KEY + ":" + getStoreName(), - PROJECT_TAG_KEY + ":" + ignored.getProjectName(), - FEATURE_SET_NAME_TAG_KEY + ":" + ignored.getFeatureSetName(), - INGESTION_JOB_NAME_KEY + ":" + c.getPipelineOptions().getJobName()); - } catch (StatsDClientException e) { - log.warn("Unable to push metrics to server", e); - } - } -} diff --git a/ingestion/src/main/java/feast/ingestion/transform/metrics/WriteFailureMetricsTransform.java b/ingestion/src/main/java/feast/ingestion/transform/metrics/WriteFailureMetricsTransform.java deleted file mode 100644 index 778515ae8a8..00000000000 --- a/ingestion/src/main/java/feast/ingestion/transform/metrics/WriteFailureMetricsTransform.java +++ /dev/null @@ -1,61 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2019 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.ingestion.transform.metrics; - -import com.google.auto.value.AutoValue; -import feast.ingestion.options.ImportOptions; -import feast.storage.api.writer.FailedElement; -import org.apache.beam.sdk.transforms.DoFn; -import org.apache.beam.sdk.transforms.PTransform; -import org.apache.beam.sdk.transforms.ParDo; -import org.apache.beam.sdk.values.PCollection; -import org.apache.beam.sdk.values.PDone; - -@AutoValue -public abstract class WriteFailureMetricsTransform - extends PTransform, PDone> { - - public abstract String getStoreName(); - - public static WriteFailureMetricsTransform create(String storeName) { - return new AutoValue_WriteFailureMetricsTransform(storeName); - } - - @Override - public PDone expand(PCollection input) { - ImportOptions options = input.getPipeline().getOptions().as(ImportOptions.class); - if ("statsd".equals(options.getMetricsExporterType())) { - input.apply( - "WriteDeadletterMetrics", - ParDo.of( - WriteDeadletterRowMetricsDoFn.newBuilder() - .setStatsdHost(options.getStatsdHost()) - .setStatsdPort(options.getStatsdPort()) - .setStoreName(getStoreName()) - .build())); - } else { - input.apply( - "Noop", - ParDo.of( - new DoFn() { - @ProcessElement - public void processElement(ProcessContext c) {} - })); - } - return PDone.in(input.getPipeline()); - } -} diff --git a/ingestion/src/main/java/feast/ingestion/transform/metrics/WriteFeatureValueMetricsDoFn.java b/ingestion/src/main/java/feast/ingestion/transform/metrics/WriteFeatureValueMetricsDoFn.java deleted file mode 100644 index 1e40e44d5cd..00000000000 --- a/ingestion/src/main/java/feast/ingestion/transform/metrics/WriteFeatureValueMetricsDoFn.java +++ /dev/null @@ -1,319 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.ingestion.transform.metrics; - -import static feast.ingestion.transform.metrics.WriteRowMetricsDoFn.*; - -import com.google.auto.value.AutoValue; -import com.timgroup.statsd.NonBlockingStatsDClient; -import com.timgroup.statsd.StatsDClient; -import feast.proto.types.FeatureRowProto.FeatureRow; -import feast.proto.types.FieldProto.Field; -import feast.proto.types.ValueProto.Value; -import java.util.*; -import java.util.Map.Entry; -import java.util.stream.Collectors; -import org.apache.beam.sdk.transforms.DoFn; -import org.apache.beam.sdk.values.KV; -import org.apache.commons.math3.stat.descriptive.rank.Percentile; -import org.slf4j.Logger; - -/** - * WriteFeatureValueMetricsDoFn accepts key value of FeatureSetRef(str) to FeatureRow(List) and - * writes a histogram of the numerical values of each feature to StatsD. - * - *

The histogram of the numerical values is represented as the following in StatsD: - * - *

    - *
  • gauge of feature_value_min - *
  • gauge of feature_value_max - *
  • gauge of feature_value_mean - *
  • gauge of feature_value_percentile_50 - *
  • gauge of feature_value_percentile_90 - *
  • gauge of feature_value_percentile_95 - *
- * - *

StatsD timing/histogram metric type is not used since it does not support negative values. - */ -@AutoValue -public abstract class WriteFeatureValueMetricsDoFn - extends DoFn>, Void> { - - abstract String getStoreName(); - - abstract String getStatsdHost(); - - abstract int getStatsdPort(); - - abstract String getMetricsNamespace(); - - static Builder newBuilder() { - return new AutoValue_WriteFeatureValueMetricsDoFn.Builder(); - } - - @AutoValue.Builder - abstract static class Builder { - - abstract Builder setStoreName(String storeName); - - abstract Builder setStatsdHost(String statsdHost); - - abstract Builder setStatsdPort(int statsdPort); - - abstract Builder setMetricsNamespace(String metricsNamespace); - - abstract WriteFeatureValueMetricsDoFn build(); - } - - private static final Logger log = - org.slf4j.LoggerFactory.getLogger(WriteFeatureValueMetricsDoFn.class); - private StatsDClient statsDClient; - public static String GAUGE_NAME_FEATURE_VALUE_MIN = "feature_value_min"; - public static String GAUGE_NAME_FEATURE_VALUE_MAX = "feature_value_max"; - public static String GAUGE_NAME_FEATURE_VALUE_MEAN = "feature_value_mean"; - public static String GAUGE_NAME_FEATURE_VALUE_PERCENTILE_25 = "feature_value_percentile_25"; - public static String GAUGE_NAME_FEATURE_VALUE_PERCENTILE_50 = "feature_value_percentile_50"; - public static String GAUGE_NAME_FEATURE_VALUE_PERCENTILE_90 = "feature_value_percentile_90"; - public static String GAUGE_NAME_FEATURE_VALUE_PERCENTILE_95 = "feature_value_percentile_95"; - public static String GAUGE_NAME_FEATURE_VALUE_PERCENTILE_99 = "feature_value_percentile_99"; - - @Setup - public void setup() { - // Note that exception may be thrown during StatsD client instantiation but no exception - // will be thrown when sending metrics (mimicking the UDP protocol behaviour). - // https://jar-download.com/artifacts/com.datadoghq/java-dogstatsd-client/2.1.1/documentation - // https://github.com/DataDog/java-dogstatsd-client#unix-domain-socket-support - try { - statsDClient = new NonBlockingStatsDClient(METRIC_PREFIX, getStatsdHost(), getStatsdPort()); - } catch (Exception e) { - log.error("StatsD client cannot be started: " + e.getMessage()); - } - } - - @Teardown - public void tearDown() { - if (statsDClient != null) { - statsDClient.close(); - } - } - - @ProcessElement - public void processElement( - ProcessContext context, - @Element KV> featureSetRefToFeatureRows) { - if (statsDClient == null) { - log.error("StatsD client is null, likely because it encounters an error during setup"); - return; - } - - String featureSetRef = featureSetRefToFeatureRows.getKey(); - if (featureSetRef == null) { - log.error( - "Feature set reference in the feature row is null. Please check the input feature rows from previous steps"); - return; - } - - String[] slashSplits = featureSetRef.split("/"); - if (slashSplits.length != 2) { - log.error( - "Skip writing feature value metrics because the feature set reference '{}' does not" - + "follow the required format /", - featureSetRef); - return; - } - String projectName = slashSplits[0]; - String featureSetName = slashSplits[1]; - - Map featureNameToStats = new HashMap<>(); - Map> featureNameToValues = new HashMap<>(); - for (FeatureRow featureRow : featureSetRefToFeatureRows.getValue()) { - for (Field field : featureRow.getFieldsList()) { - updateStats(featureNameToStats, featureNameToValues, field); - } - } - - String[] split = context.getPipelineOptions().getJobName().split("-"); - String jobNameWithoutTimestamp = - Arrays.stream(split).limit(split.length - 1).collect(Collectors.joining("-")); - - for (Entry entry : featureNameToStats.entrySet()) { - String featureName = entry.getKey(); - DoubleSummaryStatistics stats = entry.getValue(); - String[] tags = { - STORE_TAG_KEY + ":" + getStoreName(), - FEATURE_SET_PROJECT_TAG_KEY + ":" + projectName, - FEATURE_SET_NAME_TAG_KEY + ":" + featureSetName, - FEATURE_TAG_KEY + ":" + featureName, - INGESTION_JOB_NAME_KEY + ":" + jobNameWithoutTimestamp, - METRICS_NAMESPACE_KEY + ":" + getMetricsNamespace(), - }; - - // stats can return non finite values when there is no element - // or there is an element that is not a number. Metric should only be sent for finite values. - if (Double.isFinite(stats.getMin())) { - if (stats.getMin() < 0) { - // StatsD gauge will asssign a delta instead of the actual value, if there is a sign in - // the value. E.g. if the value is negative, a delta will be assigned. For this reason, - // the gauge value is set to zero beforehand. - // https://github.com/statsd/statsd/blob/master/docs/metric_types.md#gauges - statsDClient.gauge(GAUGE_NAME_FEATURE_VALUE_MIN, 0, tags); - } - statsDClient.gauge(GAUGE_NAME_FEATURE_VALUE_MIN, stats.getMin(), tags); - } - if (Double.isFinite(stats.getMax())) { - if (stats.getMax() < 0) { - statsDClient.gauge(GAUGE_NAME_FEATURE_VALUE_MAX, 0, tags); - } - statsDClient.gauge(GAUGE_NAME_FEATURE_VALUE_MAX, stats.getMax(), tags); - } - if (Double.isFinite(stats.getAverage())) { - if (stats.getAverage() < 0) { - statsDClient.gauge(GAUGE_NAME_FEATURE_VALUE_MEAN, 0, tags); - } - statsDClient.gauge(GAUGE_NAME_FEATURE_VALUE_MEAN, stats.getAverage(), tags); - } - - // For percentile calculation, Percentile class from commons-math3 from Apache is used. - // Percentile requires double[], hence the conversion below. - if (!featureNameToValues.containsKey(featureName)) { - continue; - } - List valueList = featureNameToValues.get(featureName); - if (valueList == null || valueList.size() < 1) { - continue; - } - double[] values = new double[valueList.size()]; - for (int i = 0; i < values.length; i++) { - values[i] = valueList.get(i); - } - - double p25 = new Percentile().evaluate(values, 25); - if (p25 < 0) { - statsDClient.gauge(GAUGE_NAME_FEATURE_VALUE_PERCENTILE_25, 0, tags); - } - statsDClient.gauge(GAUGE_NAME_FEATURE_VALUE_PERCENTILE_25, p25, tags); - - double p50 = new Percentile().evaluate(values, 50); - if (p50 < 0) { - statsDClient.gauge(GAUGE_NAME_FEATURE_VALUE_PERCENTILE_50, 0, tags); - } - statsDClient.gauge(GAUGE_NAME_FEATURE_VALUE_PERCENTILE_50, p50, tags); - - double p90 = new Percentile().evaluate(values, 90); - if (p90 < 0) { - statsDClient.gauge(GAUGE_NAME_FEATURE_VALUE_PERCENTILE_90, 0, tags); - } - statsDClient.gauge(GAUGE_NAME_FEATURE_VALUE_PERCENTILE_90, p90, tags); - - double p95 = new Percentile().evaluate(values, 95); - if (p95 < 0) { - statsDClient.gauge(GAUGE_NAME_FEATURE_VALUE_PERCENTILE_95, 0, tags); - } - statsDClient.gauge(GAUGE_NAME_FEATURE_VALUE_PERCENTILE_95, p95, tags); - - double p99 = new Percentile().evaluate(values, 99); - if (p99 < 0) { - statsDClient.gauge(GAUGE_NAME_FEATURE_VALUE_PERCENTILE_99, 0, tags); - } - statsDClient.gauge(GAUGE_NAME_FEATURE_VALUE_PERCENTILE_99, p99, tags); - } - } - - // Update stats and values array for the feature represented by the field. - // If the field contains non-numerical or non-boolean value, the stats and values array - // won't get updated because we are only concerned with numerical value in metrics data. - // For boolean value, true and false are treated as numerical value of 1 of 0 respectively. - private void updateStats( - Map featureNameToStats, - Map> featureNameToValues, - Field field) { - if (featureNameToStats == null || featureNameToValues == null || field == null) { - return; - } - - String featureName = field.getName(); - if (!featureNameToStats.containsKey(featureName)) { - featureNameToStats.put(featureName, new DoubleSummaryStatistics()); - } - if (!featureNameToValues.containsKey(featureName)) { - featureNameToValues.put(featureName, new ArrayList<>()); - } - - Value value = field.getValue(); - DoubleSummaryStatistics stats = featureNameToStats.get(featureName); - List values = featureNameToValues.get(featureName); - - switch (value.getValCase()) { - case INT32_VAL: - stats.accept(value.getInt32Val()); - values.add(((double) value.getInt32Val())); - break; - case INT64_VAL: - stats.accept(value.getInt64Val()); - values.add((double) value.getInt64Val()); - break; - case DOUBLE_VAL: - stats.accept(value.getDoubleVal()); - values.add(value.getDoubleVal()); - break; - case FLOAT_VAL: - stats.accept(value.getFloatVal()); - values.add((double) value.getFloatVal()); - break; - case BOOL_VAL: - stats.accept(value.getBoolVal() ? 1 : 0); - values.add(value.getBoolVal() ? 1d : 0d); - break; - case INT32_LIST_VAL: - for (Integer val : value.getInt32ListVal().getValList()) { - stats.accept(val); - values.add(((double) val)); - } - break; - case INT64_LIST_VAL: - for (Long val : value.getInt64ListVal().getValList()) { - stats.accept(val); - values.add(((double) val)); - } - break; - case DOUBLE_LIST_VAL: - for (Double val : value.getDoubleListVal().getValList()) { - stats.accept(val); - values.add(val); - } - break; - case FLOAT_LIST_VAL: - for (Float val : value.getFloatListVal().getValList()) { - stats.accept(val); - values.add(((double) val)); - } - break; - case BOOL_LIST_VAL: - for (Boolean val : value.getBoolListVal().getValList()) { - stats.accept(val ? 1 : 0); - values.add(val ? 1d : 0d); - } - break; - case BYTES_VAL: - case BYTES_LIST_VAL: - case STRING_VAL: - case STRING_LIST_VAL: - case VAL_NOT_SET: - default: - } - } -} diff --git a/ingestion/src/main/java/feast/ingestion/transform/metrics/WriteInflightMetricsTransform.java b/ingestion/src/main/java/feast/ingestion/transform/metrics/WriteInflightMetricsTransform.java deleted file mode 100644 index 602045e46d9..00000000000 --- a/ingestion/src/main/java/feast/ingestion/transform/metrics/WriteInflightMetricsTransform.java +++ /dev/null @@ -1,120 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2019 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.ingestion.transform.metrics; - -import com.google.auto.value.AutoValue; -import feast.ingestion.options.ImportOptions; -import feast.proto.types.FeatureRowProto.FeatureRow; -import org.apache.beam.sdk.metrics.Counter; -import org.apache.beam.sdk.metrics.Metrics; -import org.apache.beam.sdk.transforms.*; -import org.apache.beam.sdk.transforms.windowing.FixedWindows; -import org.apache.beam.sdk.transforms.windowing.Window; -import org.apache.beam.sdk.values.KV; -import org.apache.beam.sdk.values.PCollection; -import org.apache.beam.sdk.values.PDone; -import org.apache.beam.sdk.values.TypeDescriptors; -import org.joda.time.Duration; - -@AutoValue -public abstract class WriteInflightMetricsTransform - extends PTransform, PDone> { - - public static final String METRIC_NAMESPACE = "Inflight"; - public static final String ELEMENTS_WRITTEN_METRIC = "elements_count"; - private static final Counter elements_count = - Metrics.counter(METRIC_NAMESPACE, ELEMENTS_WRITTEN_METRIC); - - public abstract String getStoreName(); - - public static WriteInflightMetricsTransform create(String storeName) { - return new AutoValue_WriteInflightMetricsTransform(storeName); - } - - @Override - public PDone expand(PCollection input) { - ImportOptions options = input.getPipeline().getOptions().as(ImportOptions.class); - - input.apply( - "IncrementInflightElementsCounter", - MapElements.into(TypeDescriptors.booleans()) - .via( - (FeatureRow row) -> { - elements_count.inc(); - return true; - })); - - switch (options.getMetricsExporterType()) { - case "statsd": - - // Fixed window is applied so the metric collector will not be overwhelmed with the metrics - // data. For validation, only summaries of the values are usually required vs the actual - // values. - PCollection>> rowsGroupedByRef = - input - .apply( - "FixedWindow", - Window.into( - FixedWindows.of( - Duration.standardSeconds( - options.getWindowSizeInSecForFeatureValueMetric())))) - .apply( - "ConvertToKV_FeatureSetRefToFeatureRow", - ParDo.of( - new DoFn>() { - @ProcessElement - public void processElement( - ProcessContext c, @Element FeatureRow featureRow) { - c.output(KV.of(featureRow.getFeatureSet(), featureRow)); - } - })) - .apply("GroupByFeatureSetRef", GroupByKey.create()); - - rowsGroupedByRef.apply( - "WriteInflightRowMetrics", - ParDo.of( - WriteRowMetricsDoFn.newBuilder() - .setStatsdHost(options.getStatsdHost()) - .setStatsdPort(options.getStatsdPort()) - .setStoreName(getStoreName()) - .setMetricsNamespace(METRIC_NAMESPACE) - .build())); - - rowsGroupedByRef.apply( - "WriteInflightFeatureValueMetrics", - ParDo.of( - WriteFeatureValueMetricsDoFn.newBuilder() - .setStatsdHost(options.getStatsdHost()) - .setStatsdPort(options.getStatsdPort()) - .setStoreName(getStoreName()) - .setMetricsNamespace(METRIC_NAMESPACE) - .build())); - - return PDone.in(input.getPipeline()); - case "none": - default: - input.apply( - "Noop", - ParDo.of( - new DoFn() { - @ProcessElement - public void processElement(ProcessContext c) {} - })); - return PDone.in(input.getPipeline()); - } - } -} diff --git a/ingestion/src/main/java/feast/ingestion/transform/metrics/WriteRowMetricsDoFn.java b/ingestion/src/main/java/feast/ingestion/transform/metrics/WriteRowMetricsDoFn.java deleted file mode 100644 index 29597ae84f4..00000000000 --- a/ingestion/src/main/java/feast/ingestion/transform/metrics/WriteRowMetricsDoFn.java +++ /dev/null @@ -1,251 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2019 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.ingestion.transform.metrics; - -import com.google.auto.value.AutoValue; -import com.google.protobuf.util.Timestamps; -import com.timgroup.statsd.NonBlockingStatsDClient; -import com.timgroup.statsd.StatsDClient; -import feast.proto.types.FeatureRowProto.FeatureRow; -import feast.proto.types.FieldProto.Field; -import feast.proto.types.ValueProto.Value; -import feast.proto.types.ValueProto.Value.ValCase; -import java.time.Clock; -import java.util.Arrays; -import java.util.HashMap; -import java.util.Map; -import java.util.Map.Entry; -import java.util.stream.Collectors; -import javax.annotation.Nullable; -import org.apache.beam.sdk.transforms.DoFn; -import org.apache.beam.sdk.values.KV; -import org.apache.commons.lang3.ArrayUtils; -import org.apache.commons.math3.stat.descriptive.DescriptiveStatistics; -import org.slf4j.Logger; - -@AutoValue -public abstract class WriteRowMetricsDoFn extends DoFn>, Void> { - - private static final Logger log = org.slf4j.LoggerFactory.getLogger(WriteRowMetricsDoFn.class); - - public static final String METRIC_PREFIX = "feast_ingestion"; - public static final String STORE_TAG_KEY = "feast_store"; - public static final String FEATURE_SET_PROJECT_TAG_KEY = "feast_project_name"; - public static final String FEATURE_SET_NAME_TAG_KEY = "feast_featureSet_name"; - public static final String FEATURE_TAG_KEY = "feast_feature_name"; - public static final String INGESTION_JOB_NAME_KEY = "ingestion_job_name"; - public static final String METRICS_NAMESPACE_KEY = "metrics_namespace"; - - public static final String GAUGE_NAME_FEATURE_ROW_LAG_MS_MIN = "feature_row_lag_ms_min"; - public static final String GAUGE_NAME_FEATURE_ROW_LAG_MS_MAX = "feature_row_lag_ms_max"; - public static final String GAUGE_NAME_FEATURE_ROW_LAG_MS_MEAN = "feature_row_lag_ms_mean"; - public static final String GAUGE_NAME_FEATURE_ROW_LAG_MS_PERCENTILE_90 = - "feature_row_lag_ms_percentile_90"; - public static final String GAUGE_NAME_FEATURE_ROW_LAG_MS_PERCENTILE_95 = - "feature_row_lag_ms_percentile_95"; - public static final String GAUGE_NAME_FEATURE_ROW_LAG_MS_PERCENTILE_99 = - "feature_row_lag_ms_percentile_99"; - - public static final String GAUGE_NAME_FEATURE_VALUE_LAG_MS_MIN = "feature_value_lag_ms_min"; - public static final String GAUGE_NAME_FEATURE_VALUE_LAG_MS_MAX = "feature_value_lag_ms_max"; - public static final String GAUGE_NAME_FEATURE_VALUE_LAG_MS_MEAN = "feature_value_lag_ms_mean"; - public static final String GAUGE_NAME_FEATURE_VALUE_LAG_MS_PERCENTILE_90 = - "feature_value_lag_ms_percentile_90"; - public static final String GAUGE_NAME_FEATURE_VALUE_LAG_MS_PERCENTILE_95 = - "feature_value_lag_ms_percentile_95"; - public static final String GAUGE_NAME_FEATURE_VALUE_LAG_MS_PERCENTILE_99 = - "feature_value_lag_ms_percentile_99"; - - public static final String COUNT_NAME_FEATURE_ROW_INGESTED = "feature_row_ingested_count"; - public static final String COUNT_NAME_FEATURE_VALUE_MISSING = "feature_value_missing_count"; - - public abstract String getStoreName(); - - public abstract String getStatsdHost(); - - public abstract int getStatsdPort(); - - public abstract String getMetricsNamespace(); - - @Nullable - public abstract Clock getClock(); - - public static WriteRowMetricsDoFn create( - String newStoreName, String newStatsdHost, int newStatsdPort) { - return newBuilder() - .setStoreName(newStoreName) - .setStatsdHost(newStatsdHost) - .setStatsdPort(newStatsdPort) - .build(); - } - - public StatsDClient statsd; - - public static Builder newBuilder() { - return new AutoValue_WriteRowMetricsDoFn.Builder(); - } - - @AutoValue.Builder - public abstract static class Builder { - - public abstract Builder setStoreName(String storeName); - - public abstract Builder setStatsdHost(String statsdHost); - - public abstract Builder setStatsdPort(int statsdPort); - - public abstract Builder setMetricsNamespace(String metricNamespace); - - /** - * setClock will override the default system clock used to calculate feature row lag. - * - * @param clock Clock instance - */ - public abstract Builder setClock(Clock clock); - - public abstract WriteRowMetricsDoFn build(); - } - - @Setup - public void setup() { - // Note that exception may be thrown during StatsD client instantiation but no exception - // will be thrown when sending metrics (mimicking the UDP protocol behaviour). - // https://jar-download.com/artifacts/com.datadoghq/java-dogstatsd-client/2.1.1/documentation - // https://github.com/DataDog/java-dogstatsd-client#unix-domain-socket-support - try { - statsd = new NonBlockingStatsDClient(METRIC_PREFIX, getStatsdHost(), getStatsdPort()); - } catch (Exception e) { - log.error("StatsD client cannot be started: " + e.getMessage()); - } - } - - @SuppressWarnings("DuplicatedCode") - @ProcessElement - public void processElement( - ProcessContext c, @Element KV> featureSetRefToFeatureRows) { - if (statsd == null) { - log.error("StatsD client is null, likely because it encounters an error during setup"); - return; - } - - String featureSetRef = featureSetRefToFeatureRows.getKey(); - if (featureSetRef == null) { - log.error( - "Feature set reference in the feature row is null. Please check the input feature rows from previous steps"); - return; - } - String[] slashSplits = featureSetRef.split("/"); - if (slashSplits.length != 2) { - log.error( - "Skip writing feature row metrics because the feature set reference '{}' does not" - + "follow the required format /:", - featureSetRef); - return; - } - - String featureSetProject = slashSplits[0]; - String featureSetName = slashSplits[1]; - - // featureRowLagStats is stats for feature row lag for feature set "featureSetName" - DescriptiveStatistics featureRowLagStats = new DescriptiveStatistics(); - // featureNameToLagStats is stats for feature lag for all features in feature set - // "featureSetName" - Map featureNameToLagStats = new HashMap<>(); - // featureNameToMissingCount is count for "value_not_set" for all features in feature set - // "featureSetName" - Map featureNameToMissingCount = new HashMap<>(); - - for (FeatureRow featureRow : featureSetRefToFeatureRows.getValue()) { - long currentTime = getClock() == null ? System.currentTimeMillis() : getClock().millis(); - long featureRowLag = currentTime - Timestamps.toMillis(featureRow.getEventTimestamp()); - featureRowLagStats.addValue(featureRowLag); - - for (Field field : featureRow.getFieldsList()) { - String featureName = field.getName(); - Value featureValue = field.getValue(); - if (!featureNameToLagStats.containsKey(featureName)) { - // Ensure map contains the "featureName" key - featureNameToLagStats.put(featureName, new DescriptiveStatistics()); - } - if (!featureNameToMissingCount.containsKey(featureName)) { - // Ensure map contains the "featureName" key - featureNameToMissingCount.put(featureName, 0L); - } - if (featureValue.getValCase().equals(ValCase.VAL_NOT_SET)) { - featureNameToMissingCount.put( - featureName, featureNameToMissingCount.get(featureName) + 1); - } else { - featureNameToLagStats.get(featureName).addValue(featureRowLag); - } - } - } - - String[] split = c.getPipelineOptions().getJobName().split("-"); - String jobNameWithoutTimestamp = - Arrays.stream(split).limit(split.length - 1).collect(Collectors.joining("-")); - - String[] tags = { - STORE_TAG_KEY + ":" + getStoreName(), - FEATURE_SET_PROJECT_TAG_KEY + ":" + featureSetProject, - FEATURE_SET_NAME_TAG_KEY + ":" + featureSetName, - INGESTION_JOB_NAME_KEY + ":" + jobNameWithoutTimestamp, - METRICS_NAMESPACE_KEY + ":" + getMetricsNamespace(), - }; - - statsd.count(COUNT_NAME_FEATURE_ROW_INGESTED, featureRowLagStats.getN(), tags); - // DescriptiveStatistics returns invalid NaN value for getMin(), getMax(), ... when there is no - // items in the stats. - if (featureRowLagStats.getN() > 0) { - statsd.gauge(GAUGE_NAME_FEATURE_ROW_LAG_MS_MIN, featureRowLagStats.getMin(), tags); - statsd.gauge(GAUGE_NAME_FEATURE_ROW_LAG_MS_MAX, featureRowLagStats.getMax(), tags); - statsd.gauge(GAUGE_NAME_FEATURE_ROW_LAG_MS_MEAN, featureRowLagStats.getMean(), tags); - statsd.gauge( - GAUGE_NAME_FEATURE_ROW_LAG_MS_PERCENTILE_90, featureRowLagStats.getPercentile(90), tags); - statsd.gauge( - GAUGE_NAME_FEATURE_ROW_LAG_MS_PERCENTILE_95, featureRowLagStats.getPercentile(95), tags); - statsd.gauge( - GAUGE_NAME_FEATURE_ROW_LAG_MS_PERCENTILE_99, featureRowLagStats.getPercentile(99), tags); - } - - for (Entry entry : featureNameToLagStats.entrySet()) { - String featureName = entry.getKey(); - String[] tagsWithFeatureName = ArrayUtils.add(tags, FEATURE_TAG_KEY + ":" + featureName); - DescriptiveStatistics stats = entry.getValue(); - if (stats.getN() > 0) { - statsd.gauge(GAUGE_NAME_FEATURE_VALUE_LAG_MS_MIN, stats.getMin(), tagsWithFeatureName); - statsd.gauge(GAUGE_NAME_FEATURE_VALUE_LAG_MS_MAX, stats.getMax(), tagsWithFeatureName); - statsd.gauge(GAUGE_NAME_FEATURE_VALUE_LAG_MS_MEAN, stats.getMean(), tagsWithFeatureName); - statsd.gauge( - GAUGE_NAME_FEATURE_VALUE_LAG_MS_PERCENTILE_90, - stats.getPercentile(90), - tagsWithFeatureName); - statsd.gauge( - GAUGE_NAME_FEATURE_VALUE_LAG_MS_PERCENTILE_95, - stats.getPercentile(95), - tagsWithFeatureName); - statsd.gauge( - GAUGE_NAME_FEATURE_VALUE_LAG_MS_PERCENTILE_99, - stats.getPercentile(99), - tagsWithFeatureName); - } - statsd.count( - COUNT_NAME_FEATURE_VALUE_MISSING, - featureNameToMissingCount.get(featureName), - tagsWithFeatureName); - } - } -} diff --git a/ingestion/src/main/java/feast/ingestion/transform/metrics/WriteSuccessMetricsTransform.java b/ingestion/src/main/java/feast/ingestion/transform/metrics/WriteSuccessMetricsTransform.java deleted file mode 100644 index 15fe374bee3..00000000000 --- a/ingestion/src/main/java/feast/ingestion/transform/metrics/WriteSuccessMetricsTransform.java +++ /dev/null @@ -1,121 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2019 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.ingestion.transform.metrics; - -import com.google.auto.value.AutoValue; -import feast.ingestion.options.ImportOptions; -import feast.proto.types.FeatureRowProto.FeatureRow; -import org.apache.beam.sdk.metrics.Counter; -import org.apache.beam.sdk.metrics.Metrics; -import org.apache.beam.sdk.transforms.*; -import org.apache.beam.sdk.transforms.windowing.FixedWindows; -import org.apache.beam.sdk.transforms.windowing.Window; -import org.apache.beam.sdk.values.KV; -import org.apache.beam.sdk.values.PCollection; -import org.apache.beam.sdk.values.PDone; -import org.apache.beam.sdk.values.TypeDescriptors; -import org.joda.time.Duration; - -@AutoValue -public abstract class WriteSuccessMetricsTransform - extends PTransform, PDone> { - - public static final String METRIC_NAMESPACE = "WriteToStoreSuccess"; - public static final String ELEMENTS_WRITTEN_METRIC = "elements_written"; - private static final Counter elementsWritten = - Metrics.counter(METRIC_NAMESPACE, ELEMENTS_WRITTEN_METRIC); - - public abstract String getStoreName(); - - public static WriteSuccessMetricsTransform create(String storeName) { - return new AutoValue_WriteSuccessMetricsTransform(storeName); - } - - @Override - public PDone expand(PCollection input) { - ImportOptions options = input.getPipeline().getOptions().as(ImportOptions.class); - - input.apply( - "IncrementSuccessfulWriteToStoreElementsWrittenCounter", - MapElements.into(TypeDescriptors.booleans()) - .via( - (FeatureRow row) -> { - elementsWritten.inc(); - return true; - })); - - switch (options.getMetricsExporterType()) { - case "statsd": - - // Fixed window is applied so the metric collector will not be overwhelmed with the metrics - // data. For validation, only summaries of the values are usually required vs the actual - // values. - PCollection>> validRowsGroupedByRef = - input - .apply( - "FixedWindow", - Window.into( - FixedWindows.of( - Duration.standardSeconds( - options.getWindowSizeInSecForFeatureValueMetric()))) - .withAllowedLateness(Duration.millis(0))) - .apply( - "ConvertToKV_FeatureSetRefToFeatureRow", - ParDo.of( - new DoFn>() { - @ProcessElement - public void processElement( - ProcessContext c, @Element FeatureRow featureRow) { - c.output(KV.of(featureRow.getFeatureSet(), featureRow)); - } - })) - .apply("GroupByFeatureSetRef", GroupByKey.create()); - - validRowsGroupedByRef.apply( - "WriteRowMetrics", - ParDo.of( - WriteRowMetricsDoFn.newBuilder() - .setStatsdHost(options.getStatsdHost()) - .setStatsdPort(options.getStatsdPort()) - .setStoreName(getStoreName()) - .setMetricsNamespace(METRIC_NAMESPACE) - .build())); - - validRowsGroupedByRef.apply( - "WriteFeatureValueMetrics", - ParDo.of( - WriteFeatureValueMetricsDoFn.newBuilder() - .setStatsdHost(options.getStatsdHost()) - .setStatsdPort(options.getStatsdPort()) - .setStoreName(getStoreName()) - .setMetricsNamespace(METRIC_NAMESPACE) - .build())); - - return PDone.in(input.getPipeline()); - case "none": - default: - input.apply( - "Noop", - ParDo.of( - new DoFn() { - @ProcessElement - public void processElement(ProcessContext c) {} - })); - return PDone.in(input.getPipeline()); - } - } -} diff --git a/ingestion/src/main/java/feast/ingestion/transform/specs/FilterRelevantFunction.java b/ingestion/src/main/java/feast/ingestion/transform/specs/FilterRelevantFunction.java deleted file mode 100644 index 1a970643508..00000000000 --- a/ingestion/src/main/java/feast/ingestion/transform/specs/FilterRelevantFunction.java +++ /dev/null @@ -1,52 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.ingestion.transform.specs; - -import feast.common.models.Store; -import feast.proto.core.FeatureSetProto; -import feast.proto.core.SourceProto; -import feast.proto.core.StoreProto; -import java.util.List; -import org.apache.beam.sdk.transforms.ProcessFunction; -import org.apache.beam.sdk.values.KV; - -/** - * Selects only those FeatureSetSpecs that have the same source as current Job and their featureSet - * reference matches to current Job Subscriptions (extracted from Stores) - */ -public class FilterRelevantFunction - implements ProcessFunction, Boolean> { - private final List stores; - private final SourceProto.Source source; - - public FilterRelevantFunction(SourceProto.Source source, List stores) { - this.source = source; - this.stores = stores; - } - - @Override - public Boolean apply(KV input) throws Exception { - return stores.stream() - .anyMatch( - s -> - Store.isSubscribedToFeatureSet( - s.getSubscriptionsList(), - input.getValue().getProject(), - input.getValue().getName())) - && input.getValue().getSource().equals(source); - } -} diff --git a/ingestion/src/main/java/feast/ingestion/transform/specs/KafkaRecordToFeatureSetSpec.java b/ingestion/src/main/java/feast/ingestion/transform/specs/KafkaRecordToFeatureSetSpec.java deleted file mode 100644 index 47aefd0c0bb..00000000000 --- a/ingestion/src/main/java/feast/ingestion/transform/specs/KafkaRecordToFeatureSetSpec.java +++ /dev/null @@ -1,53 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.ingestion.transform.specs; - -import com.google.protobuf.InvalidProtocolBufferException; -import feast.proto.core.FeatureSetProto; -import org.apache.beam.sdk.io.kafka.KafkaRecord; -import org.apache.beam.sdk.transforms.DoFn; -import org.apache.beam.sdk.values.KV; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * Parse FeatureSetSpec from KafkaRecord. We expect message with: - * - *

- * - *

    - *
  • key: String - FeatureSet reference - *
  • value: FeatureSetSpec serialized in proto - *
- */ -public class KafkaRecordToFeatureSetSpec - extends DoFn, KV> { - private static final Logger log = LoggerFactory.getLogger(KafkaRecordToFeatureSetSpec.class); - - @ProcessElement - public void process(ProcessContext c) { - try { - FeatureSetProto.FeatureSetSpec featureSetSpec = - FeatureSetProto.FeatureSetSpec.parseFrom(c.element().getKV().getValue()); - c.output(KV.of(new String(c.element().getKV().getKey()), featureSetSpec)); - } catch (InvalidProtocolBufferException e) { - log.error( - String.format( - "Unable to decode FeatureSetSpec with reference %s", c.element().getKV().getKey())); - } - } -} diff --git a/ingestion/src/main/java/feast/ingestion/transform/specs/ReadFeatureSetSpecs.java b/ingestion/src/main/java/feast/ingestion/transform/specs/ReadFeatureSetSpecs.java deleted file mode 100644 index 7c6cfd97ae1..00000000000 --- a/ingestion/src/main/java/feast/ingestion/transform/specs/ReadFeatureSetSpecs.java +++ /dev/null @@ -1,135 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.ingestion.transform.specs; - -import static feast.ingestion.utils.SpecUtil.parseFeatureSetReference; - -import com.google.auto.value.AutoValue; -import com.google.common.collect.ImmutableMap; -import com.google.common.collect.Lists; -import feast.common.models.FeatureSetReference; -import feast.proto.core.FeatureSetProto; -import feast.proto.core.FeatureSetProto.FeatureSetSpec; -import feast.proto.core.IngestionJobProto; -import feast.proto.core.SourceProto; -import feast.proto.core.StoreProto; -import java.util.ArrayList; -import java.util.Comparator; -import java.util.List; -import org.apache.beam.sdk.coders.AvroCoder; -import org.apache.beam.sdk.coders.KvCoder; -import org.apache.beam.sdk.extensions.protobuf.ProtoCoder; -import org.apache.beam.sdk.io.kafka.KafkaIO; -import org.apache.beam.sdk.transforms.*; -import org.apache.beam.sdk.transforms.windowing.AfterPane; -import org.apache.beam.sdk.transforms.windowing.GlobalWindows; -import org.apache.beam.sdk.transforms.windowing.Repeatedly; -import org.apache.beam.sdk.transforms.windowing.Window; -import org.apache.beam.sdk.values.KV; -import org.apache.beam.sdk.values.PBegin; -import org.apache.beam.sdk.values.PCollection; -import org.apache.commons.lang3.tuple.Pair; -import org.apache.kafka.clients.consumer.ConsumerConfig; -import org.joda.time.Duration; - -/** - * Source of {@link FeatureSetSpec} - * - *

Reads from Kafka topic (from the beginning) {@link FeatureSetSpec}. - * - *

Filters only relevant Specs by {@link feast.proto.core.StoreProto.Store} subscriptions and - * {@link feast.proto.core.SourceProto.Source}. - * - *

Compacts {@link FeatureSetSpec} by reference (if it was not yet compacted in Kafka). - */ -@AutoValue -public abstract class ReadFeatureSetSpecs - extends PTransform>> { - public abstract IngestionJobProto.SpecsStreamingUpdateConfig getSpecsStreamingUpdateConfig(); - - public abstract SourceProto.Source getSource(); - - public abstract List getStores(); - - public static Builder newBuilder() { - return new AutoValue_ReadFeatureSetSpecs.Builder(); - } - - @AutoValue.Builder - public abstract static class Builder { - public abstract Builder setSpecsStreamingUpdateConfig( - IngestionJobProto.SpecsStreamingUpdateConfig config); - - public abstract Builder setSource(SourceProto.Source source); - - public abstract Builder setStores(List stores); - - public abstract ReadFeatureSetSpecs build(); - } - - @Override - public PCollection> expand(PBegin input) { - return input - .apply( - KafkaIO.readBytes() - .withBootstrapServers( - getSpecsStreamingUpdateConfig().getSource().getBootstrapServers()) - .withTopic(getSpecsStreamingUpdateConfig().getSource().getTopic()) - .withConsumerConfigUpdates( - ImmutableMap.of( - ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, - "earliest", - ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, - false))) - .apply("ParseFeatureSetSpec", ParDo.of(new KafkaRecordToFeatureSetSpec())) - .apply("OnlyRelevantSpecs", Filter.by(new FilterRelevantFunction(getSource(), getStores()))) - .apply( - Window.>into(new GlobalWindows()) - .triggering(Repeatedly.forever(AfterPane.elementCountAtLeast(1))) - .accumulatingFiredPanes() - .withAllowedLateness(Duration.ZERO)) - .apply( - Combine.perKey( - (SerializableFunction, FeatureSetSpec>) - specs -> { - ArrayList featureSetSpecs = Lists.newArrayList(specs); - featureSetSpecs.sort( - Comparator.comparing(FeatureSetSpec::getVersion).reversed()); - return featureSetSpecs.get(0); - })) - .apply("CreateFeatureSetReferenceKey", ParDo.of(new CreateFeatureSetReference())) - .setCoder( - KvCoder.of( - AvroCoder.of(FeatureSetReference.class), ProtoCoder.of(FeatureSetSpec.class))); - } - - public static class CreateFeatureSetReference - extends DoFn< - KV, - KV> { - @ProcessElement - public void process( - ProcessContext c, @Element KV input) { - Pair reference = parseFeatureSetReference(input.getKey()); - c.output( - KV.of( - FeatureSetReference.of( - reference.getLeft(), reference.getRight(), input.getValue().getVersion()), - input.getValue())); - } - } -} diff --git a/ingestion/src/main/java/feast/ingestion/transform/specs/WriteFeatureSetSpecAck.java b/ingestion/src/main/java/feast/ingestion/transform/specs/WriteFeatureSetSpecAck.java deleted file mode 100644 index 027f25a52ab..00000000000 --- a/ingestion/src/main/java/feast/ingestion/transform/specs/WriteFeatureSetSpecAck.java +++ /dev/null @@ -1,127 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.ingestion.transform.specs; - -import com.google.auto.value.AutoValue; -import feast.common.models.FeatureSetReference; -import feast.proto.core.IngestionJobProto; -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import org.apache.beam.sdk.io.kafka.KafkaIO; -import org.apache.beam.sdk.transforms.*; -import org.apache.beam.sdk.transforms.windowing.AfterPane; -import org.apache.beam.sdk.transforms.windowing.GlobalWindows; -import org.apache.beam.sdk.transforms.windowing.Repeatedly; -import org.apache.beam.sdk.transforms.windowing.Window; -import org.apache.beam.sdk.values.KV; -import org.apache.beam.sdk.values.PCollection; -import org.apache.beam.sdk.values.PDone; -import org.apache.kafka.common.serialization.ByteArraySerializer; -import org.apache.kafka.common.serialization.StringSerializer; -import org.joda.time.Duration; - -/** - * Collects output from sinks prepareWrite (several streams flatten into one). As soon as count of - * each FeatureSetReference reach getSinksCount() - it means that enough amount of sinks updated its - * state - ack is pushed. - * - *

Converts input {@link FeatureSetReference} into {@link - * feast.proto.core.IngestionJobProto.FeatureSetSpecAck} message and writes it to kafka (ack-topic). - */ -@AutoValue -public abstract class WriteFeatureSetSpecAck - extends PTransform, PDone> { - public abstract IngestionJobProto.SpecsStreamingUpdateConfig getSpecsStreamingUpdateConfig(); - - public abstract Integer getSinksCount(); - - public static Builder newBuilder() { - return new AutoValue_WriteFeatureSetSpecAck.Builder(); - } - - @AutoValue.Builder - public abstract static class Builder { - public abstract Builder setSpecsStreamingUpdateConfig( - IngestionJobProto.SpecsStreamingUpdateConfig config); - - public abstract Builder setSinksCount(Integer count); - - public abstract WriteFeatureSetSpecAck build(); - } - - @Override - public PDone expand(PCollection input) { - return input - .apply("Prepare", new PrepareWrite(getSinksCount())) - .apply("FeatureSetSpecToAckMessage", ParDo.of(new BuildAckMessage())) - .apply( - "ToKafka", - KafkaIO.write() - .withBootstrapServers( - getSpecsStreamingUpdateConfig().getAck().getBootstrapServers()) - .withTopic(getSpecsStreamingUpdateConfig().getAck().getTopic()) - .withKeySerializer(StringSerializer.class) - .withValueSerializer(ByteArraySerializer.class)); - } - - private static class BuildAckMessage extends DoFn> { - @ProcessElement - public void process(ProcessContext c) throws IOException { - ByteArrayOutputStream encodedAck = new ByteArrayOutputStream(); - - IngestionJobProto.FeatureSetSpecAck.newBuilder() - .setFeatureSetReference(c.element().getReference()) - .setJobName(c.getPipelineOptions().getJobName()) - .setFeatureSetVersion(c.element().getVersion()) - .build() - .writeTo(encodedAck); - - c.output(KV.of(c.element().getReference(), encodedAck.toByteArray())); - } - } - - /** - * Groups FeatureSetReference to generate ack only when amount of repeating elements reach - * sinksCount - */ - static class PrepareWrite - extends PTransform, PCollection> { - private final Integer sinksCount; - - PrepareWrite(Integer sinksCount) { - this.sinksCount = sinksCount; - } - - @Override - public PCollection expand(PCollection input) { - return input - .apply( - "OnEveryElementTrigger", - Window.into(new GlobalWindows()) - .accumulatingFiredPanes() - .triggering(Repeatedly.forever(AfterPane.elementCountAtLeast(1))) - .withAllowedLateness(Duration.ZERO)) - .apply("CountingReadySinks", Count.perElement()) - .apply( - "WhenAllReady", - Filter.by( - (SerializableFunction, Boolean>) - count -> count.getValue() >= sinksCount)) - .apply(Keys.create()); - } - } -} diff --git a/ingestion/src/main/java/feast/ingestion/utils/DateUtil.java b/ingestion/src/main/java/feast/ingestion/utils/DateUtil.java deleted file mode 100644 index 7a6ef429066..00000000000 --- a/ingestion/src/main/java/feast/ingestion/utils/DateUtil.java +++ /dev/null @@ -1,96 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2019 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.ingestion.utils; - -import com.google.protobuf.Timestamp; -import java.time.Instant; -import org.joda.time.DateTime; -import org.joda.time.DateTimeZone; -import org.joda.time.format.DateTimeFormat; -import org.joda.time.format.DateTimeFormatter; -import org.joda.time.format.DateTimeFormatterBuilder; -import org.joda.time.format.DateTimeParser; -import org.joda.time.format.ISODateTimeFormat; - -public class DateUtil { - - private static final DateTimeFormatter FALLBACK_TIMESTAMP_FORMAT; - - static { - DateTimeFormatterBuilder formatterBuilder = new DateTimeFormatterBuilder(); - DateTimeFormatter base = DateTimeFormat.forPattern("yyyy-MM-dd HH:mm:ss"); - DateTimeFormatter zone = DateTimeFormat.forPattern(" ZZZ"); - DateTimeParser fractionSecondParser = - new DateTimeFormatterBuilder().appendLiteral(".").appendFractionOfSecond(1, 6).toParser(); - - FALLBACK_TIMESTAMP_FORMAT = - formatterBuilder - .append(base) - .appendOptional(fractionSecondParser) - .append(zone) - .toFormatter(); - } - - public static String toString(DateTime datetime) { - return datetime.toString(ISODateTimeFormat.dateTime()); - } - - public static String toString(Timestamp timestamp) { - return toString(toDateTime(timestamp)); - } - - public static DateTime toDateTime(Timestamp timestamp) { - long millis = timestamp.getSeconds() * 1000 + (timestamp.getNanos() / 1000000); - return new DateTime(millis, DateTimeZone.UTC); - } - - public static DateTime toDateTime(String datetimeString) { - try { - return ISODateTimeFormat.dateTimeParser().parseDateTime(datetimeString); - } catch (IllegalArgumentException e) { - return DateTime.parse(datetimeString, FALLBACK_TIMESTAMP_FORMAT); - } - } - - public static Timestamp toTimestamp(DateTime datetime) { - return Timestamp.newBuilder() - .setSeconds(datetime.getMillis() / 1000) - .setNanos(datetime.getMillisOfSecond() * 1000000) - .build(); - } - - public static Timestamp toTimestamp(String datetimeString) { - return toTimestamp(toDateTime(datetimeString)); - } - - public static java.sql.Timestamp toSqlTimestamp(Timestamp timestamp) { - Instant instant = Instant.ofEpochSecond(timestamp.getSeconds(), timestamp.getNanos()); - return java.sql.Timestamp.from(instant); - } - - public static Timestamp maxTimestamp(Timestamp a, Timestamp b) { - if (a.getSeconds() != b.getSeconds()) { - return a.getSeconds() < b.getSeconds() ? b : a; - } else { - return a.getNanos() < b.getNanos() ? b : a; - } - } - - public static long toMillis(Timestamp timestamp) { - return toDateTime(timestamp).getMillis(); - } -} diff --git a/ingestion/src/main/java/feast/ingestion/utils/JsonUtil.java b/ingestion/src/main/java/feast/ingestion/utils/JsonUtil.java deleted file mode 100644 index 8fc1991b6a5..00000000000 --- a/ingestion/src/main/java/feast/ingestion/utils/JsonUtil.java +++ /dev/null @@ -1,42 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2019 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.ingestion.utils; - -import com.google.gson.Gson; -import com.google.gson.reflect.TypeToken; -import java.lang.reflect.Type; -import java.util.Collections; -import java.util.Map; - -public class JsonUtil { - - private static Gson gson = new Gson(); - - /** - * Unmarshals a given json string to map - * - * @param jsonString valid json formatted string - * @return map of keys to values in json - */ - public static Map convertJsonStringToMap(String jsonString) { - if (jsonString == null || jsonString.equals("") || jsonString.equals("{}")) { - return Collections.emptyMap(); - } - Type stringMapType = new TypeToken>() {}.getType(); - return gson.fromJson(jsonString, stringMapType); - } -} diff --git a/ingestion/src/main/java/feast/ingestion/utils/ResourceUtil.java b/ingestion/src/main/java/feast/ingestion/utils/ResourceUtil.java deleted file mode 100644 index 92912c96a5c..00000000000 --- a/ingestion/src/main/java/feast/ingestion/utils/ResourceUtil.java +++ /dev/null @@ -1,39 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2019 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.ingestion.utils; - -import com.google.common.io.Resources; -import java.nio.charset.StandardCharsets; -import org.slf4j.Logger; - -public class ResourceUtil { - private static final String DEADLETTER_SCHEMA_FILE_PATH = "schemas/deadletter_table_schema.json"; - private static final Logger log = org.slf4j.LoggerFactory.getLogger(ResourceUtil.class); - - public static String getDeadletterTableSchemaJson() { - String schemaJson = null; - try { - schemaJson = - Resources.toString( - Resources.getResource(DEADLETTER_SCHEMA_FILE_PATH), StandardCharsets.UTF_8); - } catch (Exception e) { - log.error( - "Unable to read {} file from the resources folder!", DEADLETTER_SCHEMA_FILE_PATH, e); - } - return schemaJson; - } -} diff --git a/ingestion/src/main/java/feast/ingestion/utils/SpecUtil.java b/ingestion/src/main/java/feast/ingestion/utils/SpecUtil.java deleted file mode 100644 index 512cd3752f8..00000000000 --- a/ingestion/src/main/java/feast/ingestion/utils/SpecUtil.java +++ /dev/null @@ -1,93 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2019 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.ingestion.utils; - -import com.google.protobuf.InvalidProtocolBufferException; -import com.google.protobuf.util.JsonFormat; -import feast.ingestion.values.Field; -import feast.proto.core.FeatureSetProto.EntitySpec; -import feast.proto.core.FeatureSetProto.FeatureSetSpec; -import feast.proto.core.FeatureSetProto.FeatureSpec; -import feast.proto.core.IngestionJobProto; -import feast.proto.core.IngestionJobProto.SpecsStreamingUpdateConfig; -import feast.proto.core.SourceProto.Source; -import feast.proto.core.StoreProto.Store; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import org.apache.commons.lang3.tuple.Pair; - -public class SpecUtil { - public static String PROJECT_DEFAULT_NAME = "default"; - - public static Pair parseFeatureSetReference(String reference) { - String[] split = reference.split("/", 2); - if (split.length == 1) { - return Pair.of(PROJECT_DEFAULT_NAME, split[0]); - } else { - return Pair.of(split[0], split[1]); - } - } - - public static List parseStoreJsonList(List jsonList) { - List stores = new ArrayList<>(); - for (String json : jsonList) { - Store.Builder builder = Store.newBuilder(); - try { - JsonFormat.parser().merge(json, builder); - } catch (InvalidProtocolBufferException e) { - throw new RuntimeException( - String.format("Couldn't parse StoreProto.Store object from json: %s", e.getCause())); - } - stores.add(builder.build()); - } - return stores; - } - - public static Source parseSourceJson(String jsonSource) { - Source.Builder builder = Source.newBuilder(); - try { - JsonFormat.parser().merge(jsonSource, builder); - } catch (InvalidProtocolBufferException e) { - throw new RuntimeException( - String.format("Couldn't parse SourceProto.Source object from json: %s", e.getCause())); - } - - return builder.build(); - } - - public static IngestionJobProto.SpecsStreamingUpdateConfig parseSpecsStreamingUpdateConfig( - String jsonConfig) throws InvalidProtocolBufferException { - SpecsStreamingUpdateConfig.Builder builder = SpecsStreamingUpdateConfig.newBuilder(); - JsonFormat.parser().merge(jsonConfig, builder); - return builder.build(); - } - - public static Map getFieldsByName(FeatureSetSpec featureSetSpec) { - Map fieldByName = new HashMap<>(); - for (EntitySpec entitySpec : featureSetSpec.getEntitiesList()) { - fieldByName.put( - entitySpec.getName(), new Field(entitySpec.getName(), entitySpec.getValueType())); - } - for (FeatureSpec featureSpec : featureSetSpec.getFeaturesList()) { - fieldByName.put( - featureSpec.getName(), new Field(featureSpec.getName(), featureSpec.getValueType())); - } - return fieldByName; - } -} diff --git a/ingestion/src/main/java/feast/ingestion/utils/StoreUtil.java b/ingestion/src/main/java/feast/ingestion/utils/StoreUtil.java deleted file mode 100644 index 67ea95b533f..00000000000 --- a/ingestion/src/main/java/feast/ingestion/utils/StoreUtil.java +++ /dev/null @@ -1,93 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2019 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.ingestion.utils; - -import static feast.proto.types.ValueProto.ValueType; - -import com.google.cloud.bigquery.StandardSQLTypeName; -import feast.proto.core.StoreProto.Store; -import feast.proto.core.StoreProto.Store.StoreType; -import feast.proto.types.ValueProto.ValueType.Enum; -import feast.storage.api.writer.FeatureSink; -import feast.storage.connectors.bigquery.writer.BigQueryFeatureSink; -import feast.storage.connectors.redis.writer.RedisFeatureSink; -import java.util.HashMap; -import java.util.Map; -import org.slf4j.Logger; - -// TODO: Create partitioned table by default - -/** - * This class is a utility to manage storage backends in Feast. - * - *

Examples when schemas need to be updated: - * - *

    - *
  • when a new entity is registered, a table usually needs to be created - *
  • when a new feature is registered, a column with appropriate data type usually needs to be - * created - *
- * - *

If the storage backend is a key-value or a schema-less database, however, there may not be a - * need to manage any schemas. This class will not be used in that case. - */ -public class StoreUtil { - - private static final Map VALUE_TYPE_TO_STANDARD_SQL_TYPE = - new HashMap<>(); - private static final Logger log = org.slf4j.LoggerFactory.getLogger(StoreUtil.class); - - // Column description for reserved fields - public static final String BIGQUERY_EVENT_TIMESTAMP_FIELD_DESCRIPTION = - "Event time for the FeatureRow"; - public static final String BIGQUERY_CREATED_TIMESTAMP_FIELD_DESCRIPTION = - "Processing time of the FeatureRow ingestion in Feast\""; - public static final String BIGQUERY_JOB_ID_FIELD_DESCRIPTION = - "Feast import job ID for the FeatureRow"; - - // Refer to protos/feast/core/Store.proto for the mapping definition. - static { - VALUE_TYPE_TO_STANDARD_SQL_TYPE.put(Enum.BYTES, StandardSQLTypeName.BYTES); - VALUE_TYPE_TO_STANDARD_SQL_TYPE.put(Enum.STRING, StandardSQLTypeName.STRING); - VALUE_TYPE_TO_STANDARD_SQL_TYPE.put(ValueType.Enum.INT32, StandardSQLTypeName.INT64); - VALUE_TYPE_TO_STANDARD_SQL_TYPE.put(ValueType.Enum.INT64, StandardSQLTypeName.INT64); - VALUE_TYPE_TO_STANDARD_SQL_TYPE.put(ValueType.Enum.DOUBLE, StandardSQLTypeName.FLOAT64); - VALUE_TYPE_TO_STANDARD_SQL_TYPE.put(ValueType.Enum.FLOAT, StandardSQLTypeName.FLOAT64); - VALUE_TYPE_TO_STANDARD_SQL_TYPE.put(ValueType.Enum.BOOL, StandardSQLTypeName.BOOL); - VALUE_TYPE_TO_STANDARD_SQL_TYPE.put(Enum.BYTES_LIST, StandardSQLTypeName.BYTES); - VALUE_TYPE_TO_STANDARD_SQL_TYPE.put(Enum.STRING_LIST, StandardSQLTypeName.STRING); - VALUE_TYPE_TO_STANDARD_SQL_TYPE.put(Enum.INT32_LIST, StandardSQLTypeName.INT64); - VALUE_TYPE_TO_STANDARD_SQL_TYPE.put(Enum.INT64_LIST, StandardSQLTypeName.INT64); - VALUE_TYPE_TO_STANDARD_SQL_TYPE.put(Enum.DOUBLE_LIST, StandardSQLTypeName.FLOAT64); - VALUE_TYPE_TO_STANDARD_SQL_TYPE.put(Enum.FLOAT_LIST, StandardSQLTypeName.FLOAT64); - VALUE_TYPE_TO_STANDARD_SQL_TYPE.put(Enum.BOOL_LIST, StandardSQLTypeName.BOOL); - } - - public static FeatureSink getFeatureSink(Store store) { - StoreType storeType = store.getType(); - switch (storeType) { - case REDIS_CLUSTER: - return RedisFeatureSink.fromConfig(store.getRedisClusterConfig()); - case REDIS: - return RedisFeatureSink.fromConfig(store.getRedisConfig()); - case BIGQUERY: - return BigQueryFeatureSink.fromConfig(store.getBigqueryConfig()); - default: - throw new RuntimeException(String.format("Store type '%s' is unsupported", storeType)); - } - } -} diff --git a/ingestion/src/main/java/feast/ingestion/values/FailsafeFeatureRow.java b/ingestion/src/main/java/feast/ingestion/values/FailsafeFeatureRow.java deleted file mode 100644 index 6035b8c7623..00000000000 --- a/ingestion/src/main/java/feast/ingestion/values/FailsafeFeatureRow.java +++ /dev/null @@ -1,116 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2019 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.ingestion.values; - -import com.google.common.base.MoreObjects; -import feast.ingestion.coders.FailsafeFeatureRowCoder; -import java.util.Objects; -import org.apache.avro.reflect.Nullable; -import org.apache.beam.sdk.coders.DefaultCoder; - -/** - * Adapted from: - * https://github.com/GoogleCloudPlatform/DataflowTemplates/blob/834c833c65d214a28b1f47b493c8407990c3e717/src/main/java/com/google/cloud/teleport/values/FailsafeElement.java - * - *

The {@link FailsafeFeatureRow} class holds the current value and original value of a record - * within a pipeline. This class allows pipelines to not lose valuable information about an incoming - * record throughout the processing of that record. The use of this class allows for more robust - * dead-letter strategies as the original record information is not lost throughout the pipeline and - * can be output to a dead-letter in the event of a failure during one of the pipelines transforms. - */ -@DefaultCoder(FailsafeFeatureRowCoder.class) -public class FailsafeFeatureRow { - - private final OriginalT originalPayload; - private final CurrentT payload; - @Nullable private String errorMessage; - @Nullable private String stacktrace; - - private FailsafeFeatureRow(OriginalT originalPayload, CurrentT payload) { - this.originalPayload = originalPayload; - this.payload = payload; - } - - public static FailsafeFeatureRow of( - OriginalT originalPayload, CurrentT currentPayload) { - return new FailsafeFeatureRow<>(originalPayload, currentPayload); - } - - public static FailsafeFeatureRow of( - FailsafeFeatureRow other) { - return new FailsafeFeatureRow<>(other.originalPayload, other.payload) - .setErrorMessage(other.getErrorMessage()) - .setStacktrace(other.getStacktrace()); - } - - public OriginalT getOriginalPayload() { - return originalPayload; - } - - public CurrentT getPayload() { - return payload; - } - - public String getErrorMessage() { - return errorMessage; - } - - public FailsafeFeatureRow setErrorMessage(String errorMessage) { - this.errorMessage = errorMessage; - return this; - } - - public String getStacktrace() { - return stacktrace; - } - - public FailsafeFeatureRow setStacktrace(String stacktrace) { - this.stacktrace = stacktrace; - return this; - } - - @Override - public boolean equals(Object obj) { - if (this == obj) { - return true; - } - if (obj == null || getClass() != obj.getClass()) { - return false; - } - - final FailsafeFeatureRow other = (FailsafeFeatureRow) obj; - return Objects.deepEquals(this.originalPayload, other.getOriginalPayload()) - && Objects.deepEquals(this.payload, other.getPayload()) - && Objects.deepEquals(this.errorMessage, other.getErrorMessage()) - && Objects.deepEquals(this.stacktrace, other.getStacktrace()); - } - - @Override - public int hashCode() { - return Objects.hash(originalPayload, payload, errorMessage, stacktrace); - } - - @Override - public String toString() { - return MoreObjects.toStringHelper(this) - .add("originalPayload", originalPayload) - .add("payload", payload) - .add("errorMessage", errorMessage) - .add("stacktrace", stacktrace) - .toString(); - } -} diff --git a/ingestion/src/main/java/feast/ingestion/values/FeatureSet.java b/ingestion/src/main/java/feast/ingestion/values/FeatureSet.java deleted file mode 100644 index 7395762f337..00000000000 --- a/ingestion/src/main/java/feast/ingestion/values/FeatureSet.java +++ /dev/null @@ -1,49 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2019 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.ingestion.values; - -import static feast.common.models.FeatureSet.getFeatureSetStringRef; -import static feast.ingestion.utils.SpecUtil.getFieldsByName; - -import feast.proto.core.FeatureSetProto; -import java.io.Serializable; -import java.util.Map; - -/** - * This class represents {@link feast.proto.core.FeatureSetProto.FeatureSetSpec} but contains fields - * directly accessible by name for feature validation purposes. - * - *

The use for this class is mainly for validating the Fields in FeatureRow. - */ -public class FeatureSet implements Serializable { - private final String reference; - - private final Map fields; - - public FeatureSet(FeatureSetProto.FeatureSetSpec featureSetSpec) { - this.reference = getFeatureSetStringRef(featureSetSpec); - this.fields = getFieldsByName(featureSetSpec); - } - - public String getReference() { - return reference; - } - - public Field getField(String fieldName) { - return fields.getOrDefault(fieldName, null); - } -} diff --git a/ingestion/src/main/java/feast/ingestion/values/Field.java b/ingestion/src/main/java/feast/ingestion/values/Field.java deleted file mode 100644 index 79cb83e76ad..00000000000 --- a/ingestion/src/main/java/feast/ingestion/values/Field.java +++ /dev/null @@ -1,46 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2019 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.ingestion.values; - -import feast.proto.types.ValueProto.ValueType; -import java.io.Serializable; -import org.apache.beam.sdk.coders.AvroCoder; -import org.apache.beam.sdk.coders.DefaultCoder; - -/** - * Field class represents {@link feast.proto.types.FieldProto.Field} but without value. - * - *

The use for this class is mainly for validating the Fields in FeatureRow. - */ -@DefaultCoder(AvroCoder.class) -public class Field implements Serializable { - private final String name; - private final ValueType.Enum type; - - public Field(String name, ValueType.Enum type) { - this.name = name; - this.type = type; - } - - public String getName() { - return name; - } - - public ValueType.Enum getType() { - return type; - } -} diff --git a/ingestion/src/main/proto/feast_ingestion/types/README.md b/ingestion/src/main/proto/feast_ingestion/types/README.md deleted file mode 100644 index 7736c97ba20..00000000000 --- a/ingestion/src/main/proto/feast_ingestion/types/README.md +++ /dev/null @@ -1 +0,0 @@ -Internal protobuf messages used in ingestion \ No newline at end of file diff --git a/ingestion/src/main/resources/logback.xml b/ingestion/src/main/resources/logback.xml deleted file mode 100644 index 85197f167d4..00000000000 --- a/ingestion/src/main/resources/logback.xml +++ /dev/null @@ -1,28 +0,0 @@ - - - - - - %d{HH:mm:ss} [%thread] %-5level %logger{100} - %msg%n - - - - - - - \ No newline at end of file diff --git a/ingestion/src/main/resources/schemas/deadletter_table_schema.json b/ingestion/src/main/resources/schemas/deadletter_table_schema.json deleted file mode 100644 index 92381189073..00000000000 --- a/ingestion/src/main/resources/schemas/deadletter_table_schema.json +++ /dev/null @@ -1,34 +0,0 @@ -{ - "fields": [ - { - "name": "timestamp", - "type": "TIMESTAMP", - "mode": "REQUIRED" - }, - { - "name": "job_name", - "type": "STRING", - "mode": "NULLABLE" - }, - { - "name": "transform_name", - "type": "STRING", - "mode": "NULLABLE" - }, - { - "name": "payload", - "type": "STRING", - "mode": "NULLABLE" - }, - { - "name": "error_message", - "type": "STRING", - "mode": "NULLABLE" - }, - { - "name": "stack_trace", - "type": "STRING", - "mode": "NULLABLE" - } - ] -} \ No newline at end of file diff --git a/ingestion/src/main/resources/templates/upsert_feature_row.postgres.sql.twig b/ingestion/src/main/resources/templates/upsert_feature_row.postgres.sql.twig deleted file mode 100644 index b1aa138af06..00000000000 --- a/ingestion/src/main/resources/templates/upsert_feature_row.postgres.sql.twig +++ /dev/null @@ -1,21 +0,0 @@ -INSERT INTO {{ tableName }} ( - id, - event_timestamp, - created_timestamp, - {{ join(featureNames, ', ') }} -) -VALUES ( - ?, - ?, - now(), - {% for featureName in featureNames -%} - ?{% if not(loop.last) %}, {% endif %} - {%- endfor %} -) -ON CONFLICT (id, event_timestamp) -DO UPDATE - SET created_timestamp = now(), - {%- for featureName in featureNames %} - {{ featureName }} = COALESCE(EXCLUDED.{{ featureName }}, {{ tableName }}.{{ featureName }}) - {%- if not(loop.last) %},{% endif -%} - {% endfor %} diff --git a/ingestion/src/test/java/feast/FeastMatchers.java b/ingestion/src/test/java/feast/FeastMatchers.java deleted file mode 100644 index 7c67afaf741..00000000000 --- a/ingestion/src/test/java/feast/FeastMatchers.java +++ /dev/null @@ -1,32 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2019 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast; - -import com.google.common.collect.Lists; -import org.apache.beam.sdk.transforms.SerializableFunction; -import org.junit.Assert; - -public class FeastMatchers { - - /** Can be used with the PAssert.that(..).satisfies(fn) method. */ - public static SerializableFunction, Void> hasCount(long count) { - return (Iterable iterable) -> { - Assert.assertEquals(count, Lists.newArrayList(iterable).size()); - return null; - }; - } -} diff --git a/ingestion/src/test/java/feast/ToOrderedFeatureRows.java b/ingestion/src/test/java/feast/ToOrderedFeatureRows.java deleted file mode 100644 index 35453f0754f..00000000000 --- a/ingestion/src/test/java/feast/ToOrderedFeatureRows.java +++ /dev/null @@ -1,52 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2019 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast; - -import com.google.common.collect.Lists; -import com.google.common.primitives.UnsignedBytes; -import feast.proto.types.FeatureRowExtendedProto.FeatureRowExtended; -import feast.proto.types.FeatureRowProto.FeatureRow; -import feast.proto.types.FieldProto.Field; -import java.util.List; -import org.apache.beam.sdk.transforms.MapElements; -import org.apache.beam.sdk.transforms.PTransform; -import org.apache.beam.sdk.values.PCollection; -import org.apache.beam.sdk.values.TypeDescriptor; - -public class ToOrderedFeatureRows - extends PTransform, PCollection> { - - public static FeatureRow orderedFeatureRow(FeatureRow row) { - List features = Lists.newArrayList(row.getFieldsList()); - features.sort( - (f1, f2) -> - UnsignedBytes.lexicographicalComparator().compare(f1.toByteArray(), f2.toByteArray())); - return row.toBuilder().clearFields().addAllFields(features).build(); - } - - @Override - public PCollection expand(PCollection input) { - return input - .apply( - "get rows", - MapElements.into(TypeDescriptor.of(FeatureRow.class)).via(FeatureRowExtended::getRow)) - .apply( - "normalize rows", - MapElements.into(TypeDescriptor.of(FeatureRow.class)) - .via(ToOrderedFeatureRows::orderedFeatureRow)); - } -} diff --git a/ingestion/src/test/java/feast/ingestion/ImportJobTest.java b/ingestion/src/test/java/feast/ingestion/ImportJobTest.java deleted file mode 100644 index 1be6981e8cb..00000000000 --- a/ingestion/src/test/java/feast/ingestion/ImportJobTest.java +++ /dev/null @@ -1,258 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2019 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.ingestion; - -import static feast.common.models.FeatureSet.getFeatureSetStringRef; -import static org.hamcrest.MatcherAssert.assertThat; -import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.notNullValue; - -import com.google.common.collect.ImmutableList; -import com.google.protobuf.util.JsonFormat; -import feast.ingestion.options.ImportOptions; -import feast.proto.core.FeatureSetProto.EntitySpec; -import feast.proto.core.FeatureSetProto.FeatureSet; -import feast.proto.core.FeatureSetProto.FeatureSetSpec; -import feast.proto.core.FeatureSetProto.FeatureSpec; -import feast.proto.core.IngestionJobProto; -import feast.proto.core.SourceProto.KafkaSourceConfig; -import feast.proto.core.SourceProto.Source; -import feast.proto.core.SourceProto.SourceType; -import feast.proto.core.StoreProto.Store; -import feast.proto.core.StoreProto.Store.RedisConfig; -import feast.proto.core.StoreProto.Store.StoreType; -import feast.proto.core.StoreProto.Store.Subscription; -import feast.proto.storage.RedisProto.RedisKey; -import feast.proto.types.FeatureRowProto.FeatureRow; -import feast.proto.types.FieldProto; -import feast.proto.types.ValueProto.ValueType.Enum; -import feast.test.TestUtil; -import io.lettuce.core.RedisClient; -import io.lettuce.core.RedisURI; -import io.lettuce.core.api.StatefulRedisConnection; -import io.lettuce.core.api.sync.RedisCommands; -import io.lettuce.core.codec.ByteArrayCodec; -import java.io.IOException; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; -import java.util.stream.IntStream; -import org.apache.beam.repackaged.core.org.apache.commons.lang3.tuple.Pair; -import org.apache.beam.sdk.PipelineResult; -import org.apache.beam.sdk.PipelineResult.State; -import org.apache.beam.sdk.options.PipelineOptionsFactory; -import org.apache.kafka.common.serialization.ByteArraySerializer; -import org.joda.time.Duration; -import org.junit.ClassRule; -import org.junit.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.testcontainers.containers.GenericContainer; -import org.testcontainers.containers.KafkaContainer; - -public class ImportJobTest { - - private static final Logger LOGGER = LoggerFactory.getLogger(ImportJobTest.class.getName()); - - @ClassRule public static KafkaContainer kafkaContainer = new KafkaContainer(); - - @ClassRule - public static GenericContainer redisContainer = - new GenericContainer("redis:5.0.3-alpine").withExposedPorts(6379); - - private static final String KAFKA_TOPIC = "topic_1"; - private static final String KAFKA_SPECS_TOPIC = "topic_specs_1"; - private static final String KAFKA_SPECS_ACK_TOPIC = "topic_specs_ack_1"; - - private static final long KAFKA_PUBLISH_TIMEOUT_SEC = 10; - - // No of samples of feature row that will be generated and used for testing. - // Note that larger no of samples will increase completion time for ingestion. - private static final int IMPORT_JOB_SAMPLE_FEATURE_ROW_SIZE = 128; - // Expected time taken for the import job to be ready to receive Feature Row input. - private static final int IMPORT_JOB_READY_DURATION_SEC = 10; - // The interval between checks for import job to finish writing elements to store. - private static final int IMPORT_JOB_CHECK_INTERVAL_DURATION_SEC = 5; - // Max duration to wait until the import job finishes writing to Store. - private static final int IMPORT_JOB_MAX_RUN_DURATION_SEC = 300; - - @Test - public void runPipeline_ShouldWriteToRedisCorrectlyGivenValidSpecAndFeatureRow() - throws IOException, InterruptedException { - Source featureSource = - Source.newBuilder() - .setType(SourceType.KAFKA) - .setKafkaSourceConfig( - KafkaSourceConfig.newBuilder() - .setBootstrapServers(kafkaContainer.getBootstrapServers()) - .setTopic(KAFKA_TOPIC) - .build()) - .build(); - - IngestionJobProto.SpecsStreamingUpdateConfig specsStreamingUpdateConfig = - IngestionJobProto.SpecsStreamingUpdateConfig.newBuilder() - .setSource( - KafkaSourceConfig.newBuilder() - .setBootstrapServers(kafkaContainer.getBootstrapServers()) - .setTopic(KAFKA_SPECS_TOPIC) - .build()) - .setAck( - KafkaSourceConfig.newBuilder() - .setBootstrapServers(kafkaContainer.getBootstrapServers()) - .setTopic(KAFKA_SPECS_ACK_TOPIC) - .build()) - .build(); - - FeatureSetSpec spec = - FeatureSetSpec.newBuilder() - .setName("feature_set") - .setProject("myproject") - .addEntities( - EntitySpec.newBuilder() - .setName("entity_id_primary") - .setValueType(Enum.INT32) - .build()) - .addEntities( - EntitySpec.newBuilder() - .setName("entity_id_secondary") - .setValueType(Enum.STRING) - .build()) - .addFeatures( - FeatureSpec.newBuilder() - .setName("feature_1") - .setValueType(Enum.STRING_LIST) - .build()) - .addFeatures( - FeatureSpec.newBuilder().setName("feature_2").setValueType(Enum.STRING).build()) - .addFeatures( - FeatureSpec.newBuilder().setName("feature_3").setValueType(Enum.INT64).build()) - .setSource(featureSource) - .build(); - - FeatureSet featureSet = FeatureSet.newBuilder().setSpec(spec).build(); - - Store redis = - Store.newBuilder() - .setName(StoreType.REDIS.toString()) - .setType(StoreType.REDIS) - .setRedisConfig( - RedisConfig.newBuilder() - .setHost(redisContainer.getHost()) - .setPort(redisContainer.getFirstMappedPort()) - .build()) - .addSubscriptions( - Subscription.newBuilder() - .setProject(spec.getProject()) - .setName(spec.getName()) - .build()) - .build(); - - ImportOptions options = PipelineOptionsFactory.create().as(ImportOptions.class); - - options.setSpecsStreamingUpdateConfigJson( - JsonFormat.printer().print(specsStreamingUpdateConfig)); - options.setSourceJson(JsonFormat.printer().print(featureSource)); - options.setStoresJson(Collections.singletonList(JsonFormat.printer().print(redis))); - options.setDefaultFeastProject("myproject"); - options.setProject(""); - options.setBlockOnRun(false); - - List> input = new ArrayList<>(); - Map expected = new HashMap<>(); - - LOGGER.info("Generating test data ..."); - IntStream.range(0, IMPORT_JOB_SAMPLE_FEATURE_ROW_SIZE) - .forEach( - i -> { - FeatureRow randomRow = TestUtil.createRandomFeatureRow(featureSet.getSpec()); - RedisKey redisKey = TestUtil.createRedisKey(featureSet.getSpec(), randomRow); - input.add(Pair.of("", randomRow)); - List fields = - randomRow.getFieldsList().stream() - .filter( - field -> - spec.getFeaturesList().stream() - .map(FeatureSpec::getName) - .collect(Collectors.toList()) - .contains(field.getName())) - .map( - field -> - field.toBuilder().setName(TestUtil.hash(field.getName())).build()) - .collect(Collectors.toList()); - randomRow = - randomRow - .toBuilder() - .clearFields() - .addAllFields(fields) - .clearFeatureSet() - .build(); - expected.put(redisKey, randomRow); - }); - - LOGGER.info("Starting Import Job with the following options: {}", options.toString()); - PipelineResult pipelineResult = ImportJob.runPipeline(options); - Thread.sleep(Duration.standardSeconds(IMPORT_JOB_READY_DURATION_SEC).getMillis()); - assertThat(pipelineResult.getState(), equalTo(State.RUNNING)); - - LOGGER.info("Publishing {} Feature Row messages to Kafka ...", input.size()); - TestUtil.publishToKafka( - kafkaContainer.getBootstrapServers(), - KAFKA_SPECS_TOPIC, - ImmutableList.of(Pair.of(getFeatureSetStringRef(spec), spec)), - ByteArraySerializer.class, - KAFKA_PUBLISH_TIMEOUT_SEC); - TestUtil.publishToKafka( - kafkaContainer.getBootstrapServers(), - KAFKA_TOPIC, - input, - ByteArraySerializer.class, - KAFKA_PUBLISH_TIMEOUT_SEC); - TestUtil.waitUntilAllElementsAreWrittenToStore( - pipelineResult, - Duration.standardSeconds(IMPORT_JOB_MAX_RUN_DURATION_SEC), - Duration.standardSeconds(IMPORT_JOB_CHECK_INTERVAL_DURATION_SEC)); - - LOGGER.info("Validating the actual values written to Redis ..."); - RedisClient redisClient = - RedisClient.create( - new RedisURI( - redisContainer.getHost(), - redisContainer.getFirstMappedPort(), - java.time.Duration.ofMillis(2000))); - StatefulRedisConnection connection = redisClient.connect(new ByteArrayCodec()); - RedisCommands sync = connection.sync(); - for (Map.Entry entry : expected.entrySet()) { - RedisKey key = entry.getKey(); - FeatureRow expectedValue = entry.getValue(); - - // Ensure ingested key exists. - byte[] actualByteValue = sync.get(key.toByteArray()); - assertThat("Key not found in Redis: " + key, actualByteValue, notNullValue()); - - // Ensure value is a valid serialized FeatureRow object. - FeatureRow actualValue = null; - actualValue = FeatureRow.parseFrom(actualByteValue); - - // Ensure the retrieved FeatureRow is equal to the ingested FeatureRow. - assertThat(actualValue, equalTo(expectedValue)); - } - redisClient.shutdown(); - } -} diff --git a/ingestion/src/test/java/feast/ingestion/options/BZip2CompressorTest.java b/ingestion/src/test/java/feast/ingestion/options/BZip2CompressorTest.java deleted file mode 100644 index cd03b18c793..00000000000 --- a/ingestion/src/test/java/feast/ingestion/options/BZip2CompressorTest.java +++ /dev/null @@ -1,40 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.ingestion.options; - -import java.io.BufferedReader; -import java.io.ByteArrayInputStream; -import java.io.IOException; -import java.io.InputStreamReader; -import org.apache.commons.compress.compressors.bzip2.BZip2CompressorInputStream; -import org.junit.Assert; -import org.junit.Test; - -public class BZip2CompressorTest { - - @Test - public void shouldHavBZip2CompatibleOutput() throws IOException { - BZip2Compressor compressor = new BZip2Compressor<>(String::getBytes); - String origString = "somestring"; - try (ByteArrayInputStream inputStream = - new ByteArrayInputStream(compressor.compress(origString)); - BZip2CompressorInputStream bzip2Input = new BZip2CompressorInputStream(inputStream); - BufferedReader reader = new BufferedReader(new InputStreamReader(bzip2Input))) { - Assert.assertEquals(origString, reader.readLine()); - } - } -} diff --git a/ingestion/src/test/java/feast/ingestion/options/BZip2DecompressorTest.java b/ingestion/src/test/java/feast/ingestion/options/BZip2DecompressorTest.java deleted file mode 100644 index fe7cc789d86..00000000000 --- a/ingestion/src/test/java/feast/ingestion/options/BZip2DecompressorTest.java +++ /dev/null @@ -1,48 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.ingestion.options; - -import static org.junit.Assert.*; - -import java.io.*; -import org.apache.commons.compress.compressors.bzip2.BZip2CompressorOutputStream; -import org.junit.Test; - -public class BZip2DecompressorTest { - - @Test - public void shouldDecompressBZip2Stream() throws IOException { - BZip2Decompressor decompressor = - new BZip2Decompressor<>( - inputStream -> { - BufferedReader reader = new BufferedReader(new InputStreamReader(inputStream)); - String output = reader.readLine(); - reader.close(); - return output; - }); - - String originalString = "abc"; - ByteArrayOutputStream compressedStream = new ByteArrayOutputStream(); - try (BZip2CompressorOutputStream bzip2Output = - new BZip2CompressorOutputStream(compressedStream)) { - bzip2Output.write(originalString.getBytes()); - } - - String decompressedString = decompressor.decompress(compressedStream.toByteArray()); - assertEquals(originalString, decompressedString); - } -} diff --git a/ingestion/src/test/java/feast/ingestion/options/StringListStreamConverterTest.java b/ingestion/src/test/java/feast/ingestion/options/StringListStreamConverterTest.java deleted file mode 100644 index 5ce9f054bc9..00000000000 --- a/ingestion/src/test/java/feast/ingestion/options/StringListStreamConverterTest.java +++ /dev/null @@ -1,36 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.ingestion.options; - -import static org.junit.Assert.*; - -import java.io.ByteArrayInputStream; -import java.io.IOException; -import java.io.InputStream; -import java.util.Arrays; -import org.junit.Test; - -public class StringListStreamConverterTest { - - @Test - public void shouldReadStreamAsNewlineSeparatedStrings() throws IOException { - StringListStreamConverter converter = new StringListStreamConverter(); - String originalString = "abc\ndef"; - InputStream stringStream = new ByteArrayInputStream(originalString.getBytes()); - assertEquals(Arrays.asList("abc", "def"), converter.readStream(stringStream)); - } -} diff --git a/ingestion/src/test/java/feast/ingestion/transform/FeatureRowToStoreAllocatorTest.java b/ingestion/src/test/java/feast/ingestion/transform/FeatureRowToStoreAllocatorTest.java deleted file mode 100644 index 9899e9e4e3c..00000000000 --- a/ingestion/src/test/java/feast/ingestion/transform/FeatureRowToStoreAllocatorTest.java +++ /dev/null @@ -1,157 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.ingestion.transform; - -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; -import feast.proto.core.StoreProto; -import feast.proto.core.StoreProto.Store.Subscription; -import feast.proto.types.FeatureRowProto; -import java.util.Arrays; -import java.util.List; -import java.util.Map; -import org.apache.beam.sdk.extensions.protobuf.ProtoCoder; -import org.apache.beam.sdk.testing.PAssert; -import org.apache.beam.sdk.testing.TestPipeline; -import org.apache.beam.sdk.transforms.Count; -import org.apache.beam.sdk.transforms.Create; -import org.apache.beam.sdk.values.PCollectionTuple; -import org.apache.beam.sdk.values.TupleTag; -import org.junit.Rule; -import org.junit.Test; - -public class FeatureRowToStoreAllocatorTest { - @Rule public transient TestPipeline p = TestPipeline.create(); - - private StoreProto.Store newStore(String s) { - return StoreProto.Store.newBuilder() - .addSubscriptions( - StoreProto.Store.Subscription.newBuilder().setProject("project").setName(s).build()) - .build(); - } - - private StoreProto.Store newStore(List subscriptionList) { - return StoreProto.Store.newBuilder().addAllSubscriptions(subscriptionList).build(); - } - - @Test - public void featureRowShouldBeAllocatedToStoreTagsAccordingToSubscription() { - StoreProto.Store bqOnlyStore = newStore("bq*"); - StoreProto.Store redisOnlyStore = newStore("redis*"); - StoreProto.Store anyStore = newStore("*"); - - Map> storeTags = - ImmutableMap.of( - bqOnlyStore, new TupleTag<>(), - redisOnlyStore, new TupleTag<>(), - anyStore, new TupleTag<>()); - - PCollectionTuple allocatedRows = - p.apply( - Create.of( - FeatureRowProto.FeatureRow.newBuilder().setFeatureSet("project/bq_1").build(), - FeatureRowProto.FeatureRow.newBuilder().setFeatureSet("project/bq_2").build(), - FeatureRowProto.FeatureRow.newBuilder() - .setFeatureSet("project/redis_1") - .build(), - FeatureRowProto.FeatureRow.newBuilder() - .setFeatureSet("project/redis_2") - .build(), - FeatureRowProto.FeatureRow.newBuilder() - .setFeatureSet("project/redis_3") - .build())) - .apply( - FeatureRowToStoreAllocator.newBuilder() - .setStoreTags(storeTags) - .setStores(ImmutableList.of(bqOnlyStore, redisOnlyStore, anyStore)) - .build()); - - PAssert.that( - allocatedRows - .get(storeTags.get(bqOnlyStore)) - .setCoder(ProtoCoder.of(FeatureRowProto.FeatureRow.class)) - .apply("CountBq", Count.globally())) - .containsInAnyOrder(2L); - - PAssert.that( - allocatedRows - .get(storeTags.get(redisOnlyStore)) - .setCoder(ProtoCoder.of(FeatureRowProto.FeatureRow.class)) - .apply("CountRedis", Count.globally())) - .containsInAnyOrder(3L); - - PAssert.that( - allocatedRows - .get(storeTags.get(anyStore)) - .setCoder(ProtoCoder.of(FeatureRowProto.FeatureRow.class)) - .apply("CountAll", Count.globally())) - .containsInAnyOrder(5L); - - p.run(); - } - - @Test - public void featureRowShouldBeAllocatedToStoreTagsAccordingToSubscriptionBlacklist() { - Subscription subscription1 = Subscription.newBuilder().setProject("*").setName("*").build(); - Subscription subscription2 = - Subscription.newBuilder().setProject("project1").setName("fs_2").build(); - Subscription subscription3 = - Subscription.newBuilder().setProject("project1").setName("fs_1").setExclude(true).build(); - Subscription subscription4 = - Subscription.newBuilder().setProject("project2").setName("*").setExclude(true).build(); - - List testStoreSubscriptions1 = - Arrays.asList(subscription1, subscription2, subscription3); - StoreProto.Store testStore1 = newStore(testStoreSubscriptions1); - - List testStoreSubscriptions2 = Arrays.asList(subscription1, subscription4); - StoreProto.Store testStore2 = newStore(testStoreSubscriptions2); - - Map> storeTags = - ImmutableMap.of( - testStore1, new TupleTag<>(), - testStore2, new TupleTag<>()); - - PCollectionTuple allocatedRows = - p.apply( - Create.of( - FeatureRowProto.FeatureRow.newBuilder().setFeatureSet("project1/fs_1").build(), - FeatureRowProto.FeatureRow.newBuilder().setFeatureSet("project2/fs_1").build(), - FeatureRowProto.FeatureRow.newBuilder().setFeatureSet("project2/fs_2").build())) - .apply( - FeatureRowToStoreAllocator.newBuilder() - .setStoreTags(storeTags) - .setStores(ImmutableList.of(testStore1, testStore2)) - .build()); - - PAssert.that( - allocatedRows - .get(storeTags.get(testStore1)) - .setCoder(ProtoCoder.of(FeatureRowProto.FeatureRow.class)) - .apply("CountStore1", Count.globally())) - .containsInAnyOrder(2L); - - PAssert.that( - allocatedRows - .get(storeTags.get(testStore2)) - .setCoder(ProtoCoder.of(FeatureRowProto.FeatureRow.class)) - .apply("CountStore2", Count.globally())) - .containsInAnyOrder(1L); - - p.run(); - } -} diff --git a/ingestion/src/test/java/feast/ingestion/transform/ProcessAndValidateFeatureRowsTest.java b/ingestion/src/test/java/feast/ingestion/transform/ProcessAndValidateFeatureRowsTest.java deleted file mode 100644 index 75809050f87..00000000000 --- a/ingestion/src/test/java/feast/ingestion/transform/ProcessAndValidateFeatureRowsTest.java +++ /dev/null @@ -1,302 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2019 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.ingestion.transform; - -import static feast.common.models.FeatureSet.getFeatureSetStringRef; - -import feast.proto.core.FeatureSetProto.EntitySpec; -import feast.proto.core.FeatureSetProto.FeatureSetSpec; -import feast.proto.core.FeatureSetProto.FeatureSpec; -import feast.proto.types.FeatureRowProto.FeatureRow; -import feast.proto.types.FieldProto.Field; -import feast.proto.types.ValueProto.Value; -import feast.proto.types.ValueProto.ValueType.Enum; -import feast.storage.api.writer.FailedElement; -import feast.test.TestUtil; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import org.apache.beam.sdk.extensions.protobuf.ProtoCoder; -import org.apache.beam.sdk.testing.PAssert; -import org.apache.beam.sdk.testing.TestPipeline; -import org.apache.beam.sdk.transforms.Count; -import org.apache.beam.sdk.transforms.Create; -import org.apache.beam.sdk.transforms.View; -import org.apache.beam.sdk.values.PCollectionTuple; -import org.apache.beam.sdk.values.PCollectionView; -import org.apache.beam.sdk.values.TupleTag; -import org.junit.Rule; -import org.junit.Test; - -public class ProcessAndValidateFeatureRowsTest { - - @Rule public transient TestPipeline p = TestPipeline.create(); - - private static final TupleTag SUCCESS_TAG = new TupleTag() {}; - - private static final TupleTag FAILURE_TAG = new TupleTag() {}; - - @Test - public void shouldWriteSuccessAndFailureTagsCorrectly() { - FeatureSetSpec fs1 = - FeatureSetSpec.newBuilder() - .setName("feature_set") - .setProject("myproject") - .addEntities( - EntitySpec.newBuilder() - .setName("entity_id_primary") - .setValueType(Enum.INT32) - .build()) - .addEntities( - EntitySpec.newBuilder() - .setName("entity_id_secondary") - .setValueType(Enum.STRING) - .build()) - .addFeatures( - FeatureSpec.newBuilder().setName("feature_1").setValueType(Enum.STRING).build()) - .addFeatures( - FeatureSpec.newBuilder().setName("feature_2").setValueType(Enum.INT64).build()) - .build(); - - FeatureSetSpec fs2 = - FeatureSetSpec.newBuilder() - .setName("feature_set_2") - .setProject("myproject") - .addEntities( - EntitySpec.newBuilder() - .setName("entity_id_primary") - .setValueType(Enum.INT32) - .build()) - .addEntities( - EntitySpec.newBuilder() - .setName("entity_id_secondary") - .setValueType(Enum.STRING) - .build()) - .addFeatures( - FeatureSpec.newBuilder().setName("feature_1").setValueType(Enum.STRING).build()) - .addFeatures( - FeatureSpec.newBuilder().setName("feature_2").setValueType(Enum.INT64).build()) - .build(); - - Map featureSetSpecs = new HashMap<>(); - featureSetSpecs.put("myproject/feature_set", fs1); - featureSetSpecs.put("myproject/feature_set_2", fs2); - - List input = new ArrayList<>(); - List expected = new ArrayList<>(); - - for (FeatureSetSpec featureSetSpec : featureSetSpecs.values()) { - FeatureRow randomRow = TestUtil.createRandomFeatureRow(featureSetSpec); - input.add(randomRow); - expected.add(randomRow); - } - - FeatureRow invalidRow = - FeatureRow.newBuilder() - .setFeatureSet(getFeatureSetStringRef(fs1)) - .addFields( - Field.newBuilder() - .setName("feature_1") - .setValue(Value.newBuilder().setBoolVal(false).build()) - .build()) - .build(); - - input.add(invalidRow); - - PCollectionView>> specsView = - p.apply("StaticSpecs", Create.of(featureSetSpecs)).apply(View.asMultimap()); - - PCollectionTuple output = - p.apply(Create.of(input)) - .setCoder(ProtoCoder.of(FeatureRow.class)) - .apply( - ProcessAndValidateFeatureRows.newBuilder() - .setDefaultProject("myproject") - .setFailureTag(FAILURE_TAG) - .setSuccessTag(SUCCESS_TAG) - .setFeatureSetSpecs(specsView) - .build()); - - PAssert.that(output.get(SUCCESS_TAG)).containsInAnyOrder(expected); - PAssert.that(output.get(FAILURE_TAG).apply(Count.globally())).containsInAnyOrder(1L); - - p.run(); - } - - @Test - public void shouldStripVersions() { - FeatureSetSpec fs1 = - FeatureSetSpec.newBuilder() - .setName("feature_set") - .setProject("myproject") - .addEntities( - EntitySpec.newBuilder() - .setName("entity_id_primary") - .setValueType(Enum.INT32) - .build()) - .addEntities( - EntitySpec.newBuilder() - .setName("entity_id_secondary") - .setValueType(Enum.STRING) - .build()) - .addFeatures( - FeatureSpec.newBuilder().setName("feature_1").setValueType(Enum.STRING).build()) - .addFeatures( - FeatureSpec.newBuilder().setName("feature_2").setValueType(Enum.INT64).build()) - .build(); - - Map featureSetSpecs = new HashMap<>(); - featureSetSpecs.put("myproject/feature_set", fs1); - - List input = new ArrayList<>(); - List expected = new ArrayList<>(); - - FeatureRow randomRow = TestUtil.createRandomFeatureRow(fs1); - expected.add(randomRow); - randomRow = randomRow.toBuilder().setFeatureSet("myproject/feature_set:1").build(); - input.add(randomRow); - - PCollectionView>> specsView = - p.apply("StaticSpecs", Create.of(featureSetSpecs)).apply(View.asMultimap()); - - PCollectionTuple output = - p.apply(Create.of(input)) - .setCoder(ProtoCoder.of(FeatureRow.class)) - .apply( - ProcessAndValidateFeatureRows.newBuilder() - .setDefaultProject("myproject") - .setFailureTag(FAILURE_TAG) - .setSuccessTag(SUCCESS_TAG) - .setFeatureSetSpecs(specsView) - .build()); - - PAssert.that(output.get(SUCCESS_TAG)).containsInAnyOrder(expected); - - p.run(); - } - - @Test - public void shouldApplyDefaultProject() { - FeatureSetSpec fs1 = - FeatureSetSpec.newBuilder() - .setName("feature_set") - .setProject("myproject") - .addEntities( - EntitySpec.newBuilder() - .setName("entity_id_primary") - .setValueType(Enum.INT32) - .build()) - .addEntities( - EntitySpec.newBuilder() - .setName("entity_id_secondary") - .setValueType(Enum.STRING) - .build()) - .addFeatures( - FeatureSpec.newBuilder().setName("feature_1").setValueType(Enum.STRING).build()) - .addFeatures( - FeatureSpec.newBuilder().setName("feature_2").setValueType(Enum.INT64).build()) - .build(); - - Map featureSetSpecs = new HashMap<>(); - featureSetSpecs.put("myproject/feature_set", fs1); - - List input = new ArrayList<>(); - List expected = new ArrayList<>(); - - FeatureRow randomRow = TestUtil.createRandomFeatureRow(fs1); - expected.add(randomRow); - randomRow = randomRow.toBuilder().setFeatureSet("feature_set").build(); - input.add(randomRow); - - PCollectionView>> specsView = - p.apply("StaticSpecs", Create.of(featureSetSpecs)).apply(View.asMultimap()); - - PCollectionTuple output = - p.apply(Create.of(input)) - .setCoder(ProtoCoder.of(FeatureRow.class)) - .apply( - ProcessAndValidateFeatureRows.newBuilder() - .setDefaultProject("myproject") - .setFailureTag(FAILURE_TAG) - .setSuccessTag(SUCCESS_TAG) - .setFeatureSetSpecs(specsView) - .build()); - - PAssert.that(output.get(SUCCESS_TAG)).containsInAnyOrder(expected); - - p.run(); - } - - @Test - public void shouldExcludeUnregisteredFields() { - FeatureSetSpec fs1 = - FeatureSetSpec.newBuilder() - .setName("feature_set") - .setProject("myproject") - .addEntities( - EntitySpec.newBuilder() - .setName("entity_id_primary") - .setValueType(Enum.INT32) - .build()) - .addEntities( - EntitySpec.newBuilder() - .setName("entity_id_secondary") - .setValueType(Enum.STRING) - .build()) - .addFeatures( - FeatureSpec.newBuilder().setName("feature_1").setValueType(Enum.STRING).build()) - .addFeatures( - FeatureSpec.newBuilder().setName("feature_2").setValueType(Enum.INT64).build()) - .build(); - - Map featureSetSpecs = new HashMap<>(); - featureSetSpecs.put("myproject/feature_set", fs1); - - List input = new ArrayList<>(); - List expected = new ArrayList<>(); - - FeatureRow randomRow = TestUtil.createRandomFeatureRow(fs1); - expected.add(randomRow); - input.add( - randomRow - .toBuilder() - .addFields( - Field.newBuilder() - .setName("extra") - .setValue(Value.newBuilder().setStringVal("hello"))) - .build()); - - PCollectionView>> specsView = - p.apply("StaticSpecs", Create.of(featureSetSpecs)).apply(View.asMultimap()); - - PCollectionTuple output = - p.apply(Create.of(input)) - .setCoder(ProtoCoder.of(FeatureRow.class)) - .apply( - ProcessAndValidateFeatureRows.newBuilder() - .setDefaultProject("myproject") - .setFailureTag(FAILURE_TAG) - .setSuccessTag(SUCCESS_TAG) - .setFeatureSetSpecs(specsView) - .build()); - - PAssert.that(output.get(SUCCESS_TAG)).containsInAnyOrder(expected); - - p.run(); - } -} diff --git a/ingestion/src/test/java/feast/ingestion/transform/metrics/WriteFeatureValueMetricsDoFnTest.java b/ingestion/src/test/java/feast/ingestion/transform/metrics/WriteFeatureValueMetricsDoFnTest.java deleted file mode 100644 index 54b68eb2203..00000000000 --- a/ingestion/src/test/java/feast/ingestion/transform/metrics/WriteFeatureValueMetricsDoFnTest.java +++ /dev/null @@ -1,283 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.ingestion.transform.metrics; - -import static org.junit.Assert.fail; - -import com.google.protobuf.ByteString; -import com.google.protobuf.Timestamp; -import com.google.protobuf.util.Timestamps; -import feast.proto.types.FeatureRowProto.FeatureRow; -import feast.proto.types.FeatureRowProto.FeatureRow.Builder; -import feast.proto.types.FieldProto.Field; -import feast.proto.types.ValueProto.BoolList; -import feast.proto.types.ValueProto.BytesList; -import feast.proto.types.ValueProto.DoubleList; -import feast.proto.types.ValueProto.FloatList; -import feast.proto.types.ValueProto.Int32List; -import feast.proto.types.ValueProto.Int64List; -import feast.proto.types.ValueProto.StringList; -import feast.proto.types.ValueProto.Value; -import feast.test.TestUtil.DummyStatsDServer; -import java.io.BufferedReader; -import java.io.IOException; -import java.net.URL; -import java.nio.file.Files; -import java.nio.file.Paths; -import java.time.Instant; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Map.Entry; -import org.apache.beam.sdk.options.PipelineOptions; -import org.apache.beam.sdk.options.PipelineOptionsFactory; -import org.apache.beam.sdk.testing.TestPipeline; -import org.apache.beam.sdk.transforms.Create; -import org.apache.beam.sdk.transforms.ParDo; -import org.junit.Rule; -import org.junit.Test; - -public class WriteFeatureValueMetricsDoFnTest { - - @Rule public final transient TestPipeline pipeline = TestPipeline.create(); - private static final int STATSD_SERVER_PORT = 17254; - private final DummyStatsDServer statsDServer = new DummyStatsDServer(STATSD_SERVER_PORT); - - @Test - public void shouldSendCorrectStatsDMetrics() throws IOException, InterruptedException { - PipelineOptions pipelineOptions = PipelineOptionsFactory.create(); - pipelineOptions.setJobName("job-12345678"); - - Map> input = - readTestInput("feast/ingestion/transform/WriteFeatureValueMetricsDoFnTest.input"); - List expectedLines = - readTestOutput("feast/ingestion/transform/WriteFeatureValueMetricsDoFnTest.output"); - - pipeline - .apply(Create.of(input)) - .apply( - ParDo.of( - WriteFeatureValueMetricsDoFn.newBuilder() - .setStatsdHost("localhost") - .setStatsdPort(STATSD_SERVER_PORT) - .setStoreName("store") - .setMetricsNamespace("test") - .build())); - pipeline.run(pipelineOptions).waitUntilFinish(); - // Wait until StatsD has finished processed all messages, 3 sec is a reasonable duration - // based on empirical testing. - Thread.sleep(3000); - - List actualLines = statsDServer.messagesReceived(); - for (String expected : expectedLines) { - boolean matched = false; - for (String actual : actualLines) { - if (actual.equals(expected)) { - matched = true; - break; - } - } - if (!matched) { - System.out.println("Print actual metrics output for debugging:"); - for (String line : actualLines) { - System.out.println(line); - } - fail(String.format("Expected StatsD metric not found:\n%s", expected)); - } - } - statsDServer.stop(); - } - - // Test utility method to read expected StatsD metrics output from a text file. - @SuppressWarnings("SameParameterValue") - public static List readTestOutput(String path) throws IOException { - URL url = Thread.currentThread().getContextClassLoader().getResource(path); - if (url == null) { - throw new IllegalArgumentException( - "cannot read test data, path contains null url. Path: " + path); - } - List lines = new ArrayList<>(); - try (BufferedReader reader = Files.newBufferedReader(Paths.get(url.getPath()))) { - String line = reader.readLine(); - while (line != null) { - if (line.trim().length() > 1) { - lines.add(line); - } - line = reader.readLine(); - } - } - return lines; - } - - public static Map> readTestInput(String path) throws IOException { - return readTestInput(path, null); - } - - // Test utility method to create test feature row data from a text file. - // If tsOverride is not null, all the feature row will have the same timestamp "tsOverride". - // Else if there exist a "timestamp" column with RFC3339 format, the feature row will be assigned - // that timestamp. - // Else no timestamp will be assigned (the feature row will have the default proto Timestamp - // object). - @SuppressWarnings("SameParameterValue") - public static Map> readTestInput(String path, Timestamp tsOverride) - throws IOException { - Map> data = new HashMap<>(); - URL url = Thread.currentThread().getContextClassLoader().getResource(path); - if (url == null) { - throw new IllegalArgumentException( - "cannot read test data, path contains null url. Path: " + path); - } - List lines = new ArrayList<>(); - try (BufferedReader reader = Files.newBufferedReader(Paths.get(url.getPath()))) { - String line = reader.readLine(); - while (line != null) { - lines.add(line); - line = reader.readLine(); - } - } - List colNames = new ArrayList<>(); - for (String line : lines) { - if (line.trim().length() < 1) { - continue; - } - String[] splits = line.split(","); - colNames.addAll(Arrays.asList(splits)); - - if (line.startsWith("featuresetref")) { - // Header line - colNames.addAll(Arrays.asList(splits).subList(1, splits.length)); - continue; - } - - Builder featureRowBuilder = FeatureRow.newBuilder(); - for (int i = 0; i < splits.length; i++) { - String colVal = splits[i].trim(); - if (i == 0) { - featureRowBuilder.setFeatureSet(colVal); - continue; - } - String colName = colNames.get(i); - if (colName.equals("timestamp")) { - Instant instant = Instant.parse(colVal); - featureRowBuilder.setEventTimestamp( - Timestamps.fromNanos(instant.getEpochSecond() * 1_000_000_000 + instant.getNano())); - continue; - } - - Field.Builder fieldBuilder = Field.newBuilder().setName(colName); - if (!colVal.isEmpty()) { - switch (colName) { - case "int32": - fieldBuilder.setValue(Value.newBuilder().setInt32Val((Integer.parseInt(colVal)))); - break; - case "int64": - fieldBuilder.setValue(Value.newBuilder().setInt64Val((Long.parseLong(colVal)))); - break; - case "double": - fieldBuilder.setValue(Value.newBuilder().setDoubleVal((Double.parseDouble(colVal)))); - break; - case "float": - fieldBuilder.setValue(Value.newBuilder().setFloatVal((Float.parseFloat(colVal)))); - break; - case "bool": - fieldBuilder.setValue(Value.newBuilder().setBoolVal((Boolean.parseBoolean(colVal)))); - break; - case "int32list": - List int32List = new ArrayList<>(); - for (String val : colVal.split("\\|")) { - int32List.add(Integer.parseInt(val)); - } - fieldBuilder.setValue( - Value.newBuilder().setInt32ListVal(Int32List.newBuilder().addAllVal(int32List))); - break; - case "int64list": - List int64list = new ArrayList<>(); - for (String val : colVal.split("\\|")) { - int64list.add(Long.parseLong(val)); - } - fieldBuilder.setValue( - Value.newBuilder().setInt64ListVal(Int64List.newBuilder().addAllVal(int64list))); - break; - case "doublelist": - List doubleList = new ArrayList<>(); - for (String val : colVal.split("\\|")) { - doubleList.add(Double.parseDouble(val)); - } - fieldBuilder.setValue( - Value.newBuilder() - .setDoubleListVal(DoubleList.newBuilder().addAllVal(doubleList))); - break; - case "floatlist": - List floatList = new ArrayList<>(); - for (String val : colVal.split("\\|")) { - floatList.add(Float.parseFloat(val)); - } - fieldBuilder.setValue( - Value.newBuilder().setFloatListVal(FloatList.newBuilder().addAllVal(floatList))); - break; - case "boollist": - List boolList = new ArrayList<>(); - for (String val : colVal.split("\\|")) { - boolList.add(Boolean.parseBoolean(val)); - } - fieldBuilder.setValue( - Value.newBuilder().setBoolListVal(BoolList.newBuilder().addAllVal(boolList))); - break; - case "bytes": - fieldBuilder.setValue( - Value.newBuilder().setBytesVal(ByteString.copyFromUtf8("Dummy"))); - break; - case "byteslist": - fieldBuilder.setValue( - Value.newBuilder().setBytesListVal(BytesList.getDefaultInstance())); - break; - case "string": - fieldBuilder.setValue(Value.newBuilder().setStringVal("Dummy")); - break; - case "stringlist": - fieldBuilder.setValue( - Value.newBuilder().setStringListVal(StringList.getDefaultInstance())); - break; - } - } - featureRowBuilder.addFields(fieldBuilder); - } - - if (!data.containsKey(featureRowBuilder.getFeatureSet())) { - data.put(featureRowBuilder.getFeatureSet(), new ArrayList<>()); - } - List featureRowsByFeatureSetRef = data.get(featureRowBuilder.getFeatureSet()); - if (tsOverride != null) { - featureRowBuilder.setEventTimestamp(tsOverride); - } - featureRowsByFeatureSetRef.add(featureRowBuilder.build()); - } - - // Convert List to Iterable to match the function signature in - // WriteFeatureValueMetricsDoFn - Map> dataWithIterable = new HashMap<>(); - for (Entry> entrySet : data.entrySet()) { - String key = entrySet.getKey(); - Iterable value = entrySet.getValue(); - dataWithIterable.put(key, value); - } - return dataWithIterable; - } -} diff --git a/ingestion/src/test/java/feast/ingestion/transform/metrics/WriteRowMetricsDoFnTest.java b/ingestion/src/test/java/feast/ingestion/transform/metrics/WriteRowMetricsDoFnTest.java deleted file mode 100644 index 3bed89e9b13..00000000000 --- a/ingestion/src/test/java/feast/ingestion/transform/metrics/WriteRowMetricsDoFnTest.java +++ /dev/null @@ -1,89 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.ingestion.transform.metrics; - -import static feast.ingestion.transform.metrics.WriteFeatureValueMetricsDoFnTest.readTestInput; -import static feast.ingestion.transform.metrics.WriteFeatureValueMetricsDoFnTest.readTestOutput; -import static org.junit.Assert.fail; - -import feast.proto.types.FeatureRowProto.FeatureRow; -import feast.test.TestUtil.DummyStatsDServer; -import java.io.IOException; -import java.time.Clock; -import java.time.Instant; -import java.time.ZoneId; -import java.util.List; -import java.util.Map; -import org.apache.beam.sdk.options.PipelineOptions; -import org.apache.beam.sdk.options.PipelineOptionsFactory; -import org.apache.beam.sdk.testing.TestPipeline; -import org.apache.beam.sdk.transforms.Create; -import org.apache.beam.sdk.transforms.ParDo; -import org.junit.Rule; -import org.junit.Test; - -public class WriteRowMetricsDoFnTest { - - @Rule public final transient TestPipeline pipeline = TestPipeline.create(); - private static final int STATSD_SERVER_PORT = 17255; - private final DummyStatsDServer statsDServer = new DummyStatsDServer(STATSD_SERVER_PORT); - - @Test - public void shouldSendCorrectStatsDMetrics() throws IOException, InterruptedException { - PipelineOptions pipelineOptions = PipelineOptionsFactory.create(); - pipelineOptions.setJobName("job-12345678"); - Map> input = - readTestInput("feast/ingestion/transform/WriteRowMetricsDoFnTest.input"); - List expectedLines = - readTestOutput("feast/ingestion/transform/WriteRowMetricsDoFnTest.output"); - - pipeline - .apply(Create.of(input)) - .apply( - ParDo.of( - WriteRowMetricsDoFn.newBuilder() - .setStatsdHost("localhost") - .setStatsdPort(STATSD_SERVER_PORT) - .setStoreName("store") - .setClock(Clock.fixed(Instant.ofEpochSecond(1585548645), ZoneId.of("UTC"))) - .setMetricsNamespace("test") - .build())); - pipeline.run(pipelineOptions).waitUntilFinish(); - // Wait until StatsD has finished processed all messages, 3 sec is a reasonable duration - // based on empirical testing. - Thread.sleep(3000); - - List actualLines = statsDServer.messagesReceived(); - for (String expected : expectedLines) { - boolean matched = false; - for (String actual : actualLines) { - if (actual.equals(expected)) { - matched = true; - break; - } - } - if (!matched) { - System.out.println("Print actual metrics output for debugging:"); - for (String line : actualLines) { - System.out.println(line); - } - fail(String.format("Expected StatsD metric not found:\n%s", expected)); - } - } - statsDServer.stop(); - } -} diff --git a/ingestion/src/test/java/feast/ingestion/transform/specs/FeatureSetSpecReadAndWriteTest.java b/ingestion/src/test/java/feast/ingestion/transform/specs/FeatureSetSpecReadAndWriteTest.java deleted file mode 100644 index 592adfa90a9..00000000000 --- a/ingestion/src/test/java/feast/ingestion/transform/specs/FeatureSetSpecReadAndWriteTest.java +++ /dev/null @@ -1,226 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.ingestion.transform.specs; - -import static feast.common.models.FeatureSet.getFeatureSetStringRef; -import static org.hamcrest.CoreMatchers.hasItem; -import static org.hamcrest.MatcherAssert.assertThat; - -import com.google.common.collect.ImmutableList; -import com.google.common.collect.Lists; -import com.google.protobuf.InvalidProtocolBufferException; -import feast.proto.core.FeatureSetProto; -import feast.proto.core.IngestionJobProto; -import feast.proto.core.SourceProto; -import feast.proto.core.StoreProto; -import feast.test.TestUtil; -import java.util.List; -import java.util.stream.Collectors; -import org.apache.beam.repackaged.core.org.apache.commons.lang3.tuple.Pair; -import org.apache.beam.runners.direct.DirectOptions; -import org.apache.beam.sdk.options.PipelineOptions; -import org.apache.beam.sdk.options.PipelineOptionsFactory; -import org.apache.beam.sdk.testing.TestPipeline; -import org.apache.beam.sdk.transforms.Keys; -import org.apache.kafka.clients.consumer.ConsumerRecord; -import org.apache.kafka.clients.consumer.ConsumerRecords; -import org.apache.kafka.clients.consumer.KafkaConsumer; -import org.apache.kafka.common.serialization.ByteArraySerializer; -import org.apache.kafka.common.serialization.Deserializer; -import org.junit.*; -import org.testcontainers.containers.KafkaContainer; - -public class FeatureSetSpecReadAndWriteTest { - @Rule public transient TestPipeline p = TestPipeline.fromOptions(makePipelineOptions()); - - @ClassRule public static KafkaContainer kafkaContainer = new KafkaContainer(); - - private static final String KAFKA_TOPIC = "topic"; - private static final String KAFKA_SPECS_TOPIC = "topic_specs"; - private static final String KAFKA_SPECS_ACK_TOPIC = "topic_specs_ack"; - - private static final long KAFKA_PUBLISH_TIMEOUT_SEC = 10; - private static final long KAFKA_POLL_TIMEOUT_SEC = 10; - - private KafkaConsumer consumer; - - @Before - public void setUp() { - consumer = - TestUtil.makeKafkaConsumer( - kafkaContainer.getBootstrapServers(), - KAFKA_SPECS_ACK_TOPIC, - AckMessageDeserializer.class); - } - - @After - public void tearDown() { - consumer.close(); - } - - public static PipelineOptions makePipelineOptions() { - DirectOptions options = PipelineOptionsFactory.as(DirectOptions.class); - options.setJobName("test_job"); - options.setBlockOnRun(false); - return options; - } - - @Test - public void pipelineShouldReadSpecsAndAcknowledge() throws InterruptedException { - SourceProto.Source source = - SourceProto.Source.newBuilder() - .setKafkaSourceConfig( - SourceProto.KafkaSourceConfig.newBuilder() - .setBootstrapServers(kafkaContainer.getBootstrapServers()) - .setTopic(KAFKA_TOPIC) - .build()) - .build(); - - StoreProto.Store store = - StoreProto.Store.newBuilder() - .addSubscriptions( - StoreProto.Store.Subscription.newBuilder() - .setProject("project") - .setName("*") - .build()) - .build(); - - IngestionJobProto.SpecsStreamingUpdateConfig specsStreamingUpdateConfig = - IngestionJobProto.SpecsStreamingUpdateConfig.newBuilder() - .setSource( - SourceProto.KafkaSourceConfig.newBuilder() - .setBootstrapServers(kafkaContainer.getBootstrapServers()) - .setTopic(KAFKA_SPECS_TOPIC) - .build()) - .setAck( - SourceProto.KafkaSourceConfig.newBuilder() - .setBootstrapServers(kafkaContainer.getBootstrapServers()) - .setTopic(KAFKA_SPECS_ACK_TOPIC) - .build()) - .build(); - - p.apply( - ReadFeatureSetSpecs.newBuilder() - .setSource(source) - .setStores(ImmutableList.of(store)) - .setSpecsStreamingUpdateConfig(specsStreamingUpdateConfig) - .build()) - .apply(Keys.create()) - .apply( - WriteFeatureSetSpecAck.newBuilder() - .setSinksCount(1) - .setSpecsStreamingUpdateConfig(specsStreamingUpdateConfig) - .build()); - - // specs' history is being compacted on the initial read - publishSpecToKafka("project", "fs", 1, source); - publishSpecToKafka("project", "fs", 2, source); - publishSpecToKafka("project", "fs", 3, source); - publishSpecToKafka("project", "fs_2", 2, source); - - p.run(); - Thread.sleep(10000); - - List acks = getFeatureSetSpecAcks(); - - assertThat( - acks, - hasItem( - IngestionJobProto.FeatureSetSpecAck.newBuilder() - .setJobName("test_job") - .setFeatureSetVersion(3) - .setFeatureSetReference("project/fs") - .build())); - assertThat( - acks, - hasItem( - IngestionJobProto.FeatureSetSpecAck.newBuilder() - .setJobName("test_job") - .setFeatureSetVersion(2) - .setFeatureSetReference("project/fs_2") - .build())); - - // in-flight update 1 - publishSpecToKafka("project", "fs", 4, source); - - Thread.sleep(5000); - - assertThat( - getFeatureSetSpecAcks(), - hasItem( - IngestionJobProto.FeatureSetSpecAck.newBuilder() - .setJobName("test_job") - .setFeatureSetVersion(4) - .setFeatureSetReference("project/fs") - .build())); - - // in-flight update 2 - publishSpecToKafka("project", "fs_2", 3, source); - - Thread.sleep(5000); - - assertThat( - getFeatureSetSpecAcks(), - hasItem( - IngestionJobProto.FeatureSetSpecAck.newBuilder() - .setJobName("test_job") - .setFeatureSetVersion(3) - .setFeatureSetReference("project/fs_2") - .build())); - } - - private List getFeatureSetSpecAcks() { - ConsumerRecords consumerRecords = - consumer.poll(java.time.Duration.ofSeconds(KAFKA_POLL_TIMEOUT_SEC)); - - return Lists.newArrayList(consumerRecords.records(KAFKA_SPECS_ACK_TOPIC)).stream() - .map(ConsumerRecord::value) - .collect(Collectors.toList()); - } - - private void publishSpecToKafka( - String project, String name, int version, SourceProto.Source source) { - FeatureSetProto.FeatureSetSpec spec = - FeatureSetProto.FeatureSetSpec.newBuilder() - .setProject(project) - .setName(name) - .setVersion(version) - .setSource(source) - .build(); - - TestUtil.publishToKafka( - kafkaContainer.getBootstrapServers(), - KAFKA_SPECS_TOPIC, - ImmutableList.of(Pair.of(getFeatureSetStringRef(spec), spec)), - ByteArraySerializer.class, - KAFKA_PUBLISH_TIMEOUT_SEC); - } - - public static class AckMessageDeserializer - implements Deserializer { - - @Override - public IngestionJobProto.FeatureSetSpecAck deserialize(String topic, byte[] data) { - try { - return IngestionJobProto.FeatureSetSpecAck.parseFrom(data); - } catch (InvalidProtocolBufferException e) { - e.printStackTrace(); - return null; - } - } - } -} diff --git a/ingestion/src/test/java/feast/ingestion/transform/specs/FilterRelevantTest.java b/ingestion/src/test/java/feast/ingestion/transform/specs/FilterRelevantTest.java deleted file mode 100644 index 33736dad41d..00000000000 --- a/ingestion/src/test/java/feast/ingestion/transform/specs/FilterRelevantTest.java +++ /dev/null @@ -1,109 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.ingestion.transform.specs; - -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; -import feast.proto.core.FeatureSetProto; -import feast.proto.core.SourceProto; -import feast.proto.core.StoreProto; -import org.apache.beam.sdk.testing.PAssert; -import org.apache.beam.sdk.testing.TestPipeline; -import org.apache.beam.sdk.transforms.Count; -import org.apache.beam.sdk.transforms.Create; -import org.apache.beam.sdk.transforms.Filter; -import org.apache.beam.sdk.values.KV; -import org.apache.beam.sdk.values.PCollection; -import org.junit.Rule; -import org.junit.Test; - -public class FilterRelevantTest { - @Rule public transient TestPipeline p = TestPipeline.create(); - - @Test - public void onlySpecsThatMatchStoreSubscriptionShouldPass() { - SourceProto.Source source = - SourceProto.Source.newBuilder() - .setKafkaSourceConfig( - SourceProto.KafkaSourceConfig.newBuilder() - .setBootstrapServers("localhost") - .setTopic("topic") - .build()) - .build(); - - StoreProto.Store store1 = - StoreProto.Store.newBuilder() - .addSubscriptions( - StoreProto.Store.Subscription.newBuilder() - .setProject("project") - .setName("fs*") - .build()) - .build(); - - StoreProto.Store store2 = - StoreProto.Store.newBuilder() - .addSubscriptions( - StoreProto.Store.Subscription.newBuilder() - .setProject("project_2") - .setName("fs_1") - .build()) - .build(); - - SourceProto.Source irrelevantSource = - SourceProto.Source.newBuilder() - .setKafkaSourceConfig( - SourceProto.KafkaSourceConfig.newBuilder() - .setBootstrapServers("localhost") - .setTopic("other_topic") - .build()) - .build(); - - PCollection> filtered = - p.apply( - Create.of( - ImmutableMap.of( - "project/fs_1", makeSpecBuilder(source).build(), // pass - "project/fs_2", - makeSpecBuilder(irrelevantSource).build(), // different source - "invalid/fs_3", - makeSpecBuilder(source) - .setProject("invalid") - .build(), // invalid project - "project/invalid", - makeSpecBuilder(source).setName("invalid").build(), // invalid name - "project_2/fs_1", - makeSpecBuilder(source).setProject("project_2").build() // pass - ))) - .apply(Filter.by(new FilterRelevantFunction(source, ImmutableList.of(store1, store2)))); - - PAssert.that(filtered) - .containsInAnyOrder( - KV.of("project/fs_1", makeSpecBuilder(source).build()), - KV.of("project_2/fs_1", makeSpecBuilder(source).setProject("project_2").build())); - - PAssert.that(filtered.apply(Count.globally())).containsInAnyOrder(2L); - - p.run(); - } - - private FeatureSetProto.FeatureSetSpec.Builder makeSpecBuilder(SourceProto.Source source) { - return FeatureSetProto.FeatureSetSpec.newBuilder() - .setProject("project") - .setName("fs_1") - .setSource(source); - } -} diff --git a/ingestion/src/test/java/feast/ingestion/transform/specs/WriteFeatureSetSpecAckTest.java b/ingestion/src/test/java/feast/ingestion/transform/specs/WriteFeatureSetSpecAckTest.java deleted file mode 100644 index 49baca9d938..00000000000 --- a/ingestion/src/test/java/feast/ingestion/transform/specs/WriteFeatureSetSpecAckTest.java +++ /dev/null @@ -1,71 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.ingestion.transform.specs; - -import com.google.common.collect.ImmutableList; -import feast.common.models.FeatureSetReference; -import org.apache.beam.sdk.coders.AvroCoder; -import org.apache.beam.sdk.testing.PAssert; -import org.apache.beam.sdk.testing.TestPipeline; -import org.apache.beam.sdk.testing.TestStream; -import org.apache.beam.sdk.transforms.Flatten; -import org.apache.beam.sdk.transforms.windowing.GlobalWindow; -import org.apache.beam.sdk.values.PCollection; -import org.apache.beam.sdk.values.PCollectionList; -import org.joda.time.Duration; -import org.junit.Rule; -import org.junit.Test; - -public class WriteFeatureSetSpecAckTest { - @Rule public transient TestPipeline p = TestPipeline.create(); - - @Test - public void shouldSendAckWhenAllSinksReady() { - TestStream sink1 = - TestStream.create(AvroCoder.of(FeatureSetReference.class)) - .addElements(FeatureSetReference.of("project", "fs", 1)) - .addElements(FeatureSetReference.of("project", "fs", 2)) - .addElements(FeatureSetReference.of("project", "fs", 3)) - .advanceWatermarkToInfinity(); - - TestStream sink2 = - TestStream.create(AvroCoder.of(FeatureSetReference.class)) - .addElements(FeatureSetReference.of("project", "fs_2", 1)) - .addElements(FeatureSetReference.of("project", "fs", 3)) - .advanceWatermarkToInfinity(); - - TestStream sink3 = - TestStream.create(AvroCoder.of(FeatureSetReference.class)) - .advanceProcessingTime(Duration.standardSeconds(10)) - .addElements(FeatureSetReference.of("project", "fs", 3)) - .advanceWatermarkToInfinity(); - - PCollectionList sinks = - PCollectionList.of( - ImmutableList.of( - p.apply("sink1", sink1), p.apply("sink2", sink2), p.apply("sink3", sink3))); - - PCollection grouped = - sinks.apply(Flatten.pCollections()).apply(new WriteFeatureSetSpecAck.PrepareWrite(3)); - - PAssert.that(grouped) - .inOnTimePane(GlobalWindow.INSTANCE) - .containsInAnyOrder(FeatureSetReference.of("project", "fs", 3)); - - p.run(); - } -} diff --git a/ingestion/src/test/java/feast/ingestion/utils/DateUtilTest.java b/ingestion/src/test/java/feast/ingestion/utils/DateUtilTest.java deleted file mode 100644 index 151d501a596..00000000000 --- a/ingestion/src/test/java/feast/ingestion/utils/DateUtilTest.java +++ /dev/null @@ -1,62 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2019 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.ingestion.utils; - -import static org.hamcrest.MatcherAssert.assertThat; -import static org.hamcrest.Matchers.*; - -import com.google.protobuf.Timestamp; -import junit.framework.TestCase; -import org.joda.time.DateTime; - -public class DateUtilTest extends TestCase { - public void testStringToTimestamp() { - Timestamp timestamp1 = DateUtil.toTimestamp("2018-07-03T15:09:34.123888999Z"); - Timestamp timestamp2 = DateUtil.toTimestamp("2018-07-03T15:09:34.123000000Z"); - Timestamp timestamp3 = DateUtil.toTimestamp("2018-07-03T15:09:34.124000000Z"); - // we are okay with only millisecond granularity - - assertThat(timestamp1, is(equalTo(timestamp2))); - assertThat(timestamp1, is(not(equalTo(timestamp3)))); - } - - public void testBigqueryTimestampStringToTimestamp() { - Timestamp timestamp = DateUtil.toTimestamp("2018-10-23 00:00:00 UTC"); - Timestamp timestamp2 = DateUtil.toTimestamp("2018-10-23T00:00:00.000Z"); - - assertThat(timestamp, equalTo(timestamp2)); - } - - public void testBigqueryTimestampWithFractionSecondStringToTimestamp() { - Timestamp timestamp = DateUtil.toTimestamp("2018-10-23 00:00:00.123456 UTC"); - Timestamp timestamp2 = DateUtil.toTimestamp("2018-10-23T00:00:00.123456Z"); - - assertThat(timestamp, equalTo(timestamp2)); - } - - public void testTimestampToDateTime() { - Timestamp timestamp1 = DateUtil.toTimestamp("2018-07-03T15:09:34.123888999Z"); - DateTime datetime = DateUtil.toDateTime(timestamp1); - assertThat(2018, is(equalTo(datetime.getYear()))); - assertThat(7, is(equalTo(datetime.getMonthOfYear()))); - assertThat(3, is(equalTo(datetime.getDayOfMonth()))); - assertThat(15, is(equalTo(datetime.getHourOfDay()))); - assertThat(9, is(equalTo(datetime.getMinuteOfHour()))); - assertThat(34, is(equalTo(datetime.getSecondOfMinute()))); - assertThat(123, is(equalTo(datetime.getMillisOfSecond()))); - } -} diff --git a/ingestion/src/test/java/feast/ingestion/utils/JsonUtilTest.java b/ingestion/src/test/java/feast/ingestion/utils/JsonUtilTest.java deleted file mode 100644 index 62c74dfc345..00000000000 --- a/ingestion/src/test/java/feast/ingestion/utils/JsonUtilTest.java +++ /dev/null @@ -1,43 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2019 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.ingestion.utils; - -import static org.hamcrest.Matchers.equalTo; -import static org.junit.Assert.assertThat; - -import java.util.Collections; -import java.util.HashMap; -import java.util.Map; -import org.junit.Test; - -public class JsonUtilTest { - - @Test - public void convertJsonStringToMapShouldConvertJsonStringToMap() { - String input = "{\"key\": \"value\"}"; - Map expected = new HashMap<>(); - expected.put("key", "value"); - assertThat(JsonUtil.convertJsonStringToMap(input), equalTo(expected)); - } - - @Test - public void convertJsonStringToMapShouldReturnEmptyMapForEmptyJson() { - String input = "{}"; - Map expected = Collections.emptyMap(); - assertThat(JsonUtil.convertJsonStringToMap(input), equalTo(expected)); - } -} diff --git a/ingestion/src/test/java/feast/test/TestUtil.java b/ingestion/src/test/java/feast/test/TestUtil.java deleted file mode 100644 index 0634d26cc9d..00000000000 --- a/ingestion/src/test/java/feast/test/TestUtil.java +++ /dev/null @@ -1,421 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2019 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.test; - -import static feast.common.models.FeatureSet.getFeatureSetStringRef; - -import com.google.common.collect.ImmutableList; -import com.google.common.hash.Hashing; -import com.google.protobuf.ByteString; -import com.google.protobuf.Message; -import com.google.protobuf.Timestamp; -import feast.ingestion.transform.metrics.WriteSuccessMetricsTransform; -import feast.proto.core.FeatureSetProto.FeatureSet; -import feast.proto.core.FeatureSetProto.FeatureSetSpec; -import feast.proto.storage.RedisProto.RedisKey; -import feast.proto.types.FeatureRowProto.FeatureRow; -import feast.proto.types.FeatureRowProto.FeatureRow.Builder; -import feast.proto.types.FieldProto.Field; -import feast.proto.types.ValueProto.*; -import java.net.DatagramPacket; -import java.net.DatagramSocket; -import java.net.SocketException; -import java.nio.charset.StandardCharsets; -import java.time.Instant; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.Properties; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.ThreadLocalRandom; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; -import org.apache.beam.repackaged.core.org.apache.commons.lang3.tuple.Pair; -import org.apache.beam.sdk.PipelineResult; -import org.apache.beam.sdk.PipelineResult.State; -import org.apache.beam.sdk.metrics.MetricResult; -import org.apache.beam.sdk.metrics.MetricResults; -import org.apache.commons.lang3.RandomStringUtils; -import org.apache.kafka.clients.consumer.ConsumerConfig; -import org.apache.kafka.clients.consumer.KafkaConsumer; -import org.apache.kafka.clients.producer.KafkaProducer; -import org.apache.kafka.clients.producer.Producer; -import org.apache.kafka.clients.producer.ProducerConfig; -import org.apache.kafka.clients.producer.ProducerRecord; -import org.apache.kafka.common.serialization.StringDeserializer; -import org.apache.kafka.common.serialization.StringSerializer; -import org.joda.time.Duration; - -@SuppressWarnings("WeakerAccess") -public class TestUtil { - - /** - * Publish test Feature Row messages to a running Kafka broker - * - * @param bootstrapServers e.g. localhost:9092 - * @param topic e.g. my_topic - * @param messages e.g. list of Feature Row - * @param valueSerializer in Feast this valueSerializer should be "ByteArraySerializer.class" - * @param publishTimeoutSec duration to wait for publish operation (of each message) to succeed - */ - public static void publishToKafka( - String bootstrapServers, - String topic, - List> messages, - Class valueSerializer, - long publishTimeoutSec) { - - Properties prop = new Properties(); - prop.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapServers); - prop.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class); - prop.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, valueSerializer); - Producer producer = new KafkaProducer<>(prop); - - for (Pair featureRow : messages) { - ProducerRecord record = - new ProducerRecord<>(topic, featureRow.getLeft(), featureRow.getRight().toByteArray()); - try { - producer.send(record).get(publishTimeoutSec, TimeUnit.SECONDS); - } catch (InterruptedException | ExecutionException | TimeoutException e) { - e.printStackTrace(); - } - } - - producer.close(); - } - - public static KafkaConsumer makeKafkaConsumer( - String bootstrapServers, String topic, Class valueDeserializer) { - Properties prop = new Properties(); - prop.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapServers); - prop.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class); - prop.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, valueDeserializer); - prop.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest"); - prop.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, false); - prop.put(ConsumerConfig.GROUP_ID_CONFIG, "test"); - - KafkaConsumer consumer = new KafkaConsumer<>(prop); - - consumer.subscribe(ImmutableList.of(topic)); - return consumer; - } - - /** - * Create a Feature Row with random value according to the FeatureSetSpec - * - *

See {@link #createRandomFeatureRow(FeatureSetSpec, int)} - * - * @param featureSetSpec {@link FeatureSetSpec} - * @return {@link FeatureRow} - */ - public static FeatureRow createRandomFeatureRow(FeatureSetSpec featureSetSpec) { - ThreadLocalRandom random = ThreadLocalRandom.current(); - int randomStringSizeMaxSize = 12; - return createRandomFeatureRow(featureSetSpec, random.nextInt(0, randomStringSizeMaxSize) + 4); - } - - /** - * Create a Feature Row with random value according to the FeatureSet. - * - *

The Feature Row created contains fields according to the entities and features defined in - * FeatureSet, matching the value type of the field, with randomized value for testing. - * - * @param featureSetSpec {@link FeatureSetSpec} - * @param randomStringSize number of characters for the generated random string - * @return {@link FeatureRow} - */ - public static FeatureRow createRandomFeatureRow( - FeatureSetSpec featureSetSpec, int randomStringSize) { - - Instant time = Instant.now(); - Timestamp timestamp = - Timestamp.newBuilder().setSeconds(time.getEpochSecond()).setNanos(time.getNano()).build(); - - Builder builder = - FeatureRow.newBuilder() - .setFeatureSet(getFeatureSetStringRef(featureSetSpec)) - .setEventTimestamp(timestamp); - - featureSetSpec - .getEntitiesList() - .forEach( - field -> { - builder.addFields( - Field.newBuilder() - .setName(field.getName()) - .setValue(createRandomValue(field.getValueType(), randomStringSize)) - .build()); - }); - - featureSetSpec - .getFeaturesList() - .forEach( - field -> { - builder.addFields( - Field.newBuilder() - .setName(field.getName()) - .setValue(createRandomValue(field.getValueType(), randomStringSize)) - .build()); - }); - - return builder.build(); - } - - /** - * Create a random Feast {@link Value} of {@link ValueType.Enum}. - * - * @param type {@link ValueType.Enum} - * @param randomStringSize number of characters for the generated random string - * @return {@link Value} - */ - public static Value createRandomValue(ValueType.Enum type, int randomStringSize) { - Value.Builder builder = Value.newBuilder(); - ThreadLocalRandom random = ThreadLocalRandom.current(); - - switch (type) { - case INVALID: - case UNRECOGNIZED: - throw new IllegalArgumentException("Invalid ValueType: " + type); - case BYTES: - builder.setBytesVal( - ByteString.copyFrom(RandomStringUtils.randomAlphanumeric(randomStringSize).getBytes())); - break; - case STRING: - builder.setStringVal(RandomStringUtils.randomAlphanumeric(randomStringSize)); - break; - case INT32: - builder.setInt32Val(random.nextInt()); - break; - case INT64: - builder.setInt64Val(random.nextLong()); - break; - case DOUBLE: - builder.setDoubleVal(random.nextDouble()); - break; - case FLOAT: - builder.setFloatVal(random.nextFloat()); - break; - case BOOL: - builder.setBoolVal(random.nextBoolean()); - break; - case BYTES_LIST: - builder.setBytesListVal( - BytesList.newBuilder() - .addVal( - ByteString.copyFrom( - RandomStringUtils.randomAlphanumeric(randomStringSize).getBytes())) - .build()); - break; - case STRING_LIST: - builder.setStringListVal( - StringList.newBuilder() - .addVal(RandomStringUtils.randomAlphanumeric(randomStringSize)) - .build()); - break; - case INT32_LIST: - builder.setInt32ListVal(Int32List.newBuilder().addVal(random.nextInt()).build()); - break; - case INT64_LIST: - builder.setInt64ListVal(Int64List.newBuilder().addVal(random.nextLong()).build()); - break; - case DOUBLE_LIST: - builder.setDoubleListVal(DoubleList.newBuilder().addVal(random.nextDouble()).build()); - break; - case FLOAT_LIST: - builder.setFloatListVal(FloatList.newBuilder().addVal(random.nextFloat()).build()); - break; - case BOOL_LIST: - builder.setBoolListVal(BoolList.newBuilder().addVal(random.nextBoolean()).build()); - break; - } - return builder.build(); - } - - /** - * Create {@link RedisKey} from {@link FeatureSet} and {@link FeatureRow}. - * - *

The entities in the created {@link RedisKey} will contain the value with matching field name - * in the {@link FeatureRow} - * - * @param featureSetSpec {@link FeatureSetSpec} - * @param row {@link FeatureSet} - * @return {@link RedisKey} - */ - public static RedisKey createRedisKey(FeatureSetSpec featureSetSpec, FeatureRow row) { - RedisKey.Builder builder = - RedisKey.newBuilder().setFeatureSet(getFeatureSetStringRef(featureSetSpec)); - featureSetSpec - .getEntitiesList() - .forEach( - entityField -> - row.getFieldsList().stream() - .filter(rowField -> rowField.getName().equals(entityField.getName())) - .findFirst() - .ifPresent(builder::addEntities)); - return builder.build(); - } - - // Modified version of - // https://github.com/tim-group/java-statsd-client/blob/master/src/test/java/com/timgroup/statsd/NonBlockingStatsDClientTest.java - @SuppressWarnings("CatchMayIgnoreException") - public static class DummyStatsDServer { - - private final List messagesReceived = new ArrayList(); - private final DatagramSocket server; - - public DummyStatsDServer(int port) { - try { - server = new DatagramSocket(port); - } catch (SocketException e) { - throw new IllegalStateException(e); - } - new Thread( - () -> { - try { - while (true) { - final DatagramPacket packet = new DatagramPacket(new byte[65535], 65535); - server.receive(packet); - messagesReceived.add( - new String(packet.getData(), StandardCharsets.UTF_8).trim() + "\n"); - // The sleep duration here is shorter than that used in waitForMessage() at - // 50ms. - // Otherwise sometimes some messages seem to be lost, leading to flaky tests. - Thread.sleep(15L); - } - - } catch (Exception e) { - } - }) - .start(); - } - - public void stop() { - server.close(); - } - - public void waitForMessage() { - while (messagesReceived.isEmpty()) { - try { - Thread.sleep(50L); - } catch (InterruptedException e) { - } - } - } - - public List messagesReceived() { - List out = new ArrayList<>(); - for (String msg : messagesReceived) { - String[] lines = msg.split("\n"); - out.addAll(Arrays.asList(lines)); - } - return out; - } - } - - /** - * Create a field object with given name and type. - * - * @param name of the field. - * @param value of the field. Should be compatible with the valuetype given. - * @param valueType type of the field. - * @return Field object - */ - public static Field field(String name, Object value, ValueType.Enum valueType) { - Field.Builder fieldBuilder = Field.newBuilder().setName(name); - switch (valueType) { - case INT32: - return fieldBuilder.setValue(Value.newBuilder().setInt32Val((int) value)).build(); - case INT64: - return fieldBuilder.setValue(Value.newBuilder().setInt64Val((int) value)).build(); - case FLOAT: - return fieldBuilder.setValue(Value.newBuilder().setFloatVal((float) value)).build(); - case DOUBLE: - return fieldBuilder.setValue(Value.newBuilder().setDoubleVal((double) value)).build(); - case STRING: - return fieldBuilder.setValue(Value.newBuilder().setStringVal((String) value)).build(); - default: - throw new IllegalStateException("Unexpected valueType: " + value.getClass()); - } - } - - /** - * This blocking method waits until an ImportJob pipeline has written all elements to the store. - * - *

The pipeline must be in the RUNNING state before calling this method. - * - * @param pipelineResult result of running the Pipeline - * @param maxWaitDuration wait until this max amount of duration - * @throws InterruptedException if the thread is interruped while waiting - */ - public static void waitUntilAllElementsAreWrittenToStore( - PipelineResult pipelineResult, Duration maxWaitDuration, Duration checkInterval) - throws InterruptedException { - if (pipelineResult.getState().isTerminal()) { - return; - } - - if (!pipelineResult.getState().equals(State.RUNNING)) { - throw new IllegalArgumentException( - "Pipeline must be in RUNNING state before calling this method."); - } - - MetricResults metricResults; - try { - metricResults = pipelineResult.metrics(); - } catch (UnsupportedOperationException e) { - // Runner does not support metrics so we just wait as long as we are allowed to. - Thread.sleep(maxWaitDuration.getMillis()); - return; - } - - String writeToStoreMetric = - WriteSuccessMetricsTransform.METRIC_NAMESPACE - + ":" - + WriteSuccessMetricsTransform.ELEMENTS_WRITTEN_METRIC; - long committed = 0; - long maxSystemTimeMillis = System.currentTimeMillis() + maxWaitDuration.getMillis(); - - while (System.currentTimeMillis() <= maxSystemTimeMillis) { - Thread.sleep(checkInterval.getMillis()); - - for (MetricResult metricResult : metricResults.allMetrics().getCounters()) { - // We are only concerned with the metric: count of elements that have been - // written to the store. - if (!metricResult.getName().toString().contains(writeToStoreMetric)) { - continue; - } - try { - // If between check interval, no more changes in the no of committed elements - // we can assume the pipeline has finished writing all the elements to store. - if (metricResult.getCommitted() == committed) { - return; - } - committed = metricResult.getCommitted(); - break; - } catch (UnsupportedOperationException e) { - // Runner does not support committed metrics so we just wait as long as we are allowed to. - Thread.sleep(maxWaitDuration.getMillis()); - return; - } - } - } - } - - public static String hash(String input) { - return Hashing.murmur3_32().hashString(input, StandardCharsets.UTF_8).toString(); - } -} diff --git a/ingestion/src/test/resources/feast/ingestion/transform/WriteFeatureValueMetricsDoFnTest.README b/ingestion/src/test/resources/feast/ingestion/transform/WriteFeatureValueMetricsDoFnTest.README deleted file mode 100644 index 3c8759d1702..00000000000 --- a/ingestion/src/test/resources/feast/ingestion/transform/WriteFeatureValueMetricsDoFnTest.README +++ /dev/null @@ -1,9 +0,0 @@ -WriteFeatureValueMetricsDoFnTest.input file contains data that can be read by test utility -into map of FeatureSetRef -> [FeatureRow]. In the first row, the cell value corresponds to the -field name in the FeatureRow. This should not be changed as the test utility derives the value -type from this name. Empty value in the cell is a value that is not set. For list type, the values -of different element is separated by the '|' character. - -WriteFeatureValueMetricsDoFnTest.output file contains lines of expected StatsD metrics that should -be sent when WriteFeatureValueMetricsDoFn runs. It can be checked against the actual outputted -StatsD metrics to test for correctness. diff --git a/ingestion/src/test/resources/feast/ingestion/transform/WriteFeatureValueMetricsDoFnTest.input b/ingestion/src/test/resources/feast/ingestion/transform/WriteFeatureValueMetricsDoFnTest.input deleted file mode 100644 index 42731b9fe1c..00000000000 --- a/ingestion/src/test/resources/feast/ingestion/transform/WriteFeatureValueMetricsDoFnTest.input +++ /dev/null @@ -1,4 +0,0 @@ -featuresetref,int32,int64,double,float,bool,int32list,int64list,doublelist,floatlist,boollist,bytes,byteslist,string,stringlist -project/featureset,1,5,8,5,true,1|4|3,5|1|12,5|7|3,-2.0,true|false,,,, -project/featureset,5,-10,8,10.0,true,1|12|5,,,-1.0|-3.0,false|true,,,, -project/featureset,6,-4,8,0.0,true,2,2|5,,,true|false,,,, \ No newline at end of file diff --git a/ingestion/src/test/resources/feast/ingestion/transform/WriteFeatureValueMetricsDoFnTest.output b/ingestion/src/test/resources/feast/ingestion/transform/WriteFeatureValueMetricsDoFnTest.output deleted file mode 100644 index 372be5f88eb..00000000000 --- a/ingestion/src/test/resources/feast/ingestion/transform/WriteFeatureValueMetricsDoFnTest.output +++ /dev/null @@ -1,66 +0,0 @@ -feast_ingestion.feature_value_min:1|g|#metrics_namespace:test,ingestion_job_name:job,feast_feature_name:int32,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store -feast_ingestion.feature_value_max:6|g|#metrics_namespace:test,ingestion_job_name:job,feast_feature_name:int32,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store -feast_ingestion.feature_value_mean:4|g|#metrics_namespace:test,ingestion_job_name:job,feast_feature_name:int32,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store -feast_ingestion.feature_value_percentile_50:5|g|#metrics_namespace:test,ingestion_job_name:job,feast_feature_name:int32,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store -feast_ingestion.feature_value_percentile_90:6|g|#metrics_namespace:test,ingestion_job_name:job,feast_feature_name:int32,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store - -feast_ingestion.feature_value_min:0|g|#metrics_namespace:test,ingestion_job_name:job,feast_feature_name:int64,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store -feast_ingestion.feature_value_min:-10|g|#metrics_namespace:test,ingestion_job_name:job,feast_feature_name:int64,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store -feast_ingestion.feature_value_max:5|g|#metrics_namespace:test,ingestion_job_name:job,feast_feature_name:int64,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store -feast_ingestion.feature_value_mean:0|g|#metrics_namespace:test,ingestion_job_name:job,feast_feature_name:int64,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store -feast_ingestion.feature_value_mean:-3|g|#metrics_namespace:test,ingestion_job_name:job,feast_feature_name:int64,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store -feast_ingestion.feature_value_percentile_50:-4|g|#metrics_namespace:test,ingestion_job_name:job,feast_feature_name:int64,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store -feast_ingestion.feature_value_percentile_90:5|g|#metrics_namespace:test,ingestion_job_name:job,feast_feature_name:int64,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store - -feast_ingestion.feature_value_min:8|g|#metrics_namespace:test,ingestion_job_name:job,feast_feature_name:double,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store -feast_ingestion.feature_value_max:8|g|#metrics_namespace:test,ingestion_job_name:job,feast_feature_name:double,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store -feast_ingestion.feature_value_mean:8|g|#metrics_namespace:test,ingestion_job_name:job,feast_feature_name:double,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store -feast_ingestion.feature_value_percentile_50:8|g|#metrics_namespace:test,ingestion_job_name:job,feast_feature_name:double,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store -feast_ingestion.feature_value_percentile_90:8|g|#metrics_namespace:test,ingestion_job_name:job,feast_feature_name:double,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store - -feast_ingestion.feature_value_min:0|g|#metrics_namespace:test,ingestion_job_name:job,feast_feature_name:float,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store -feast_ingestion.feature_value_max:10|g|#metrics_namespace:test,ingestion_job_name:job,feast_feature_name:float,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store -feast_ingestion.feature_value_mean:5|g|#metrics_namespace:test,ingestion_job_name:job,feast_feature_name:float,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store -feast_ingestion.feature_value_percentile_50:5|g|#metrics_namespace:test,ingestion_job_name:job,feast_feature_name:float,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store -feast_ingestion.feature_value_percentile_90:10|g|#metrics_namespace:test,ingestion_job_name:job,feast_feature_name:float,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store - -feast_ingestion.feature_value_min:1|g|#metrics_namespace:test,ingestion_job_name:job,feast_feature_name:bool,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store -feast_ingestion.feature_value_max:1|g|#metrics_namespace:test,ingestion_job_name:job,feast_feature_name:bool,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store -feast_ingestion.feature_value_mean:1|g|#metrics_namespace:test,ingestion_job_name:job,feast_feature_name:bool,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store -feast_ingestion.feature_value_percentile_50:1|g|#metrics_namespace:test,ingestion_job_name:job,feast_feature_name:bool,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store -feast_ingestion.feature_value_percentile_90:1|g|#metrics_namespace:test,ingestion_job_name:job,feast_feature_name:bool,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store - -feast_ingestion.feature_value_min:1|g|#metrics_namespace:test,ingestion_job_name:job,feast_feature_name:int32list,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store -feast_ingestion.feature_value_max:12|g|#metrics_namespace:test,ingestion_job_name:job,feast_feature_name:int32list,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store -feast_ingestion.feature_value_mean:4|g|#metrics_namespace:test,ingestion_job_name:job,feast_feature_name:int32list,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store -feast_ingestion.feature_value_percentile_50:3|g|#metrics_namespace:test,ingestion_job_name:job,feast_feature_name:int32list,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store -feast_ingestion.feature_value_percentile_90:12|g|#metrics_namespace:test,ingestion_job_name:job,feast_feature_name:int32list,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store - -feast_ingestion.feature_value_min:1|g|#metrics_namespace:test,ingestion_job_name:job,feast_feature_name:int64list,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store -feast_ingestion.feature_value_max:12|g|#metrics_namespace:test,ingestion_job_name:job,feast_feature_name:int64list,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store -feast_ingestion.feature_value_mean:5|g|#metrics_namespace:test,ingestion_job_name:job,feast_feature_name:int64list,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store -feast_ingestion.feature_value_percentile_50:5|g|#metrics_namespace:test,ingestion_job_name:job,feast_feature_name:int64list,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store -feast_ingestion.feature_value_percentile_90:12|g|#metrics_namespace:test,ingestion_job_name:job,feast_feature_name:int64list,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store - -feast_ingestion.feature_value_min:3|g|#metrics_namespace:test,ingestion_job_name:job,feast_feature_name:doublelist,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store -feast_ingestion.feature_value_max:7|g|#metrics_namespace:test,ingestion_job_name:job,feast_feature_name:doublelist,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store -feast_ingestion.feature_value_mean:5|g|#metrics_namespace:test,ingestion_job_name:job,feast_feature_name:doublelist,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store -feast_ingestion.feature_value_percentile_50:5|g|#metrics_namespace:test,ingestion_job_name:job,feast_feature_name:doublelist,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store -feast_ingestion.feature_value_percentile_90:7|g|#metrics_namespace:test,ingestion_job_name:job,feast_feature_name:doublelist,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store - -feast_ingestion.feature_value_min:0|g|#metrics_namespace:test,ingestion_job_name:job,feast_feature_name:floatlist,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store -feast_ingestion.feature_value_min:-3|g|#metrics_namespace:test,ingestion_job_name:job,feast_feature_name:floatlist,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store -feast_ingestion.feature_value_max:0|g|#metrics_namespace:test,ingestion_job_name:job,feast_feature_name:floatlist,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store -feast_ingestion.feature_value_max:-1|g|#metrics_namespace:test,ingestion_job_name:job,feast_feature_name:floatlist,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store -feast_ingestion.feature_value_mean:0|g|#metrics_namespace:test,ingestion_job_name:job,feast_feature_name:floatlist,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store -feast_ingestion.feature_value_mean:-2|g|#metrics_namespace:test,ingestion_job_name:job,feast_feature_name:floatlist,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store -feast_ingestion.feature_value_percentile_50:0|g|#metrics_namespace:test,ingestion_job_name:job,feast_feature_name:floatlist,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store -feast_ingestion.feature_value_percentile_50:-2|g|#metrics_namespace:test,ingestion_job_name:job,feast_feature_name:floatlist,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store -feast_ingestion.feature_value_percentile_90:0|g|#metrics_namespace:test,ingestion_job_name:job,feast_feature_name:floatlist,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store -feast_ingestion.feature_value_percentile_90:-1|g|#metrics_namespace:test,ingestion_job_name:job,feast_feature_name:floatlist,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store - -feast_ingestion.feature_value_min:0|g|#metrics_namespace:test,ingestion_job_name:job,feast_feature_name:boollist,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store -feast_ingestion.feature_value_max:1|g|#metrics_namespace:test,ingestion_job_name:job,feast_feature_name:boollist,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store -feast_ingestion.feature_value_mean:0.5|g|#metrics_namespace:test,ingestion_job_name:job,feast_feature_name:boollist,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store -feast_ingestion.feature_value_percentile_50:0.5|g|#metrics_namespace:test,ingestion_job_name:job,feast_feature_name:boollist,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store -feast_ingestion.feature_value_percentile_90:1|g|#metrics_namespace:test,ingestion_job_name:job,feast_feature_name:boollist,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store \ No newline at end of file diff --git a/ingestion/src/test/resources/feast/ingestion/transform/WriteRowMetricsDoFnTest.input b/ingestion/src/test/resources/feast/ingestion/transform/WriteRowMetricsDoFnTest.input deleted file mode 100644 index c5543d2889d..00000000000 --- a/ingestion/src/test/resources/feast/ingestion/transform/WriteRowMetricsDoFnTest.input +++ /dev/null @@ -1,4 +0,0 @@ -featuresetref,int32,int64,timestamp -project/featureset,1,5,2020-03-30T06:10:38Z -project/featureset,5,8,2020-03-30T06:10:43Z -project/featureset,6,,2020-03-30T06:10:42Z \ No newline at end of file diff --git a/ingestion/src/test/resources/feast/ingestion/transform/WriteRowMetricsDoFnTest.output b/ingestion/src/test/resources/feast/ingestion/transform/WriteRowMetricsDoFnTest.output deleted file mode 100644 index bbc20411c74..00000000000 --- a/ingestion/src/test/resources/feast/ingestion/transform/WriteRowMetricsDoFnTest.output +++ /dev/null @@ -1,23 +0,0 @@ -feast_ingestion.feature_row_ingested_count:3|c|#metrics_namespace:test,ingestion_job_name:job,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store -feast_ingestion.feature_row_lag_ms_min:2000|g|#metrics_namespace:test,ingestion_job_name:job,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store -feast_ingestion.feature_row_lag_ms_max:7000|g|#metrics_namespace:test,ingestion_job_name:job,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store -feast_ingestion.feature_row_lag_ms_mean:4000|g|#metrics_namespace:test,ingestion_job_name:job,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store -feast_ingestion.feature_row_lag_ms_percentile_90:7000|g|#metrics_namespace:test,ingestion_job_name:job,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store -feast_ingestion.feature_row_lag_ms_percentile_95:7000|g|#metrics_namespace:test,ingestion_job_name:job,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store -feast_ingestion.feature_row_lag_ms_percentile_99:7000|g|#metrics_namespace:test,ingestion_job_name:job,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store - -feast_ingestion.feature_value_lag_ms_min:2000|g|#feast_feature_name:int32,metrics_namespace:test,ingestion_job_name:job,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store -feast_ingestion.feature_value_lag_ms_max:7000|g|#feast_feature_name:int32,metrics_namespace:test,ingestion_job_name:job,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store -feast_ingestion.feature_value_lag_ms_mean:4000|g|#feast_feature_name:int32,metrics_namespace:test,ingestion_job_name:job,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store -feast_ingestion.feature_value_lag_ms_percentile_90:7000|g|#feast_feature_name:int32,metrics_namespace:test,ingestion_job_name:job,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store -feast_ingestion.feature_value_lag_ms_percentile_95:7000|g|#feast_feature_name:int32,metrics_namespace:test,ingestion_job_name:job,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store -feast_ingestion.feature_value_lag_ms_percentile_99:7000|g|#feast_feature_name:int32,metrics_namespace:test,ingestion_job_name:job,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store -feast_ingestion.feature_value_missing_count:0|c|#feast_feature_name:int32,metrics_namespace:test,ingestion_job_name:job,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store - -feast_ingestion.feature_value_lag_ms_min:2000|g|#feast_feature_name:int64,metrics_namespace:test,ingestion_job_name:job,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store -feast_ingestion.feature_value_lag_ms_max:7000|g|#feast_feature_name:int64,metrics_namespace:test,ingestion_job_name:job,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store -feast_ingestion.feature_value_lag_ms_mean:4500|g|#feast_feature_name:int64,metrics_namespace:test,ingestion_job_name:job,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store -feast_ingestion.feature_value_lag_ms_percentile_90:7000|g|#feast_feature_name:int64,metrics_namespace:test,ingestion_job_name:job,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store -feast_ingestion.feature_value_lag_ms_percentile_95:7000|g|#feast_feature_name:int64,metrics_namespace:test,ingestion_job_name:job,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store -feast_ingestion.feature_value_lag_ms_percentile_99:7000|g|#feast_feature_name:int64,metrics_namespace:test,ingestion_job_name:job,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store -feast_ingestion.feature_value_missing_count:1|c|#feast_feature_name:int64,metrics_namespace:test,ingestion_job_name:job,feast_featureSet_name:featureset,feast_project_name:project,feast_store:store \ No newline at end of file diff --git a/ingestion/src/test/resources/logback-test.xml b/ingestion/src/test/resources/logback-test.xml deleted file mode 100644 index 14fa4086115..00000000000 --- a/ingestion/src/test/resources/logback-test.xml +++ /dev/null @@ -1,35 +0,0 @@ - - - - - - - - - - - - - %d{HH:mm:ss} [%thread] %-5level %logger{50} - %msg%n - - - - - - - diff --git a/job-controller/pom.xml b/job-controller/pom.xml deleted file mode 100644 index b2fab14d84a..00000000000 --- a/job-controller/pom.xml +++ /dev/null @@ -1,221 +0,0 @@ - - - - 4.0.0 - - - dev.feast - feast-parent - ${revision} - - - Feast Job Controller - Feature ingestion controller - feast-job-controller - - - - - org.apache.maven.plugins - maven-compiler-plugin - - 11 - - - - - org.jacoco - jacoco-maven-plugin - - - - org.springframework.boot - spring-boot-maven-plugin - - false - - - - build-info - - build-info - - - - - - - - - - dev.feast - feast-ingestion - ${project.version} - - - - org.slf4j - slf4j-simple - - - - - dev.feast - feast-common - ${project.version} - - - - javax.inject - javax.inject - 1 - - - - org.springframework.boot - spring-boot-starter-web - - - - org.springframework.boot - spring-boot-starter-log4j2 - - - org.apache.logging.log4j - log4j-web - - - net.devh - grpc-server-spring-boot-starter - ${grpc.spring.boot.starter.version} - - - - org.springframework.boot - spring-boot-starter-actuator - - - - - org.springframework.boot - spring-boot-configuration-processor - - - - io.grpc - grpc-services - - - - io.grpc - grpc-stub - - - - com.google.protobuf - protobuf-java-util - - - - com.google.guava - guava - - - - com.google.code.gson - gson - 2.8.5 - - - com.google.api-client - google-api-client - 1.30.9 - - - com.google.apis - google-api-services-dataflow - v1b3-rev20200305-1.30.9 - - - - org.springframework.kafka - spring-kafka - - - - - org.projectlombok - lombok - ${lombok.version} - - - - io.prometheus - simpleclient - - - - io.prometheus - simpleclient_servlet - - - com.google.api.client - google-api-client-googleapis-auth-oauth - 1.2.3-alpha - - - - javax.xml.bind - jaxb-api - - - - org.apache.bval - bval-jsr - 2.0.4 - - - - - com.jayway.jsonpath - json-path-assert - 2.2.0 - test - - - dev.feast - feast-core - ${project.version} - test - - - dev.feast - feast-common-test - ${project.version} - test - - - org.postgresql - postgresql - test - true - - - diff --git a/job-controller/src/main/java/feast/jobcontroller/JobControllerApplication.java b/job-controller/src/main/java/feast/jobcontroller/JobControllerApplication.java deleted file mode 100644 index d3e6e311ec2..00000000000 --- a/job-controller/src/main/java/feast/jobcontroller/JobControllerApplication.java +++ /dev/null @@ -1,47 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2019 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.jobcontroller; - -import feast.jobcontroller.config.FeastProperties; -import lombok.extern.slf4j.Slf4j; -import org.springframework.boot.SpringApplication; -import org.springframework.boot.actuate.autoconfigure.security.servlet.ManagementWebSecurityAutoConfiguration; -import org.springframework.boot.autoconfigure.SpringBootApplication; -import org.springframework.boot.autoconfigure.jdbc.DataSourceAutoConfiguration; -import org.springframework.boot.autoconfigure.jdbc.DataSourceTransactionManagerAutoConfiguration; -import org.springframework.boot.autoconfigure.orm.jpa.HibernateJpaAutoConfiguration; -import org.springframework.boot.autoconfigure.security.servlet.SecurityAutoConfiguration; -import org.springframework.boot.context.properties.EnableConfigurationProperties; -import org.springframework.scheduling.annotation.EnableScheduling; - -@EnableScheduling -@SpringBootApplication( - exclude = { - DataSourceAutoConfiguration.class, - DataSourceTransactionManagerAutoConfiguration.class, - HibernateJpaAutoConfiguration.class, - // TODO: Disables spring security. Remove when implementing security for JobController. - SecurityAutoConfiguration.class, - ManagementWebSecurityAutoConfiguration.class, - }) -@EnableConfigurationProperties(FeastProperties.class) -@Slf4j -public class JobControllerApplication { - public static void main(String[] args) { - SpringApplication.run(JobControllerApplication.class, args); - } -} diff --git a/job-controller/src/main/java/feast/jobcontroller/config/FeastProperties.java b/job-controller/src/main/java/feast/jobcontroller/config/FeastProperties.java deleted file mode 100644 index a7ce8d76073..00000000000 --- a/job-controller/src/main/java/feast/jobcontroller/config/FeastProperties.java +++ /dev/null @@ -1,319 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2019 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.jobcontroller.config; - -import feast.common.logging.config.LoggingProperties; -import feast.common.validators.OneOfStrings; -import feast.proto.core.StoreProto; -import java.net.InetAddress; -import java.net.UnknownHostException; -import java.util.*; -import javax.annotation.PostConstruct; -import javax.validation.*; -import javax.validation.constraints.NotBlank; -import javax.validation.constraints.NotNull; -import javax.validation.constraints.Positive; -import lombok.Getter; -import lombok.Setter; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.context.properties.ConfigurationProperties; -import org.springframework.boot.info.BuildProperties; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.ComponentScan; -import org.springframework.context.annotation.Configuration; - -@Getter -@Setter -@Configuration -@ComponentScan("feast.common.logging") -@ConfigurationProperties(prefix = "feast", ignoreInvalidFields = true) -public class FeastProperties { - - /** - * Instantiates a new Feast properties. - * - * @param buildProperties Feast build properties - */ - @Autowired - public FeastProperties(BuildProperties buildProperties) { - setVersion(buildProperties.getVersion()); - } - - /** Instantiates a new Feast properties. */ - public FeastProperties() {} - - /* Feast Job Controller Build Version */ - @NotBlank private String version = "unknown"; - - /* Feast Core Address */ - @NotBlank private String coreHost; - private Integer corePort; - - /* Population job properties */ - @NotNull private JobProperties jobs; - - @NotNull - /* Feast Kafka stream properties */ - private StreamProperties stream; - - /* Feast Audit Logging properties */ - @NotNull private LoggingProperties logging; - - @Bean - LoggingProperties loggingProperties() { - return getLogging(); - } - - /** Feast job properties. These properties are used for ingestion jobs. */ - @Getter - @Setter - public static class JobProperties { - /* Toggle for enabling/disabling job management */ - private Boolean enabled = true; - - @NotBlank - /* The active Apache Beam runner name. This name references one instance of the Runner class */ - private String activeRunner; - - /* Job Controller related properties */ - private ControllerProperties controller; - - @Getter - @Setter - public static class ControllerProperties { - /* If true only one IngestionJob would be created per source with all subscribed stores in it */ - private Boolean consolidateJobsPerSource = false; - - /* Labels to identify jobs managed by this job controller */ - private Map jobSelector = new HashMap<>(); - - /* Selectors to define featureSets that are responsibility of current JobManager */ - private List featureSetSelector = new ArrayList<>(); - - /* Specify names of stores that must be used by current JobManager */ - private List whitelistedStores = new ArrayList<>(); - - /** - * Similarly to Store's subscription this selector defines set of FeatureSets. All FeatureSets - * that match both project and name will be selected. Project and name may use * - */ - @Getter - @Setter - public static class FeatureSetSelector { - private String project; - private String name; - - public StoreProto.Store.Subscription toSubscription() { - return StoreProto.Store.Subscription.newBuilder() - .setName(this.name) - .setProject(this.project) - .build(); - } - } - } - - /** List of configured job runners. */ - private List runners = new ArrayList<>(); - - /** - * Gets a {@link Runner} instance of the active runner - * - * @return the active runner - */ - public Runner getActiveRunner() { - for (Runner runner : getRunners()) { - if (activeRunner.equals(runner.getName())) { - return runner; - } - } - throw new RuntimeException( - String.format( - "Active runner is misconfigured. Could not find runner: %s.", activeRunner)); - } - - /** Job Runner class. */ - @Getter - @Setter - public static class Runner { - - /** Job runner name. This must be unique. */ - String name; - - /** Job runner type DirectRunner, DataflowRunner currently supported */ - String type; - - /** - * Job runner configuration options. See the following for options - * https://api.docs.feast.dev/grpc/feast.core.pb.html#Runner - */ - Map options = new HashMap<>(); - - /** - * Gets the job runner type as an enum. - * - * @return Returns the job runner type as {@link feast.jobcontroller.runner.Runner} - */ - public feast.jobcontroller.runner.Runner getType() { - return feast.jobcontroller.runner.Runner.fromString(type); - } - } - - @NotNull - /* Population job metric properties */ - private MetricsProperties metrics; - - @NotNull - /* Prefix for JobId to separate consumer groups for independent jobs running in parallel */ - private String jobIdPrefix; - - /* Timeout in seconds for each attempt to update or submit a new job to the runner */ - @Positive private long jobUpdateTimeoutSeconds; - - /* Job update polling interval in millisecond. How frequently Feast will update running jobs. */ - @Positive private long pollingIntervalMilliseconds; - } - - /** Properties used to configure Feast's managed Kafka feature stream. */ - @Getter - @Setter - public static class StreamProperties { - - /* Feature stream type. Only "kafka" is supported. */ - @OneOfStrings({"kafka"}) - @NotBlank - private String type; - - /* Feature stream options */ - @NotNull private FeatureStreamOptions options; - - /* FeatureSetSpec stream options - communication channel between SpecService and IngestionJob - * to update Spec inside job w/o restart */ - @NotNull private FeatureSetSpecStreamProperties specsOptions; - - /** Feature stream options */ - @Getter - @Setter - public static class FeatureStreamOptions { - - /* Kafka topic to use for feature sets without source topics. */ - @NotBlank private String topic = "feast-features"; - - /** - * Comma separated list of Kafka bootstrap servers. Used for feature sets without a defined - * source. - */ - @NotBlank private String bootstrapServers = "localhost:9092"; - - /* Defines the number of copies of managed feature stream Kafka. */ - @Positive private short replicationFactor = 1; - - /* Number of Kafka partitions to to use for managed feature stream. */ - @Positive private int partitions = 1; - } - - @Getter - @Setter - public static class FeatureSetSpecStreamProperties { - /* Kafka topic to send feature set spec to ingestion streaming job */ - @NotBlank private String specsTopic = "feast-feature-set-specs"; - - /* Kafka topic to receive acknowledgment from ingestion job on successful processing of new specs */ - @NotBlank private String specsAckTopic = "feast-feature-set-specs-ack"; - - /* Notify jobs interval in millisecond. - How frequently Feast will check on Pending FeatureSets and publish them to kafka. */ - @Positive private long notifyIntervalMilliseconds; - } - } - - /** Feast population job metrics */ - @Getter - @Setter - public static class MetricsProperties { - - /* Population job metrics enabled */ - private boolean enabled; - - /* Metric type. Possible options: statsd */ - @OneOfStrings({"statsd"}) - @NotBlank - private String type; - - /* Host of metric sink */ - private String host; - - /* Port of metric sink */ - @Positive private int port; - } - - /** - * Validates all FeastProperties. This method runs after properties have been initialized and - * individually and conditionally validates each class. - */ - @PostConstruct - public void validate() { - ValidatorFactory factory = Validation.buildDefaultValidatorFactory(); - Validator validator = factory.getValidator(); - - // Validate root fields in FeastProperties - Set> violations = validator.validate(this); - if (!violations.isEmpty()) { - throw new ConstraintViolationException(violations); - } - - // Validate Stream properties - Set> streamPropertyViolations = - validator.validate(getStream()); - if (!streamPropertyViolations.isEmpty()) { - throw new ConstraintViolationException(streamPropertyViolations); - } - - // Validate Stream Options - Set> featureStreamOptionsViolations = - validator.validate(getStream().getOptions()); - if (!featureStreamOptionsViolations.isEmpty()) { - throw new ConstraintViolationException(featureStreamOptionsViolations); - } - - // Validate JobProperties - Set> jobPropertiesViolations = validator.validate(getJobs()); - if (!jobPropertiesViolations.isEmpty()) { - throw new ConstraintViolationException(jobPropertiesViolations); - } - - // Validate MetricsProperties - if (getJobs().getMetrics().isEnabled()) { - Set> jobMetricViolations = - validator.validate(getJobs().getMetrics()); - if (!jobMetricViolations.isEmpty()) { - throw new ConstraintViolationException(jobMetricViolations); - } - // Additional custom check for hostname value because there is no built-in Spring annotation - // to validate the value is a DNS resolvable hostname or an IP address. - try { - //noinspection ResultOfMethodCallIgnored - InetAddress.getByName(getJobs().getMetrics().getHost()); - } catch (UnknownHostException e) { - throw new IllegalArgumentException( - "Invalid config value for feast.jobs.metrics.host: " - + getJobs().getMetrics().getHost() - + ". Make sure it is a valid IP address or DNS hostname e.g. localhost or 10.128.10.40. Error detail: " - + e.getMessage()); - } - } - } -} diff --git a/job-controller/src/main/java/feast/jobcontroller/config/FeatureStreamConfig.java b/job-controller/src/main/java/feast/jobcontroller/config/FeatureStreamConfig.java deleted file mode 100644 index c7eb63f9842..00000000000 --- a/job-controller/src/main/java/feast/jobcontroller/config/FeatureStreamConfig.java +++ /dev/null @@ -1,162 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2019 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.jobcontroller.config; - -import feast.common.util.KafkaSerialization; -import feast.proto.core.FeatureSetProto; -import feast.proto.core.IngestionJobProto; -import java.util.HashMap; -import java.util.Map; -import lombok.extern.slf4j.Slf4j; -import org.apache.kafka.clients.admin.AdminClientConfig; -import org.apache.kafka.clients.admin.NewTopic; -import org.apache.kafka.clients.consumer.ConsumerConfig; -import org.apache.kafka.clients.producer.ProducerConfig; -import org.apache.kafka.common.config.TopicConfig; -import org.apache.kafka.common.serialization.StringDeserializer; -import org.apache.kafka.common.serialization.StringSerializer; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.kafka.config.ConcurrentKafkaListenerContainerFactory; -import org.springframework.kafka.config.KafkaListenerContainerFactory; -import org.springframework.kafka.core.*; -import org.springframework.kafka.listener.ConcurrentMessageListenerContainer; - -@Slf4j -@Configuration -public class FeatureStreamConfig { - - String DEFAULT_KAFKA_REQUEST_TIMEOUT_MS_CONFIG = "15000"; - int DEFAULT_SPECS_TOPIC_PARTITIONING = 1; - short DEFAULT_SPECS_TOPIC_REPLICATION = 1; - - @Bean - public KafkaAdmin admin(FeastProperties feastProperties) { - String bootstrapServers = feastProperties.getStream().getOptions().getBootstrapServers(); - - Map configs = new HashMap<>(); - configs.put(AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapServers); - configs.put( - AdminClientConfig.REQUEST_TIMEOUT_MS_CONFIG, DEFAULT_KAFKA_REQUEST_TIMEOUT_MS_CONFIG); - return new KafkaAdmin(configs); - } - - @Bean - public NewTopic featureRowsTopic(FeastProperties feastProperties) { - FeastProperties.StreamProperties streamProperties = feastProperties.getStream(); - - return new NewTopic( - streamProperties.getOptions().getTopic(), - streamProperties.getOptions().getPartitions(), - streamProperties.getOptions().getReplicationFactor()); - } - - @Bean - public NewTopic featureSetSpecsTopic(FeastProperties feastProperties) { - FeastProperties.StreamProperties streamProperties = feastProperties.getStream(); - Map configs = new HashMap<>(); - configs.put(TopicConfig.CLEANUP_POLICY_CONFIG, TopicConfig.CLEANUP_POLICY_COMPACT); - - NewTopic topic = - new NewTopic( - streamProperties.getSpecsOptions().getSpecsTopic(), - DEFAULT_SPECS_TOPIC_PARTITIONING, - DEFAULT_SPECS_TOPIC_REPLICATION); - - topic.configs(configs); - return topic; - } - - @Bean - public NewTopic featureSetSpecsAckTopic(FeastProperties feastProperties) { - FeastProperties.StreamProperties streamProperties = feastProperties.getStream(); - - return new NewTopic( - streamProperties.getSpecsOptions().getSpecsAckTopic(), - DEFAULT_SPECS_TOPIC_PARTITIONING, - (short) 1); - } - - /** - * Creates kafka publisher for sending FeatureSetSpecs to ingestion job. Uses ProtoSerializer to - * serialize FeatureSetSpec. - * - * @param feastProperties - * @return - */ - @Bean - public KafkaTemplate specKafkaTemplate( - FeastProperties feastProperties) { - FeastProperties.StreamProperties streamProperties = feastProperties.getStream(); - Map props = new HashMap<>(); - - props.put( - ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, - streamProperties.getOptions().getBootstrapServers()); - - KafkaTemplate t = - new KafkaTemplate<>( - new DefaultKafkaProducerFactory<>( - props, new StringSerializer(), new KafkaSerialization.ProtoSerializer<>())); - t.setDefaultTopic(streamProperties.getSpecsOptions().getSpecsTopic()); - return t; - } - - /** - * Set configured consumerFactory for specs acknowledgment topic (see ackConsumerFactory) as - * default for KafkaListener. - * - * @param consumerFactory - * @return - */ - @Bean - KafkaListenerContainerFactory< - ConcurrentMessageListenerContainer> - kafkaAckListenerContainerFactory( - ConsumerFactory consumerFactory) { - ConcurrentKafkaListenerContainerFactory factory = - new ConcurrentKafkaListenerContainerFactory<>(); - factory.setConsumerFactory(consumerFactory); - return factory; - } - - /** - * Prepares kafka consumer (by configuring ConsumerFactory) to receive acknowledgments from - * IngestionJob on successful updates of FeatureSetSpecs. - * - * @param feastProperties - * @return ConsumerFactory for FeatureSetSpecAck - */ - @Bean - public ConsumerFactory ackConsumerFactory( - FeastProperties feastProperties) { - FeastProperties.StreamProperties streamProperties = feastProperties.getStream(); - Map props = new HashMap<>(); - - props.put( - ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, - streamProperties.getOptions().getBootstrapServers()); - props.put( - ConsumerConfig.GROUP_ID_CONFIG, - String.format("core-service-%s", feastProperties.getVersion())); - - return new DefaultKafkaConsumerFactory<>( - props, - new StringDeserializer(), - new KafkaSerialization.ProtoDeserializer<>(IngestionJobProto.FeatureSetSpecAck.parser())); - } -} diff --git a/job-controller/src/main/java/feast/jobcontroller/config/JobControllerConfig.java b/job-controller/src/main/java/feast/jobcontroller/config/JobControllerConfig.java deleted file mode 100644 index 82819022af8..00000000000 --- a/job-controller/src/main/java/feast/jobcontroller/config/JobControllerConfig.java +++ /dev/null @@ -1,158 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2019 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.jobcontroller.config; - -import com.google.gson.Gson; -import com.google.protobuf.InvalidProtocolBufferException; -import com.google.protobuf.util.JsonFormat; -import feast.jobcontroller.dao.JobRepository; -import feast.jobcontroller.runner.ConsolidatedJobStrategy; -import feast.jobcontroller.runner.JobGroupingStrategy; -import feast.jobcontroller.runner.JobManager; -import feast.jobcontroller.runner.JobPerStoreStrategy; -import feast.jobcontroller.runner.dataflow.DataflowJobManager; -import feast.jobcontroller.runner.direct.DirectJobRegistry; -import feast.jobcontroller.runner.direct.DirectRunnerJobManager; -import feast.proto.core.CoreServiceGrpc; -import feast.proto.core.IngestionJobProto; -import feast.proto.core.RunnerProto.DataflowRunnerConfigOptions; -import feast.proto.core.RunnerProto.DirectRunnerConfigOptions; -import feast.proto.core.SourceProto; -import io.grpc.CallCredentials; -import io.grpc.ManagedChannel; -import io.grpc.ManagedChannelBuilder; -import java.util.Map; -import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.ObjectProvider; -import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; - -/** Beans for job management */ -@Slf4j -@Configuration -public class JobControllerConfig { - private final Gson gson = new Gson(); - - /** - * Create SpecsStreamingUpdateConfig, which is used to set up communications (bi-directional - * channel) to send new FeatureSetSpec to IngestionJob and receive acknowledgments. - * - * @param feastProperties feast config properties - */ - @Bean - public IngestionJobProto.SpecsStreamingUpdateConfig createSpecsStreamingUpdateConfig( - FeastProperties feastProperties) { - FeastProperties.StreamProperties streamProperties = feastProperties.getStream(); - - return IngestionJobProto.SpecsStreamingUpdateConfig.newBuilder() - .setSource( - SourceProto.KafkaSourceConfig.newBuilder() - .setBootstrapServers(streamProperties.getOptions().getBootstrapServers()) - .setTopic(streamProperties.getSpecsOptions().getSpecsTopic()) - .build()) - .setAck( - SourceProto.KafkaSourceConfig.newBuilder() - .setBootstrapServers(streamProperties.getOptions().getBootstrapServers()) - .setTopic(streamProperties.getSpecsOptions().getSpecsAckTopic())) - .build(); - } - - /** - * Returns Grouping Strategy which is responsible for how Ingestion would be split across job - * instances (or how Sources and Stores would be grouped together). Choosing strategy depends on - * FeastProperties config "feast.jobs.consolidate-jobs-per-source". - * - * @param feastProperties feast config properties - * @param jobRepository repository required by strategy - * @return JobGroupingStrategy - */ - @Bean - public JobGroupingStrategy getJobGroupingStrategy( - FeastProperties feastProperties, JobRepository jobRepository) { - Boolean shouldConsolidateJobs = - feastProperties.getJobs().getController().getConsolidateJobsPerSource(); - FeastProperties.JobProperties jobProperties = feastProperties.getJobs(); - if (shouldConsolidateJobs) { - return new ConsolidatedJobStrategy(jobRepository, jobProperties); - } else { - return new JobPerStoreStrategy(jobRepository, jobProperties); - } - } - - /** - * Get a JobManager according to the runner type and Dataflow configuration. - * - * @param feastProperties feast config properties - */ - @Bean - @ConditionalOnMissingBean - public JobManager getJobManager( - FeastProperties feastProperties, - IngestionJobProto.SpecsStreamingUpdateConfig specsStreamingUpdateConfig) - throws InvalidProtocolBufferException { - - FeastProperties.JobProperties jobProperties = feastProperties.getJobs(); - FeastProperties.JobProperties.Runner runner = jobProperties.getActiveRunner(); - Map runnerConfigOptions = runner.getOptions(); - - FeastProperties.MetricsProperties metrics = jobProperties.getMetrics(); - String configJson = gson.toJson(runnerConfigOptions); - - switch (runner.getType()) { - case DATAFLOW: - DataflowRunnerConfigOptions.Builder dataflowRunnerConfigOptions = - DataflowRunnerConfigOptions.newBuilder(); - JsonFormat.parser().merge(configJson, dataflowRunnerConfigOptions); - return DataflowJobManager.of( - dataflowRunnerConfigOptions.build(), - metrics, - specsStreamingUpdateConfig, - jobProperties.getController().getJobSelector()); - case DIRECT: - DirectRunnerConfigOptions.Builder directRunnerConfigOptions = - DirectRunnerConfigOptions.newBuilder(); - JsonFormat.parser().merge(configJson, directRunnerConfigOptions); - return new DirectRunnerJobManager( - directRunnerConfigOptions.build(), - new DirectJobRegistry(), - metrics, - specsStreamingUpdateConfig); - default: - throw new IllegalArgumentException("Unsupported runner: " + runner); - } - } - - @Bean - public CoreServiceGrpc.CoreServiceBlockingStub coreService( - FeastProperties feastProperties, ObjectProvider callCredentials) { - ManagedChannel channel = - ManagedChannelBuilder.forAddress( - feastProperties.getCoreHost(), feastProperties.getCorePort()) - .usePlaintext() - .build(); - CallCredentials creds = callCredentials.getIfAvailable(); - - CoreServiceGrpc.CoreServiceBlockingStub blockingStub; - if (creds != null) { - blockingStub = CoreServiceGrpc.newBlockingStub(channel).withCallCredentials(creds); - } else { - blockingStub = CoreServiceGrpc.newBlockingStub(channel); - } - return blockingStub; - } -} diff --git a/job-controller/src/main/java/feast/jobcontroller/dao/InMemoryJobRepository.java b/job-controller/src/main/java/feast/jobcontroller/dao/InMemoryJobRepository.java deleted file mode 100644 index 97d21cf3ace..00000000000 --- a/job-controller/src/main/java/feast/jobcontroller/dao/InMemoryJobRepository.java +++ /dev/null @@ -1,122 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.jobcontroller.dao; - -import com.google.common.collect.Lists; -import feast.common.models.FeatureSetReference; -import feast.jobcontroller.model.Job; -import feast.jobcontroller.model.JobStatus; -import feast.jobcontroller.runner.JobManager; -import feast.proto.core.SourceProto; -import java.util.*; -import java.util.function.Predicate; -import java.util.stream.Collectors; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Component; - -/** - * Keeps state of all jobs managed by current application in memory. On start loads persistent state - * through JobManager from JobManager backend. - * - *

E.g., with Dataflow runner all jobs with their state stored on Google's side and accessible - * via API. So we don't need to persist this state ourselves. Instead we just fetch it once and then - * we apply same changes to dataflow (via JobManager) and to InMemoryRepository to keep them in - * sync. - * - *

Provides flexible access to objects via JPA-like filtering API. - */ -@Component -public class InMemoryJobRepository implements JobRepository { - private final JobManager jobManager; - - /** Internal storage for all jobs mapped by their Id */ - private Map storage; - - @Autowired - public InMemoryJobRepository(JobManager jobManager) { - this.jobManager = jobManager; - - this.storage = - this.jobManager.listRunningJobs().stream().collect(Collectors.toMap(Job::getId, j -> j)); - } - - /** - * Returns single job that has given source, store with given name ans its status is not in given - * statuses. We expect this parameters to specify only one RUNNING job (most of the time). But in - * case there're many - we return latest updated one. - * - * @return job that matches given parameters if it's present - */ - @Override - public Optional findFirstBySourceAndStoreNameAndStatusNotInOrderByLastUpdatedDesc( - SourceProto.Source source, String storeName, Collection statuses) { - return this.storage.values().stream() - .filter( - j -> - j.getSource().equals(source) - && (storeName == null || j.getStores().containsKey(storeName)) - && (!statuses.contains(j.getStatus()))) - .max(Comparator.comparing(Job::getLastUpdated)); - } - - private List findWithFilter(Predicate p) { - return this.storage.values().stream().filter(p).collect(Collectors.toList()); - } - - /** Find Jobs that have given status */ - @Override - public List findByStatus(JobStatus status) { - return this.findWithFilter(j -> j.getStatus().equals(status)); - } - - /** - * Find Jobs that have given FeatureSet (specified by {@link FeatureSetReference} allocated to it. - */ - @Override - public List findByFeatureSetReference(FeatureSetReference reference) { - return this.findWithFilter(j -> j.getFeatureSetDeliveryStatuses().containsKey(reference)); - } - - /** Find Jobs that have one of the stores with given name */ - @Override - public List findByJobStoreName(String storeName) { - return this.findWithFilter(j -> j.getStores().containsKey(storeName)); - } - - /** Find by Job's Id */ - @Override - public Optional findById(String jobId) { - return Optional.ofNullable(this.storage.get(jobId)); - } - - @Override - public List findAll() { - return Lists.newArrayList(this.storage.values()); - } - - @Override - public void add(Job job) { - job.preSave(); - - this.storage.put(job.getId(), job); - } - - @Override - public void deleteAll() { - this.storage.clear(); - } -} diff --git a/job-controller/src/main/java/feast/jobcontroller/dao/JobRepository.java b/job-controller/src/main/java/feast/jobcontroller/dao/JobRepository.java deleted file mode 100644 index c70716b1696..00000000000 --- a/job-controller/src/main/java/feast/jobcontroller/dao/JobRepository.java +++ /dev/null @@ -1,46 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2019 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.jobcontroller.dao; - -import feast.common.models.FeatureSetReference; -import feast.jobcontroller.model.Job; -import feast.jobcontroller.model.JobStatus; -import feast.proto.core.SourceProto; -import java.util.Collection; -import java.util.List; -import java.util.Optional; - -public interface JobRepository { - Optional findFirstBySourceAndStoreNameAndStatusNotInOrderByLastUpdatedDesc( - SourceProto.Source source, String storeName, Collection statuses); - - List findByStatus(JobStatus status); - - // find jobs that have given FeatureSet allocated to them - List findByFeatureSetReference(FeatureSetReference reference); - - // find jobs that have at least one store with given name - List findByJobStoreName(String storeName); - - Optional findById(String jobId); - - List findAll(); - - void add(Job job); - - void deleteAll(); -} diff --git a/job-controller/src/main/java/feast/jobcontroller/exception/JobExecutionException.java b/job-controller/src/main/java/feast/jobcontroller/exception/JobExecutionException.java deleted file mode 100644 index 035640c323c..00000000000 --- a/job-controller/src/main/java/feast/jobcontroller/exception/JobExecutionException.java +++ /dev/null @@ -1,32 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2019 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.jobcontroller.exception; - -/** Exception thrown when a request for job execution fails. */ -public class JobExecutionException extends RuntimeException { - public JobExecutionException() { - super(); - } - - public JobExecutionException(String message) { - super(message); - } - - public JobExecutionException(String message, Throwable cause) { - super(message, cause); - } -} diff --git a/job-controller/src/main/java/feast/jobcontroller/exception/JobMonitoringException.java b/job-controller/src/main/java/feast/jobcontroller/exception/JobMonitoringException.java deleted file mode 100644 index 1a8ec749af2..00000000000 --- a/job-controller/src/main/java/feast/jobcontroller/exception/JobMonitoringException.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2019 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.jobcontroller.exception; - -/** Exception thrown when error happen during job monitoring. */ -public class JobMonitoringException extends RuntimeException { - - public JobMonitoringException() { - super(); - } - - public JobMonitoringException(String message) { - super(message); - } - - public JobMonitoringException(String message, Throwable cause) { - super(message, cause); - } -} diff --git a/job-controller/src/main/java/feast/jobcontroller/grpc/HealthServiceImpl.java b/job-controller/src/main/java/feast/jobcontroller/grpc/HealthServiceImpl.java deleted file mode 100644 index 19724517a3a..00000000000 --- a/job-controller/src/main/java/feast/jobcontroller/grpc/HealthServiceImpl.java +++ /dev/null @@ -1,55 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.jobcontroller.grpc; - -import feast.proto.core.CoreServiceGrpc; -import feast.proto.core.CoreServiceProto; -import io.grpc.Status; -import io.grpc.health.v1.HealthGrpc.HealthImplBase; -import io.grpc.health.v1.HealthProto.HealthCheckRequest; -import io.grpc.health.v1.HealthProto.HealthCheckResponse; -import io.grpc.health.v1.HealthProto.ServingStatus; -import io.grpc.stub.StreamObserver; -import lombok.extern.slf4j.Slf4j; -import net.devh.boot.grpc.server.service.GrpcService; -import org.springframework.beans.factory.annotation.Autowired; - -@Slf4j -@GrpcService -public class HealthServiceImpl extends HealthImplBase { - private final CoreServiceGrpc.CoreServiceBlockingStub specService; - - @Autowired - public HealthServiceImpl(CoreServiceGrpc.CoreServiceBlockingStub specService) { - this.specService = specService; - } - - @Override - public void check( - HealthCheckRequest request, StreamObserver responseObserver) { - try { - specService.listProjects(CoreServiceProto.ListProjectsRequest.newBuilder().build()); - responseObserver.onNext( - HealthCheckResponse.newBuilder().setStatus(ServingStatus.SERVING).build()); - responseObserver.onCompleted(); - } catch (Exception e) { - log.error("Health Check: unable to retrieve projects.\nError: %s", e); - responseObserver.onError( - Status.INTERNAL.withDescription(e.getMessage()).withCause(e).asRuntimeException()); - } - } -} diff --git a/job-controller/src/main/java/feast/jobcontroller/grpc/JobControllerServiceImpl.java b/job-controller/src/main/java/feast/jobcontroller/grpc/JobControllerServiceImpl.java deleted file mode 100644 index f24afe8506b..00000000000 --- a/job-controller/src/main/java/feast/jobcontroller/grpc/JobControllerServiceImpl.java +++ /dev/null @@ -1,107 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2019 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.jobcontroller.grpc; - -import com.google.api.gax.rpc.InvalidArgumentException; -import feast.common.logging.interceptors.GrpcMessageInterceptor; -import feast.jobcontroller.service.JobService; -import feast.proto.core.CoreServiceProto.*; -import feast.proto.core.JobControllerServiceGrpc.JobControllerServiceImplBase; -import io.grpc.Status; -import io.grpc.stub.StreamObserver; -import java.util.NoSuchElementException; -import lombok.extern.slf4j.Slf4j; -import net.devh.boot.grpc.server.service.GrpcService; -import org.springframework.beans.factory.annotation.Autowired; - -/** Implementation of the feast core GRPC service. */ -@Slf4j -@GrpcService(interceptors = {GrpcMessageInterceptor.class}) -public class JobControllerServiceImpl extends JobControllerServiceImplBase { - - private JobService jobService; - - @Autowired - public JobControllerServiceImpl(JobService jobService) { - this.jobService = jobService; - } - - @Override - public void listIngestionJobs( - ListIngestionJobsRequest request, - StreamObserver responseObserver) { - try { - ListIngestionJobsResponse response = this.jobService.listJobs(request); - responseObserver.onNext(response); - responseObserver.onCompleted(); - } catch (InvalidArgumentException e) { - log.error("Received an invalid request on calling listIngestionJobs method:", e); - responseObserver.onError( - Status.INVALID_ARGUMENT.withDescription(e.getMessage()).withCause(e).asException()); - } catch (Exception e) { - log.error("Unexpected exception on calling listIngestionJobs method:", e); - responseObserver.onError( - Status.INTERNAL.withDescription(e.getMessage()).withCause(e).asRuntimeException()); - } - } - - @Override - public void restartIngestionJob( - RestartIngestionJobRequest request, - StreamObserver responseObserver) { - try { - RestartIngestionJobResponse response = this.jobService.restartJob(request); - responseObserver.onNext(response); - responseObserver.onCompleted(); - } catch (NoSuchElementException e) { - log.error( - "Attempted to restart an nonexistent job on calling restartIngestionJob method:", e); - responseObserver.onError( - Status.NOT_FOUND.withDescription(e.getMessage()).withCause(e).asException()); - } catch (UnsupportedOperationException e) { - log.error("Recieved an unsupported request on calling restartIngestionJob method:", e); - responseObserver.onError( - Status.FAILED_PRECONDITION.withDescription(e.getMessage()).withCause(e).asException()); - } catch (Exception e) { - log.error("Unexpected exception on calling restartIngestionJob method:", e); - responseObserver.onError( - Status.INTERNAL.withDescription(e.getMessage()).withCause(e).asRuntimeException()); - } - } - - @Override - public void stopIngestionJob( - StopIngestionJobRequest request, StreamObserver responseObserver) { - try { - StopIngestionJobResponse response = this.jobService.stopJob(request); - responseObserver.onNext(response); - responseObserver.onCompleted(); - } catch (NoSuchElementException e) { - log.error("Attempted to stop an nonexistent job on calling stopIngestionJob method:", e); - responseObserver.onError( - Status.NOT_FOUND.withDescription(e.getMessage()).withCause(e).asException()); - } catch (UnsupportedOperationException e) { - log.error("Recieved an unsupported request on calling stopIngestionJob method:", e); - responseObserver.onError( - Status.FAILED_PRECONDITION.withDescription(e.getMessage()).withCause(e).asException()); - } catch (Exception e) { - log.error("Unexpected exception on calling stopIngestionJob method:", e); - responseObserver.onError( - Status.INTERNAL.withDescription(e.getMessage()).withCause(e).asRuntimeException()); - } - } -} diff --git a/job-controller/src/main/java/feast/jobcontroller/model/FeatureSetDeliveryStatus.java b/job-controller/src/main/java/feast/jobcontroller/model/FeatureSetDeliveryStatus.java deleted file mode 100644 index e7300f3be2d..00000000000 --- a/job-controller/src/main/java/feast/jobcontroller/model/FeatureSetDeliveryStatus.java +++ /dev/null @@ -1,74 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.jobcontroller.model; - -import com.google.common.base.Objects; -import feast.common.models.FeatureSetReference; -import feast.proto.core.FeatureSetProto.FeatureSetJobDeliveryStatus; - -/** - * Data class that represents connection between {@link Job} and FeatureSet. For all FeatureSets - * allocated to Job FeatureSetDeliveryStatus must be created and added to Job's - * featureSetDeliveryStatuses map. FeatureSet is determined by {@link FeatureSetReference}. Stores - * delivery status and latest delivered version. - */ -public class FeatureSetDeliveryStatus { - private final FeatureSetReference featureSetReference; - private FeatureSetJobDeliveryStatus deliveryStatus; - private int deliveredVersion; - - public FeatureSetDeliveryStatus(FeatureSetReference featureSetReference) { - this.featureSetReference = featureSetReference; - } - - public FeatureSetDeliveryStatus setDeliveryStatus(FeatureSetJobDeliveryStatus deliveryStatus) { - this.deliveryStatus = deliveryStatus; - return this; - } - - public FeatureSetDeliveryStatus setDeliveredVersion(int deliveredVersion) { - this.deliveredVersion = deliveredVersion; - return this; - } - - public FeatureSetJobDeliveryStatus getDeliveryStatus() { - return deliveryStatus; - } - - public FeatureSetReference getFeatureSetReference() { - return featureSetReference; - } - - public int getDeliveredVersion() { - return deliveredVersion; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - FeatureSetDeliveryStatus that = (FeatureSetDeliveryStatus) o; - return deliveredVersion == that.deliveredVersion - && Objects.equal(this.featureSetReference, that.featureSetReference) - && deliveryStatus == that.deliveryStatus; - } - - @Override - public int hashCode() { - return Objects.hashCode(this.featureSetReference); - } -} diff --git a/job-controller/src/main/java/feast/jobcontroller/model/Job.java b/job-controller/src/main/java/feast/jobcontroller/model/Job.java deleted file mode 100644 index cf592d4567c..00000000000 --- a/job-controller/src/main/java/feast/jobcontroller/model/Job.java +++ /dev/null @@ -1,205 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2019 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.jobcontroller.model; - -import com.google.auto.value.AutoValue; -import feast.common.models.FeatureSetReference; -import feast.proto.core.*; -import java.util.*; - -/** Contains information about a run job. */ -@AutoValue -public abstract class Job { - private Date created; - private Date lastUpdated; - - private String extId; - private JobStatus status = JobStatus.UNKNOWN; - - // Internal job name. Generated by feast ingestion upon invocation. - public abstract String getId(); - - // External job id, generated by the runner and retrieved by feast. - // Used internally for job management. - public String getExtId() { - return extId; - } - - public JobStatus getStatus() { - return status; - } - - // Source type and config, derived from job's source and stored as inline fields. - public abstract SourceProto.Source getSource(); - - // Sinks - public abstract Map getStores(); - - // Allocated FeatureSets' delivery statuses - public abstract Map - getFeatureSetDeliveryStatuses(); - - // Job's labels - public abstract Map getLabels(); - - public static Builder builder() { - return new AutoValue_Job.Builder() - .setFeatureSetDeliveryStatuses(new HashMap<>()) - .setStores(new HashMap<>()) - .setLabels(new HashMap<>()); - } - - @AutoValue.Builder - public interface Builder { - Builder setId(String id); - - Builder setSource(SourceProto.Source source); - - Builder setStores(Map stores); - - Builder setFeatureSetDeliveryStatuses( - Map statuses); - - Builder setLabels(Map labels); - - Job build(); - } - - public Date getCreated() { - return created; - } - - public Date getLastUpdated() { - return lastUpdated; - } - - public void preSave() { - if (this.created == null) { - this.created = new Date(); - } - this.lastUpdated = new Date(); - } - - public void setExtId(String extId) { - this.extId = extId; - } - - public void setStatus(JobStatus status) { - this.status = status; - } - - public void setCreated(Date created) { - this.created = created; - this.lastUpdated = created; - } - - public boolean hasTerminated() { - return getStatus().isTerminal(); - } - - public boolean isRunning() { - return getStatus() == JobStatus.RUNNING; - } - - public boolean isDeployed() { - return getExtId() != null && !getExtId().isEmpty(); - } - - public void addAllStores(Set stores) { - for (var store : stores) { - this.getStores().put(store.getName(), store); - } - } - - public void addAllFeatureSets(Set featureSets) { - for (FeatureSetProto.FeatureSet fs : featureSets) { - FeatureSetReference ref = - FeatureSetReference.of(fs.getSpec().getProject(), fs.getSpec().getName()); - FeatureSetDeliveryStatus status = new FeatureSetDeliveryStatus(ref); - - if (fs.getMeta().getStatus() == FeatureSetProto.FeatureSetStatus.STATUS_READY) { - // Feature Set was already delivered to previous generation of the job - // (another words, it exists in kafka) - // so we expect Job will ack latest version based on history from kafka topic - status.setDeliveredVersion(fs.getSpec().getVersion()); - } - status.setDeliveryStatus(FeatureSetProto.FeatureSetJobDeliveryStatus.STATUS_IN_PROGRESS); - this.getFeatureSetDeliveryStatuses().put(ref, status); - } - } - - /** - * Convert a job model to ingestion job proto - * - * @return Ingestion Job proto derieved from the given job - */ - public IngestionJobProto.IngestionJob toProto() { - - // convert featuresets of job to protos - List featureSetReferences = new ArrayList<>(); - - for (FeatureSetReference featureSetReference : this.getFeatureSetDeliveryStatuses().keySet()) { - featureSetReferences.add( - FeatureSetReferenceProto.FeatureSetReference.newBuilder() - .setName(featureSetReference.getFeatureSetName()) - .setProject(featureSetReference.getProjectName()) - .build()); - } - - // build ingestion job proto with job data - IngestionJobProto.IngestionJob ingestJob = - IngestionJobProto.IngestionJob.newBuilder() - .setId(this.getId()) - .setExternalId(this.getExtId()) - .setStatus(this.getStatus().toProto()) - .setSource(this.getSource()) - .addAllStores(new HashSet<>(this.getStores().values())) - .addAllFeatureSetReferences(featureSetReferences) - .build(); - - return ingestJob; - } - - public Job cloneWithIdAndLabels(String newJobId, Map labels) { - return Job.builder() - .setSource(this.getSource()) - .setFeatureSetDeliveryStatuses(new HashMap<>(this.getFeatureSetDeliveryStatuses())) - .setStores(new HashMap<>(this.getStores())) - .setId(newJobId) - .setLabels(labels) - .build(); - } - - @Override - public int hashCode() { - return Objects.hash(getSource(), getStores()); - } - - @Override - public boolean equals(Object obj) { - if (this == obj) return true; - if (!super.equals(obj)) return false; - if (getClass() != obj.getClass()) return false; - Job other = (Job) obj; - if (!this.getSource().equals(other.getSource())) { - return false; - } else if (!this.getStores().equals(other.getStores())) { - return false; - } - return true; - } -} diff --git a/job-controller/src/main/java/feast/jobcontroller/model/JobStatus.java b/job-controller/src/main/java/feast/jobcontroller/model/JobStatus.java deleted file mode 100644 index 3e8824c743d..00000000000 --- a/job-controller/src/main/java/feast/jobcontroller/model/JobStatus.java +++ /dev/null @@ -1,110 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2019 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.jobcontroller.model; - -import feast.proto.core.IngestionJobProto.IngestionJobStatus; -import java.util.Map; -import java.util.Set; - -public enum JobStatus { - /** Job status is not known. */ - UNKNOWN, - - /** Import job is submitted to runner and currently pending for executing */ - PENDING, - - /** Import job is currently running in the runner */ - RUNNING, - - /** Runner’s reported the import job has completed (applicable to batch job) */ - COMPLETED, - - /** When user sent abort command, but it's still running */ - ABORTING, - - /** User initiated abort job */ - ABORTED, - - /** - * Runner’s reported that the import job failed to run or there is a failure during job - * submission. - */ - ERROR, - - /** job has been suspended and waiting for cleanup */ - SUSPENDING, - - /** job has been suspended */ - SUSPENDED; - - private static final Set TERMINAL_STATES = Set.of(COMPLETED, ABORTED, ERROR); - - /** - * Get the set of terminal job states. - * - *

A terminal job state is final and will not change to any other state. - * - * @return set of terminal job states. - */ - public static Set getTerminalStates() { - return TERMINAL_STATES; - } - - private static final Set TRANSITIONAL_STATES = Set.of(PENDING, ABORTING, SUSPENDING); - - /** - * Get Transitional Job Status states. Transitional states are assigned to jobs that are - * transitioning to a more stable state (ie SUSPENDED, ABORTED etc.) - * - * @return set of transitional Job Status states. - */ - public static Set getTransitionalStates() { - return TRANSITIONAL_STATES; - } - - /** @return true if this {@code JobStatus} is a terminal state. */ - public boolean isTerminal() { - return getTerminalStates().contains(this); - } - - /** @return true if this {@code JobStatus} is a transitional state. */ - public boolean isTransitional() { - return getTransitionalStates().contains(this); - } - - private static final Map INGESTION_JOB_STATUS_MAP = - Map.of( - JobStatus.UNKNOWN, IngestionJobStatus.UNKNOWN, - JobStatus.PENDING, IngestionJobStatus.PENDING, - JobStatus.RUNNING, IngestionJobStatus.RUNNING, - JobStatus.COMPLETED, IngestionJobStatus.COMPLETED, - JobStatus.ABORTING, IngestionJobStatus.ABORTING, - JobStatus.ABORTED, IngestionJobStatus.ABORTED, - JobStatus.ERROR, IngestionJobStatus.ERROR, - JobStatus.SUSPENDING, IngestionJobStatus.SUSPENDING, - JobStatus.SUSPENDED, IngestionJobStatus.SUSPENDED); - - /** - * Convert a Job Status to Ingestion Job Status proto - * - * @return IngestionJobStatus proto derived from this job status - */ - public IngestionJobStatus toProto() { - // maps job models job status to ingestion job status - return INGESTION_JOB_STATUS_MAP.get(this); - } -} diff --git a/job-controller/src/main/java/feast/jobcontroller/runner/ConsolidatedJobStrategy.java b/job-controller/src/main/java/feast/jobcontroller/runner/ConsolidatedJobStrategy.java deleted file mode 100644 index 23b1d3251c6..00000000000 --- a/job-controller/src/main/java/feast/jobcontroller/runner/ConsolidatedJobStrategy.java +++ /dev/null @@ -1,101 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.jobcontroller.runner; - -import feast.jobcontroller.config.FeastProperties.JobProperties; -import feast.jobcontroller.dao.JobRepository; -import feast.jobcontroller.model.Job; -import feast.jobcontroller.model.JobStatus; -import feast.proto.core.SourceProto; -import feast.proto.core.StoreProto; -import java.time.Instant; -import java.util.Map; -import java.util.Objects; -import java.util.Set; -import java.util.stream.Collectors; -import java.util.stream.Stream; -import org.apache.commons.lang3.tuple.Pair; - -/** - * In this strategy one Ingestion Job per source is created. All stores that subscribed to - * FeatureSets from this source will be included as sinks in this consolidated Job. - * - *

JobId will contain only source parameters (type + config). StoreName will remain empty in Job - * table. - */ -public class ConsolidatedJobStrategy implements JobGroupingStrategy { - private final JobRepository jobRepository; - private final JobProperties jobProperties; - - public ConsolidatedJobStrategy(JobRepository jobRepository, JobProperties jobProperties) { - this.jobRepository = jobRepository; - this.jobProperties = jobProperties; - } - - @Override - public Job getOrCreateJob( - SourceProto.Source source, Set stores, Map labels) { - return jobRepository - .findFirstBySourceAndStoreNameAndStatusNotInOrderByLastUpdatedDesc( - source, null, JobStatus.getTerminalStates()) - .orElseGet( - () -> - Job.builder() - .setId(createJobId(source)) - .setSource(source) - .setStores( - stores.stream() - .collect(Collectors.toMap(StoreProto.Store::getName, s -> s))) - .setLabels(labels) - .build()); - } - - private String createJobId(SourceProto.Source source) { - String dateSuffix = String.valueOf(Instant.now().toEpochMilli()); - String jobId = - String.format( - "%s-%d-%s", - source.getType().getValueDescriptor().getName(), - Objects.hash( - source.getKafkaSourceConfig().getBootstrapServers(), - source.getKafkaSourceConfig().getTopic()), - dateSuffix); - if (this.jobProperties.getJobIdPrefix() != null - && !this.jobProperties.getJobIdPrefix().isEmpty()) { - jobId = this.jobProperties.getJobIdPrefix() + "-" + jobId; - } - return jobId.replaceAll("_store", "-").toLowerCase(); - } - - @Override - public String createJobId(Job job) { - return createJobId(job.getSource()); - } - - @Override - public Iterable>> collectSingleJobInput( - Stream> stream) { - Map> map = - stream.collect( - Collectors.groupingBy( - Pair::getLeft, Collectors.mapping(Pair::getRight, Collectors.toSet()))); - - return map.entrySet().stream() - .map(e -> Pair.of(e.getKey(), e.getValue())) - .collect(Collectors.toList()); - } -} diff --git a/job-controller/src/main/java/feast/jobcontroller/runner/JobGroupingStrategy.java b/job-controller/src/main/java/feast/jobcontroller/runner/JobGroupingStrategy.java deleted file mode 100644 index 0a672808ab5..00000000000 --- a/job-controller/src/main/java/feast/jobcontroller/runner/JobGroupingStrategy.java +++ /dev/null @@ -1,40 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.jobcontroller.runner; - -import feast.jobcontroller.model.Job; -import feast.proto.core.SourceProto; -import feast.proto.core.StoreProto; -import java.util.Map; -import java.util.Set; -import java.util.stream.Stream; -import org.apache.commons.lang3.tuple.Pair; - -/** - * Strategy interface that defines how responsibility for sources and stores will be distributed - * across Ingestion Jobs. - */ -public interface JobGroupingStrategy { - /** Get the non terminated ingestion job ingesting for given source and stores. */ - Job getOrCreateJob( - SourceProto.Source source, Set stores, Map labels); - /** Create unique JobId that would be used as key in communications with JobRunner */ - String createJobId(Job job); - /* Distribute given sources and stores across jobs. One yielded Pair - one created Job **/ - Iterable>> collectSingleJobInput( - Stream> stream); -} diff --git a/job-controller/src/main/java/feast/jobcontroller/runner/JobManager.java b/job-controller/src/main/java/feast/jobcontroller/runner/JobManager.java deleted file mode 100644 index c9cbaf42a0a..00000000000 --- a/job-controller/src/main/java/feast/jobcontroller/runner/JobManager.java +++ /dev/null @@ -1,81 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2019 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.jobcontroller.runner; - -import feast.jobcontroller.model.Job; -import feast.jobcontroller.model.JobStatus; -import java.util.List; - -public interface JobManager { - - /** - * Get Runner Type - * - * @return runner type - */ - Runner getRunnerType(); - - /** - * Start an import job. The JobManager should also attach external id that is specific to - * JobManager implementation - * - * @param job job to start - * @return Running Job with extId set. - */ - Job startJob(Job job); - - /** - * Update already running job with new set of features to ingest. - * - * @param job job of target job to change - * @return Job - */ - Job updateJob(Job job); - - /** - * Abort a job given runner-specific job ID. - * - * @param job to abort. - * @return The Aborting Job - */ - Job abortJob(Job job); - - /** - * Restart an job. If job is an terminated state, will simply start the job. Might cause data to - * be lost during when restarting running jobs in some implementations. Refer to on docs the - * specific implementation. - * - * @param job job to restart - * @return the restarted job - */ - Job restartJob(Job job); - - /** - * Get status of a job given runner-specific job ID. - * - * @param job job. - * @return job status. - */ - JobStatus getJobStatus(Job job); - - /** - * List of RUNNING jobs - * - * @return list of jobs - */ - List listRunningJobs(); -} diff --git a/job-controller/src/main/java/feast/jobcontroller/runner/JobPerStoreStrategy.java b/job-controller/src/main/java/feast/jobcontroller/runner/JobPerStoreStrategy.java deleted file mode 100644 index 738ef897944..00000000000 --- a/job-controller/src/main/java/feast/jobcontroller/runner/JobPerStoreStrategy.java +++ /dev/null @@ -1,101 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.jobcontroller.runner; - -import com.google.common.collect.Lists; -import feast.jobcontroller.config.FeastProperties.JobProperties; -import feast.jobcontroller.dao.JobRepository; -import feast.jobcontroller.model.Job; -import feast.jobcontroller.model.JobStatus; -import feast.proto.core.SourceProto; -import feast.proto.core.StoreProto; -import java.time.Instant; -import java.util.ArrayList; -import java.util.Map; -import java.util.Objects; -import java.util.Set; -import java.util.stream.Collectors; -import java.util.stream.Stream; -import org.apache.commons.lang3.tuple.Pair; - -/** - * In this strategy one job per Source-Store pair is created. - * - *

JobId is generated accordingly from Source (type+config) and StoreName. - */ -public class JobPerStoreStrategy implements JobGroupingStrategy { - private final JobRepository jobRepository; - private final JobProperties jobProperties; - - public JobPerStoreStrategy(JobRepository jobRepository, JobProperties jobProperties) { - this.jobRepository = jobRepository; - this.jobProperties = jobProperties; - } - - @Override - public Job getOrCreateJob( - SourceProto.Source source, Set stores, Map labels) { - ArrayList storesList = Lists.newArrayList(stores); - if (storesList.size() != 1) { - throw new RuntimeException("Only one store is acceptable in JobPerStore Strategy"); - } - StoreProto.Store store = storesList.get(0); - - return jobRepository - .findFirstBySourceAndStoreNameAndStatusNotInOrderByLastUpdatedDesc( - source, store.getName(), JobStatus.getTerminalStates()) - .orElseGet( - () -> - Job.builder() - .setId(createJobId(source, stores)) - .setSource(source) - .setStores( - stores.stream() - .collect(Collectors.toMap(StoreProto.Store::getName, s -> s))) - .setLabels(labels) - .build()); - } - - private String createJobId(SourceProto.Source source, Iterable stores) { - String dateSuffix = String.valueOf(Instant.now().toEpochMilli()); - String jobId = - String.format( - "%s-%d-to-%s-%s", - source.getType().getValueDescriptor().getName(), - Objects.hash( - source.getKafkaSourceConfig().getBootstrapServers(), - source.getKafkaSourceConfig().getTopic()), - Lists.newArrayList(stores).get(0).getName(), - dateSuffix); - if (this.jobProperties.getJobIdPrefix() != null - && !this.jobProperties.getJobIdPrefix().isEmpty()) { - jobId = this.jobProperties.getJobIdPrefix() + "-" + jobId; - } - return jobId.replaceAll("_store", "-").toLowerCase(); - } - - @Override - public String createJobId(Job job) { - return createJobId(job.getSource(), job.getStores().values()); - } - - @Override - public Iterable>> collectSingleJobInput( - Stream> stream) { - return stream.map(p -> Pair.of(p.getLeft(), Set.of(p.getRight()))).collect(Collectors.toList()); - } -} diff --git a/job-controller/src/main/java/feast/jobcontroller/runner/Runner.java b/job-controller/src/main/java/feast/jobcontroller/runner/Runner.java deleted file mode 100644 index 8db14b5d2e4..00000000000 --- a/job-controller/src/main/java/feast/jobcontroller/runner/Runner.java +++ /dev/null @@ -1,52 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2019 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.jobcontroller.runner; - -import java.util.NoSuchElementException; - -/** - * An Apache Beam Runner, for which Feast Core supports managing ingestion jobs. - * - * @see Beam Runners - */ -public enum Runner { - DATAFLOW("DataflowRunner"), - FLINK("FlinkRunner"), - DIRECT("DirectRunner"); - - private final String humanName; - - Runner(String humanName) { - this.humanName = humanName; - } - - /** Returns the human readable name of this runner, usable in logging, config files, etc. */ - @Override - public String toString() { - return humanName; - } - - /** Parses a runner from its human readable name. */ - public static Runner fromString(String humanName) { - for (Runner r : Runner.values()) { - if (r.toString().equals(humanName)) { - return r; - } - } - throw new NoSuchElementException("Unknown Runner value: " + humanName); - } -} diff --git a/job-controller/src/main/java/feast/jobcontroller/runner/dataflow/DataflowJobManager.java b/job-controller/src/main/java/feast/jobcontroller/runner/dataflow/DataflowJobManager.java deleted file mode 100644 index 38be7522fd2..00000000000 --- a/job-controller/src/main/java/feast/jobcontroller/runner/dataflow/DataflowJobManager.java +++ /dev/null @@ -1,409 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2019 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.jobcontroller.runner.dataflow; - -import static feast.ingestion.utils.SpecUtil.parseSourceJson; -import static feast.ingestion.utils.SpecUtil.parseStoreJsonList; -import static feast.jobcontroller.util.PipelineUtil.detectClassPathResourcesToStage; - -import com.google.api.client.auth.oauth2.Credential; -import com.google.api.client.googleapis.auth.oauth2.GoogleCredential; -import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport; -import com.google.api.client.json.jackson2.JacksonFactory; -import com.google.api.services.dataflow.Dataflow; -import com.google.api.services.dataflow.DataflowScopes; -import com.google.common.base.Strings; -import com.google.protobuf.InvalidProtocolBufferException; -import com.google.protobuf.util.JsonFormat; -import feast.common.models.FeatureSetReference; -import feast.ingestion.ImportJob; -import feast.ingestion.options.ImportOptions; -import feast.jobcontroller.config.FeastProperties.MetricsProperties; -import feast.jobcontroller.exception.JobExecutionException; -import feast.jobcontroller.model.Job; -import feast.jobcontroller.model.JobStatus; -import feast.jobcontroller.runner.JobManager; -import feast.jobcontroller.runner.Runner; -import feast.proto.core.IngestionJobProto; -import feast.proto.core.RunnerProto.DataflowRunnerConfigOptions; -import feast.proto.core.SourceProto; -import feast.proto.core.StoreProto; -import java.io.IOException; -import java.security.GeneralSecurityException; -import java.util.*; -import java.util.stream.Collectors; -import lombok.extern.slf4j.Slf4j; -import org.apache.beam.runners.dataflow.DataflowPipelineJob; -import org.apache.beam.runners.dataflow.DataflowRunner; -import org.apache.beam.sdk.PipelineResult.State; -import org.apache.beam.sdk.options.PipelineOptionsFactory; -import org.joda.time.DateTime; - -@Slf4j -public class DataflowJobManager implements JobManager { - - private final Runner RUNNER_TYPE = Runner.DATAFLOW; - - private final String projectId; - private final String location; - private final Dataflow dataflow; - private final DataflowRunnerConfig defaultOptions; - private final MetricsProperties metrics; - private final IngestionJobProto.SpecsStreamingUpdateConfig specsStreamingUpdateConfig; - private final Map jobSelector; - - public static DataflowJobManager of( - DataflowRunnerConfigOptions runnerConfigOptions, - MetricsProperties metricsProperties, - IngestionJobProto.SpecsStreamingUpdateConfig specsStreamingUpdateConfig, - Map jobSelector) { - - Dataflow dataflow; - try { - dataflow = - new Dataflow( - GoogleNetHttpTransport.newTrustedTransport(), - JacksonFactory.getDefaultInstance(), - getGoogleCredential()); - } catch (GeneralSecurityException e) { - throw new IllegalStateException("Security exception while connecting to Dataflow API", e); - } catch (IOException e) { - throw new IllegalStateException("Unable to initialize DataflowJobManager", e); - } - - return new DataflowJobManager( - runnerConfigOptions, metricsProperties, specsStreamingUpdateConfig, jobSelector, dataflow); - } - - DataflowJobManager( - DataflowRunnerConfigOptions runnerConfigOptions, - MetricsProperties metricsProperties, - IngestionJobProto.SpecsStreamingUpdateConfig specsStreamingUpdateConfig, - Map jobSelector, - Dataflow dataflow) { - - defaultOptions = new DataflowRunnerConfig(runnerConfigOptions); - this.dataflow = dataflow; - this.metrics = metricsProperties; - this.projectId = defaultOptions.getProject(); - this.location = defaultOptions.getRegion(); - this.specsStreamingUpdateConfig = specsStreamingUpdateConfig; - this.jobSelector = jobSelector; - } - - private static Credential getGoogleCredential() { - GoogleCredential credential = null; - try { - credential = GoogleCredential.getApplicationDefault().createScoped(DataflowScopes.all()); - } catch (IOException e) { - throw new IllegalStateException( - "Unable to find credential required for Dataflow monitoring API", e); - } - return credential; - } - - @Override - public Runner getRunnerType() { - return RUNNER_TYPE; - } - - @Override - public Job startJob(Job job) { - try { - String extId = - submitDataflowJob( - job.getId(), - job.getSource(), - new HashSet<>(job.getStores().values()), - job.getLabels(), - false); - job.setExtId(extId); - return job; - - } catch (RuntimeException e) { - log.error(e.getMessage()); - if (e.getCause() instanceof InvalidProtocolBufferException) { - throw new IllegalArgumentException( - String.format( - "DataflowJobManager failed to START job with id '%s' because the job" - + "has an invalid spec. Please check the FeatureSet, Source and Store specs. Actual error message: %s", - job.getId(), e.getMessage())); - } - - throw e; - } - } - - /** - * Drain existing job. Replacement will be created on next run (when job gracefully stop) - * - * @param job job of target job to change - * @return same job as input - */ - @Override - public Job updateJob(Job job) { - abortJob(job); - return job; - } - - /** - * Abort an existing Dataflow job. Streaming Dataflow jobs are always drained, not cancelled. - * - * @param job to abort. - * @return The aborted Job. - */ - @Override - public Job abortJob(Job job) { - String dataflowJobId = job.getExtId(); - try { - com.google.api.services.dataflow.model.Job dataflowJob = - dataflow.projects().locations().jobs().get(projectId, location, dataflowJobId).execute(); - com.google.api.services.dataflow.model.Job content = - new com.google.api.services.dataflow.model.Job(); - if (dataflowJob.getType().equals(DataflowJobType.JOB_TYPE_BATCH.toString())) { - content.setRequestedState(DataflowJobState.JOB_STATE_CANCELLED.toString()); - } else if (dataflowJob.getType().equals(DataflowJobType.JOB_TYPE_STREAMING.toString())) { - content.setRequestedState(DataflowJobState.JOB_STATE_DRAINING.toString()); - } - dataflow - .projects() - .locations() - .jobs() - .update(projectId, location, dataflowJobId, content) - .execute(); - } catch (Exception e) { - log.error("Unable to drain job with id: {}, cause: {}", dataflowJobId, e.getMessage()); - throw new RuntimeException( - Strings.lenientFormat("Unable to drain job with id: %s", dataflowJobId), e); - } - - return job; - } - - /** - * Restart a Dataflow job. Dataflow should ensure continuity such that no data should be lost - * during the restart operation. - * - * @param job job to restart - * @return the restarted job - */ - @Override - public Job restartJob(Job job) { - if (job.getStatus().isTerminal()) { - // job yet not running: just start job - return this.startJob(job); - } else { - // job is running - updating the job without changing the job has - // the effect of restarting the job - return this.updateJob(job); - } - } - - /** - * Get status of a dataflow job with given id and try to map it into Feast's JobStatus. - * - * @param job Job containing dataflow job id - * @return status of the job, or return {@link JobStatus#UNKNOWN} if error happens. - */ - @Override - public JobStatus getJobStatus(Job job) { - try { - com.google.api.services.dataflow.model.Job dataflowJob = - dataflow.projects().locations().jobs().get(projectId, location, job.getExtId()).execute(); - return DataflowJobStateMapper.map(dataflowJob.getCurrentState()); - } catch (Exception e) { - log.error( - "Unable to retrieve status of a dataflow job with id : {}\ncause: {}", - job.getExtId(), - e.getMessage()); - } - return JobStatus.UNKNOWN; - } - - @Override - public List listRunningJobs() { - List jobs; - - try { - jobs = - dataflow - .projects() - .locations() - .jobs() - .list(projectId, location) - .setFilter("ACTIVE") - .execute() - .getJobs(); - } catch (IOException e) { - throw new RuntimeException( - String.format("Unable to retrieve list of jobs from dataflow: %s", e.getMessage())); - } - - if (jobs == null) { - return Collections.emptyList(); - } - - return jobs.stream() - .map( - dfJob -> { - try { - return dataflow - .projects() - .locations() - .jobs() - .get(projectId, location, dfJob.getId()) - .setView("JOB_VIEW_ALL") - .execute(); - } catch (IOException e) { - log.error( - "Job's detailed info {} couldn't be loaded from Dataflow: {}", - dfJob.getId(), - e.getMessage()); - return null; - } - }) - .filter( - dfJob -> - dfJob != null - && (dfJob.getLabels() != null || this.jobSelector.isEmpty()) - && this.jobSelector.entrySet().stream() - .allMatch( - entry -> - dfJob - .getLabels() - .getOrDefault(entry.getKey(), "") - .equals(entry.getValue()))) - .map( - dfJob -> { - Map options = - (Map) - dfJob.getEnvironment().getSdkPipelineOptions().get("options"); - - List stores = - parseStoreJsonList((List) options.get("storesJson")); - SourceProto.Source source = parseSourceJson((String) options.get("sourceJson")); - - Job job = - Job.builder() - .setId((String) options.get("jobName")) - .setSource(source) - .setStores( - stores.stream() - .collect(Collectors.toMap(StoreProto.Store::getName, s -> s))) - .setLabels( - dfJob.getLabels() == null - ? new HashMap<>() - : new HashMap<>(dfJob.getLabels())) - .build(); - - job.setExtId(dfJob.getId()); - job.setStatus(JobStatus.RUNNING); - if (dfJob.getCreateTime() != null) { - job.setCreated(DateTime.parse(dfJob.getCreateTime()).toDate()); - } - - return job; - }) - .collect(Collectors.toList()); - } - - private String submitDataflowJob( - String jobName, - SourceProto.Source source, - Set sinks, - Map labels, - boolean update) { - try { - ImportOptions pipelineOptions = getPipelineOptions(jobName, source, sinks, labels, update); - DataflowPipelineJob pipelineResult = runPipeline(pipelineOptions); - String jobId = waitForJobToRun(pipelineResult); - return jobId; - } catch (Exception e) { - log.error("Error submitting job", e); - throw new JobExecutionException(String.format("Error running ingestion job: %s", e), e); - } - } - - private ImportOptions getPipelineOptions( - String jobName, - SourceProto.Source source, - Set sinks, - Map labels, - boolean update) - throws IOException, IllegalAccessException { - ImportOptions pipelineOptions = - PipelineOptionsFactory.fromArgs(defaultOptions.toArgs()).as(ImportOptions.class); - - JsonFormat.Printer jsonPrinter = JsonFormat.printer(); - List storesJson = new ArrayList<>(); - for (StoreProto.Store sink : sinks) { - String print = jsonPrinter.print(sink); - storesJson.add(print); - } - - pipelineOptions.setSpecsStreamingUpdateConfigJson( - jsonPrinter.print(specsStreamingUpdateConfig)); - pipelineOptions.setSourceJson(jsonPrinter.print(source)); - pipelineOptions.setStoresJson(storesJson); - pipelineOptions.setProject(projectId); - pipelineOptions.setDefaultFeastProject(FeatureSetReference.PROJECT_DEFAULT_NAME); - pipelineOptions.setUpdate(update); - pipelineOptions.setRunner(DataflowRunner.class); - pipelineOptions.setJobName(jobName); - pipelineOptions.setFilesToStage( - detectClassPathResourcesToStage(DataflowRunner.class.getClassLoader())); - - // Merge common labels with job's labels - Map mergedLabels = new HashMap<>(defaultOptions.getLabels()); - labels.forEach(mergedLabels::put); - pipelineOptions.setLabels(mergedLabels); - - if (metrics.isEnabled()) { - pipelineOptions.setMetricsExporterType(metrics.getType()); - if (metrics.getType().equals("statsd")) { - pipelineOptions.setStatsdHost(metrics.getHost()); - pipelineOptions.setStatsdPort(metrics.getPort()); - } - } - return pipelineOptions; - } - - public DataflowPipelineJob runPipeline(ImportOptions pipelineOptions) throws IOException { - return (DataflowPipelineJob) ImportJob.runPipeline(pipelineOptions); - } - - private String waitForJobToRun(DataflowPipelineJob pipelineResult) - throws RuntimeException, InterruptedException { - // TODO: add timeout - while (true) { - State state = pipelineResult.getState(); - if (state.isTerminal()) { - String dataflowDashboardUrl = - String.format( - "https://console.cloud.google.com/dataflow/jobsDetail/locations/%s/jobs/%s", - location, pipelineResult.getJobId()); - throw new RuntimeException( - String.format( - "Failed to submit dataflow job, job state is %s. Refer to the dataflow dashboard for more information: %s", - state.toString(), dataflowDashboardUrl)); - } else if (state.equals(State.RUNNING)) { - return pipelineResult.getJobId(); - } - Thread.sleep(2000); - } - } -} diff --git a/job-controller/src/main/java/feast/jobcontroller/runner/dataflow/DataflowJobState.java b/job-controller/src/main/java/feast/jobcontroller/runner/dataflow/DataflowJobState.java deleted file mode 100644 index e66ed701e54..00000000000 --- a/job-controller/src/main/java/feast/jobcontroller/runner/dataflow/DataflowJobState.java +++ /dev/null @@ -1,31 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2019 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.jobcontroller.runner.dataflow; - -public enum DataflowJobState { - JOB_STATE_UNKNOWN, - JOB_STATE_STOPPED, - JOB_STATE_RUNNING, - JOB_STATE_DONE, - JOB_STATE_FAILED, - JOB_STATE_CANCELLED, - JOB_STATE_UPDATED, - JOB_STATE_DRAINING, - JOB_STATE_DRAINED, - JOB_STATE_PENDING, - JOB_STATE_CANCELLING -} diff --git a/job-controller/src/main/java/feast/jobcontroller/runner/dataflow/DataflowJobStateMapper.java b/job-controller/src/main/java/feast/jobcontroller/runner/dataflow/DataflowJobStateMapper.java deleted file mode 100644 index fc0ad632b69..00000000000 --- a/job-controller/src/main/java/feast/jobcontroller/runner/dataflow/DataflowJobStateMapper.java +++ /dev/null @@ -1,58 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2019 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.jobcontroller.runner.dataflow; - -import static feast.jobcontroller.runner.dataflow.DataflowJobState.*; - -import feast.jobcontroller.model.JobStatus; -import java.util.HashMap; -import java.util.Map; - -public class DataflowJobStateMapper { - private static final Map DATAFLOW_TO_FEAST_JOB_STATUS; - - static { - DATAFLOW_TO_FEAST_JOB_STATUS = new HashMap<>(); - DATAFLOW_TO_FEAST_JOB_STATUS.put(JOB_STATE_UNKNOWN, JobStatus.UNKNOWN); - // Dataflow: JOB_STATE_STOPPED indicates that the job has not yet started to run. - DATAFLOW_TO_FEAST_JOB_STATUS.put(JOB_STATE_STOPPED, JobStatus.PENDING); - DATAFLOW_TO_FEAST_JOB_STATUS.put(JOB_STATE_PENDING, JobStatus.PENDING); - DATAFLOW_TO_FEAST_JOB_STATUS.put(JOB_STATE_RUNNING, JobStatus.RUNNING); - DATAFLOW_TO_FEAST_JOB_STATUS.put(JOB_STATE_UPDATED, JobStatus.RUNNING); - DATAFLOW_TO_FEAST_JOB_STATUS.put(JOB_STATE_DRAINING, JobStatus.ABORTING); - DATAFLOW_TO_FEAST_JOB_STATUS.put(JOB_STATE_CANCELLING, JobStatus.ABORTING); - DATAFLOW_TO_FEAST_JOB_STATUS.put(JOB_STATE_DRAINED, JobStatus.ABORTED); - DATAFLOW_TO_FEAST_JOB_STATUS.put(JOB_STATE_CANCELLED, JobStatus.ABORTED); - DATAFLOW_TO_FEAST_JOB_STATUS.put(JOB_STATE_FAILED, JobStatus.ERROR); - DATAFLOW_TO_FEAST_JOB_STATUS.put(JOB_STATE_DONE, JobStatus.COMPLETED); - } - - /** - * Map a string containing Dataflow's JobState into Feast's JobStatus - * - * @param jobState Dataflow JobState - * @return JobStatus. - * @throws IllegalArgumentException if jobState is invalid. - */ - public static JobStatus map(String jobState) { - DataflowJobState dfJobState = DataflowJobState.valueOf(jobState); - if (DATAFLOW_TO_FEAST_JOB_STATUS.containsKey(dfJobState)) { - return DATAFLOW_TO_FEAST_JOB_STATUS.get(dfJobState); - } - throw new IllegalArgumentException("Unknown job state: " + jobState); - } -} diff --git a/job-controller/src/main/java/feast/jobcontroller/runner/dataflow/DataflowJobType.java b/job-controller/src/main/java/feast/jobcontroller/runner/dataflow/DataflowJobType.java deleted file mode 100644 index 1f9c75ecc55..00000000000 --- a/job-controller/src/main/java/feast/jobcontroller/runner/dataflow/DataflowJobType.java +++ /dev/null @@ -1,22 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2019 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.jobcontroller.runner.dataflow; - -public enum DataflowJobType { - JOB_TYPE_BATCH, - JOB_TYPE_STREAMING -} diff --git a/job-controller/src/main/java/feast/jobcontroller/runner/dataflow/DataflowRunnerConfig.java b/job-controller/src/main/java/feast/jobcontroller/runner/dataflow/DataflowRunnerConfig.java deleted file mode 100644 index 7e348997287..00000000000 --- a/job-controller/src/main/java/feast/jobcontroller/runner/dataflow/DataflowRunnerConfig.java +++ /dev/null @@ -1,135 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2019 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.jobcontroller.runner.dataflow; - -import feast.common.validators.OneOfStrings; -import feast.jobcontroller.runner.option.RunnerConfig; -import feast.proto.core.RunnerProto.DataflowRunnerConfigOptions; -import java.util.Map; -import java.util.Set; -import javax.validation.*; -import javax.validation.constraints.NotBlank; -import lombok.Getter; -import lombok.Setter; - -/** DataflowRunnerConfig contains configuration fields for the Dataflow job runner. */ -@Getter -@Setter -public class DataflowRunnerConfig extends RunnerConfig { - - public DataflowRunnerConfig(DataflowRunnerConfigOptions runnerConfigOptions) { - this.project = runnerConfigOptions.getProject(); - this.region = runnerConfigOptions.getRegion(); - this.workerZone = runnerConfigOptions.getWorkerZone(); - this.serviceAccount = runnerConfigOptions.getServiceAccount(); - this.network = runnerConfigOptions.getNetwork(); - this.subnetwork = runnerConfigOptions.getSubnetwork(); - this.workerMachineType = runnerConfigOptions.getWorkerMachineType(); - this.autoscalingAlgorithm = runnerConfigOptions.getAutoscalingAlgorithm(); - this.usePublicIps = runnerConfigOptions.getUsePublicIps(); - this.tempLocation = runnerConfigOptions.getTempLocation(); - this.maxNumWorkers = runnerConfigOptions.getMaxNumWorkers(); - this.deadLetterTableSpec = runnerConfigOptions.getDeadLetterTableSpec(); - this.diskSizeGb = runnerConfigOptions.getDiskSizeGb(); - this.labels = runnerConfigOptions.getLabelsMap(); - this.enableStreamingEngine = runnerConfigOptions.getEnableStreamingEngine(); - this.workerDiskType = runnerConfigOptions.getWorkerDiskType(); - this.kafkaConsumerProperties = runnerConfigOptions.getKafkaConsumerPropertiesMap(); - validate(); - } - - /* Project id to use when launching jobs. */ - @NotBlank public String project; - - /* The Google Compute Engine region for creating Dataflow jobs. */ - @OneOfStrings({ - "us-west1", - "us-central1", - "us-east1", - "us-east4", - "northamerica-northeast1", - "europe-west1", - "europe-west2", - "europe-west3", - "europe-west4", - "asia-southeast1", - "asia-east1", - "asia-northeast1", - "australia-southeast1" - }) - @NotBlank - public String region; - - /* GCP availability zone for operations. */ - @NotBlank public String workerZone; - - /* Run the job as a specific service account, instead of the default GCE robot. */ - public String serviceAccount; - - /* GCE network for launching workers. */ - @NotBlank public String network; - - /* GCE subnetwork for launching workers. */ - @NotBlank public String subnetwork; - - /* Machine type to create Dataflow worker VMs as. */ - public String workerMachineType; - - /* The autoscaling algorithm to use for the workerpool. */ - public String autoscalingAlgorithm; - - /* Specifies whether worker pools should be started with public IP addresses. */ - public Boolean usePublicIps; - - /** - * A pipeline level default location for storing temporary files. Support Google Cloud Storage - * locations, e.g. gs://bucket/object - */ - @NotBlank public String tempLocation; - - /* The maximum number of workers to use for the workerpool. */ - public Integer maxNumWorkers; - - /* BigQuery table specification, e.g. PROJECT_ID:DATASET_ID.PROJECT_ID */ - public String deadLetterTableSpec; - - /* Disk size to use on each remote Compute Engine worker instance */ - public Integer diskSizeGb; - - public Map labels; - - /* If true job will be run on StreamingEngine instead of VMs */ - public Boolean enableStreamingEngine; - - /* Type of persistent disk to be used by workers */ - public String workerDiskType; - - /* Kafka Consumer Config Properties used in FeatureRow Consumer */ - public Map kafkaConsumerProperties; - - /** Validates Dataflow runner configuration options */ - public void validate() { - ValidatorFactory factory = Validation.buildDefaultValidatorFactory(); - Validator validator = factory.getValidator(); - - Set> dataflowRunnerConfigViolation = - validator.validate(this); - if (!dataflowRunnerConfigViolation.isEmpty()) { - throw new ConstraintViolationException(dataflowRunnerConfigViolation); - } - } -} diff --git a/job-controller/src/main/java/feast/jobcontroller/runner/direct/DirectJob.java b/job-controller/src/main/java/feast/jobcontroller/runner/direct/DirectJob.java deleted file mode 100644 index 50eafc3aee0..00000000000 --- a/job-controller/src/main/java/feast/jobcontroller/runner/direct/DirectJob.java +++ /dev/null @@ -1,40 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2019 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.jobcontroller.runner.direct; - -import java.io.IOException; -import lombok.AllArgsConstructor; -import lombok.Getter; -import org.apache.beam.sdk.PipelineResult; - -@Getter -@AllArgsConstructor -public class DirectJob { - - private String jobId; - private PipelineResult pipelineResult; - - /** - * Abort the job, if the state is not terminal. If the job has already concluded, this method will - * do nothing. - */ - public void abort() throws IOException { - if (!pipelineResult.getState().isTerminal()) { - pipelineResult.cancel(); - } - } -} diff --git a/job-controller/src/main/java/feast/jobcontroller/runner/direct/DirectJobRegistry.java b/job-controller/src/main/java/feast/jobcontroller/runner/direct/DirectJobRegistry.java deleted file mode 100644 index f3cd2a4485f..00000000000 --- a/job-controller/src/main/java/feast/jobcontroller/runner/direct/DirectJobRegistry.java +++ /dev/null @@ -1,79 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2019 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.jobcontroller.runner.direct; - -import com.google.common.base.Strings; -import java.io.IOException; -import java.util.HashMap; -import java.util.Map; -import javax.inject.Singleton; -import lombok.extern.slf4j.Slf4j; - -@Slf4j -@Singleton -public class DirectJobRegistry { - - private Map jobs; - - public DirectJobRegistry() { - this.jobs = new HashMap<>(); - } - - /** - * Add the given job to the registry. - * - * @param job containing the job id, - */ - public void add(DirectJob job) { - if (jobs.containsKey(job.getJobId())) { - throw new IllegalArgumentException( - Strings.lenientFormat("Job with id %s already exists and is running", job.getJobId())); - } - jobs.put(job.getJobId(), job); - } - - /** - * Get DirectJob corresponding to the given ID - * - * @param id of the job to retrieve - * @return DirectJob - */ - public DirectJob get(String id) { - return jobs.getOrDefault(id, null); - } - - /** - * Remove DirectJob corresponding to the given ID - * - * @param id of the job to remove - */ - public void remove(String id) { - jobs.remove(id); - } - - /** Kill all child jobs when the registry is garbage collected */ - @Override - public void finalize() { - for (DirectJob job : this.jobs.values()) { - try { - job.getPipelineResult().cancel(); - } catch (IOException e) { - log.error("Failed to stop job", e); - } - } - } -} diff --git a/job-controller/src/main/java/feast/jobcontroller/runner/direct/DirectJobStateMapper.java b/job-controller/src/main/java/feast/jobcontroller/runner/direct/DirectJobStateMapper.java deleted file mode 100644 index 016484ff667..00000000000 --- a/job-controller/src/main/java/feast/jobcontroller/runner/direct/DirectJobStateMapper.java +++ /dev/null @@ -1,48 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2019 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.jobcontroller.runner.direct; - -import feast.jobcontroller.model.JobStatus; -import java.util.HashMap; -import java.util.Map; -import org.apache.beam.sdk.PipelineResult.State; - -public class DirectJobStateMapper { - - private static final Map BEAM_TO_FEAT_JOB_STATUS; - - static { - BEAM_TO_FEAT_JOB_STATUS = new HashMap<>(); - BEAM_TO_FEAT_JOB_STATUS.put(State.FAILED, JobStatus.ERROR); - BEAM_TO_FEAT_JOB_STATUS.put(State.RUNNING, JobStatus.RUNNING); - BEAM_TO_FEAT_JOB_STATUS.put(State.UNKNOWN, JobStatus.UNKNOWN); - BEAM_TO_FEAT_JOB_STATUS.put(State.CANCELLED, JobStatus.ABORTED); - BEAM_TO_FEAT_JOB_STATUS.put(State.DONE, JobStatus.COMPLETED); - BEAM_TO_FEAT_JOB_STATUS.put(State.STOPPED, JobStatus.ABORTED); - BEAM_TO_FEAT_JOB_STATUS.put(State.UPDATED, JobStatus.RUNNING); - } - - /** - * Map a dataflow job state to Feast's JobStatus - * - * @param jobState beam PipelineResult State - * @return JobStatus - */ - public static JobStatus map(State jobState) { - return BEAM_TO_FEAT_JOB_STATUS.get(jobState); - } -} diff --git a/job-controller/src/main/java/feast/jobcontroller/runner/direct/DirectRunnerConfig.java b/job-controller/src/main/java/feast/jobcontroller/runner/direct/DirectRunnerConfig.java deleted file mode 100644 index d4b518d26e6..00000000000 --- a/job-controller/src/main/java/feast/jobcontroller/runner/direct/DirectRunnerConfig.java +++ /dev/null @@ -1,39 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.jobcontroller.runner.direct; - -import feast.jobcontroller.runner.option.RunnerConfig; -import feast.proto.core.RunnerProto.DirectRunnerConfigOptions; - -public class DirectRunnerConfig extends RunnerConfig { - /** - * Controls the amount of target parallelism the DirectRunner will use. Defaults to the greater of - * the number of available processors and 3. Must be a value greater than zero. - */ - public Integer targetParallelism; - - /* BigQuery table specification, e.g. PROJECT_ID:DATASET_ID.PROJECT_ID */ - public String deadletterTableSpec; - - public String tempLocation; - - public DirectRunnerConfig(DirectRunnerConfigOptions runnerConfigOptions) { - this.deadletterTableSpec = runnerConfigOptions.getDeadLetterTableSpec(); - this.targetParallelism = runnerConfigOptions.getTargetParallelism(); - this.tempLocation = runnerConfigOptions.getTempLocation(); - } -} diff --git a/job-controller/src/main/java/feast/jobcontroller/runner/direct/DirectRunnerJobManager.java b/job-controller/src/main/java/feast/jobcontroller/runner/direct/DirectRunnerJobManager.java deleted file mode 100644 index 358d2c29e54..00000000000 --- a/job-controller/src/main/java/feast/jobcontroller/runner/direct/DirectRunnerJobManager.java +++ /dev/null @@ -1,203 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2019 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.jobcontroller.runner.direct; - -import com.google.common.base.Strings; -import com.google.protobuf.util.JsonFormat; -import feast.common.models.FeatureSetReference; -import feast.ingestion.ImportJob; -import feast.ingestion.options.ImportOptions; -import feast.jobcontroller.config.FeastProperties.MetricsProperties; -import feast.jobcontroller.exception.JobExecutionException; -import feast.jobcontroller.model.Job; -import feast.jobcontroller.model.JobStatus; -import feast.jobcontroller.runner.JobManager; -import feast.jobcontroller.runner.Runner; -import feast.proto.core.IngestionJobProto; -import feast.proto.core.RunnerProto.DirectRunnerConfigOptions; -import feast.proto.core.SourceProto; -import feast.proto.core.StoreProto; -import java.io.IOException; -import java.util.*; -import lombok.extern.slf4j.Slf4j; -import org.apache.beam.runners.direct.DirectRunner; -import org.apache.beam.sdk.PipelineResult; -import org.apache.beam.sdk.options.PipelineOptionsFactory; - -@Slf4j -public class DirectRunnerJobManager implements JobManager { - - private final Runner RUNNER_TYPE = Runner.DIRECT; - - private DirectRunnerConfig defaultOptions; - private final DirectJobRegistry jobs; - private MetricsProperties metrics; - private final IngestionJobProto.SpecsStreamingUpdateConfig specsStreamingUpdateConfig; - - public DirectRunnerJobManager( - DirectRunnerConfigOptions directRunnerConfigOptions, - DirectJobRegistry jobs, - MetricsProperties metricsProperties, - IngestionJobProto.SpecsStreamingUpdateConfig specsStreamingUpdateConfig) { - this.defaultOptions = new DirectRunnerConfig(directRunnerConfigOptions); - this.jobs = jobs; - this.metrics = metricsProperties; - this.specsStreamingUpdateConfig = specsStreamingUpdateConfig; - } - - @Override - public Runner getRunnerType() { - return RUNNER_TYPE; - } - - /** - * Start a direct runner job. - * - * @param job Job to start - */ - @Override - public Job startJob(Job job) { - try { - ImportOptions pipelineOptions = - getPipelineOptions(job.getId(), job.getSource(), new HashSet<>(job.getStores().values())); - PipelineResult pipelineResult = runPipeline(pipelineOptions); - DirectJob directJob = new DirectJob(job.getId(), pipelineResult); - jobs.add(directJob); - job.setExtId(job.getId()); - return job; - } catch (Exception e) { - log.error("Error submitting job", e); - throw new JobExecutionException(String.format("Error running ingestion job: %s", e), e); - } - } - - private ImportOptions getPipelineOptions( - String jobName, SourceProto.Source source, Set sinks) - throws IOException, IllegalAccessException { - ImportOptions pipelineOptions = - PipelineOptionsFactory.fromArgs(defaultOptions.toArgs()).as(ImportOptions.class); - - JsonFormat.Printer printer = JsonFormat.printer(); - List storesJson = new ArrayList<>(); - for (StoreProto.Store sink : sinks) { - String print = printer.print(sink); - storesJson.add(print); - } - - pipelineOptions.setSpecsStreamingUpdateConfigJson(printer.print(specsStreamingUpdateConfig)); - pipelineOptions.setSourceJson(printer.print(source)); - pipelineOptions.setJobName(jobName); - pipelineOptions.setStoresJson(storesJson); - pipelineOptions.setRunner(DirectRunner.class); - pipelineOptions.setDefaultFeastProject(FeatureSetReference.PROJECT_DEFAULT_NAME); - pipelineOptions.setProject(""); // set to default value to satisfy validation - if (metrics.isEnabled()) { - pipelineOptions.setMetricsExporterType(metrics.getType()); - if (metrics.getType().equals("statsd")) { - pipelineOptions.setStatsdHost(metrics.getHost()); - pipelineOptions.setStatsdPort(metrics.getPort()); - } - } - pipelineOptions.setBlockOnRun(false); - return pipelineOptions; - } - - /** - * Stops an existing job and restarts a new job in its place as a proxy for job updates. Note that - * since we do not maintain a consumer group across the two jobs and the old job is not drained, - * some data may be lost. - * - *

As a rule of thumb, direct jobs in feast should only be used for testing. - * - * @param job job of target job to change - * @return jobId of the job - */ - @Override - public Job updateJob(Job job) { - try { - return startJob(abortJob(job)); - } catch (JobExecutionException e) { - throw new JobExecutionException(String.format("Error running ingestion job: %s", e), e); - } - } - - /** - * Abort the direct runner job,removing it from the direct jobs registry. - * - * @param job to abort. - * @return The aborted Job - */ - @Override - public Job abortJob(Job job) { - DirectJob directJob = jobs.get(job.getExtId()); - if (directJob != null) { - try { - directJob.abort(); - } catch (IOException e) { - throw new RuntimeException( - Strings.lenientFormat("Unable to abort DirectRunner job %s", job.getExtId(), e)); - } - jobs.remove(job.getExtId()); - } - - return job; - } - - public PipelineResult runPipeline(ImportOptions pipelineOptions) throws IOException { - return ImportJob.runPipeline(pipelineOptions); - } - - /** - * Restart a direct runner job. Note that some data will be temporarily lost during when - * restarting running direct runner jobs. See {#link {@link #updateJob(Job)} for more info. - * - * @param job job to restart - * @return the restarted job - */ - @Override - public Job restartJob(Job job) { - if (job.getStatus().isTerminal()) { - // job yet not running: just start job - return this.startJob(job); - } else { - // job is running - updating the job without changing the job has - // the effect of restarting the job. - return this.updateJob(job); - } - } - - /** - * Gets the state of the direct runner job. Direct runner jobs only have 2 states: RUNNING and - * ABORTED. - * - * @param job Job of the desired job. - * @return JobStatus of the job. - */ - @Override - public JobStatus getJobStatus(Job job) { - DirectJob directJob = jobs.get(job.getId()); - if (directJob == null) { - return JobStatus.ABORTED; - } - return DirectJobStateMapper.map(directJob.getPipelineResult().getState()); - } - - @Override - public List listRunningJobs() { - return Collections.emptyList(); - } -} diff --git a/job-controller/src/main/java/feast/jobcontroller/runner/option/FeatureSetJsonByteConverter.java b/job-controller/src/main/java/feast/jobcontroller/runner/option/FeatureSetJsonByteConverter.java deleted file mode 100644 index 426bd372124..00000000000 --- a/job-controller/src/main/java/feast/jobcontroller/runner/option/FeatureSetJsonByteConverter.java +++ /dev/null @@ -1,47 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.jobcontroller.runner.option; - -import com.google.protobuf.InvalidProtocolBufferException; -import com.google.protobuf.util.JsonFormat; -import feast.ingestion.options.OptionByteConverter; -import feast.proto.core.FeatureSetProto; -import java.util.ArrayList; -import java.util.List; - -public class FeatureSetJsonByteConverter - implements OptionByteConverter> { - - /** - * Convert list of feature sets to json strings joined by new line, represented as byte arrays - * - * @param featureSets List of feature set protobufs - * @return Byte array representation of the json strings - * @throws InvalidProtocolBufferException - */ - @Override - public byte[] toByte(List featureSets) - throws InvalidProtocolBufferException { - JsonFormat.Printer printer = - JsonFormat.printer().omittingInsignificantWhitespace().printingEnumsAsInts(); - List featureSetsJson = new ArrayList<>(); - for (FeatureSetProto.FeatureSet featureSet : featureSets) { - featureSetsJson.add(printer.print(featureSet.getSpec())); - } - return String.join("\n", featureSetsJson).getBytes(); - } -} diff --git a/job-controller/src/main/java/feast/jobcontroller/runner/option/RunnerConfig.java b/job-controller/src/main/java/feast/jobcontroller/runner/option/RunnerConfig.java deleted file mode 100644 index 336f133efce..00000000000 --- a/job-controller/src/main/java/feast/jobcontroller/runner/option/RunnerConfig.java +++ /dev/null @@ -1,75 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.jobcontroller.runner.option; - -import feast.jobcontroller.util.TypeConversion; -import java.lang.reflect.Field; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; - -/** - * Value class containing the application-default configuration for a runner. When a job is started - * by jobcontroller, all fields in the object will be converted into --key=value args to seed the - * beam pipeline options. - */ -public abstract class RunnerConfig { - - /** - * Converts the public-access fields in this class to a list of --key=value args to be passed to a - * {@link org.apache.beam.sdk.options.PipelineOptionsFactory}. - * - *

Ignores values that are proto-default (e.g. empty string, 0). - * - * @return Array of string args in the format --key=value. - * @throws IllegalAccessException - */ - public String[] toArgs() throws IllegalAccessException { - List args = new ArrayList<>(); - for (Field field : this.getClass().getFields()) { - if (field.get(this) == null) { - continue; - } - Class type = field.getType(); - if (Map.class.equals(type)) { - String jsonString = - TypeConversion.convertMapToJsonString((Map) field.get(this)); - args.add(String.format("--%s=%s", field.getName(), jsonString)); - continue; - } - - if (String.class.equals(type)) { - String val = (String) field.get(this); - if (!val.equals("")) { - args.add(String.format("--%s=%s", field.getName(), val)); - } - continue; - } - - if (Integer.class.equals(type)) { - Integer val = (Integer) field.get(this); - if (val != 0) { - args.add(String.format("--%s=%d", field.getName(), val)); - } - continue; - } - - args.add(String.format("--%s=%s", field.getName(), field.get(this))); - } - return args.toArray(String[]::new); - } -} diff --git a/job-controller/src/main/java/feast/jobcontroller/runner/task/CreateJobTask.java b/job-controller/src/main/java/feast/jobcontroller/runner/task/CreateJobTask.java deleted file mode 100644 index cadadc0de8c..00000000000 --- a/job-controller/src/main/java/feast/jobcontroller/runner/task/CreateJobTask.java +++ /dev/null @@ -1,63 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.jobcontroller.runner.task; - -import feast.common.logging.AuditLogger; -import feast.common.logging.entry.LogResource.ResourceType; -import feast.jobcontroller.model.Job; -import feast.jobcontroller.model.JobStatus; -import feast.jobcontroller.runner.JobManager; -import lombok.extern.slf4j.Slf4j; -import org.slf4j.event.Level; - -/** Task that starts recently created {@link Job} by using {@link JobManager}. */ -@Slf4j -public class CreateJobTask extends JobTask { - - public CreateJobTask(Job job, JobManager jobManager) { - super(job, jobManager); - } - - @Override - public Job call() { - try { - String runnerName = jobManager.getRunnerType().toString(); - changeJobStatus(JobStatus.PENDING); - - // Start job with jobManager. - job = jobManager.startJob(job); - - log.info(String.format("Build graph and submitting to %s", runnerName)); - AuditLogger.logAction(Level.INFO, JobTasks.CREATE.name(), ResourceType.JOB, job.getId()); - - // Check for expected external job id - if (job.getExtId().isEmpty()) { - throw new RuntimeException( - String.format( - "Could not submit job %s: unable to retrieve job external id", job.getId())); - } - - log.info( - String.format("Job submitted to runner %s with ext id %s.", runnerName, job.getExtId())); - changeJobStatus(JobStatus.RUNNING); - return job; - } catch (Exception e) { - handleException(e); - return job; - } - } -} diff --git a/job-controller/src/main/java/feast/jobcontroller/runner/task/JobTask.java b/job-controller/src/main/java/feast/jobcontroller/runner/task/JobTask.java deleted file mode 100644 index 294e078e58f..00000000000 --- a/job-controller/src/main/java/feast/jobcontroller/runner/task/JobTask.java +++ /dev/null @@ -1,68 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.jobcontroller.runner.task; - -import feast.common.logging.AuditLogger; -import feast.common.logging.entry.LogResource.ResourceType; -import feast.jobcontroller.model.Job; -import feast.jobcontroller.model.JobStatus; -import feast.jobcontroller.runner.JobManager; -import java.util.concurrent.Callable; -import lombok.Getter; -import lombok.Setter; -import lombok.extern.slf4j.Slf4j; -import org.slf4j.event.Level; - -@Getter -@Setter -@Slf4j -public abstract class JobTask implements Callable { - protected Job job; - protected JobManager jobManager; - - public JobTask(Job job, JobManager jobManager) { - this.job = job; - this.jobManager = jobManager; - } - - @Override - public abstract Job call() throws RuntimeException; - - /** - * Change Job Status to the given status and logs changes in Job Status to audit and normal log. - */ - protected void changeJobStatus(JobStatus newStatus) { - JobStatus currentStatus = job.getStatus(); - if (currentStatus != newStatus) { - job.setStatus(newStatus); - log.info( - String.format("Job status updated: changed from %s to %s", currentStatus, newStatus)); - - AuditLogger.logTransition(Level.INFO, newStatus.name(), ResourceType.JOB, job.getId()); - log.info("test"); - } - } - - /** - * Handle Exception when executing JobTask by transition Job to ERROR status and logging exception - */ - protected void handleException(Exception e) { - log.error("Unexpected exception performing JobTask: %s", e.getMessage()); - e.printStackTrace(); - changeJobStatus(JobStatus.ERROR); - } -} diff --git a/job-controller/src/main/java/feast/jobcontroller/runner/task/JobTasks.java b/job-controller/src/main/java/feast/jobcontroller/runner/task/JobTasks.java deleted file mode 100644 index f913b9b56c8..00000000000 --- a/job-controller/src/main/java/feast/jobcontroller/runner/task/JobTasks.java +++ /dev/null @@ -1,25 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.jobcontroller.runner.task; - -/** Enum listing of the available Job Tasks to perform on Jobs */ -public enum JobTasks { - CREATE, - UPDATE_STATUS, - RESTART, - ABORT, -} diff --git a/job-controller/src/main/java/feast/jobcontroller/runner/task/RestartJobTask.java b/job-controller/src/main/java/feast/jobcontroller/runner/task/RestartJobTask.java deleted file mode 100644 index 39183d24aa6..00000000000 --- a/job-controller/src/main/java/feast/jobcontroller/runner/task/RestartJobTask.java +++ /dev/null @@ -1,49 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.jobcontroller.runner.task; - -import feast.common.logging.AuditLogger; -import feast.common.logging.entry.LogResource.ResourceType; -import feast.jobcontroller.model.Job; -import feast.jobcontroller.model.JobStatus; -import feast.jobcontroller.runner.JobManager; -import lombok.extern.slf4j.Slf4j; -import org.slf4j.event.Level; - -/** Task that restarts given {@link Job} by restarting it in {@link JobManager} */ -@Slf4j -public class RestartJobTask extends JobTask { - public RestartJobTask(Job job, JobManager jobManager) { - super(job, jobManager); - } - - @Override - public Job call() { - try { - // abort job and expect replacement will be spawned - job = jobManager.abortJob(job); - log.info("Restart job {} for runner {}", job.getId(), jobManager.getRunnerType().toString()); - AuditLogger.logAction(Level.INFO, JobTasks.ABORT.name(), ResourceType.JOB, job.getId()); - - changeJobStatus(JobStatus.ABORTING); - return job; - } catch (Exception e) { - handleException(e); - return job; - } - } -} diff --git a/job-controller/src/main/java/feast/jobcontroller/runner/task/TerminateJobTask.java b/job-controller/src/main/java/feast/jobcontroller/runner/task/TerminateJobTask.java deleted file mode 100644 index ade6ebc9599..00000000000 --- a/job-controller/src/main/java/feast/jobcontroller/runner/task/TerminateJobTask.java +++ /dev/null @@ -1,50 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.jobcontroller.runner.task; - -import feast.common.logging.AuditLogger; -import feast.common.logging.entry.LogResource.ResourceType; -import feast.jobcontroller.model.Job; -import feast.jobcontroller.model.JobStatus; -import feast.jobcontroller.runner.JobManager; -import lombok.extern.slf4j.Slf4j; -import org.slf4j.event.Level; - -/** Task to terminate given {@link Job} by using {@link JobManager} */ -@Slf4j -public class TerminateJobTask extends JobTask { - public TerminateJobTask(Job job, JobManager jobManager) { - super(job, jobManager); - } - - @Override - public Job call() { - try { - job = jobManager.abortJob(job); - log.info( - String.format( - "Aborted job %s for runner %s", job.getId(), jobManager.getRunnerType().toString())); - AuditLogger.logAction(Level.INFO, JobTasks.ABORT.name(), ResourceType.JOB, job.getId()); - - changeJobStatus(JobStatus.ABORTING); - return job; - } catch (Exception e) { - handleException(e); - return job; - } - } -} diff --git a/job-controller/src/main/java/feast/jobcontroller/runner/task/UpdateJobStatusTask.java b/job-controller/src/main/java/feast/jobcontroller/runner/task/UpdateJobStatusTask.java deleted file mode 100644 index 9a5f4ade8b1..00000000000 --- a/job-controller/src/main/java/feast/jobcontroller/runner/task/UpdateJobStatusTask.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.jobcontroller.runner.task; - -import feast.jobcontroller.model.Job; -import feast.jobcontroller.model.JobStatus; -import feast.jobcontroller.runner.JobManager; - -/** - * Task that retrieves status from {@link JobManager} on given {@link Job} and update the job - * accordingly in-place - */ -public class UpdateJobStatusTask extends JobTask { - public UpdateJobStatusTask(Job job, JobManager jobManager) { - super(job, jobManager); - } - - @Override - public Job call() { - try { - JobStatus newStatus = jobManager.getJobStatus(job); - changeJobStatus(newStatus); - - return job; - } catch (Exception e) { - handleException(e); - return job; - } - } -} diff --git a/job-controller/src/main/java/feast/jobcontroller/service/JobControllerService.java b/job-controller/src/main/java/feast/jobcontroller/service/JobControllerService.java deleted file mode 100644 index d2e7dbd0394..00000000000 --- a/job-controller/src/main/java/feast/jobcontroller/service/JobControllerService.java +++ /dev/null @@ -1,561 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2019 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.jobcontroller.service; - -import static feast.common.models.Store.isSubscribedToFeatureSet; - -import com.google.common.collect.Sets; -import feast.common.models.FeatureSetReference; -import feast.jobcontroller.config.FeastProperties; -import feast.jobcontroller.config.FeastProperties.JobProperties; -import feast.jobcontroller.dao.JobRepository; -import feast.jobcontroller.model.FeatureSetDeliveryStatus; -import feast.jobcontroller.model.Job; -import feast.jobcontroller.model.JobStatus; -import feast.jobcontroller.runner.JobGroupingStrategy; -import feast.jobcontroller.runner.JobManager; -import feast.jobcontroller.runner.task.CreateJobTask; -import feast.jobcontroller.runner.task.JobTask; -import feast.jobcontroller.runner.task.TerminateJobTask; -import feast.jobcontroller.runner.task.UpdateJobStatusTask; -import feast.proto.core.*; -import feast.proto.core.CoreServiceProto.ListStoresRequest.Filter; -import feast.proto.core.CoreServiceProto.ListStoresResponse; -import feast.proto.core.FeatureSetProto.FeatureSet; -import feast.proto.core.FeatureSetProto.FeatureSetSpec; -import feast.proto.core.SourceProto.Source; -import feast.proto.core.StoreProto.Store; -import io.grpc.StatusRuntimeException; -import java.util.*; -import java.util.concurrent.*; -import java.util.stream.Collectors; -import java.util.stream.Stream; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.tuple.Pair; -import org.apache.kafka.clients.consumer.ConsumerRecord; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; -import org.springframework.kafka.annotation.KafkaListener; -import org.springframework.kafka.core.KafkaTemplate; -import org.springframework.scheduling.annotation.Scheduled; -import org.springframework.stereotype.Service; - -@Slf4j -@Service -@ConditionalOnProperty("feast.jobs.enabled") -public class JobControllerService { - - private final int SPEC_PUBLISHING_TIMEOUT_SECONDS = 5; - public static final String VERSION_LABEL = "feast_version"; - - private final JobRepository jobRepository; - private final CoreServiceGrpc.CoreServiceBlockingStub specService; - private final JobManager jobManager; - private final JobProperties jobProperties; - private final JobGroupingStrategy groupingStrategy; - private final KafkaTemplate specPublisher; - private final List featureSetSubscriptions; - private final List whitelistedStores; - private final Map jobLabels; - private final String currentVersion; - - @Autowired - public JobControllerService( - JobRepository jobRepository, - CoreServiceGrpc.CoreServiceBlockingStub specService, - JobManager jobManager, - FeastProperties feastProperties, - JobGroupingStrategy groupingStrategy, - KafkaTemplate specPublisher) { - this.jobRepository = jobRepository; - this.specService = specService; - this.jobManager = jobManager; - this.jobProperties = feastProperties.getJobs(); - this.specPublisher = specPublisher; - this.groupingStrategy = groupingStrategy; - this.featureSetSubscriptions = - feastProperties.getJobs().getController().getFeatureSetSelector().stream() - .map(JobProperties.ControllerProperties.FeatureSetSelector::toSubscription) - .collect(Collectors.toList()); - this.whitelistedStores = feastProperties.getJobs().getController().getWhitelistedStores(); - this.currentVersion = feastProperties.getVersion(); - this.jobLabels = new HashMap<>(feastProperties.getJobs().getController().getJobSelector()); - this.jobLabels.put(VERSION_LABEL, getVersionLabel()); - } - - /** - * Generate version label value that must conform to regular expression - * [\\p{Ll}\\p{Lo}\\p{N}_-]{0,63} from current version of Feast. - * - * @return lower-cased version with replaced dots - */ - private String getVersionLabel() { - return this.currentVersion - .replace(".", "-") - .toLowerCase() - .substring(0, Math.min(this.currentVersion.length(), 63)); - } - - /** - * Poll does the following: - * - *

1) Checks DB and extracts jobs that have to run based on the specs available - * - *

2) Does a diff with the current set of jobs, starts/updates/stops job(s) if necessary - * - *

3) Updates job object in DB with status, feature sets - * - *

4) Updates Feature set statuses - */ - @Scheduled(fixedDelayString = "${feast.jobs.polling_interval_milliseconds}") - public void Poll() { - log.info("Polling for new jobs..."); - Iterable>> sourceStoreMappings = getSourceToStoreMappings(); - List jobUpdateTasks = makeJobUpdateTasks(sourceStoreMappings); - - if (jobUpdateTasks.isEmpty()) { - log.info("No jobs found."); - return; - } - - log.info("Creating/Updating {} jobs...", jobUpdateTasks.size()); - startOrUpdateJobs(jobUpdateTasks); - } - - void startOrUpdateJobs(List tasks) { - ExecutorService executorService = Executors.newFixedThreadPool(tasks.size()); - ExecutorCompletionService ecs = new ExecutorCompletionService<>(executorService); - tasks.forEach(ecs::submit); - - int completedTasks = 0; - List processedJobs = new ArrayList<>(); - while (completedTasks < tasks.size()) { - try { - Job job = ecs.take().get(jobProperties.getJobUpdateTimeoutSeconds(), TimeUnit.SECONDS); - if (job != null) { - processedJobs.add(job); - } - } catch (ExecutionException | InterruptedException | TimeoutException e) { - log.warn("Unable to start or update job: {}", e.getMessage()); - e.printStackTrace(); - } - completedTasks++; - } - processedJobs.forEach(jobRepository::add); - executorService.shutdown(); - } - - /** - * Makes Job Update Tasks required to reconcile the current ingestion jobs with the given source - * to store map. Compares the current ingestion jobs and source to store mapping to determine - * which whether jobs have started/stopped/updated in terms of Job Update tasks. Only tries to - * stop ingestion jobs its the required ingestions to maintained ingestion jobs are already - * RUNNING. - * - * @param sourceToStores a iterable of source to stores pairs where ingestion jobs would have to - * be maintained for ingestion to work correctly. - * @return list of job update tasks required to reconcile the current ingestion jobs to the state - * that is defined by sourceStoreMap. - */ - List makeJobUpdateTasks(Iterable>> sourceToStores) { - List jobTasks = new LinkedList<>(); - // Ensure a running job for each source to store mapping - List activeJobs = new LinkedList<>(); - boolean isSafeToStopJobs = true; - - for (Pair> mapping : sourceToStores) { - Source source = mapping.getKey(); - Set stores = mapping.getValue(); - - Job job = groupingStrategy.getOrCreateJob(source, stores, this.jobLabels); - - if (job.isDeployed()) { - if (!job.isRunning()) { - jobTasks.add(new UpdateJobStatusTask(job, jobManager)); - - // Mark that it is not safe to stop jobs without disrupting ingestion - isSafeToStopJobs = false; - continue; - } - - if (jobRequiresUpgrade(job, stores) && job.isRunning()) { - // Since we want to upgrade job without downtime - // it would make sense to spawn clone of current job - // and terminate old version on the next Poll. - // Both jobs should be in the same consumer group and not conflict with each other - job = job.cloneWithIdAndLabels(groupingStrategy.createJobId(job), this.jobLabels); - job.addAllStores(stores); - - isSafeToStopJobs = false; - - jobTasks.add(new CreateJobTask(job, jobManager)); - } else { - jobTasks.add(new UpdateJobStatusTask(job, jobManager)); - } - } else { - job.addAllFeatureSets( - stores.stream() - .flatMap(s -> getFeatureSetsForStore(s).stream()) - .filter(fs -> fs.getSpec().getSource().equals(source)) - .collect(Collectors.toSet())); - - jobTasks.add(new CreateJobTask(job, jobManager)); - } - - // Record the job as required to safeguard it from getting stopped - activeJobs.add(job); - } - // Stop extra jobs that are not required to maintain ingestion when safe - if (isSafeToStopJobs) { - getExtraJobs(activeJobs) - .forEach( - extraJob -> { - jobTasks.add(new TerminateJobTask(extraJob, jobManager)); - }); - } - - return jobTasks; - } - - /** - * Decides whether we need to upgrade (restart) given job. Since we send updated FeatureSets to - * IngestionJob via Kafka, and there's only one source per job (if it change - new job would be - * created) main trigger that can cause upgrade here are stores: new stores can be added, or - * existing stores will change subscriptions. Another trigger is release of new version: current - * version is being compared with job's version stored in labels. - * - * @param job {@link Job} to check - * @param stores Set of {@link Store} new version of stores (vs current version job.getStores()) - * @return boolean - need to upgrade - */ - private boolean jobRequiresUpgrade(Job job, Set stores) { - // if store subscriptions have changed - if (!Sets.newHashSet(stores).equals(Sets.newHashSet(job.getStores().values()))) { - return true; - } - - if (!getVersionLabel().equals(job.getLabels().get(VERSION_LABEL))) { - return true; - } - - return false; - } - - /** - * Connects given {@link FeatureSet} with Jobs by creating {@link FeatureSetDeliveryStatus}. This - * connection represents responsibility of the job to handle allocated FeatureSet. We use this - * connection {@link FeatureSetDeliveryStatus} to monitor Ingestion of specific FeatureSet and - * Specs delivery status. - * - *

Only after this connection is created FeatureSetSpec could be sent to IngestionJob. - * - * @param featureSet featureSet {@link FeatureSet} to find jobs and allocate - */ - FeatureSet allocateFeatureSetToJobs(FeatureSet featureSet) { - FeatureSetReference ref = - FeatureSetReference.of(featureSet.getSpec().getProject(), featureSet.getSpec().getName()); - Set confirmedJobIds = new HashSet<>(); - - Stream> jobArgsStream = - getAllStores().stream() - .filter( - s -> - isSubscribedToFeatureSet( - s.getSubscriptionsList(), - featureSet.getSpec().getProject(), - featureSet.getSpec().getName())) - .map(s -> Pair.of(featureSet.getSpec().getSource(), s)); - - // Add featureSet to allocated job if not allocated before - for (Pair> jobArgs : groupingStrategy.collectSingleJobInput(jobArgsStream)) { - Job job = - groupingStrategy.getOrCreateJob( - jobArgs.getLeft(), jobArgs.getRight(), Collections.emptyMap()); - if (!job.isRunning()) { - continue; - } - - FeatureSetDeliveryStatus status = new FeatureSetDeliveryStatus(ref); - status.setDeliveryStatus(FeatureSetProto.FeatureSetJobDeliveryStatus.STATUS_IN_PROGRESS); - status.setDeliveredVersion(0); - - Map deliveryStatuses = - job.getFeatureSetDeliveryStatuses(); - - if (!deliveryStatuses.containsKey(ref)) { - deliveryStatuses.put(status.getFeatureSetReference(), status); - } - - confirmedJobIds.add(job.getId()); - } - - // remove from other jobs that was not confirmed - for (Job job : jobRepository.findByFeatureSetReference(ref)) { - if (!confirmedJobIds.contains(job.getId())) { - job.getFeatureSetDeliveryStatuses().remove(ref); - } - } - return featureSet; - } - - /** Get running extra ingestion jobs that have ids not in keepJobs */ - private Collection getExtraJobs(List keepJobs) { - List runningJobs = jobRepository.findByStatus(JobStatus.RUNNING); - Map extraJobMap = - runningJobs.stream().collect(Collectors.toMap(job -> job.getId(), job -> job)); - keepJobs.forEach(job -> extraJobMap.remove(job.getId())); - return extraJobMap.values(); - } - - private List getAllStores() { - ListStoresResponse listStoresResponse; - try { - listStoresResponse = - specService.listStores( - CoreServiceProto.ListStoresRequest.newBuilder() - .setFilter(Filter.newBuilder().build()) - .build()); - } catch (StatusRuntimeException e) { - log.error("Core Service is unavailable. Reason: {}", e.getMessage()); - return Collections.emptyList(); - } - - return listStoresResponse.getStoreList().stream() - .filter(s -> this.whitelistedStores.contains(s.getName())) - .collect(Collectors.toList()); - } - - /** - * Generate a source to stores mapping. The resulting iterable yields pairs of Source and - * Set-of-stores to create one ingestion job per each pair. - * - * @return a Map from source to stores. - */ - Iterable>> getSourceToStoreMappings() { - // build mapping from source to store. - // compile a set of sources via subscribed FeatureSets of stores. - Stream> distinctPairs = - getAllStores().stream() - .flatMap( - store -> - getFeatureSetsForStore(store).stream() - .map(f -> f.getSpec().getSource()) - .map(source -> Pair.of(source, store))) - .distinct(); - return groupingStrategy.collectSingleJobInput(distinctPairs); - } - - /** - * Get the FeatureSets that the given store subscribes to. - * - * @param store to get subscribed FeatureSets for - * @return list of FeatureSets that the store subscribes to. - */ - List getFeatureSetsForStore(Store store) { - return store.getSubscriptionsList().stream() - .flatMap( - subscription -> { - return specService - .listFeatureSets( - CoreServiceProto.ListFeatureSetsRequest.newBuilder() - .setFilter( - CoreServiceProto.ListFeatureSetsRequest.Filter.newBuilder() - .setProject(subscription.getProject()) - .setFeatureSetName(subscription.getName()) - .build()) - .build()) - .getFeatureSetsList().stream() - .filter( - f -> - this.featureSetSubscriptions.isEmpty() - || isSubscribedToFeatureSet( - this.featureSetSubscriptions, - f.getSpec().getProject(), - f.getSpec().getName())); - }) - .distinct() - .collect(Collectors.toList()); - } - - @Scheduled(fixedDelayString = "${feast.stream.specsOptions.notifyIntervalMilliseconds}") - public void notifyJobsWhenFeatureSetUpdated() { - List pendingFeatureSets; - try { - pendingFeatureSets = - specService - .listFeatureSets( - CoreServiceProto.ListFeatureSetsRequest.newBuilder() - .setFilter( - CoreServiceProto.ListFeatureSetsRequest.Filter.newBuilder() - .setProject("*") - .setFeatureSetName("*") - .setStatus(FeatureSetProto.FeatureSetStatus.STATUS_PENDING) - .build()) - .build()) - .getFeatureSetsList(); - } catch (StatusRuntimeException e) { - log.error("Core Service is unavailable. Reason: {}", e.getMessage()); - return; - } - - pendingFeatureSets.stream() - .map(this::allocateFeatureSetToJobs) - .map( - fs -> { - FeatureSetReference ref = - FeatureSetReference.of(fs.getSpec().getProject(), fs.getSpec().getName()); - List deliveryStatuses = - jobRepository.findByFeatureSetReference(ref).stream() - .filter(Job::isRunning) - .flatMap(job -> job.getFeatureSetDeliveryStatuses().values().stream()) - .filter(jobStatus -> jobStatus.getFeatureSetReference().equals(ref)) - .collect(Collectors.toList()); - - return Pair.of(fs, deliveryStatuses); - }) - .filter( - pair -> - pair.getRight().size() > 0 - && pair.getRight().stream() - .anyMatch( - jobStatus -> - jobStatus.getDeliveredVersion() - < pair.getLeft().getSpec().getVersion())) - .forEach( - pair -> { - FeatureSet fs = pair.getLeft(); - List deliveryStatuses = pair.getRight(); - - FeatureSetReference ref = - FeatureSetReference.of(fs.getSpec().getProject(), fs.getSpec().getName()); - - log.info("Sending new FeatureSet {} to Ingestion", ref); - - // Sending latest version of FeatureSet to all currently running IngestionJobs - // (there's one topic for all sets). - // All related jobs would apply new FeatureSet on the fly. - // In case kafka doesn't respond within SPEC_PUBLISHING_TIMEOUT_SECONDS we will try - // again later. - try { - specPublisher - .sendDefault(ref.getReference(), fs.getSpec()) - .get(SPEC_PUBLISHING_TIMEOUT_SECONDS, TimeUnit.SECONDS); - } catch (Exception e) { - log.error( - "Error occurred while sending FeatureSetSpec to kafka. Cause {}." - + " Will retry later", - e.getMessage()); - return; - } - - // Updating delivery status for related jobs (that are currently using this - // FeatureSet). - // We now set status to IN_PROGRESS, so listenAckFromJobs would be able to - // monitor delivery progress for each new version. - deliveryStatuses.forEach( - jobStatus -> { - jobStatus.setDeliveryStatus( - FeatureSetProto.FeatureSetJobDeliveryStatus.STATUS_IN_PROGRESS); - jobStatus.setDeliveredVersion(fs.getSpec().getVersion()); - }); - }); - } - - /** - * Listener for ACK messages coming from IngestionJob when FeatureSetSpec is installed (in - * pipeline). - * - *

Updates FeatureSetJobStatus for respected FeatureSet (selected by reference) and Job (select - * by Id). - * - *

When all related (running) to FeatureSet jobs are updated - FeatureSet receives READY status - * - * @param record ConsumerRecord with key: FeatureSet reference and value: Ack message - */ - @KafkaListener( - topics = {"${feast.stream.specsOptions.specsAckTopic}"}, - containerFactory = "kafkaAckListenerContainerFactory") - public void listenAckFromJobs( - ConsumerRecord record) { - String setReference = record.key(); - FeatureSetReference ref = FeatureSetReference.parse(setReference); - FeatureSet featureSet; - try { - featureSet = - specService - .getFeatureSet( - CoreServiceProto.GetFeatureSetRequest.newBuilder() - .setProject(ref.getProjectName()) - .setName(ref.getFeatureSetName()) - .build()) - .getFeatureSet(); - } catch (StatusRuntimeException e) { - log.error("Core Service is unavailable. Reason: {}", e.getMessage()); - return; - } - - if (featureSet == null) { - log.warn( - String.format("ACKListener received message for unknown FeatureSet %s", setReference)); - return; - } - - int ackVersion = record.value().getFeatureSetVersion(); - - if (featureSet.getSpec().getVersion() != ackVersion) { - log.warn( - String.format( - "ACKListener received outdated ack for %s. Current %d, Received %d", - setReference, featureSet.getSpec().getVersion(), ackVersion)); - return; - } - - log.info("Updating featureSet {} delivery statuses.", ref); - - jobRepository - .findById(record.value().getJobName()) - .map(j -> j.getFeatureSetDeliveryStatuses().get(ref)) - .filter(deliveryStatus -> deliveryStatus.getDeliveredVersion() == ackVersion) - .ifPresent( - deliveryStatus -> - deliveryStatus.setDeliveryStatus( - FeatureSetProto.FeatureSetJobDeliveryStatus.STATUS_DELIVERED)); - - boolean allDelivered = - jobRepository.findByFeatureSetReference(ref).stream() - .filter(Job::isRunning) - .map(j -> j.getFeatureSetDeliveryStatuses().get(ref)) - .allMatch( - js -> - js.getDeliveryStatus() - .equals(FeatureSetProto.FeatureSetJobDeliveryStatus.STATUS_DELIVERED)); - - if (allDelivered) { - log.info("FeatureSet {} update is completely delivered", ref); - - specService.updateFeatureSetStatus( - CoreServiceProto.UpdateFeatureSetStatusRequest.newBuilder() - .setReference( - FeatureSetReferenceProto.FeatureSetReference.newBuilder() - .setName(ref.getFeatureSetName()) - .setProject(ref.getProjectName()) - .build()) - .setStatus(FeatureSetProto.FeatureSetStatus.STATUS_READY) - .build()); - } - } -} diff --git a/job-controller/src/main/java/feast/jobcontroller/service/JobService.java b/job-controller/src/main/java/feast/jobcontroller/service/JobService.java deleted file mode 100644 index 7b99fbf733d..00000000000 --- a/job-controller/src/main/java/feast/jobcontroller/service/JobService.java +++ /dev/null @@ -1,215 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.jobcontroller.service; - -import feast.common.models.FeatureSetReference; -import feast.jobcontroller.dao.JobRepository; -import feast.jobcontroller.model.Job; -import feast.jobcontroller.model.JobStatus; -import feast.jobcontroller.runner.JobManager; -import feast.jobcontroller.runner.task.RestartJobTask; -import feast.jobcontroller.runner.task.TerminateJobTask; -import feast.proto.core.CoreServiceProto.*; -import feast.proto.core.FeatureSetReferenceProto; -import feast.proto.core.IngestionJobProto; -import java.util.*; -import java.util.stream.Collectors; -import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Service; -import org.springframework.transaction.annotation.Transactional; - -/** A Job Management Service that allows users to manage Feast ingestion jobs. */ -@Slf4j -@Service -public class JobService { - private final JobRepository jobRepository; - private final JobManager jobManager; - - @Autowired - public JobService(JobRepository jobRepository, JobManager jobManager) { - this.jobRepository = jobRepository; - this.jobManager = jobManager; - } - - // region Job Service API - - /** - * List Ingestion Jobs in Feast matching the given request. See CoreService protobuf documentation - * for more detailed documentation. - * - * @param request list ingestion jobs request specifying which jobs to include - * @throws IllegalArgumentException when given filter in a unsupported configuration - * @return list ingestion jobs response - */ - @Transactional(readOnly = true) - public ListIngestionJobsResponse listJobs(ListIngestionJobsRequest request) { - Set matchingJobIds = new HashSet<>(); - - // check that filter specified and not empty - if (request.hasFilter() - && !(request.getFilter().getId().isEmpty() - && request.getFilter().getStoreName().isEmpty() - && !request.getFilter().hasFeatureSetReference())) { - // filter jobs based on request filter - ListIngestionJobsRequest.Filter filter = request.getFilter(); - - // for proto3, default value for missing values: - // - numeric values (ie int) is zero - // - strings is empty string - if (!filter.getId().isEmpty()) { - // get by id: no more filters required: found job - Optional job = this.jobRepository.findById(filter.getId()); - if (job.isPresent()) { - matchingJobIds.add(filter.getId()); - } - } else { - // multiple filters can apply together in an 'and' operation - if (!filter.getStoreName().isEmpty()) { - // find jobs by name - List jobs = this.jobRepository.findByJobStoreName(filter.getStoreName()); - Set jobIds = jobs.stream().map(Job::getId).collect(Collectors.toSet()); - matchingJobIds = this.mergeResults(matchingJobIds, jobIds); - } - if (filter.hasFeatureSetReference()) { - // find a matching featuresets for reference - FeatureSetReferenceProto.FeatureSetReference fsReference = - filter.getFeatureSetReference(); - - // find jobs for the matching featuresets - Collection matchingJobs = - this.jobRepository.findByFeatureSetReference( - FeatureSetReference.of(fsReference.getProject(), fsReference.getName())); - List jobIds = - matchingJobs.stream() - .filter(job -> job.getStatus().equals(JobStatus.RUNNING)) - .map(Job::getId) - .collect(Collectors.toList()); - matchingJobIds = this.mergeResults(matchingJobIds, jobIds); - } - } - } else { - // no or empty filter: match all jobs - matchingJobIds = - this.jobRepository.findAll().stream().map(Job::getId).collect(Collectors.toSet()); - } - - // convert matching job models to ingestion job protos - List ingestJobs = new ArrayList<>(); - for (String jobId : matchingJobIds) { - Job job = this.jobRepository.findById(jobId).orElseThrow(); - // job that failed on start won't be converted toProto successfully - // and they're irrelevant here - if (job.getStatus() == JobStatus.ERROR) { - continue; - } - ingestJobs.add(job.toProto()); - } - - // pack jobs into response - return ListIngestionJobsResponse.newBuilder().addAllJobs(ingestJobs).build(); - } - - /** - * Restart (Aborts) the ingestion job matching the given restart request. See CoreService protobuf - * documentation for more detailed documentation. - * - * @param request restart ingestion job request specifying which job to stop - * @throws NoSuchElementException when restart job request requests to restart a nonexistent job. - * @throws UnsupportedOperationException when job to be restarted is in an unsupported status - */ - @Transactional - public RestartIngestionJobResponse restartJob(RestartIngestionJobRequest request) { - String jobId = request.getId(); - - Job job = - this.jobRepository - .findById(jobId) - .orElseThrow( - () -> - new NoSuchElementException( - "Attempted to restart nonexistent job with id: " + jobId)); - - // check job status is valid for restarting - JobStatus status = job.getStatus(); - if (status.isTransitional() || status.isTerminal() || status == JobStatus.UNKNOWN) { - throw new UnsupportedOperationException( - "Restarting a job with a transitional, terminal or unknown status is unsupported"); - } - - // restart job by running job task - new RestartJobTask(job, this.jobManager).call(); - - // update job model in job repository - this.jobRepository.add(job); - - return RestartIngestionJobResponse.newBuilder().build(); - } - - /** - * Stops (Aborts) the ingestion job matching the given stop request. See CoreService protobuf - * documentation for more detailed documentation. - * - * @param request stop ingestion job request specifying which job to stop - * @throws NoSuchElementException when stop job request requests to stop a nonexistent job. - * @throws UnsupportedOperationException when job to be stopped is in an unsupported status - */ - @Transactional - public StopIngestionJobResponse stopJob(StopIngestionJobRequest request) { - String jobId = request.getId(); - - Job job = - this.jobRepository - .findById(jobId) - .orElseThrow( - () -> - new NoSuchElementException( - "Attempted to stop nonexistent job with id: " + jobId)); - - // check job status is valid for stopping - JobStatus status = job.getStatus(); - if (status.isTerminal()) { - // do nothing - job is already stopped - return StopIngestionJobResponse.newBuilder().build(); - } else if (status.isTransitional() || status == JobStatus.UNKNOWN) { - throw new UnsupportedOperationException( - "Stopping a job with a transitional or unknown status is unsupported"); - } - - // stop job with job task - new TerminateJobTask(job, this.jobManager).call(); - - // update job model in job repository - this.jobRepository.add(job); - - return StopIngestionJobResponse.newBuilder().build(); - } - - // endregion - // region Private Utility Methods - - private Set mergeResults(Set results, Collection newResults) { - if (results.size() <= 0) { - // no existing results: copy over new results - results.addAll(newResults); - } else { - // and operation: keep results that exist in both existing and new results - results.retainAll(newResults); - } - return results; - } -} diff --git a/job-controller/src/main/java/feast/jobcontroller/util/PackageUtil.java b/job-controller/src/main/java/feast/jobcontroller/util/PackageUtil.java deleted file mode 100644 index 3eb4673413c..00000000000 --- a/job-controller/src/main/java/feast/jobcontroller/util/PackageUtil.java +++ /dev/null @@ -1,144 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2019 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.jobcontroller.util; - -import java.io.File; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.net.URL; -import java.nio.file.Files; -import java.nio.file.Paths; -import java.util.Enumeration; -import java.util.jar.JarEntry; -import java.util.jar.JarFile; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -@SuppressWarnings("WeakerAccess") -public class PackageUtil { - - // TODO: Unit tests for PackageUtil - - private static Logger LOG = LoggerFactory.getLogger(PackageUtil.class); - - /** - * Get a local file path from a URL that reference classes or a resource located in Spring Boot - * packaged jar. - * - *

The packaged jar will be extracted, if needed, in order to get a file path that directly - * points to the resource location. Note that the extraction process can take several minutes to - * complete. - * - *

One use case of this function is to detect the class path of resources to stage when using - * Dataflow runner. The resource URL however is in "jar:file:" format, which cannot be handled by - * default in Apache Beam. - * - * 

-   * 
-   * URL url = new URL("jar:file:/tmp/springexample/target/spring-example-1.0-SNAPSHOT.jar!/BOOT-INF/lib/beam-sdks-java-core-2.16.0.jar!/");
-   * 
-   * String resolvedPath = resolveSpringBootPackageClasspath(url);
-   * // resolvedPath should point to "/tmp/springexample/target/spring-example-1.0-SNAPSHOT/BOOT-INF/lib/beam-sdks-java-core-2.16.0.jar"
-   * // Note that spring-example-1.0-SNAPSHOT.jar is extracted in the process.
-   *
- * - * @param url Location of the resource or classes to resolve, must start with "jar:file:". - * @return Local file path that points to the resource file. - * @throws IOException If read or write error occurs during the resolve process. - */ - public static String resolveSpringBootPackageClasspath(URL url) throws IOException { - if (!url.toString().startsWith("jar:file:")) { - throw new IllegalArgumentException("URL must start with 'jar:file:'"); - } - - String path = url.toString().substring(9).replaceAll("!/", "/"); - if (path.endsWith("/")) { - path = path.substring(0, path.length() - 1); - } - - if (path.contains(".jar/BOOT-INF/")) { - String jarPath = path.substring(0, path.indexOf(".jar/BOOT-INF/") + 4); - String extractedJarPath = jarPath.substring(0, jarPath.length() - 4); - - if (Files.notExists(Paths.get(extractedJarPath))) { - LOG.info( - "Extracting '{}' to '{}' so we can get a local file path for the resource.", - jarPath, - extractedJarPath); - extractJar(jarPath, extractedJarPath); - } - path = path.replace(".jar/BOOT-INF/", "/BOOT-INF/"); - } - - return path; - } - - // TODO: extractJar() currently is quite slow because it only uses a single core to extract the - // jar. Extracting a jar packaged by Spring boot, for example, can take more than 5 minutes. - // One - // way to speed it up is to parallelize the extraction. - - /** - * Extract contents of a jar file to an output directory. - * - *

Adapted from: https://stackoverflow.com/a/1529707/3949303 - * - * @param jarPath File path of the jar file to extract. - * @param destDirPath Destination directory to extract the jar content, will be created if not - * exists. - * @throws IOException If error occured when reading or writing files. - */ - public static void extractJar(String jarPath, String destDirPath) throws IOException { - File destDirFile = new File(destDirPath); - - if (destDirFile.exists() && !destDirFile.isDirectory()) { - throw new IOException(destDirPath + " must be a directory path"); - } - - if (!destDirFile.exists()) { - if (!destDirFile.mkdirs()) { - throw new IOException("Failed to create directory: " + destDirPath); - } - } - - JarFile jar = new JarFile(jarPath); - Enumeration enumEntries = jar.entries(); - - while (enumEntries.hasMoreElements()) { - JarEntry jarEntry = (JarEntry) enumEntries.nextElement(); - File outFile = new File(destDirPath + File.separator + jarEntry.getName()); - - if (jarEntry.isDirectory()) { - if (!outFile.mkdir()) { - throw new IOException("Failed to created directory: " + outFile); - } - continue; - } - - InputStream is = jar.getInputStream(jarEntry); - FileOutputStream fos = new FileOutputStream(outFile); - while (is.available() > 0) { - fos.write(is.read()); - } - fos.close(); - is.close(); - } - - jar.close(); - } -} diff --git a/job-controller/src/main/java/feast/jobcontroller/util/PipelineUtil.java b/job-controller/src/main/java/feast/jobcontroller/util/PipelineUtil.java deleted file mode 100644 index 2cec65062a4..00000000000 --- a/job-controller/src/main/java/feast/jobcontroller/util/PipelineUtil.java +++ /dev/null @@ -1,75 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2019 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.jobcontroller.util; - -import static feast.jobcontroller.util.PackageUtil.resolveSpringBootPackageClasspath; - -import java.io.File; -import java.io.IOException; -import java.net.URISyntaxException; -import java.net.URL; -import java.net.URLClassLoader; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.stream.Collectors; - -public class PipelineUtil { - - /** - * Attempts to detect all the resources the class loader has access to. This does not recurse to - * class loader parents stopping it from pulling in resources from the system class loader. - * - *

This method extends this implemention - * https://github.com/apache/beam/blob/01726e9c62313749f9ea7c93063a1178abd1a8db/runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/PipelineResources.java#L51 - * to support URL that starts with "jar:file:", usually coming from a packaged Spring Boot jar. - * - * @param classLoader The URLClassLoader to use to detect resources to stage. - * @return A list of absolute paths to the resources the class loader uses. - * @throws IllegalArgumentException If either the class loader is not a URLClassLoader or one of - * the resources the class loader exposes is not a file resource. - * @throws IOException If there is an error in reading or writing files. - */ - public static List detectClassPathResourcesToStage(ClassLoader classLoader) - throws IOException { - if (!(classLoader instanceof URLClassLoader)) { - return getClasspathFiles(); - } - - List files = new ArrayList<>(); - for (URL url : ((URLClassLoader) classLoader).getURLs()) { - if (url.toString().startsWith("jar:file:")) { - files.add(resolveSpringBootPackageClasspath(url)); - continue; - } - - try { - files.add(new File(url.toURI()).getAbsolutePath()); - } catch (IllegalArgumentException | URISyntaxException e) { - String message = String.format("Unable to convert url (%s) to file.", url); - throw new IllegalArgumentException(message, e); - } - } - return files; - } - - private static List getClasspathFiles() { - return Arrays.stream(System.getProperty("java.class.path").split(File.pathSeparator)) - .map(entry -> new File(entry).getPath()) - .collect(Collectors.toList()); - } -} diff --git a/job-controller/src/main/java/feast/jobcontroller/util/TypeConversion.java b/job-controller/src/main/java/feast/jobcontroller/util/TypeConversion.java deleted file mode 100644 index 92b3c8dd6f7..00000000000 --- a/job-controller/src/main/java/feast/jobcontroller/util/TypeConversion.java +++ /dev/null @@ -1,73 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2019 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.jobcontroller.util; - -import com.google.gson.Gson; -import com.google.gson.reflect.TypeToken; -import java.lang.reflect.Type; -import java.util.*; - -public class TypeConversion { - private static Gson gson = new Gson(); - - /** - * Convert a java data object to protobuf Timestamp object - * - * @param ts timestamp - * @return protobuf.Timestamp object of the given timestamp - */ - public static com.google.protobuf.Timestamp convertTimestamp(Date ts) { - return com.google.protobuf.Timestamp.newBuilder().setSeconds(ts.getTime() / 1000).build(); - } - - /** - * Convert a string of comma-separated strings to list of strings - * - * @param tags comma separated tags - * @return list of tags - */ - public static List convertTagStringToList(String tags) { - if (tags == null || tags.isEmpty()) { - return Collections.emptyList(); - } - return Arrays.asList(tags.split(",")); - } - - /** - * Unmarshals a given json string to map - * - * @param jsonString valid json formatted string - * @return map of keys to values in json - */ - public static Map convertJsonStringToMap(String jsonString) { - if (jsonString == null || jsonString.equals("") || jsonString.equals("{}")) { - return Collections.emptyMap(); - } - Type stringMapType = new TypeToken>() {}.getType(); - return gson.fromJson(jsonString, stringMapType); - } - - /** - * Marshals a given map into its corresponding json string - * - * @param map - * @return json string corresponding to given map - */ - public static String convertMapToJsonString(Map map) { - return gson.toJson(map); - } -} diff --git a/job-controller/src/main/resources/application.yml b/job-controller/src/main/resources/application.yml deleted file mode 100644 index 5ddbb1e97d3..00000000000 --- a/job-controller/src/main/resources/application.yml +++ /dev/null @@ -1,151 +0,0 @@ -# -# Copyright 2018 The Feast Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# - -feast: - core-host: localhost - core-port: 6565 - - jobs: - # Enabling JobManagement - enabled: true - - # Configurable Prefix for JobId to allow jobs working in parallel to separate their Kafka consumer groups - job_id_prefix: "" - - # Job update polling interval in milliseconds: how often Feast checks if new jobs should be sent to the runner. - polling_interval_milliseconds: 60000 - - # Timeout in seconds for each attempt to update or submit a new job to the runner. - job_update_timeout_seconds: 240 - - # Name of the active runner in "runners" that should be used. Only a single runner can be active at one time. - active_runner: direct - - # List of runner configurations. Please see protos/feast/core/Runner.proto for more details - # Alternatively see the following for options https://api.docs.feast.dev/grpc/feast.core.pb.html#Runner - runners: - - name: direct - type: DirectRunner - options: - tempLocation: gs://bucket/tempLocation - - - name: dataflow - type: DataflowRunner - options: - project: my_gcp_project - region: asia-east1 - workerZone: asia-east1-a - tempLocation: gs://bucket/tempLocation - network: default - subnetwork: regions/asia-east1/subnetworks/mysubnetwork - maxNumWorkers: 1 - enableStreamingEngine: false - workerDiskType: compute.googleapis.com/projects/asia-east1-a/diskTypes/pd-ssd - autoscalingAlgorithm: THROUGHPUT_BASED - usePublicIps: false - workerMachineType: n1-standard-1 - deadLetterTableSpec: project_id:dataset_id.table_id - kafkaConsumerProperties: - "[max.poll.records]": "50000" - "[receive.buffer.bytes]": "33554432" - - # Configuration options for metric collection for all ingestion jobs - metrics: - # Enable metrics pushing for all ingestion jobs. - enabled: false - # Type of metrics sink. Only statsd is currently supported. - type: statsd - # Host of the metrics sink. - host: localhost - # Port of the metrics sink. - port: 9125 - - controller: - # if true one job per source with many stores would be created - # if false one job per source-store pair would be created - consolidate-jobs-per-source: false - - # labels (map) that being assigned to job on creation. - # And also used to determine jobs that are being managed by current application - # among all running jobs - jobSelector: - application: feast - - # Specify feature sets that should be handled by current instance of JobManager - featureSetSelector: - - project: "*" - name: "*" - # Stores names that are enabled on current instance of JobManager - whitelisted-stores: - - historical - - online - - online_cluster - - stream: - # Feature stream type. Only kafka is supported. - type: kafka - # Feature stream options. - # See the following for options https://api.docs.feast.dev/grpc/feast.core.pb.html#KafkaSourceConfig - options: - topic: feast-features - bootstrapServers: localhost:9092 - replicationFactor: 1 - partitions: 1 - specsOptions: - specsTopic: feast-specs - specsAckTopic: feast-specs-ack - notifyIntervalMilliseconds: 1000 - - logging: - # Audit logging provides a machine readable structured JSON log that can give better - # insight into what is happening in Feast. - audit: - # Whether audit logging is enabled. - enabled: true - # Whether to enable message level (ie request/response) audit logging - messageLogging: - enabled: false - # Logging forwarder currently provides a machine readable structured JSON log to an - # external fluentd service that can give better insight into what is happening in Feast. - # Accepts console / fluentd as destination - destination: console - fluentdHost: localhost - fluentdPort: 24224 - -grpc: - server: - # The port that Feast Core gRPC service listens on - port: 6570 - security: - enabled: false - certificateChain: server.crt - privateKey: server.key - -management: - metrics: - export: - simple: - enabled: false - statsd: - enabled: true - host: ${STATSD_HOST:localhost} - port: ${STATSD_PORT:8125} - -server: - # The port number on which the Tomcat webserver that serves REST API endpoints should listen - # Set default value avoiding conflicts with core & serving - port: ${SERVER_PORT:8082} \ No newline at end of file diff --git a/job-controller/src/main/resources/banner.txt b/job-controller/src/main/resources/banner.txt deleted file mode 100644 index 3ae34145853..00000000000 --- a/job-controller/src/main/resources/banner.txt +++ /dev/null @@ -1,21 +0,0 @@ - -███████╗███████╗ █████╗ ███████╗████████╗ -██╔════╝██╔════╝██╔══██╗██╔════╝╚══██╔══╝ -█████╗ █████╗ ███████║███████╗ ██║ -██╔══╝ ██╔══╝ ██╔══██║╚════██║ ██║ -██║ ███████╗██║ ██║███████║ ██║ -╚═╝ ╚══════╝╚═╝ ╚═╝╚══════╝ ╚═╝ - - ██╗ ██████╗ ██████╗ - ██║██╔═══██╗██╔══██╗ - ██║██║ ██║██████╔╝ -██ ██║██║ ██║██╔══██╗ -╚█████╔╝╚██████╔╝██████╔╝ - ╚════╝ ╚═════╝ ╚═════╝ - - ██████╗ ██████╗ ███╗ ██╗████████╗██████╗ ██████╗ ██╗ ██╗ ███████╗██████╗ -██╔════╝██╔═══██╗████╗ ██║╚══██╔══╝██╔══██╗██╔═══██╗██║ ██║ ██╔════╝██╔══██╗ -██║ ██║ ██║██╔██╗ ██║ ██║ ██████╔╝██║ ██║██║ ██║ █████╗ ██████╔╝ -██║ ██║ ██║██║╚██╗██║ ██║ ██╔══██╗██║ ██║██║ ██║ ██╔══╝ ██╔══██╗ -╚██████╗╚██████╔╝██║ ╚████║ ██║ ██║ ██║╚██████╔╝███████╗███████╗███████╗██║ ██║ - ╚═════╝ ╚═════╝ ╚═╝ ╚═══╝ ╚═╝ ╚═╝ ╚═╝ ╚═════╝ ╚══════╝╚══════╝╚══════╝╚═╝ ╚═╝ diff --git a/job-controller/src/test/java/feast/jobcontroller/model/JobStatusTest.java b/job-controller/src/test/java/feast/jobcontroller/model/JobStatusTest.java deleted file mode 100644 index 884f9b73ec7..00000000000 --- a/job-controller/src/test/java/feast/jobcontroller/model/JobStatusTest.java +++ /dev/null @@ -1,45 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.jobcontroller.model; - -import static org.hamcrest.MatcherAssert.assertThat; -import static org.hamcrest.Matchers.is; - -import org.junit.jupiter.api.Test; - -public class JobStatusTest { - - @Test - public void isTerminalReturnsTrueForJobStatusWithTerminalState() { - JobStatus.getTerminalStates() - .forEach( - status -> { - assertThat(status.isTerminal(), is(true)); - assertThat(status.isTransitional(), is(false)); - }); - } - - @Test - public void isTransitionalReturnsTrueForJobStatusWithTransitionalState() { - JobStatus.getTransitionalStates() - .forEach( - status -> { - assertThat(status.isTransitional(), is(true)); - assertThat(status.isTerminal(), is(false)); - }); - } -} diff --git a/job-controller/src/test/java/feast/jobcontroller/runner/RunnerTest.java b/job-controller/src/test/java/feast/jobcontroller/runner/RunnerTest.java deleted file mode 100644 index 98c8da3798f..00000000000 --- a/job-controller/src/test/java/feast/jobcontroller/runner/RunnerTest.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.jobcontroller.runner; - -import static org.hamcrest.MatcherAssert.assertThat; -import static org.hamcrest.Matchers.is; -import static org.junit.jupiter.api.Assertions.assertThrows; - -import java.util.NoSuchElementException; -import org.junit.jupiter.api.Test; - -public class RunnerTest { - - @Test - public void toStringReturnsHumanReadableName() { - assertThat(Runner.DATAFLOW.toString(), is("DataflowRunner")); - } - - @Test - public void fromStringLoadsValueFromHumanReadableName() { - var humanName = Runner.DATAFLOW.toString(); - assertThat(Runner.fromString(humanName), is(Runner.DATAFLOW)); - } - - @Test - public void fromStringThrowsNoSuchElementExceptionForUnknownValue() { - assertThrows( - NoSuchElementException.class, () -> Runner.fromString("this is not a valid Runner")); - } -} diff --git a/job-controller/src/test/java/feast/jobcontroller/runner/dataflow/DataflowJobManagerTest.java b/job-controller/src/test/java/feast/jobcontroller/runner/dataflow/DataflowJobManagerTest.java deleted file mode 100644 index 7cf922febbc..00000000000 --- a/job-controller/src/test/java/feast/jobcontroller/runner/dataflow/DataflowJobManagerTest.java +++ /dev/null @@ -1,347 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2019 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.jobcontroller.runner.dataflow; - -import static org.hamcrest.MatcherAssert.assertThat; -import static org.hamcrest.Matchers.*; -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.*; - -import com.google.api.services.dataflow.Dataflow; -import com.google.api.services.dataflow.model.Environment; -import com.google.api.services.dataflow.model.ListJobsResponse; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; -import com.google.common.collect.Lists; -import com.google.protobuf.util.JsonFormat; -import com.google.protobuf.util.JsonFormat.Printer; -import feast.ingestion.options.ImportOptions; -import feast.jobcontroller.config.FeastProperties; -import feast.jobcontroller.exception.JobExecutionException; -import feast.jobcontroller.model.Job; -import feast.proto.core.IngestionJobProto; -import feast.proto.core.RunnerProto.DataflowRunnerConfigOptions; -import feast.proto.core.RunnerProto.DataflowRunnerConfigOptions.Builder; -import feast.proto.core.SourceProto; -import feast.proto.core.SourceProto.KafkaSourceConfig; -import feast.proto.core.SourceProto.SourceType; -import feast.proto.core.StoreProto; -import feast.proto.core.StoreProto.Store.RedisConfig; -import feast.proto.core.StoreProto.Store.StoreType; -import feast.proto.core.StoreProto.Store.Subscription; -import java.io.IOException; -import java.util.List; -import lombok.SneakyThrows; -import org.apache.beam.runners.dataflow.DataflowPipelineJob; -import org.apache.beam.runners.dataflow.DataflowRunner; -import org.apache.beam.sdk.PipelineResult.State; -import org.apache.beam.sdk.options.PipelineOptionsFactory; -import org.joda.time.DateTime; -import org.joda.time.LocalDateTime; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.mockito.ArgumentCaptor; -import org.mockito.Mockito; - -public class DataflowJobManagerTest { - - private Dataflow dataflow; - - private DataflowRunnerConfigOptions defaults; - private IngestionJobProto.SpecsStreamingUpdateConfig specsStreamingUpdateConfig; - private DataflowJobManager dfJobManager; - - private StoreProto.Store store; - private SourceProto.Source source; - - @BeforeEach - public void setUp() { - Builder optionsBuilder = DataflowRunnerConfigOptions.newBuilder(); - optionsBuilder.setProject("project"); - optionsBuilder.setRegion("region"); - optionsBuilder.setWorkerZone("zone"); - optionsBuilder.setTempLocation("tempLocation"); - optionsBuilder.setNetwork("network"); - optionsBuilder.setSubnetwork("subnetwork"); - optionsBuilder.putLabels("orchestrator", "feast"); - defaults = optionsBuilder.build(); - FeastProperties.MetricsProperties metricsProperties = new FeastProperties.MetricsProperties(); - metricsProperties.setEnabled(false); - - dataflow = mock(Dataflow.class, RETURNS_DEEP_STUBS); - - specsStreamingUpdateConfig = - IngestionJobProto.SpecsStreamingUpdateConfig.newBuilder() - .setSource( - KafkaSourceConfig.newBuilder() - .setTopic("specs_topic") - .setBootstrapServers("servers:9092") - .build()) - .build(); - - store = - StoreProto.Store.newBuilder() - .setName("SERVING") - .setType(StoreType.REDIS) - .setRedisConfig(RedisConfig.newBuilder().setHost("localhost").setPort(6379).build()) - .addSubscriptions(Subscription.newBuilder().setProject("*").setName("*").build()) - .build(); - - source = - SourceProto.Source.newBuilder() - .setType(SourceType.KAFKA) - .setKafkaSourceConfig( - KafkaSourceConfig.newBuilder() - .setTopic("topic") - .setBootstrapServers("servers:9092") - .build()) - .build(); - - dfJobManager = - new DataflowJobManager( - defaults, - metricsProperties, - specsStreamingUpdateConfig, - ImmutableMap.of("application", "feast"), - dataflow); - dfJobManager = spy(dfJobManager); - } - - @Test - public void shouldStartJobWithCorrectPipelineOptions() throws IOException { - Printer printer = JsonFormat.printer(); - String expectedExtJobId = "feast-job-0"; - String jobName = "job"; - - ImportOptions expectedPipelineOptions = - PipelineOptionsFactory.fromArgs("").as(ImportOptions.class); - expectedPipelineOptions.setRunner(DataflowRunner.class); - expectedPipelineOptions.setProject("project"); - expectedPipelineOptions.setRegion("region"); - expectedPipelineOptions.setUpdate(false); - expectedPipelineOptions.setAppName("DataflowJobManager"); - expectedPipelineOptions.setLabels(defaults.getLabelsMap()); - expectedPipelineOptions.setJobName(jobName); - expectedPipelineOptions.setStoresJson(Lists.newArrayList(printer.print(store))); - expectedPipelineOptions.setSourceJson(printer.print(source)); - - ArgumentCaptor captor = ArgumentCaptor.forClass(ImportOptions.class); - - DataflowPipelineJob mockPipelineResult = Mockito.mock(DataflowPipelineJob.class); - when(mockPipelineResult.getState()).thenReturn(State.RUNNING); - when(mockPipelineResult.getJobId()).thenReturn(expectedExtJobId); - - doReturn(mockPipelineResult).when(dfJobManager).runPipeline(any()); - - Job job = - Job.builder() - .setId(jobName) - .setSource(source) - .setStores(ImmutableMap.of(store.getName(), store)) - .build(); - Job actual = dfJobManager.startJob(job); - - verify(dfJobManager, times(1)).runPipeline(captor.capture()); - ImportOptions actualPipelineOptions = captor.getValue(); - - expectedPipelineOptions.setOptionsId( - actualPipelineOptions.getOptionsId()); // avoid comparing this value - - // We only check that we are calling getFilesToStage() manually, because the automatic approach - // throws an error: https://github.com/feast-dev/feast/pull/291 i.e. do not check for the actual - // files that are staged - assertThat( - "filesToStage in pipelineOptions should not be null, job manager should set it.", - actualPipelineOptions.getFilesToStage() != null); - assertThat( - "filesToStage in pipelineOptions should contain at least 1 item", - actualPipelineOptions.getFilesToStage().size() > 0); - // Assume the files that are staged are correct - expectedPipelineOptions.setFilesToStage(actualPipelineOptions.getFilesToStage()); - - assertThat( - actualPipelineOptions.getDeadLetterTableSpec(), - equalTo(expectedPipelineOptions.getDeadLetterTableSpec())); - assertThat( - actualPipelineOptions.getStatsdHost(), equalTo(expectedPipelineOptions.getStatsdHost())); - assertThat( - actualPipelineOptions.getMetricsExporterType(), - equalTo(expectedPipelineOptions.getMetricsExporterType())); - assertThat( - actualPipelineOptions.getStoresJson(), equalTo(expectedPipelineOptions.getStoresJson())); - assertThat( - actualPipelineOptions.getSourceJson(), equalTo(expectedPipelineOptions.getSourceJson())); - assertThat( - actualPipelineOptions.getSpecsStreamingUpdateConfigJson(), - equalTo(printer.print(specsStreamingUpdateConfig))); - assertThat(actual.getExtId(), equalTo(expectedExtJobId)); - } - - @Test - public void shouldThrowExceptionWhenJobStateTerminal() throws IOException { - dfJobManager = Mockito.spy(dfJobManager); - - DataflowPipelineJob mockPipelineResult = Mockito.mock(DataflowPipelineJob.class); - when(mockPipelineResult.getState()).thenReturn(State.FAILED); - - doReturn(mockPipelineResult).when(dfJobManager).runPipeline(any()); - - Job job = - Job.builder() - .setId("job") - .setSource(source) - .setStores(ImmutableMap.of(store.getName(), store)) - .build(); - assertThrows(JobExecutionException.class, () -> dfJobManager.startJob(job)); - } - - @Test - @SneakyThrows - public void shouldRetrieveRunningJobsFromDataflow() { - when(dataflow - .projects() - .locations() - .jobs() - .list("project", "region") - .setFilter("ACTIVE") - .execute()) - .thenReturn( - new ListJobsResponse() - .setJobs( - ImmutableList.of( - new com.google.api.services.dataflow.model.Job().setId("job-1"), - new com.google.api.services.dataflow.model.Job().setId("job-2")))); - - // Job doesn't have required labels, should be skipped - when(dataflow - .projects() - .locations() - .jobs() - .get("project", "region", "job-1") - .setView("JOB_VIEW_ALL") - .execute()) - .thenReturn(new com.google.api.services.dataflow.model.Job()); - - Printer jsonPrinter = JsonFormat.printer(); - - LocalDateTime created = DateTime.now().toLocalDateTime(); - - when(dataflow - .projects() - .locations() - .jobs() - .get("project", "region", "job-2") - .setView("JOB_VIEW_ALL") - .execute()) - .thenReturn( - new com.google.api.services.dataflow.model.Job() - .setLabels(ImmutableMap.of("application", "feast")) - .setId("job-2") - .setCreateTime(created.toString()) - .setEnvironment( - new Environment() - .setSdkPipelineOptions( - ImmutableMap.of( - "options", - ImmutableMap.of( - "jobName", "kafka-to-redis", - "sourceJson", jsonPrinter.print(source), - "storesJson", ImmutableList.of(jsonPrinter.print(store))))))); - - List jobs = dfJobManager.listRunningJobs(); - - assertThat(jobs, hasSize(1)); - assertThat( - jobs, - hasItem( - allOf( - hasProperty("id", equalTo("kafka-to-redis")), - hasProperty("source", equalTo(source)), - hasProperty("stores", hasValue(store)), - hasProperty("extId", equalTo("job-2")), - hasProperty("created", equalTo(created.toDate())), - hasProperty("lastUpdated", equalTo(created.toDate())), - hasProperty("labels", hasEntry("application", "feast"))))); - } - - @Test - @SneakyThrows - public void shouldHandleNullResponseFromDataflow() { - when(dataflow - .projects() - .locations() - .jobs() - .list("project", "region") - .setFilter("ACTIVE") - .execute() - .getJobs()) - .thenReturn(null); - - assertThat(dfJobManager.listRunningJobs(), hasSize(0)); - } - - @Test - @SneakyThrows - public void shouldRetrieveRunningJobsWithoutLabels() { - when(dataflow - .projects() - .locations() - .jobs() - .list("project", "region") - .setFilter("ACTIVE") - .execute()) - .thenReturn( - new ListJobsResponse() - .setJobs( - ImmutableList.of( - new com.google.api.services.dataflow.model.Job().setId("job-1")))); - - Printer jsonPrinter = JsonFormat.printer(); - - // job with no labels - when(dataflow - .projects() - .locations() - .jobs() - .get("project", "region", "job-1") - .setView("JOB_VIEW_ALL") - .execute()) - .thenReturn( - new com.google.api.services.dataflow.model.Job() - .setId("job-1") - .setEnvironment( - new Environment() - .setSdkPipelineOptions( - ImmutableMap.of( - "options", - ImmutableMap.of( - "jobName", "kafka-to-redis", - "sourceJson", jsonPrinter.print(source), - "storesJson", ImmutableList.of(jsonPrinter.print(store))))))); - - FeastProperties.MetricsProperties metricsProperties = new FeastProperties.MetricsProperties(); - metricsProperties.setEnabled(false); - - dfJobManager = - new DataflowJobManager( - defaults, metricsProperties, specsStreamingUpdateConfig, ImmutableMap.of(), dataflow); - - List jobs = dfJobManager.listRunningJobs(); - assertThat(jobs, hasItem(hasProperty("id", equalTo("kafka-to-redis")))); - } -} diff --git a/job-controller/src/test/java/feast/jobcontroller/runner/dataflow/DataflowJobStateMapperTest.java b/job-controller/src/test/java/feast/jobcontroller/runner/dataflow/DataflowJobStateMapperTest.java deleted file mode 100644 index 62d1d6e59f7..00000000000 --- a/job-controller/src/test/java/feast/jobcontroller/runner/dataflow/DataflowJobStateMapperTest.java +++ /dev/null @@ -1,31 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2019 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.jobcontroller.runner.dataflow; - -import static org.junit.jupiter.api.Assertions.assertThrows; - -import org.junit.jupiter.api.Test; - -public class DataflowJobStateMapperTest { - - private DataflowJobStateMapper mapper = new DataflowJobStateMapper(); - - @Test - public void shouldThrowIllegalArgumentExceptionForInvalidString() { - assertThrows(IllegalArgumentException.class, () -> mapper.map("INVALID_STATE")); - } -} diff --git a/job-controller/src/test/java/feast/jobcontroller/runner/dataflow/DataflowRunnerConfigTest.java b/job-controller/src/test/java/feast/jobcontroller/runner/dataflow/DataflowRunnerConfigTest.java deleted file mode 100644 index f444af519aa..00000000000 --- a/job-controller/src/test/java/feast/jobcontroller/runner/dataflow/DataflowRunnerConfigTest.java +++ /dev/null @@ -1,126 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.jobcontroller.runner.dataflow; - -import static org.hamcrest.CoreMatchers.equalTo; -import static org.hamcrest.MatcherAssert.assertThat; -import static org.hamcrest.Matchers.containsInAnyOrder; - -import com.google.common.collect.Lists; -import feast.ingestion.options.ImportOptions; -import feast.proto.core.RunnerProto.DataflowRunnerConfigOptions; -import java.util.Arrays; -import java.util.List; -import org.apache.beam.sdk.options.PipelineOptionsFactory; -import org.junit.jupiter.api.Test; - -public class DataflowRunnerConfigTest { - @Test - public void shouldConvertToPipelineArgs() throws IllegalAccessException { - DataflowRunnerConfigOptions opts = - DataflowRunnerConfigOptions.newBuilder() - .setProject("my-project") - .setRegion("asia-east1") - .setWorkerZone("asia-east1-a") - .setEnableStreamingEngine(true) - .setWorkerDiskType("pd-ssd") - .setTempLocation("gs://bucket/tempLocation") - .setNetwork("default") - .setSubnetwork("regions/asia-east1/subnetworks/mysubnetwork") - .setMaxNumWorkers(1) - .setAutoscalingAlgorithm("THROUGHPUT_BASED") - .setUsePublicIps(false) - .setWorkerMachineType("n1-standard-1") - .setDeadLetterTableSpec("project_id:dataset_id.table_id") - .setDiskSizeGb(100) - .putLabels("key", "value") - .putKafkaConsumerProperties("max.poll.records", "1000") - .putKafkaConsumerProperties("receive.buffer.bytes", "1000000") - .build(); - - DataflowRunnerConfig dataflowRunnerConfig = new DataflowRunnerConfig(opts); - List args = Lists.newArrayList(dataflowRunnerConfig.toArgs()); - String[] expectedArgs = - Arrays.asList( - "--project=my-project", - "--region=asia-east1", - "--workerZone=asia-east1-a", - "--tempLocation=gs://bucket/tempLocation", - "--network=default", - "--subnetwork=regions/asia-east1/subnetworks/mysubnetwork", - "--maxNumWorkers=1", - "--autoscalingAlgorithm=THROUGHPUT_BASED", - "--usePublicIps=false", - "--workerMachineType=n1-standard-1", - "--deadLetterTableSpec=project_id:dataset_id.table_id", - "--diskSizeGb=100", - "--labels={\"key\":\"value\"}", - "--kafkaConsumerProperties={\"max.poll.records\":\"1000\",\"receive.buffer.bytes\":\"1000000\"}", - "--enableStreamingEngine=true", - "--workerDiskType=pd-ssd") - .toArray(String[]::new); - - assertThat(args.size(), equalTo(expectedArgs.length)); - assertThat(args, containsInAnyOrder(expectedArgs)); - - ImportOptions pipelineOptions = - PipelineOptionsFactory.fromArgs(dataflowRunnerConfig.toArgs()).as(ImportOptions.class); - - assertThat( - pipelineOptions.getKafkaConsumerProperties(), - equalTo(opts.getKafkaConsumerPropertiesMap())); - } - - @Test - public void shouldIgnoreOptionalArguments() throws IllegalAccessException { - DataflowRunnerConfigOptions opts = - DataflowRunnerConfigOptions.newBuilder() - .setProject("my-project") - .setRegion("asia-east1") - .setWorkerZone("asia-east1-a") - .setTempLocation("gs://bucket/tempLocation") - .setNetwork("default") - .setSubnetwork("regions/asia-east1/subnetworks/mysubnetwork") - .setMaxNumWorkers(1) - .setAutoscalingAlgorithm("THROUGHPUT_BASED") - .setUsePublicIps(false) - .setWorkerMachineType("n1-standard-1") - .build(); - - DataflowRunnerConfig dataflowRunnerConfig = new DataflowRunnerConfig(opts); - List args = Lists.newArrayList(dataflowRunnerConfig.toArgs()); - String[] expectedArgs = - Arrays.asList( - "--project=my-project", - "--region=asia-east1", - "--workerZone=asia-east1-a", - "--tempLocation=gs://bucket/tempLocation", - "--network=default", - "--subnetwork=regions/asia-east1/subnetworks/mysubnetwork", - "--maxNumWorkers=1", - "--autoscalingAlgorithm=THROUGHPUT_BASED", - "--usePublicIps=false", - "--workerMachineType=n1-standard-1", - "--labels={}", - "--kafkaConsumerProperties={}", - "--enableStreamingEngine=false") - .toArray(String[]::new); - - assertThat(args.size(), equalTo(expectedArgs.length)); - assertThat(args, containsInAnyOrder(expectedArgs)); - } -} diff --git a/job-controller/src/test/java/feast/jobcontroller/runner/direct/DirectRunnerConfigTest.java b/job-controller/src/test/java/feast/jobcontroller/runner/direct/DirectRunnerConfigTest.java deleted file mode 100644 index 60f1c7fe695..00000000000 --- a/job-controller/src/test/java/feast/jobcontroller/runner/direct/DirectRunnerConfigTest.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.jobcontroller.runner.direct; - -import static org.hamcrest.MatcherAssert.assertThat; -import static org.hamcrest.Matchers.containsInAnyOrder; -import static org.hamcrest.Matchers.equalTo; - -import com.google.common.collect.Lists; -import feast.proto.core.RunnerProto.DirectRunnerConfigOptions; -import java.util.List; -import org.junit.jupiter.api.Test; - -public class DirectRunnerConfigTest { - @Test - public void shouldConvertToPipelineArgs() throws IllegalAccessException { - DirectRunnerConfigOptions opts = - DirectRunnerConfigOptions.newBuilder() - .setTargetParallelism(1) - .setDeadLetterTableSpec("project_id:dataset_id.table_id") - .build(); - DirectRunnerConfig directRunnerConfig = new DirectRunnerConfig(opts); - List args = Lists.newArrayList(directRunnerConfig.toArgs()); - assertThat(args.size(), equalTo(2)); - assertThat( - args, - containsInAnyOrder( - "--targetParallelism=1", "--deadletterTableSpec=project_id:dataset_id.table_id")); - } -} diff --git a/job-controller/src/test/java/feast/jobcontroller/runner/direct/DirectRunnerJobManagerTest.java b/job-controller/src/test/java/feast/jobcontroller/runner/direct/DirectRunnerJobManagerTest.java deleted file mode 100644 index 60bbd927c5c..00000000000 --- a/job-controller/src/test/java/feast/jobcontroller/runner/direct/DirectRunnerJobManagerTest.java +++ /dev/null @@ -1,179 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2019 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.jobcontroller.runner.direct; - -import static org.hamcrest.MatcherAssert.assertThat; -import static org.hamcrest.Matchers.equalTo; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.*; -import static org.mockito.MockitoAnnotations.initMocks; - -import com.google.common.collect.ImmutableMap; -import com.google.common.collect.Lists; -import com.google.protobuf.util.JsonFormat; -import com.google.protobuf.util.JsonFormat.Printer; -import feast.ingestion.options.ImportOptions; -import feast.jobcontroller.config.FeastProperties.MetricsProperties; -import feast.jobcontroller.model.Job; -import feast.jobcontroller.model.JobStatus; -import feast.proto.core.IngestionJobProto; -import feast.proto.core.RunnerProto.DirectRunnerConfigOptions; -import feast.proto.core.SourceProto; -import feast.proto.core.SourceProto.KafkaSourceConfig; -import feast.proto.core.SourceProto.SourceType; -import feast.proto.core.StoreProto; -import feast.proto.core.StoreProto.Store.RedisConfig; -import feast.proto.core.StoreProto.Store.StoreType; -import feast.proto.core.StoreProto.Store.Subscription; -import java.io.IOException; -import org.apache.beam.runners.direct.DirectRunner; -import org.apache.beam.sdk.PipelineResult; -import org.apache.beam.sdk.options.PipelineOptionsFactory; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.mockito.ArgumentCaptor; -import org.mockito.Mock; -import org.mockito.Mockito; - -public class DirectRunnerJobManagerTest { - @Mock private DirectJobRegistry directJobRegistry; - - private DirectRunnerJobManager drJobManager; - private DirectRunnerConfigOptions defaults; - private IngestionJobProto.SpecsStreamingUpdateConfig specsStreamingUpdateConfig; - - private StoreProto.Store store; - private SourceProto.Source source; - - @BeforeEach - public void setUp() { - initMocks(this); - defaults = DirectRunnerConfigOptions.newBuilder().setTargetParallelism(1).build(); - MetricsProperties metricsProperties = new MetricsProperties(); - metricsProperties.setEnabled(false); - - specsStreamingUpdateConfig = - IngestionJobProto.SpecsStreamingUpdateConfig.newBuilder() - .setSource( - KafkaSourceConfig.newBuilder() - .setTopic("specs_topic") - .setBootstrapServers("servers:9092") - .build()) - .build(); - - store = - StoreProto.Store.newBuilder() - .setName("SERVING") - .setType(StoreType.REDIS) - .setRedisConfig(RedisConfig.newBuilder().setHost("localhost").setPort(6379).build()) - .addSubscriptions(Subscription.newBuilder().setProject("*").setName("*").build()) - .build(); - - source = - SourceProto.Source.newBuilder() - .setType(SourceType.KAFKA) - .setKafkaSourceConfig( - KafkaSourceConfig.newBuilder() - .setTopic("topic") - .setBootstrapServers("servers:9092") - .build()) - .build(); - - drJobManager = - new DirectRunnerJobManager( - defaults, directJobRegistry, metricsProperties, specsStreamingUpdateConfig); - drJobManager = Mockito.spy(drJobManager); - } - - @Test - public void shouldStartDirectJobAndRegisterPipelineResult() throws IOException { - Printer printer = JsonFormat.printer(); - - String expectedJobId = "feast-job-0"; - ImportOptions expectedPipelineOptions = - PipelineOptionsFactory.fromArgs("").as(ImportOptions.class); - expectedPipelineOptions.setJobName(expectedJobId); - expectedPipelineOptions.setAppName("DirectRunnerJobManager"); - expectedPipelineOptions.setRunner(DirectRunner.class); - expectedPipelineOptions.setBlockOnRun(false); - expectedPipelineOptions.setTargetParallelism(1); - expectedPipelineOptions.setStoresJson(Lists.newArrayList(printer.print(store))); - expectedPipelineOptions.setProject(""); - expectedPipelineOptions.setSourceJson(printer.print(source)); - - ArgumentCaptor pipelineOptionsCaptor = - ArgumentCaptor.forClass(ImportOptions.class); - ArgumentCaptor directJobCaptor = ArgumentCaptor.forClass(DirectJob.class); - - PipelineResult mockPipelineResult = Mockito.mock(PipelineResult.class); - doReturn(mockPipelineResult).when(drJobManager).runPipeline(any()); - - Job job = - Job.builder() - .setSource(source) - .setStores(ImmutableMap.of(store.getName(), store)) - .setId(expectedJobId) - .build(); - Job actual = drJobManager.startJob(job); - - verify(drJobManager, times(1)).runPipeline(pipelineOptionsCaptor.capture()); - verify(directJobRegistry, times(1)).add(directJobCaptor.capture()); - - ImportOptions actualPipelineOptions = pipelineOptionsCaptor.getValue(); - DirectJob jobStarted = directJobCaptor.getValue(); - expectedPipelineOptions.setOptionsId( - actualPipelineOptions.getOptionsId()); // avoid comparing this value - - assertThat( - actualPipelineOptions.getDeadLetterTableSpec(), - equalTo(expectedPipelineOptions.getDeadLetterTableSpec())); - assertThat( - actualPipelineOptions.getStatsdHost(), equalTo(expectedPipelineOptions.getStatsdHost())); - assertThat( - actualPipelineOptions.getMetricsExporterType(), - equalTo(expectedPipelineOptions.getMetricsExporterType())); - assertThat( - actualPipelineOptions.getStoresJson(), equalTo(expectedPipelineOptions.getStoresJson())); - assertThat( - actualPipelineOptions.getSourceJson(), equalTo(expectedPipelineOptions.getSourceJson())); - assertThat( - actualPipelineOptions.getSpecsStreamingUpdateConfigJson(), - equalTo(printer.print(specsStreamingUpdateConfig))); - - assertThat(jobStarted.getPipelineResult(), equalTo(mockPipelineResult)); - assertThat(jobStarted.getJobId(), equalTo(expectedJobId)); - assertThat(actual.getExtId(), equalTo(expectedJobId)); - } - - @Test - public void shouldAbortJobThenRemoveFromRegistry() throws IOException { - Job job = - Job.builder() - .setSource(source) - .setStores(ImmutableMap.of(store.getName(), store)) - .setId("id") - .build(); - job.setExtId("ext1"); - job.setStatus(JobStatus.RUNNING); - - DirectJob directJob = Mockito.mock(DirectJob.class); - when(directJobRegistry.get("ext1")).thenReturn(directJob); - job = drJobManager.abortJob(job); - verify(directJob, times(1)).abort(); - verify(directJobRegistry, times(1)).remove("ext1"); - } -} diff --git a/job-controller/src/test/java/feast/jobcontroller/runner/option/FeatureSetJsonByteConverterTest.java b/job-controller/src/test/java/feast/jobcontroller/runner/option/FeatureSetJsonByteConverterTest.java deleted file mode 100644 index b319a649e85..00000000000 --- a/job-controller/src/test/java/feast/jobcontroller/runner/option/FeatureSetJsonByteConverterTest.java +++ /dev/null @@ -1,81 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.jobcontroller.runner.option; - -import static org.junit.Assert.assertEquals; - -import com.google.protobuf.InvalidProtocolBufferException; -import feast.proto.core.FeatureSetProto; -import feast.proto.core.SourceProto; -import feast.proto.types.ValueProto; -import java.util.List; -import java.util.stream.Collectors; -import java.util.stream.IntStream; -import org.junit.jupiter.api.Test; - -public class FeatureSetJsonByteConverterTest { - - private FeatureSetProto.FeatureSet newFeatureSet(Integer numberOfFeatures) { - List features = - IntStream.range(1, numberOfFeatures + 1) - .mapToObj( - i -> - FeatureSetProto.FeatureSpec.newBuilder() - .setValueType(ValueProto.ValueType.Enum.FLOAT) - .setName("feature".concat(Integer.toString(i))) - .build()) - .collect(Collectors.toList()); - - return FeatureSetProto.FeatureSet.newBuilder() - .setSpec( - FeatureSetProto.FeatureSetSpec.newBuilder() - .setSource( - SourceProto.Source.newBuilder() - .setType(SourceProto.SourceType.KAFKA) - .setKafkaSourceConfig( - SourceProto.KafkaSourceConfig.newBuilder() - .setBootstrapServers("somebrokers:9092") - .setTopic("sometopic"))) - .addAllFeatures(features) - .addEntities( - FeatureSetProto.EntitySpec.newBuilder() - .setName("entity") - .setValueType(ValueProto.ValueType.Enum.STRING))) - .build(); - } - - @Test - public void shouldConvertFeatureSetsAsJsonStringBytes() throws InvalidProtocolBufferException { - int nrOfFeatureSet = 1; - int nrOfFeatures = 1; - List featureSets = - IntStream.range(1, nrOfFeatureSet + 1) - .mapToObj(i -> newFeatureSet(nrOfFeatures)) - .collect(Collectors.toList()); - - String expectedOutputString = - "{\"entities\":[{\"name\":\"entity\",\"valueType\":2}]," - + "\"features\":[{\"name\":\"feature1\",\"valueType\":6}]," - + "\"source\":{" - + "\"type\":1," - + "\"kafkaSourceConfig\":{" - + "\"bootstrapServers\":\"somebrokers:9092\"," - + "\"topic\":\"sometopic\"}}}"; - FeatureSetJsonByteConverter byteConverter = new FeatureSetJsonByteConverter(); - assertEquals(expectedOutputString, new String(byteConverter.toByte(featureSets))); - } -} diff --git a/job-controller/src/test/java/feast/jobcontroller/runner/task/JobTasksTest.java b/job-controller/src/test/java/feast/jobcontroller/runner/task/JobTasksTest.java deleted file mode 100644 index 4e8aaa4459a..00000000000 --- a/job-controller/src/test/java/feast/jobcontroller/runner/task/JobTasksTest.java +++ /dev/null @@ -1,153 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2019 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.jobcontroller.runner.task; - -import static org.hamcrest.MatcherAssert.assertThat; -import static org.hamcrest.beans.HasPropertyWithValue.hasProperty; -import static org.hamcrest.core.IsEqual.equalTo; -import static org.mockito.Mockito.*; -import static org.mockito.MockitoAnnotations.initMocks; - -import com.google.common.collect.ImmutableMap; -import feast.common.util.TestUtil; -import feast.jobcontroller.model.Job; -import feast.jobcontroller.model.JobStatus; -import feast.jobcontroller.runner.JobManager; -import feast.jobcontroller.runner.Runner; -import feast.proto.core.SourceProto; -import feast.proto.core.SourceProto.KafkaSourceConfig; -import feast.proto.core.SourceProto.SourceType; -import feast.proto.core.StoreProto; -import feast.proto.core.StoreProto.Store.RedisConfig; -import feast.proto.core.StoreProto.Store.StoreType; -import feast.proto.core.StoreProto.Store.Subscription; -import lombok.SneakyThrows; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.mockito.Mock; - -public class JobTasksTest { - private static final Runner RUNNER = Runner.DATAFLOW; - - @Mock private JobManager jobManager; - - private StoreProto.Store store; - private SourceProto.Source source; - - @BeforeEach - public void setUp() { - initMocks(this); - when(jobManager.getRunnerType()).thenReturn(RUNNER); - - store = - StoreProto.Store.newBuilder() - .setName("test") - .setType(StoreType.REDIS) - .setRedisConfig(RedisConfig.newBuilder().build()) - .addSubscriptions(Subscription.newBuilder().setProject("*").setName("*").build()) - .build(); - - source = - SourceProto.Source.newBuilder() - .setType(SourceType.KAFKA) - .setKafkaSourceConfig( - KafkaSourceConfig.newBuilder() - .setTopic("topic") - .setBootstrapServers("servers:9092") - .build()) - .build(); - TestUtil.setupAuditLogger(); - } - - @SneakyThrows - Job makeJob(String extId, JobStatus status) { - Job job = - Job.builder() - .setId("job") - .setSource(source) - .setStores(ImmutableMap.of(store.getName(), store)) - .build(); - job.setStatus(status); - job.setExtId(extId); - return job; - } - - CreateJobTask makeCreateTask(Job currentJob) { - return new CreateJobTask(currentJob, jobManager); - } - - UpdateJobStatusTask makeCheckStatusTask(Job currentJob) { - return new UpdateJobStatusTask(currentJob, jobManager); - } - - TerminateJobTask makeTerminateTask(Job currentJob) { - return new TerminateJobTask(currentJob, jobManager); - } - - @Test - public void shouldCreateJobIfNotPresent() { - Job expectedInput = makeJob("ext", JobStatus.PENDING); - - CreateJobTask task = makeCreateTask(expectedInput); - - when(jobManager.startJob(expectedInput)).thenReturn(makeJob("ext", JobStatus.RUNNING)); - - Job actual = task.call(); - assertThat(actual, hasProperty("status", equalTo(JobStatus.RUNNING))); - } - - @Test - public void shouldUpdateJobStatusIfNotCreateOrUpdate() { - Job originalJob = makeJob("ext", JobStatus.RUNNING); - JobTask jobUpdateTask = makeCheckStatusTask(originalJob); - - when(jobManager.getJobStatus(originalJob)).thenReturn(JobStatus.ABORTING); - Job updated = jobUpdateTask.call(); - - assertThat(updated.getStatus(), equalTo(JobStatus.ABORTING)); - } - - @Test - public void shouldReturnJobWithErrorStatusIfFailedToSubmit() { - Job expectedInput = makeJob("", JobStatus.PENDING); - - CreateJobTask jobUpdateTask = makeCreateTask(expectedInput); - - Job expected = makeJob("", JobStatus.ERROR); - - when(jobManager.startJob(expectedInput)) - .thenThrow(new RuntimeException("Something went wrong")); - - Job actual = jobUpdateTask.call(); - assertThat(actual, hasProperty("status", equalTo(JobStatus.ERROR))); - } - - @Test - public void shouldStopJobIfTargetStatusIsAbort() { - Job originalJob = makeJob("ext", JobStatus.RUNNING); - JobTask jobUpdateTask = makeTerminateTask(originalJob); - - Job expected = makeJob("ext", JobStatus.ABORTING); - - when(jobManager.getJobStatus(originalJob)).thenReturn(JobStatus.ABORTING); - when(jobManager.abortJob(originalJob)).thenReturn(expected); - - Job actual = jobUpdateTask.call(); - verify(jobManager, times(1)).abortJob(originalJob); - assertThat(actual, hasProperty("status", equalTo(JobStatus.ABORTING))); - } -} diff --git a/job-controller/src/test/java/feast/jobcontroller/service/FakeJobManager.java b/job-controller/src/test/java/feast/jobcontroller/service/FakeJobManager.java deleted file mode 100644 index 22c56fc6fc0..00000000000 --- a/job-controller/src/test/java/feast/jobcontroller/service/FakeJobManager.java +++ /dev/null @@ -1,85 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.jobcontroller.service; - -import com.google.common.collect.Lists; -import feast.jobcontroller.model.Job; -import feast.jobcontroller.model.JobStatus; -import feast.jobcontroller.runner.JobManager; -import feast.jobcontroller.runner.Runner; -import java.util.*; - -public class FakeJobManager implements JobManager { - private final Map state; - - public FakeJobManager() { - state = new HashMap<>(); - } - - @Override - public Runner getRunnerType() { - return Runner.DIRECT; - } - - @Override - public Job startJob(Job job) { - String extId = UUID.randomUUID().toString(); - job.setExtId(extId); - job.setStatus(JobStatus.RUNNING); - state.put(extId, job); - return job; - } - - @Override - public Job updateJob(Job job) { - return job; - } - - @Override - public Job abortJob(Job job) { - job.setStatus(JobStatus.ABORTING); - state.remove(job.getExtId()); - return job; - } - - @Override - public Job restartJob(Job job) { - return abortJob(job); - } - - @Override - public JobStatus getJobStatus(Job job) { - if (state.containsKey(job.getExtId())) { - return JobStatus.RUNNING; - } - - return JobStatus.ABORTED; - } - - @Override - public List listRunningJobs() { - return Collections.emptyList(); - } - - public List getAllJobs() { - return Lists.newArrayList(state.values()); - } - - public void cleanAll() { - state.clear(); - } -} diff --git a/job-controller/src/test/java/feast/jobcontroller/service/JobControllerIT.java b/job-controller/src/test/java/feast/jobcontroller/service/JobControllerIT.java deleted file mode 100644 index 877ac9c1abf..00000000000 --- a/job-controller/src/test/java/feast/jobcontroller/service/JobControllerIT.java +++ /dev/null @@ -1,449 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.jobcontroller.service; - -import static org.awaitility.Awaitility.await; -import static org.hamcrest.CoreMatchers.*; -import static org.hamcrest.MatcherAssert.assertThat; -import static org.hamcrest.beans.HasPropertyWithValue.hasProperty; -import static org.hamcrest.collection.IsCollectionWithSize.hasSize; -import static org.hamcrest.collection.IsIterableContainingInAnyOrder.containsInAnyOrder; -import static org.hamcrest.collection.IsMapContaining.hasEntry; -import static org.hamcrest.collection.IsMapWithSize.aMapWithSize; -import static org.hamcrest.core.AllOf.allOf; -import static org.hamcrest.number.OrderingComparison.greaterThan; - -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; -import com.google.protobuf.InvalidProtocolBufferException; -import feast.common.it.*; -import feast.common.util.KafkaSerialization; -import feast.jobcontroller.config.FeastProperties; -import feast.jobcontroller.dao.JobRepository; -import feast.jobcontroller.model.Job; -import feast.jobcontroller.model.JobStatus; -import feast.jobcontroller.runner.JobManager; -import feast.proto.core.*; -import feast.proto.types.ValueProto; -import io.grpc.ManagedChannel; -import io.grpc.ManagedChannelBuilder; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import lombok.SneakyThrows; -import org.apache.kafka.clients.consumer.ConsumerRecord; -import org.apache.kafka.clients.producer.ProducerConfig; -import org.apache.kafka.common.serialization.StringSerializer; -import org.junit.jupiter.api.*; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.boot.test.context.TestConfiguration; -import org.springframework.context.annotation.Bean; -import org.springframework.kafka.annotation.KafkaListener; -import org.springframework.kafka.core.DefaultKafkaProducerFactory; -import org.springframework.kafka.core.KafkaTemplate; -import org.springframework.test.context.DynamicPropertyRegistry; -import org.springframework.test.context.DynamicPropertySource; -import org.springframework.util.SocketUtils; - -@SpringBootTest( - properties = { - "feast.jobs.enabled=true", - "feast.jobs.job_id_prefix=test-prefix", - "feast.jobs.polling_interval_milliseconds=1000", - "feast.stream.specsOptions.notifyIntervalMilliseconds=1000", - "feast.jobs.controller.consolidate-jobs-per-source=true", - "feast.jobs.controller.feature-set-selector[0].name=test", - "feast.jobs.controller.feature-set-selector[0].project=default", - "feast.jobs.controller.whitelisted-stores[0]=test-store", - "feast.jobs.controller.whitelisted-stores[1]=new-store", - "feast.version=1.0.0" - }) -public class JobControllerIT extends BaseIT { - @Autowired private FakeJobManager jobManager; - - @Autowired private JobRepository jobRepository; - - @Autowired KafkaTemplate ackPublisher; - - static CoreServiceGrpc.CoreServiceBlockingStub stub; - static List specsMailbox = new ArrayList<>(); - static SimpleCoreClient coreApiClient; - static SimpleJcClient apiClient; - - static int corePort = SocketUtils.findAvailableTcpPort(); - static ExternalApp coreApp = - ExternalApp.builder() - .setSpringApplication(feast.core.CoreApplication.class) - .setName("it-core") - .setGRPCPort(corePort) - .setPostgreSQL(postgreSQLContainer) - .build(); - - @DynamicPropertySource - static void properties(DynamicPropertyRegistry registry) { - registry.add("feast.core-port", () -> corePort); - } - - @BeforeAll - public static void globalSetUp(@Value("${grpc.server.port}") int localPort) { - coreApp.start(); - - ManagedChannel channel = - ManagedChannelBuilder.forAddress("localhost", corePort).usePlaintext().build(); - stub = CoreServiceGrpc.newBlockingStub(channel); - coreApiClient = new SimpleCoreClient(stub); - - apiClient = - new SimpleJcClient( - JobControllerServiceGrpc.newBlockingStub( - ManagedChannelBuilder.forAddress("localhost", localPort).usePlaintext().build())); - } - - @AfterAll - public static void globalTearDown() { - coreApp.stop(); - } - - @BeforeEach - public void setUp(TestInfo testInfo) { - coreApiClient.updateStore(DataGenerator.getDefaultStore()); - - specsMailbox = new ArrayList<>(); - - if (!isSequentialTest(testInfo)) { - jobManager.cleanAll(); - jobRepository.deleteAll(); - } - } - - @KafkaListener( - topics = {"${feast.stream.specsOptions.specsTopic}"}, - containerFactory = "testListenerContainerFactory") - public void listenSpecs(ConsumerRecord record) - throws InvalidProtocolBufferException { - FeatureSetProto.FeatureSetSpec featureSetSpec = - FeatureSetProto.FeatureSetSpec.parseFrom(record.value()); - specsMailbox.add(featureSetSpec); - } - - @Test - @SneakyThrows - public void shouldCreateJobForNewSource() { - coreApiClient.simpleApplyFeatureSet( - DataGenerator.createFeatureSet(DataGenerator.getDefaultSource(), "default", "test")); - - List featureSets = coreApiClient.simpleListFeatureSets("*"); - assertThat(featureSets.size(), equalTo(1)); - - await() - .until( - jobManager::getAllJobs, - containsInAnyOrder( - allOf( - hasProperty("id", containsString("kafka-1422433213")), - hasProperty("stores", aMapWithSize(1)), - hasProperty("featureSetDeliveryStatuses", aMapWithSize(1))))); - - // verify stay stable - Job job = jobManager.getAllJobs().get(0); - Thread.sleep(3000); - - assertThat( - jobManager.getAllJobs(), containsInAnyOrder(hasProperty("id", equalTo(job.getId())))); - assertThat( - jobManager.getAllJobs(), - containsInAnyOrder(hasProperty("id", containsString("test-prefix")))); - } - - @Test - public void shouldUpgradeJobWhenStoreChanged() { - coreApiClient.simpleApplyFeatureSet( - DataGenerator.createFeatureSet(DataGenerator.getDefaultSource(), "default", "test")); - - await().until(jobManager::getAllJobs, hasSize(1)); - - coreApiClient.updateStore( - DataGenerator.createStore( - "new-store", - StoreProto.Store.StoreType.REDIS, - ImmutableList.of(DataGenerator.getDefaultSubscription()))); - - await() - .until( - jobManager::getAllJobs, - containsInAnyOrder( - allOf( - hasProperty("stores", aMapWithSize(2)), - hasProperty("featureSetDeliveryStatuses", aMapWithSize(1))))); - - await().until(jobManager::getAllJobs, hasSize(1)); - } - - @Test - public void shouldRestoreJobThatStopped() { - coreApiClient.simpleApplyFeatureSet( - DataGenerator.createFeatureSet(DataGenerator.getDefaultSource(), "default", "test")); - - await().until(() -> jobRepository.findByStatus(JobStatus.RUNNING), hasSize(1)); - Job job = jobRepository.findByStatus(JobStatus.RUNNING).get(0); - - List ingestionJobs = apiClient.listIngestionJobs(); - assertThat(ingestionJobs, hasSize(1)); - assertThat(ingestionJobs, containsInAnyOrder(hasProperty("id", equalTo(job.getId())))); - - apiClient.restartIngestionJob(ingestionJobs.get(0).getId()); - - await().until(() -> jobManager.getJobStatus(job), equalTo(JobStatus.ABORTED)); - - await() - .until( - apiClient::listIngestionJobs, - hasItem( - allOf( - hasProperty("status", equalTo(IngestionJobProto.IngestionJobStatus.RUNNING)), - hasProperty("id", not(ingestionJobs.get(0).getId()))))); - } - - @Test - @SneakyThrows - public void shouldNotCreateJobForUnwantedFeatureSet() { - coreApiClient.simpleApplyFeatureSet( - DataGenerator.createFeatureSet(DataGenerator.getDefaultSource(), "default", "other")); - - Thread.sleep(2000); - - assertThat(jobManager.getAllJobs(), hasSize(0)); - } - - @Test - @SneakyThrows - public void shouldRestartJobWithOldVersion() { - coreApiClient.simpleApplyFeatureSet( - DataGenerator.createFeatureSet(DataGenerator.getDefaultSource(), "default", "test")); - - Job job = - Job.builder() - .setSource(DataGenerator.getDefaultSource()) - .setStores( - ImmutableMap.of( - DataGenerator.getDefaultStore().getName(), DataGenerator.getDefaultStore())) - .setId("some-running-id") - .setLabels(ImmutableMap.of(JobControllerService.VERSION_LABEL, "0-9-9")) - .build(); - - jobManager.startJob(job); - jobRepository.add(job); - - await().until(() -> jobManager.getJobStatus(job), equalTo(JobStatus.ABORTED)); - - Job replacement = jobRepository.findByStatus(JobStatus.RUNNING).get(0); - assertThat(replacement.getSource(), equalTo(job.getSource())); - assertThat(replacement.getStores(), equalTo(job.getStores())); - assertThat(replacement.getLabels(), hasEntry(JobControllerService.VERSION_LABEL, "1-0-0")); - } - - @TestMethodOrder(MethodOrderer.OrderAnnotation.class) - @Nested - class SpecNotificationFlow extends SequentialFlow { - Job job; - - @AfterAll - public void tearDown() { - jobManager.cleanAll(); - jobRepository.deleteAll(); - } - - @Test - @Order(1) - public void shouldSendNewSpec() { - jobManager.cleanAll(); - jobRepository.deleteAll(); - - job = - Job.builder() - .setSource(DataGenerator.getDefaultSource()) - .setStores( - ImmutableMap.of( - DataGenerator.getDefaultStore().getName(), DataGenerator.getDefaultStore())) - .setId("some-running-id") - .setLabels(ImmutableMap.of(JobControllerService.VERSION_LABEL, "1-0-0")) - .build(); - - jobManager.startJob(job); - jobRepository.add(job); - - coreApiClient.simpleApplyFeatureSet( - DataGenerator.createFeatureSet( - DataGenerator.getDefaultSource(), - "default", - "test", - ImmutableMap.of("entity", ValueProto.ValueType.Enum.BOOL), - ImmutableMap.of())); - - FeatureSetProto.FeatureSet featureSet = coreApiClient.simpleGetFeatureSet("default", "test"); - - assertThat( - featureSet.getMeta().getStatus(), - equalTo(FeatureSetProto.FeatureSetStatus.STATUS_PENDING)); - - await().until(() -> specsMailbox, hasSize(1)); - - assertThat( - specsMailbox.get(0), - allOf( - hasProperty("project", equalTo("default")), - hasProperty("name", equalTo("test")), - hasProperty("entitiesList", hasSize(1)), - hasProperty("version", equalTo(1)))); - - assertThat(jobRepository.findByStatus(JobStatus.RUNNING), hasSize(1)); - } - - @Test - @Order(2) - public void shouldUpdateSpec() { - coreApiClient.simpleApplyFeatureSet( - DataGenerator.createFeatureSet( - DataGenerator.getDefaultSource(), - "default", - "test", - ImmutableMap.of("entity", ValueProto.ValueType.Enum.BOOL), - ImmutableMap.of("feature", ValueProto.ValueType.Enum.INT32))); - - await().until(() -> specsMailbox, hasSize(1)); - - assertThat( - specsMailbox.get(0), - allOf( - hasProperty("project", equalTo("default")), - hasProperty("name", equalTo("test")), - hasProperty("featuresList", hasSize(1)), - hasProperty("version", equalTo(2)))); - } - - @Test - @Order(3) - public void shouldIgnoreOutdatedACKs() throws InterruptedException { - ackPublisher.sendDefault( - "default/test", - IngestionJobProto.FeatureSetSpecAck.newBuilder() - .setFeatureSetVersion(1) - .setJobName(job.getId()) - .setFeatureSetReference("default/test") - .build()); - - // time to process - Thread.sleep(1000); - - FeatureSetProto.FeatureSet featureSet = coreApiClient.simpleGetFeatureSet("default", "test"); - - assertThat( - featureSet.getMeta().getStatus(), - equalTo(FeatureSetProto.FeatureSetStatus.STATUS_PENDING)); - } - - @Test - @Order(4) - public void shouldUpdateDeliveryStatus() { - ackPublisher.sendDefault( - "default/test", - IngestionJobProto.FeatureSetSpecAck.newBuilder() - .setFeatureSetVersion(2) - .setJobName(job.getId()) - .setFeatureSetReference("default/test") - .build()); - - await() - .until( - () -> coreApiClient.simpleGetFeatureSet("default", "test").getMeta().getStatus(), - equalTo(FeatureSetProto.FeatureSetStatus.STATUS_READY)); - } - - @Test - @Order(5) - public void shouldReallocateFeatureSetAfterSourceChanged() { - assertThat(jobManager.getJobStatus(job), equalTo(JobStatus.RUNNING)); - - coreApiClient.simpleApplyFeatureSet( - DataGenerator.createFeatureSet( - DataGenerator.createSource("localhost", "newTopic"), - "default", - "test", - ImmutableMap.of("entity", ValueProto.ValueType.Enum.BOOL), - ImmutableMap.of("feature", ValueProto.ValueType.Enum.INT32))); - - await().until(() -> jobManager.getJobStatus(job), equalTo(JobStatus.ABORTED)); - - await().until(() -> jobRepository.findByStatus(JobStatus.RUNNING), hasSize(1)); - - await().until(() -> specsMailbox, hasSize(greaterThan(0))); - - assertThat( - specsMailbox.get(0), - allOf( - hasProperty("project", equalTo("default")), - hasProperty("name", equalTo("test")), - hasProperty("version", equalTo(3)))); - } - - @Test - @Order(6) - public void shouldUpdateStatusAfterACKFromNewJob() { - job = jobRepository.findByStatus(JobStatus.RUNNING).get(0); - - ackPublisher.sendDefault( - "default/test", - IngestionJobProto.FeatureSetSpecAck.newBuilder() - .setFeatureSetVersion(3) - .setJobName(job.getId()) - .setFeatureSetReference("default/test") - .build()); - - await() - .until( - () -> coreApiClient.simpleGetFeatureSet("default", "test").getMeta().getStatus(), - equalTo(FeatureSetProto.FeatureSetStatus.STATUS_READY)); - } - } - - @TestConfiguration - public static class TestConfig extends BaseIT.BaseTestConfig { - @Bean - public JobManager getJobManager() { - return new FakeJobManager(); - } - - @Bean - public KafkaTemplate specAckKafkaTemplate( - FeastProperties feastProperties) { - FeastProperties.StreamProperties streamProperties = feastProperties.getStream(); - Map props = new HashMap<>(); - - props.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, kafka.getBootstrapServers()); - - KafkaTemplate t = - new KafkaTemplate<>( - new DefaultKafkaProducerFactory<>( - props, new StringSerializer(), new KafkaSerialization.ProtoSerializer<>())); - t.setDefaultTopic(streamProperties.getSpecsOptions().getSpecsAckTopic()); - return t; - } - } -} diff --git a/job-controller/src/test/java/feast/jobcontroller/service/JobControllerServiceTest.java b/job-controller/src/test/java/feast/jobcontroller/service/JobControllerServiceTest.java deleted file mode 100644 index 1fe59253b37..00000000000 --- a/job-controller/src/test/java/feast/jobcontroller/service/JobControllerServiceTest.java +++ /dev/null @@ -1,439 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2019 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.jobcontroller.service; - -import static org.hamcrest.MatcherAssert.assertThat; -import static org.hamcrest.Matchers.containsInAnyOrder; -import static org.hamcrest.Matchers.hasSize; -import static org.hamcrest.beans.HasPropertyWithValue.hasProperty; -import static org.hamcrest.core.Is.isA; -import static org.hamcrest.core.IsIterableContaining.hasItem; -import static org.hamcrest.core.StringContains.containsString; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; -import static org.mockito.MockitoAnnotations.initMocks; - -import com.google.common.collect.*; -import com.google.protobuf.InvalidProtocolBufferException; -import feast.common.it.DataGenerator; -import feast.jobcontroller.config.FeastProperties; -import feast.jobcontroller.config.FeastProperties.JobProperties; -import feast.jobcontroller.dao.InMemoryJobRepository; -import feast.jobcontroller.dao.JobRepository; -import feast.jobcontroller.model.Job; -import feast.jobcontroller.model.JobStatus; -import feast.jobcontroller.runner.ConsolidatedJobStrategy; -import feast.jobcontroller.runner.JobManager; -import feast.jobcontroller.runner.JobPerStoreStrategy; -import feast.jobcontroller.runner.task.CreateJobTask; -import feast.jobcontroller.runner.task.JobTask; -import feast.jobcontroller.runner.task.UpdateJobStatusTask; -import feast.proto.core.CoreServiceGrpc; -import feast.proto.core.CoreServiceProto; -import feast.proto.core.CoreServiceProto.ListFeatureSetsRequest.Filter; -import feast.proto.core.CoreServiceProto.ListFeatureSetsResponse; -import feast.proto.core.CoreServiceProto.ListStoresResponse; -import feast.proto.core.FeatureSetProto.FeatureSet; -import feast.proto.core.SourceProto.Source; -import feast.proto.core.StoreProto.Store; -import java.util.*; -import lombok.SneakyThrows; -import org.apache.commons.lang3.tuple.Pair; -import org.apache.commons.lang3.tuple.Triple; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.mockito.Mock; -import org.springframework.kafka.core.KafkaTemplate; - -public class JobControllerServiceTest { - - JobRepository jobRepository; - - @Mock CoreServiceGrpc.CoreServiceBlockingStub specService; - - private FeastProperties feastProperties; - private JobControllerService controllerWithConsolidation; - private JobControllerService controllerWithJobPerStore; - - @BeforeEach - public void setUp() { - initMocks(this); - feastProperties = new FeastProperties(); - JobProperties jobProperties = new JobProperties(); - jobProperties.setJobUpdateTimeoutSeconds(5); - jobProperties.setJobIdPrefix("test-prefix"); - - FeastProperties.JobProperties.ControllerProperties.FeatureSetSelector selector = - new FeastProperties.JobProperties.ControllerProperties.FeatureSetSelector(); - selector.setName("fs*"); - selector.setProject("*"); - - FeastProperties.JobProperties.ControllerProperties controllerProperties = - new FeastProperties.JobProperties.ControllerProperties(); - controllerProperties.setFeatureSetSelector(ImmutableList.of(selector)); - controllerProperties.setWhitelistedStores( - ImmutableList.of("test-store", "test", "test-1", "test-2", "normal-store")); - controllerProperties.setJobSelector(ImmutableMap.of("application", "feast")); - - jobProperties.setController(controllerProperties); - feastProperties.setJobs(jobProperties); - feastProperties.setVersion("1.0.0"); - - JobManager jobManager = mock(JobManager.class); - - when(jobManager.listRunningJobs()).thenReturn(Collections.emptyList()); - jobRepository = new InMemoryJobRepository(jobManager); - - controllerWithConsolidation = - new JobControllerService( - jobRepository, - specService, - jobManager, - feastProperties, - new ConsolidatedJobStrategy(jobRepository, feastProperties.getJobs()), - mock(KafkaTemplate.class)); - - controllerWithJobPerStore = - new JobControllerService( - jobRepository, - specService, - jobManager, - feastProperties, - new JobPerStoreStrategy(jobRepository, feastProperties.getJobs()), - mock(KafkaTemplate.class)); - } - - @Test - public void shouldDoNothingIfNoStoresFound() { - when(specService.listStores(any())).thenReturn(ListStoresResponse.newBuilder().build()); - - List jobTasks = - controllerWithConsolidation.makeJobUpdateTasks( - controllerWithConsolidation.getSourceToStoreMappings()); - - assertThat(jobTasks, hasSize(0)); - } - - @Test - public void shouldDoNothingIfNoMatchingFeatureSetsFound() throws InvalidProtocolBufferException { - Store storeSpec = DataGenerator.getDefaultStore(); - - when(specService.listStores(any())) - .thenReturn(ListStoresResponse.newBuilder().addStore(storeSpec).build()); - when(specService.listFeatureSets( - CoreServiceProto.ListFeatureSetsRequest.newBuilder() - .setFilter(Filter.newBuilder().setProject("*").setFeatureSetName("*").build()) - .build())) - .thenReturn(ListFeatureSetsResponse.newBuilder().build()); - - List jobTasks = - controllerWithConsolidation.makeJobUpdateTasks( - controllerWithConsolidation.getSourceToStoreMappings()); - - assertThat(jobTasks, hasSize(0)); - } - - @Test - @SneakyThrows - public void shouldGroupJobsBySource() { - Store store = - DataGenerator.createStore( - "test", Store.StoreType.REDIS, ImmutableList.of(Triple.of("project1", "*", false))); - - Source source1 = DataGenerator.createSource("servers:9092", "topic"); - Source source2 = DataGenerator.createSource("others.servers:9092", "topic"); - - FeatureSet featureSet1 = DataGenerator.createFeatureSet(source1, "project1", "fs1"); - FeatureSet featureSet2 = DataGenerator.createFeatureSet(source2, "project1", "fs2"); - - when(specService.listFeatureSets( - CoreServiceProto.ListFeatureSetsRequest.newBuilder() - .setFilter( - Filter.newBuilder().setFeatureSetName("*").setProject("project1").build()) - .build())) - .thenReturn( - ListFeatureSetsResponse.newBuilder() - .addAllFeatureSets(Lists.newArrayList(featureSet1, featureSet2)) - .build()); - when(specService.listStores(any())) - .thenReturn(ListStoresResponse.newBuilder().addStore(store).build()); - - ArrayList>> pairs = - Lists.newArrayList(controllerWithConsolidation.getSourceToStoreMappings()); - - assertThat(pairs, hasSize(2)); - assertThat(pairs, hasItem(Pair.of(source1, Sets.newHashSet(store)))); - assertThat(pairs, hasItem(Pair.of(source2, Sets.newHashSet(store)))); - } - - @Test - @SneakyThrows - public void shouldUseStoreSubscriptionToMapStore() { - Store store1 = - DataGenerator.createStore( - "test", Store.StoreType.REDIS, ImmutableList.of(Triple.of("*", "features1", false))); - - Store store2 = - DataGenerator.createStore( - "test", Store.StoreType.REDIS, ImmutableList.of(Triple.of("*", "features2", false))); - - Source source1 = DataGenerator.createSource("servers:9092", "topic"); - Source source2 = DataGenerator.createSource("other.servers:9092", "topic"); - - FeatureSet featureSet1 = DataGenerator.createFeatureSet(source1, "default", "fs1"); - FeatureSet featureSet2 = DataGenerator.createFeatureSet(source2, "default", "fs2"); - - when(specService.listFeatureSets( - CoreServiceProto.ListFeatureSetsRequest.newBuilder() - .setFilter( - Filter.newBuilder().setFeatureSetName("features1").setProject("*").build()) - .build())) - .thenReturn( - ListFeatureSetsResponse.newBuilder() - .addAllFeatureSets(Lists.newArrayList(featureSet1)) - .build()); - - when(specService.listFeatureSets( - CoreServiceProto.ListFeatureSetsRequest.newBuilder() - .setFilter( - Filter.newBuilder().setFeatureSetName("features2").setProject("*").build()) - .build())) - .thenReturn( - ListFeatureSetsResponse.newBuilder() - .addAllFeatureSets(Lists.newArrayList(featureSet2)) - .build()); - - when(specService.listStores(any())) - .thenReturn(ListStoresResponse.newBuilder().addStore(store1).addStore(store2).build()); - - ArrayList>> pairs = - Lists.newArrayList(controllerWithConsolidation.getSourceToStoreMappings()); - - assertThat(pairs, hasSize(2)); - assertThat(pairs, hasItem(Pair.of(source1, Sets.newHashSet(store1)))); - assertThat(pairs, hasItem(Pair.of(source2, Sets.newHashSet(store2)))); - } - - @Test - public void shouldCheckStatusOfAbortingJob() { - Source source = DataGenerator.createSource("kafka:9092", "topic"); - Store store = DataGenerator.getDefaultStore(); - - Job job = - Job.builder() - .setSource(source) - .setStores(ImmutableMap.of(store.getName(), store)) - .setId("some-id") - .build(); - job.setStatus(JobStatus.ABORTING); - job.setExtId("extId"); - - jobRepository.add(job); - - List tasks = - controllerWithConsolidation.makeJobUpdateTasks( - ImmutableList.of(Pair.of(source, ImmutableSet.of(store)))); - - assertThat("CheckStatus is expected", tasks.get(0) instanceof UpdateJobStatusTask); - } - - @Test - public void shouldUpgradeJobWhenNeeded() { - Source source = DataGenerator.createSource("kafka:9092", "topic"); - Store store = DataGenerator.getDefaultStore(); - - Job job = Job.builder().setSource(source).setId("some-id").build(); - - job.setStatus(JobStatus.RUNNING); - job.setExtId("extId"); - - jobRepository.add(job); - - List tasks = - controllerWithConsolidation.makeJobUpdateTasks( - ImmutableList.of(Pair.of(source, ImmutableSet.of(store)))); - - assertThat("CreateTask is expected", tasks.get(0) instanceof CreateJobTask); - } - - @Test - @SneakyThrows - public void shouldCreateJobIfNoRunning() { - Source source = DataGenerator.createSource("kafka:9092", "topic"); - Store store = DataGenerator.getDefaultStore(); - - when(specService.listFeatureSets( - CoreServiceProto.ListFeatureSetsRequest.newBuilder() - .setFilter(Filter.newBuilder().setFeatureSetName("*").setProject("*").build()) - .build())) - .thenReturn(ListFeatureSetsResponse.newBuilder().build()); - - List tasks = - controllerWithConsolidation.makeJobUpdateTasks( - ImmutableList.of(Pair.of(source, ImmutableSet.of(store)))); - - assertThat(tasks.get(0).getJob().getId(), containsString("test-prefix")); - assertThat("CreateTask is expected", tasks.get(0) instanceof CreateJobTask); - } - - @Test - public void shouldCreateJobPerStore() throws InvalidProtocolBufferException { - Store store1 = - DataGenerator.createStore( - "test-1", Store.StoreType.REDIS, ImmutableList.of(Triple.of("*", "*", false))); - Store store2 = - DataGenerator.createStore( - "test-2", Store.StoreType.REDIS, ImmutableList.of(Triple.of("*", "*", false))); - - Source source = DataGenerator.createSource("servers:9092", "topic"); - - FeatureSet featureSet = DataGenerator.createFeatureSet(source, "default", "fs1"); - - when(specService.listFeatureSets( - CoreServiceProto.ListFeatureSetsRequest.newBuilder() - .setFilter(Filter.newBuilder().setFeatureSetName("*").setProject("*").build()) - .build())) - .thenReturn( - ListFeatureSetsResponse.newBuilder() - .addAllFeatureSets(Lists.newArrayList(featureSet)) - .build()); - when(specService.listStores(any())) - .thenReturn(ListStoresResponse.newBuilder().addStore(store1).addStore(store2).build()); - - List jobTasks = - controllerWithJobPerStore.makeJobUpdateTasks( - controllerWithJobPerStore.getSourceToStoreMappings()); - - int hash = - Objects.hash( - source.getKafkaSourceConfig().getBootstrapServers(), - source.getKafkaSourceConfig().getTopic()); - - assertThat(jobTasks, hasSize(2)); - assertThat( - jobTasks, - hasItem( - hasProperty( - "job", - hasProperty("id", containsString(String.format("kafka-%d-to-test-1", hash)))))); - assertThat( - jobTasks, - hasItem( - hasProperty( - "job", - hasProperty("id", containsString(String.format("kafka-%d-to-test-2", hash)))))); - } - - @Test - public void shouldCloneRunningJobOnUpgrade() throws InvalidProtocolBufferException { - Store store1 = - DataGenerator.createStore( - "test-1", Store.StoreType.REDIS, ImmutableList.of(Triple.of("*", "*", false))); - Store store2 = - DataGenerator.createStore( - "test-2", Store.StoreType.REDIS, ImmutableList.of(Triple.of("*", "*", false))); - - Source source = DataGenerator.createSource("servers:9092", "topic"); - - Job existingJob = - Job.builder() - .setId("some-id") - .setSource(source) - .setStores(ImmutableMap.of(store1.getName(), store1)) - .build(); - - existingJob.setExtId("extId"); - existingJob.setStatus(JobStatus.RUNNING); - - jobRepository.add(existingJob); - - List jobTasks = - controllerWithConsolidation.makeJobUpdateTasks( - ImmutableList.of(Pair.of(source, ImmutableSet.of(store1, store2)))); - - assertThat(jobTasks, hasSize(1)); - assertThat(jobTasks, hasItem(isA(CreateJobTask.class))); - } - - @Test - @SneakyThrows - public void shouldSelectOnlyFeatureSetsThatJobManagerSubscribedTo() { - Store store = DataGenerator.getDefaultStore(); - Source source = DataGenerator.getDefaultSource(); - - FeatureSet featureSet1 = DataGenerator.createFeatureSet(source, "default", "fs1"); - FeatureSet featureSet2 = DataGenerator.createFeatureSet(source, "project", "fs3"); - FeatureSet featureSet3 = DataGenerator.createFeatureSet(source, "default", "not-fs"); - - when(specService.listFeatureSets( - CoreServiceProto.ListFeatureSetsRequest.newBuilder() - .setFilter(Filter.newBuilder().setFeatureSetName("*").setProject("*").build()) - .build())) - .thenReturn( - ListFeatureSetsResponse.newBuilder() - .addAllFeatureSets(Lists.newArrayList(featureSet1, featureSet2, featureSet3)) - .build()); - - List featureSetsForStore = - controllerWithConsolidation.getFeatureSetsForStore(store); - assertThat(featureSetsForStore, containsInAnyOrder(featureSet1, featureSet2)); - } - - @Test - @SneakyThrows - public void shouldSelectOnlyStoresThatWhitelisted() { - Store store1 = - DataGenerator.createStore( - "normal-store", - Store.StoreType.REDIS, - ImmutableList.of(Triple.of("project1", "*", false))); - Store store2 = - DataGenerator.createStore( - "blacklisted-store", - Store.StoreType.REDIS, - ImmutableList.of(Triple.of("project2", "*", false))); - - Source source1 = DataGenerator.createSource("source-1", "topic"); - Source source2 = DataGenerator.createSource("source-2", "topic"); - - FeatureSet featureSet1 = DataGenerator.createFeatureSet(source1, "default", "fs1"); - FeatureSet featureSet2 = DataGenerator.createFeatureSet(source2, "project", "fs3"); - - when(specService.listStores(any())) - .thenReturn(ListStoresResponse.newBuilder().addStore(store1).addStore(store2).build()); - - when(specService.listFeatureSets( - CoreServiceProto.ListFeatureSetsRequest.newBuilder() - .setFilter( - Filter.newBuilder().setProject("project1").setFeatureSetName("*").build()) - .build())) - .thenReturn(ListFeatureSetsResponse.newBuilder().addFeatureSets(featureSet1).build()); - - when(specService.listFeatureSets( - CoreServiceProto.ListFeatureSetsRequest.newBuilder() - .setFilter( - Filter.newBuilder().setProject("project2").setFeatureSetName("*").build()) - .build())) - .thenReturn(ListFeatureSetsResponse.newBuilder().addFeatureSets(featureSet2).build()); - - ArrayList>> pairs = - Lists.newArrayList(controllerWithConsolidation.getSourceToStoreMappings()); - - assertThat(pairs, containsInAnyOrder(Pair.of(source1, ImmutableSet.of(store1)))); - } -} diff --git a/job-controller/src/test/java/feast/jobcontroller/service/JobServiceIT.java b/job-controller/src/test/java/feast/jobcontroller/service/JobServiceIT.java deleted file mode 100644 index 3a6e2a23966..00000000000 --- a/job-controller/src/test/java/feast/jobcontroller/service/JobServiceIT.java +++ /dev/null @@ -1,212 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.jobcontroller.service; - -import static org.hamcrest.CoreMatchers.equalTo; -import static org.hamcrest.MatcherAssert.assertThat; -import static org.hamcrest.beans.HasPropertyWithValue.hasProperty; -import static org.hamcrest.collection.IsCollectionWithSize.hasSize; -import static org.hamcrest.collection.IsIterableContainingInAnyOrder.containsInAnyOrder; - -import com.google.common.collect.ImmutableMap; -import feast.common.it.BaseIT; -import feast.common.it.DataGenerator; -import feast.common.models.FeatureSetReference; -import feast.jobcontroller.dao.JobRepository; -import feast.jobcontroller.model.FeatureSetDeliveryStatus; -import feast.jobcontroller.model.Job; -import feast.jobcontroller.model.JobStatus; -import feast.jobcontroller.runner.JobManager; -import feast.proto.core.CoreServiceProto; -import feast.proto.core.FeatureSetReferenceProto; -import feast.proto.core.JobControllerServiceGrpc; -import io.grpc.ManagedChannel; -import io.grpc.ManagedChannelBuilder; -import io.grpc.StatusRuntimeException; -import org.junit.jupiter.api.*; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.boot.test.context.TestConfiguration; -import org.springframework.context.annotation.Bean; - -@SpringBootTest() -public class JobServiceIT extends BaseIT { - @Autowired private FakeJobManager jobManager; - - @Autowired private JobRepository jobRepository; - - static JobControllerServiceGrpc.JobControllerServiceBlockingStub stub; - - Job job; - - @BeforeAll - public static void globalSetUp(@Value("${grpc.server.port}") int port) { - ManagedChannel channel = - ManagedChannelBuilder.forAddress("localhost", port).usePlaintext().build(); - stub = JobControllerServiceGrpc.newBlockingStub(channel); - } - - private Job createJobWithId(String jobId) { - return Job.builder() - .setId(jobId) - .setSource(DataGenerator.getDefaultSource()) - .setStores( - ImmutableMap.of( - DataGenerator.getDefaultStore().getName(), DataGenerator.getDefaultStore())) - .build(); - } - - @BeforeEach - public void createJob() { - this.job = createJobWithId("some-id"); - this.jobManager.startJob(job); - this.jobRepository.add(job); - } - - @AfterEach - public void tearDown() { - this.jobManager.cleanAll(); - this.jobRepository.deleteAll(); - } - - @Test - public void shouldReturnListOfJobsById() { - CoreServiceProto.ListIngestionJobsRequest.Filter filter = - CoreServiceProto.ListIngestionJobsRequest.Filter.newBuilder() - .setId(this.job.getId()) - .build(); - - assertReturnsJob(filter); - CoreServiceProto.ListIngestionJobsRequest request; - - // list with no filter - request = CoreServiceProto.ListIngestionJobsRequest.newBuilder().build(); - assertThat( - stub.listIngestionJobs(request).getJobsList(), - containsInAnyOrder(hasProperty("id", equalTo(this.job.getId())))); - - // list with empty filter - filter = CoreServiceProto.ListIngestionJobsRequest.Filter.newBuilder().build(); - assertReturnsJob(filter); - } - - @Test - public void shouldReturnListOfJobsByStoreName() { - CoreServiceProto.ListIngestionJobsRequest.Filter filter = - CoreServiceProto.ListIngestionJobsRequest.Filter.newBuilder() - .setStoreName(DataGenerator.getDefaultStore().getName()) - .build(); - - assertReturnsJob(filter); - } - - private void assertReturnsJob(CoreServiceProto.ListIngestionJobsRequest.Filter filter) { - CoreServiceProto.ListIngestionJobsRequest request = - CoreServiceProto.ListIngestionJobsRequest.newBuilder().setFilter(filter).build(); - - assertThat( - stub.listIngestionJobs(request).getJobsList(), - containsInAnyOrder(hasProperty("id", equalTo(this.job.getId())))); - } - - @Test - public void shouldReturnListOfJobsByByFeatureSetReference() { - FeatureSetReference ref = FeatureSetReference.of("default", "fs"); - FeatureSetDeliveryStatus featureSetDeliveryStatus = new FeatureSetDeliveryStatus(ref); - - this.job.getFeatureSetDeliveryStatuses().put(ref, featureSetDeliveryStatus); - - // list job by feature set reference: name and project - CoreServiceProto.ListIngestionJobsRequest.Filter filter = - CoreServiceProto.ListIngestionJobsRequest.Filter.newBuilder() - .setFeatureSetReference( - FeatureSetReferenceProto.FeatureSetReference.newBuilder() - .setProject(ref.getProjectName()) - .setName(ref.getFeatureSetName()) - .build()) - .build(); - - assertReturnsJob(filter); - - // list job by feature set reference: name - filter = - CoreServiceProto.ListIngestionJobsRequest.Filter.newBuilder() - .setFeatureSetReference( - FeatureSetReferenceProto.FeatureSetReference.newBuilder() - .setName(ref.getFeatureSetName()) - .build()) - .build(); - assertReturnsJob(filter); - } - - @Test - public void shouldStopJobById() { - CoreServiceProto.StopIngestionJobRequest request = - CoreServiceProto.StopIngestionJobRequest.newBuilder().setId(this.job.getId()).build(); - - stub.stopIngestionJob(request); - - assertThat(this.job.getStatus(), equalTo(JobStatus.ABORTING)); - assertThat(this.jobManager.getAllJobs(), hasSize(0)); - } - - @Test - public void shouldNotStopJobInTransition() { - this.job.setStatus(JobStatus.UNKNOWN); - - CoreServiceProto.StopIngestionJobRequest request = - CoreServiceProto.StopIngestionJobRequest.newBuilder().setId(this.job.getId()).build(); - - Assertions.assertThrows(StatusRuntimeException.class, () -> stub.stopIngestionJob(request)); - } - - @Test - public void shouldNotStopUnknownJob() { - CoreServiceProto.StopIngestionJobRequest request = - CoreServiceProto.StopIngestionJobRequest.newBuilder().setId("unknown-id").build(); - - Assertions.assertThrows(StatusRuntimeException.class, () -> stub.stopIngestionJob(request)); - } - - @Test - public void shouldRestartJob() { - CoreServiceProto.RestartIngestionJobRequest request = - CoreServiceProto.RestartIngestionJobRequest.newBuilder().setId(this.job.getId()).build(); - stub.restartIngestionJob(request); - - assertThat(this.job.getStatus(), equalTo(JobStatus.ABORTING)); - } - - @Test - public void shouldNotRestartJobInTransition() { - this.job.setStatus(JobStatus.UNKNOWN); - - CoreServiceProto.RestartIngestionJobRequest request = - CoreServiceProto.RestartIngestionJobRequest.newBuilder().setId(this.job.getId()).build(); - - Assertions.assertThrows(StatusRuntimeException.class, () -> stub.restartIngestionJob(request)); - } - - @TestConfiguration - public static class TestConfig extends BaseIT.BaseTestConfig { - @Bean - public JobManager getJobManager() { - return new FakeJobManager(); - } - } -} diff --git a/job-controller/src/test/resources/application-it-core.yml b/job-controller/src/test/resources/application-it-core.yml deleted file mode 100644 index 6072acd39e5..00000000000 --- a/job-controller/src/test/resources/application-it-core.yml +++ /dev/null @@ -1,65 +0,0 @@ -feast: - stream: - # Feature stream type. Only kafka is supported. - type: kafka - # Feature stream options. - # See the following for options https://api.docs.feast.dev/grpc/feast.core.pb.html#KafkaSourceConfig - options: - topic: feast-features - bootstrapServers: localhost:9092 - replicationFactor: 1 - partitions: 1 - specsOptions: - specsTopic: feast-specs - specsAckTopic: feast-specs-ack - notifyIntervalMilliseconds: 1000 - - security: - authentication: - enabled: false - provider: jwt - options: - jwkEndpointURI: "https://www.googleapis.com/oauth2/v3/certs" - - authorization: - enabled: false - provider: http - options: - authorizationUrl: http://localhost:8082 - subjectClaim: email - - logging: - # Audit logging provides a machine readable structured JSON log that can give better - # insight into what is happening in Feast. - audit: - # Whether audit logging is enabled. - enabled: true - # Whether to enable message level (ie request/response) audit logging - messageLoggingEnabled: false - -grpc: - server: - security: - enabled: false - -spring: - jpa: - properties.hibernate: - format_sql: true - event: - merge: - entity_copy_observer: allow - hibernate.naming.physical-strategy=org.hibernate.boot.model.naming: PhysicalNamingStrategyStandardImpl - hibernate.ddl-auto: none - datasource: - driverClassName: org.postgresql.Driver - -management: - metrics: - export: - simple: - enabled: false - statsd: - enabled: true - host: ${STATSD_HOST:localhost} - port: ${STATSD_PORT:8125} diff --git a/job-controller/src/test/resources/application-it.properties b/job-controller/src/test/resources/application-it.properties deleted file mode 100644 index ccb339fa9ab..00000000000 --- a/job-controller/src/test/resources/application-it.properties +++ /dev/null @@ -1,31 +0,0 @@ -# -# Copyright 2018 The Feast Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# -grpc.server.port=6666 - -feast.security.authentication.enabled = false -feast.security.authorization.enabled = false - -feast.jobs.enabled=false - -spring.jpa.properties.hibernate.format_sql=true -spring.jpa.properties.hibernate.show_sql=false -spring.jpa.hibernate.naming.physical-strategy=org.springframework.boot.orm.jpa.hibernate.SpringPhysicalNamingStrategy -spring.jpa.hibernate.ddl-auto=none - -spring.datasource.hikari.maximum-pool-size=100 -spring.main.allow-bean-definition-overriding=true - diff --git a/job-controller/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker b/job-controller/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker deleted file mode 100644 index ca6ee9cea8e..00000000000 --- a/job-controller/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker +++ /dev/null @@ -1 +0,0 @@ -mock-maker-inline \ No newline at end of file diff --git a/pom.xml b/pom.xml index 409e60e6f4e..49fc65454eb 100644 --- a/pom.xml +++ b/pom.xml @@ -41,7 +41,7 @@ - 0.8.0 + 0.9.0 https://github.com/feast-dev/feast UTF-8 @@ -54,7 +54,6 @@ 5.2.7.RELEASE 5.3.0.RELEASE 2.9.0.RELEASE - 2.22.0 1.111.1 0.8.0 1.9.10 @@ -106,6 +105,7 @@ ${maven.multiModuleProjectDirectory} false + false feast.common.auth.providers.http.client @@ -487,7 +487,11 @@ pom import - + + com.squareup.okio + okio + 1.17.2 + javax.xml.bind jaxb-api diff --git a/protos/feast/core/CoreService.proto b/protos/feast/core/CoreService.proto index 5e7eb824183..e9b7b4c43ba 100644 --- a/protos/feast/core/CoreService.proto +++ b/protos/feast/core/CoreService.proto @@ -24,41 +24,23 @@ option java_package = "feast.proto.core"; import "google/protobuf/timestamp.proto"; import "tensorflow_metadata/proto/v0/statistics.proto"; import "feast/core/Entity.proto"; -import "feast/core/FeatureSet.proto"; +import "feast/core/Feature.proto"; import "feast/core/FeatureTable.proto"; import "feast/core/Store.proto"; -import "feast/core/FeatureSetReference.proto"; -import "feast/core/IngestionJob.proto"; service CoreService { // Retrieve version information about this Feast deployment rpc GetFeastCoreVersion (GetFeastCoreVersionRequest) returns (GetFeastCoreVersionResponse); - // Returns a specific feature set - rpc GetFeatureSet (GetFeatureSetRequest) returns (GetFeatureSetResponse); - // Returns a specific entity rpc GetEntity (GetEntityRequest) returns (GetEntityResponse); - // Retrieve feature set details given a filter. - // - // Returns all feature sets matching that filter. If none are found, - // an empty list will be returned. - // If no filter is provided in the request, the response will contain all the feature - // sets currently stored in the registry. - rpc ListFeatureSets (ListFeatureSetsRequest) returns (ListFeatureSetsResponse); - // Returns all feature references and respective features matching that filter. If none are found // an empty map will be returned // If no filter is provided in the request, the response will contain all the features // currently stored in the default project. rpc ListFeatures (ListFeaturesRequest) returns (ListFeaturesResponse); - // Get feature statistics computed over the data in the batch stores. - // - // Returns a dataset containing TFDV statistics mapped to each valid historical store. - rpc GetFeatureStatistics (GetFeatureStatisticsRequest) returns (GetFeatureStatisticsResponse); - // Retrieve store details given a filter. // // Returns all stores matching that filter. If none are found, an empty list will be returned. @@ -66,16 +48,6 @@ service CoreService { // stored in the registry. rpc ListStores (ListStoresRequest) returns (ListStoresResponse); - // Create or update and existing feature set. - // - // This function is idempotent - it will not create a new feature set if schema does not change. - // Schema changes will update the feature set if the changes are valid. - // All changes except the following are valid: - // - Changes to feature set id (name, project) - // - Changes to entities - // - Changes to feature name and type - rpc ApplyFeatureSet (ApplyFeatureSetRequest) returns (ApplyFeatureSetResponse); - // Create or update and existing entity. // // This function is idempotent - it will not create a new entity if schema does not change. @@ -98,8 +70,8 @@ service CoreService { rpc UpdateStore (UpdateStoreRequest) returns (UpdateStoreResponse); // Creates a project. Projects serve as namespaces within which resources like features will be - // created. Feature set names as must be unique within a project while field (Feature/Entity) names - // must be unique within a Feature Set. Project names themselves must be globally unique. + // created. Feature table names as must be unique within a project while field (Feature/Entity) names + // must be unique within a Feature Table. Project names themselves must be globally unique. rpc CreateProject (CreateProjectRequest) returns (CreateProjectResponse); // Archives a project. Archived projects will continue to exist and function, but won't be visible @@ -111,9 +83,6 @@ service CoreService { // Lists all projects active projects. rpc ListProjects (ListProjectsRequest) returns (ListProjectsResponse); - // Internal API for Job Controller to update featureSet's status once responsible ingestion job is running - rpc UpdateFeatureSetStatus (UpdateFeatureSetStatusRequest) returns (UpdateFeatureSetStatusResponse); - /* Feature Tables */ // Create or update an existing feature table. // This function is idempotent - it will not create a new feature table if the schema does not change. @@ -139,76 +108,6 @@ service CoreService { } -service JobControllerService { - // List Ingestion Jobs given an optional filter. - // Returns allow ingestions matching the given request filter. - // Returns all ingestion jobs if no filter is provided. - // Returns an empty list if no ingestion jobs match the filter. - rpc ListIngestionJobs (ListIngestionJobsRequest) returns (ListIngestionJobsResponse); - - // Restart an Ingestion Job. Restarts the ingestion job with the given job id. - // NOTE: Data might be lost during the restart for some job runners. - // Does not support stopping a job in a transitional (ie pending, suspending, aborting), - // terminal state (ie suspended or aborted) or unknown status - rpc RestartIngestionJob (RestartIngestionJobRequest) returns (RestartIngestionJobResponse); - - // Stop an Ingestion Job. Stop (Aborts) the ingestion job with the given job id. - // Does nothing if the target job if already in a terminal state (ie suspended or aborted). - // Does not support stopping a job in a transitional (ie pending, suspending, aborting) or unknown status - rpc StopIngestionJob (StopIngestionJobRequest) returns (StopIngestionJobResponse); -} - -// Request for a single feature set -message GetFeatureSetRequest { - // Name of project the feature set belongs to. If omitted will default to 'default' project. - string project = 3; - - // Name of feature set (required). - string name = 1; -} - -// Response containing a single feature set -message GetFeatureSetResponse { - feast.core.FeatureSet feature_set = 1; -} - -// Retrieves details for all versions of a specific feature set -message ListFeatureSetsRequest { - Filter filter = 1; - - message Filter { - // Name of project that the feature sets belongs to. This can be one of - // - [project_name] - // - * - // If an asterisk is provided, filtering on projects will be disabled. All projects will - // be matched. It is NOT possible to provide an asterisk with a string in order to do - // pattern matching. - // If unspecified this field will default to the default project 'default'. - string project = 3; - - // Name of the desired feature set. Asterisks can be used as wildcards in the name. - // Matching on names is only permitted if a specific project is defined. It is disallowed - // If the project name is set to "*" - // e.g. - // - * can be used to match all feature sets - // - my-feature-set* can be used to match all features prefixed by "my-feature-set" - // - my-feature-set-6 can be used to select a single feature set - string feature_set_name = 1; - - // User defined metadata for feature set. - // Feature sets with all matching labels will be returned. - map labels = 4; - - // Filter by FeatureSet's current status - // Project and Feature Set name still must be specified (could be "*") - FeatureSetStatus status = 5; - } -} - -message ListFeatureSetsResponse { - repeated feast.core.FeatureSet feature_sets = 1; -} - // Request for a single entity message GetEntityRequest { // Name of entity (required). @@ -250,10 +149,10 @@ message ListFeaturesRequest { map labels = 1; // List of entities contained within the featureSet that the feature belongs to. - // Only feature sets with these entities will be searched for features. + // Only feature tables with these entities will be searched for features. repeated string entities = 2; - // Name of project that the feature sets belongs to. Filtering on projects is disabled. + // Name of project that the feature tables belongs to. Filtering on projects is disabled. // It is NOT possible to provide an asterisk with a string in order to do pattern matching. // If unspecified this field will default to the default project 'default'. string project = 3; @@ -263,7 +162,9 @@ message ListFeaturesRequest { } message ListFeaturesResponse { - map features = 1; + reserved 1; + + map features = 2; } message ListStoresRequest { @@ -292,33 +193,6 @@ message ApplyEntityResponse { feast.core.Entity entity = 1; } -message ApplyFeatureSetRequest { - // Feature set version - // If project is unspecified, will default to 'default' project. - // If project specified does not exist, the project would be automatically created. - feast.core.FeatureSet feature_set = 1; -} - -message ApplyFeatureSetResponse { - // TODO: 0 should correspond to invalid rather than NO_CHANGE - enum Status { - // Latest feature set is consistent with provided feature set - NO_CHANGE = 0; - - // New feature set created - CREATED = 1; - - // Error occurred while trying to apply changes - ERROR = 2; - - // Changes detected and updated successfully - UPDATED = 3; - } - - feast.core.FeatureSet feature_set = 1; - Status status = 2; -} - message GetFeastCoreVersionRequest { } @@ -372,90 +246,6 @@ message ListProjectsResponse { repeated string projects = 1; } -// Request for listing ingestion jobs -message ListIngestionJobsRequest { - Filter filter = 1; - - message Filter { - // Filter by Job ID assigned by Feast - string id = 1; - // Filter by ingestion job target feature set. - FeatureSetReference feature_set_reference = 2; - // Filter by Name of store - string store_name = 3; - } -} - -// Response from listing ingestion jobs -message ListIngestionJobsResponse { - repeated IngestionJob jobs = 1; -} - -// Request to restart ingestion job -message RestartIngestionJobRequest { - // Job ID assigned by Feast - string id = 1; -} - -// Response from restartingan injestion job -message RestartIngestionJobResponse {} - - -// Request to stop ingestion job -message StopIngestionJobRequest { - // Job ID assigned by Feast - string id = 1; -} - -// Request from stopping an ingestion job -message StopIngestionJobResponse {} - -message GetFeatureStatisticsRequest { - // Feature set to retrieve the statistics for. A fully qualified feature set - // id in the format of project/feature_set must be provided. - string feature_set_id = 1; - - // Optional filter which filters returned statistics by selected features. These - // features must be present in the data that is being processed. - repeated string features = 2; - - // Optional filter to select store over which the statistics will retrieved. - // Only historical stores are allowed. - string store = 3; - - // Optional start and end dates over which to filter statistical data - // Start date is inclusive, but end date is not. - // Only dates are supported, not times. - // Cannot be used with dataset_ids. - // If this period spans multiple days, unaggregatable statistics will be dropped. - google.protobuf.Timestamp start_date = 4; - google.protobuf.Timestamp end_date = 5; - - // Optional list of ingestion Ids by which to filter data before - // retrieving statistics. - // Cannot be used with the date ranges - // If multiple dataset ids are provided, unaggregatable statistics will be dropped. - repeated string ingestion_ids = 6; - - // Setting this flag to true will force a recalculation of statistics and overwrite results currently in the - // cache, if any. - bool force_refresh = 7; -} - -message GetFeatureStatisticsResponse { - // Contains statistics for the requested data. - // Due to the limitations of TFDV and Facets, only a single dataset can be returned in, - // despite the message being of list type. - tensorflow.metadata.v0.DatasetFeatureStatisticsList dataset_feature_statistics_list = 1; -} - -message UpdateFeatureSetStatusRequest { - // FeatureSetReference of FeatureSet to update - FeatureSetReference reference = 1; - // Target status - FeatureSetStatus status = 2; -} - message UpdateFeatureSetStatusResponse {} message ApplyFeatureTableRequest { diff --git a/protos/feast/core/FeatureSet.proto b/protos/feast/core/FeatureSet.proto deleted file mode 100644 index de22388ce76..00000000000 --- a/protos/feast/core/FeatureSet.proto +++ /dev/null @@ -1,161 +0,0 @@ -// -// * Copyright 2019 The Feast Authors -// * -// * Licensed under the Apache License, Version 2.0 (the "License"); -// * you may not use this file except in compliance with the License. -// * You may obtain a copy of the License at -// * -// * https://www.apache.org/licenses/LICENSE-2.0 -// * -// * Unless required by applicable law or agreed to in writing, software -// * distributed under the License is distributed on an "AS IS" BASIS, -// * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// * See the License for the specific language governing permissions and -// * limitations under the License. -// - -syntax = "proto3"; -package feast.core; -option java_package = "feast.proto.core"; -option java_outer_classname = "FeatureSetProto"; -option go_package = "github.com/feast-dev/feast/sdk/go/protos/feast/core"; - -import "feast/types/Value.proto"; -import "feast/core/Source.proto"; -import "google/protobuf/duration.proto"; -import "google/protobuf/timestamp.proto"; -import "tensorflow_metadata/proto/v0/schema.proto"; - -message FeatureSet { - // User-specified specifications of this feature set. - FeatureSetSpec spec = 1; - // System-populated metadata for this feature set. - FeatureSetMeta meta = 2; -} - -message FeatureSetSpec { - // Name of project that this feature set belongs to. - string project = 7; - - // Name of the feature set. Must be unique. - string name = 1; - - // Feature set version was removed in v0.5.0. - reserved 2; - - // List of entities contained within this featureSet. - // This allows the feature to be used during joins between feature sets. - // If the featureSet is ingested into a store that supports keys, this value - // will be made a key. - repeated EntitySpec entities = 3; - - // List of features contained within this featureSet. - repeated FeatureSpec features = 4; - - // Features in this feature set will only be retrieved if they are found - // after [time - max_age]. Missing or older feature values will be returned - // as nulls and indicated to end user - google.protobuf.Duration max_age = 5; - - // Optional. Source on which feature rows can be found. - // If not set, source will be set to the default value configured in Feast Core. - Source source = 6; - - // User defined metadata - map labels = 8; - - // Read-only self-incrementing version that increases monotonically - // when changes are made to a feature set - int32 version = 9; -} - -message EntitySpec { - // Name of the entity. - string name = 1; - - // Value type of the entity. - feast.types.ValueType.Enum value_type = 2; -} - -message FeatureSpec { - // Name of the feature. - string name = 1; - - // Value type of the feature. - feast.types.ValueType.Enum value_type = 2; - - // Reserve field numbers 15 and below for fields that will almost always be set - // https://developers.google.com/protocol-buffers/docs/proto3#assigning-field-numbers - reserved 3 to 15; - - // Labels for user defined metadata on a feature - map labels = 16; - - // Reserved for fundamental future additions less noisy in the schema that TFDV stats fields - reserved 17 to 29; - - // presence_constraints, shape_type and domain_info are referenced from: - // https://github.com/tensorflow/metadata/blob/36f65d1268cbc92cdbcf812ee03dcf47fb53b91e/tensorflow_metadata/proto/v0/schema.proto#L107 - - oneof presence_constraints { - // Constraints on the presence of this feature in the examples. - tensorflow.metadata.v0.FeaturePresence presence = 30; - // Only used in the context of a "group" context, e.g., inside a sequence. - tensorflow.metadata.v0.FeaturePresenceWithinGroup group_presence = 31; - } - - // The shape of the feature which governs the number of values that appear in - // each example. - oneof shape_type { - // The feature has a fixed shape corresponding to a multi-dimensional - // tensor. - tensorflow.metadata.v0.FixedShape shape = 32; - // The feature doesn't have a well defined shape. All we know are limits on - // the minimum and maximum number of values. - tensorflow.metadata.v0.ValueCount value_count = 33; - } - - // Domain for the values of the feature. - oneof domain_info { - // Reference to a domain defined at the schema level. - string domain = 34; - // Inline definitions of domains. - tensorflow.metadata.v0.IntDomain int_domain = 35; - tensorflow.metadata.v0.FloatDomain float_domain = 36; - tensorflow.metadata.v0.StringDomain string_domain = 37; - tensorflow.metadata.v0.BoolDomain bool_domain = 38; - tensorflow.metadata.v0.StructDomain struct_domain = 39; - // Supported semantic domains. - tensorflow.metadata.v0.NaturalLanguageDomain natural_language_domain = 40; - tensorflow.metadata.v0.ImageDomain image_domain = 41; - tensorflow.metadata.v0.MIDDomain mid_domain = 42; - tensorflow.metadata.v0.URLDomain url_domain = 43; - tensorflow.metadata.v0.TimeDomain time_domain = 44; - tensorflow.metadata.v0.TimeOfDayDomain time_of_day_domain = 45; - } -} - -message FeatureSetMeta { - // Created timestamp of this specific feature set. - google.protobuf.Timestamp created_timestamp = 1; - - // Status of the feature set. - // Used to indicate whether the feature set is ready for consumption or ingestion. - // Currently supports 2 states: - // 1) STATUS_PENDING - A feature set is in pending state if Feast has not spun up the jobs - // necessary to push rows for this feature set to stores subscribing to this feature set. - // 2) STATUS_READY - Feature set is ready for consumption or ingestion - FeatureSetStatus status = 2; -} - -enum FeatureSetStatus { - STATUS_INVALID = 0; - STATUS_PENDING = 1; - STATUS_JOB_STARTING = 3; - STATUS_READY = 2; -} - -enum FeatureSetJobDeliveryStatus { - STATUS_IN_PROGRESS = 0; - STATUS_DELIVERED = 1; -} \ No newline at end of file diff --git a/protos/feast/core/FeatureSetReference.proto b/protos/feast/core/FeatureSetReference.proto deleted file mode 100644 index 85762512ca6..00000000000 --- a/protos/feast/core/FeatureSetReference.proto +++ /dev/null @@ -1,33 +0,0 @@ -// -// Copyright 2020 The Feast Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// - -syntax = "proto3"; - -package feast.core; - -option go_package = "github.com/feast-dev/feast/sdk/go/protos/feast/core"; -option java_outer_classname = "FeatureSetReferenceProto"; -option java_package = "feast.proto.core"; - -// Defines a composite key that refers to a unique FeatureSet -message FeatureSetReference { - // Name of the project - string project = 1; - // Name of the FeatureSet - string name = 2; - // Feature set version was removed in v0.5.0. - reserved 3; -} diff --git a/protos/feast/core/FeatureTable.proto b/protos/feast/core/FeatureTable.proto index 279072fccfc..13780f0aecd 100644 --- a/protos/feast/core/FeatureTable.proto +++ b/protos/feast/core/FeatureTable.proto @@ -37,7 +37,7 @@ message FeatureTable { } message FeatureTableSpec { - // Name of the feature set. Must be unique. Not updated. + // Name of the feature table. Must be unique. Not updated. string name = 1; // List names of entities to associate with the Features defined in this diff --git a/protos/feast/core/IngestionJob.proto b/protos/feast/core/IngestionJob.proto deleted file mode 100644 index d14be75ce9a..00000000000 --- a/protos/feast/core/IngestionJob.proto +++ /dev/null @@ -1,83 +0,0 @@ -// -// Copyright 2020 The Feast Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// - -syntax = "proto3"; - -package feast.core; - -option go_package = "github.com/feast-dev/feast/sdk/go/protos/feast/core"; -option java_outer_classname = "IngestionJobProto"; -option java_package = "feast.proto.core"; - -import "feast/core/FeatureSetReference.proto"; -import "feast/core/Store.proto"; -import "feast/core/Source.proto"; - -// Represents Feast Injestion Job -message IngestionJob { - // Job ID assigned by Feast - string id = 1; - // External job ID specific to the runner. - // For DirectRunner jobs, this is identical to id. For DataflowRunner jobs, this refers to the Dataflow job ID. - string external_id = 2; - IngestionJobStatus status = 3; - // List of feature sets was removed. Use list of references instead - reserved 4; - // Source this job is reading from. - feast.core.Source source = 5; - // Store this job is writing to. - repeated feast.core.Store stores = 6; - - // List of Feature Set References - repeated feast.core.FeatureSetReference feature_set_references = 7; -} - -// Status of a Feast Ingestion Job -enum IngestionJobStatus { - // Job status is not known. - UNKNOWN = 0; - // Import job is submitted to runner and currently pending for executing - PENDING = 1; - // Import job is currently running in the runner - RUNNING = 2; - // Runner's reported the import job has completed (applicable to batch job) - COMPLETED = 3; - // When user sent abort command, but it's still running - ABORTING = 4; - // User initiated abort job - ABORTED = 5; - // Runner's reported that the import job failed to run or there is a failure during job - ERROR = 6; - // job has been suspended and waiting for cleanup - SUSPENDING = 7; - // job has been suspended - SUSPENDED = 8; -} - -// Config for bi-directional communication channel between Core Service and Ingestion Job -message SpecsStreamingUpdateConfig { - // out-channel for publishing new FeatureSetSpecs (by Core). - // IngestionJob use it as source of existing FeatureSetSpecs and new real-time updates - feast.core.KafkaSourceConfig source = 1; - // ack-channel for sending acknowledgments when new FeatureSetSpecs is installed in Job - feast.core.KafkaSourceConfig ack = 2; -} - -message FeatureSetSpecAck { - string feature_set_reference = 1; - int32 feature_set_version = 2; - string job_name = 3; -} \ No newline at end of file diff --git a/protos/feast/core/JobService.proto b/protos/feast/core/JobService.proto index d3924ecc71f..d79cb9e104a 100644 --- a/protos/feast/core/JobService.proto +++ b/protos/feast/core/JobService.proto @@ -72,23 +72,32 @@ message Job { JobType type = 2; // Current job status JobStatus status = 3; + // Deterministic hash of the Job + string hash = 4; + // Start time of the Job + google.protobuf.Timestamp start_time = 5; message RetrievalJobMeta { - string output_location = 4; + string output_location = 1; } message OfflineToOnlineMeta { + string table_name = 1; } message StreamToOnlineMeta { + string table_name = 1; } // JobType specific metadata on the job oneof meta { - RetrievalJobMeta retrieval = 5; - OfflineToOnlineMeta batch_ingestion = 6; - StreamToOnlineMeta stream_ingestion = 7; + RetrievalJobMeta retrieval = 6; + OfflineToOnlineMeta batch_ingestion = 7; + StreamToOnlineMeta stream_ingestion = 8; } + + // Path to Spark job logs, if available + string log_uri = 9; } // Ingest data from offline store into online store @@ -105,8 +114,17 @@ message StartOfflineToOnlineIngestionJobRequest { } message StartOfflineToOnlineIngestionJobResponse { - // Job ID assigned by Feast - string id = 1; + // Job ID assigned by Feast + string id = 1; + + // Job start time + google.protobuf.Timestamp job_start_time = 2; + + // Feature table associated with the job + string table_name = 3; + + // Path to Spark job logs, if available + string log_uri = 4; } message GetHistoricalFeaturesRequest { @@ -134,9 +152,18 @@ message GetHistoricalFeaturesRequest { } message GetHistoricalFeaturesResponse { - // Export Job with ID assigned by Feast - string id = 1; - string output_file_uri = 2; + // Export Job with ID assigned by Feast + string id = 1; + + // Uri to the join result output file + string output_file_uri = 2; + + // Job start time + google.protobuf.Timestamp job_start_time = 3; + + // Path to Spark job logs, if available + string log_uri = 4; + } message StartStreamToOnlineIngestionJobRequest { @@ -146,12 +173,22 @@ message StartStreamToOnlineIngestionJobRequest { } message StartStreamToOnlineIngestionJobResponse { - // Job ID assigned by Feast - string id = 1; + // Job ID assigned by Feast + string id = 1; + + // Job start time + google.protobuf.Timestamp job_start_time = 2; + + // Feature table associated with the job + string table_name = 3; + + // Path to Spark job logs, if available + string log_uri = 4; } message ListJobsRequest { bool include_terminated = 1; + string table_name = 2; } message ListJobsResponse { diff --git a/protos/feast/core/Runner.proto b/protos/feast/core/Runner.proto deleted file mode 100644 index ff9cfe6ea7a..00000000000 --- a/protos/feast/core/Runner.proto +++ /dev/null @@ -1,92 +0,0 @@ -// -// * Copyright 2020 The Feast Authors -// * -// * Licensed under the Apache License, Version 2.0 (the "License"); -// * you may not use this file except in compliance with the License. -// * You may obtain a copy of the License at -// * -// * https://www.apache.org/licenses/LICENSE-2.0 -// * -// * Unless required by applicable law or agreed to in writing, software -// * distributed under the License is distributed on an "AS IS" BASIS, -// * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// * See the License for the specific language governing permissions and -// * limitations under the License. -// - -syntax = "proto3"; -package feast.core; - -option java_package = "feast.proto.core"; -option java_outer_classname = "RunnerProto"; -option go_package = "github.com/feast-dev/feast/sdk/go/protos/feast/core"; - -message DirectRunnerConfigOptions { - /** - * Controls the amount of target parallelism the DirectRunner will use. - * Defaults to the greater of the number of available processors and 3. Must be a value - * greater than zero. - */ - int32 targetParallelism = 1; - - /* BigQuery table specification, e.g. PROJECT_ID:DATASET_ID.PROJECT_ID */ - string deadLetterTableSpec = 2; - - // A pipeline level default location for storing temporary files. - // Support Google Cloud Storage locations or local path - string tempLocation = 3; -} - -message DataflowRunnerConfigOptions { - /* Project id to use when launching jobs. */ - string project = 1; - - /* The Google Compute Engine region for creating Dataflow jobs. */ - string region = 2; - - /* GCP availability zone for operations. */ - string workerZone = 3; - - /* Run the job as a specific service account, instead of the default GCE robot. */ - string serviceAccount = 4; - - /* GCE network for launching workers. */ - string network = 5; - - /* GCE subnetwork for launching workers. e.g. regions/asia-east1/subnetworks/mysubnetwork */ - string subnetwork = 6; - - /* Machine type to create Dataflow worker VMs as. */ - string workerMachineType = 7; - - /* The autoscaling algorithm to use for the workerpool. */ - string autoscalingAlgorithm = 8; - - /* Specifies whether worker pools should be started with public IP addresses. */ - bool usePublicIps = 9; - - // A pipeline level default location for storing temporary files. Support Google Cloud Storage locations, - // e.g. gs://bucket/object - string tempLocation = 10; - - /* The maximum number of workers to use for the workerpool. */ - int32 maxNumWorkers = 11; - - /* BigQuery table specification, e.g. PROJECT_ID:DATASET_ID.PROJECT_ID */ - string deadLetterTableSpec = 12; - - /* Labels to apply to the dataflow job */ - map labels = 13; - - /* Disk size to use on each remote Compute Engine worker instance */ - int32 diskSizeGb = 14; - - /* Run job on Dataflow Streaming Engine instead of creating worker VMs */ - bool enableStreamingEngine = 15; - - /* Type of persistent disk to be used by workers */ - string workerDiskType = 16; - - /* Kafka consumer configuration properties */ - map kafkaConsumerProperties = 17; -} \ No newline at end of file diff --git a/protos/feast/core/Source.proto b/protos/feast/core/Source.proto deleted file mode 100644 index 9dcbf2fa056..00000000000 --- a/protos/feast/core/Source.proto +++ /dev/null @@ -1,53 +0,0 @@ -// -// * Copyright 2019 The Feast Authors -// * -// * Licensed under the Apache License, Version 2.0 (the "License"); -// * you may not use this file except in compliance with the License. -// * You may obtain a copy of the License at -// * -// * https://www.apache.org/licenses/LICENSE-2.0 -// * -// * Unless required by applicable law or agreed to in writing, software -// * distributed under the License is distributed on an "AS IS" BASIS, -// * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// * See the License for the specific language governing permissions and -// * limitations under the License. -// - -syntax = "proto3"; -package feast.core; - -option java_package = "feast.proto.core"; -option java_outer_classname = "SourceProto"; -option go_package = "github.com/feast-dev/feast/sdk/go/protos/feast/core"; - - -message Source { - - // The kind of data source Feast should connect to in order to retrieve FeatureRow value - SourceType type = 1; - - // Source specific configuration - oneof source_config { - KafkaSourceConfig kafka_source_config = 2; - } -} - -enum SourceType { - INVALID = 0; - KAFKA = 1; -} - -message KafkaSourceConfig { - // Comma separated list of Kafka bootstrap servers. Used for feature sets without a defined source host[:port]] - string bootstrap_servers = 1; - - // Kafka topic to use for feature sets without user defined topics - string topic = 2; - - // Number of Kafka partitions to to use for managed feature stream. - int32 partitions = 3; - - // Defines the number of copies of managed feature stream Kafka. - int32 replicationFactor = 4; -} \ No newline at end of file diff --git a/protos/feast/core/Store.proto b/protos/feast/core/Store.proto index 53b258d264a..41a76a11c2c 100644 --- a/protos/feast/core/Store.proto +++ b/protos/feast/core/Store.proto @@ -26,12 +26,12 @@ option go_package = "github.com/feast-dev/feast/sdk/go/protos/feast/core"; // The way FeatureRow is encoded and decoded when it is written to and read from // the Store depends on the type of the Store. // -// For example, a FeatureRow will materialize as a row in a table in -// BigQuery but it will materialize as a key, value pair element in Redis. -// message Store { enum StoreType { + // These positions should not be reused. + reserved 2, 3, 12, 13; + INVALID = 0; // Redis stores a FeatureRow element as a key, value pair. @@ -41,62 +41,11 @@ message Store { // - value: STRING // // Encodings: - // - key: byte array of RedisKey (refer to feast.storage.RedisKey) - // - value: byte array of FeatureRow (refer to feast.types.FeatureRow) + // - key: byte array of RedisKey (refer to feast.storage.RedisKeyV2) + // - value: Redis hashmap // REDIS = 1; - // BigQuery stores a FeatureRow element as a row in a BigQuery table. - // - // Table name is derived is the same as the feature set name. - // - // The entities and features in a FeatureSetSpec corresponds to the - // fields in the BigQuery table (these make up the BigQuery schema). - // The name of the entity spec and feature spec corresponds to the column - // names, and the value_type of entity spec and feature spec corresponds - // to BigQuery standard SQL data type of the column. - // - // The following BigQuery fields are reserved for Feast internal use. - // Ingestion of entity or feature spec with names identical - // to the following field names will raise an exception during ingestion. - // - // column_name | column_data_type | description - // ====================|==================|================================ - // - event_timestamp | TIMESTAMP | event time of the FeatureRow - // - created_timestamp | TIMESTAMP | processing time of the ingestion of the FeatureRow - // - ingestion_id | STRING | unique id identifying groups of rows that have been ingested together - // - job_id | STRING | identifier for the job that writes the FeatureRow to the corresponding BigQuery table - // - // BigQuery table created will be partitioned by the field "event_timestamp" - // of the FeatureRow (https://cloud.google.com/bigquery/docs/partitioned-tables). - // - // The following table shows how ValueType in Feast is mapped to - // BigQuery Standard SQL data types - // (https://cloud.google.com/bigquery/docs/reference/standard-sql/data-types): - // - // BYTES : BYTES - // STRING : STRING - // INT32 : INT64 - // INT64 : IN64 - // DOUBLE : FLOAT64 - // FLOAT : FLOAT64 - // BOOL : BOOL - // BYTES_LIST : ARRAY - // STRING_LIST : ARRAY - // INT32_LIST : ARRAY - // INT64_LIST : ARRAY - // DOUBLE_LIST : ARRAY - // FLOAT_LIST : ARRAY - // BOOL_LIST : ARRAY - // - // The column mode in BigQuery is set to "Nullable" such that unset Value - // in a FeatureRow corresponds to NULL value in BigQuery. - // - BIGQUERY = 2; - - // Unsupported in Feast 0.3 - CASSANDRA = 3; - REDIS_CLUSTER = 4; } @@ -114,21 +63,6 @@ message Store { bool ssl = 6; } - message BigQueryConfig { - string project_id = 1; - string dataset_id = 2; - string staging_location = 3; - int32 initial_retry_delay_seconds = 4; - int32 total_timeout_seconds = 5; - // Required. Frequency of running BQ load job and flushing all collected rows to BQ table - int32 write_triggering_frequency_seconds = 6; - } - - message CassandraConfig { - string host = 1; - int32 port = 2; - } - message RedisClusterConfig { // List of Redis Uri for all the nodes in Redis Cluster, comma separated. Eg. host1:6379, host2:6379 string connection_string = 1; @@ -144,6 +78,14 @@ message Store { // Optional. This would be the fallback prefix to use if enable_fallback is true. string fallback_prefix = 7; + // Optional. Priority of nodes when reading from cluster + enum ReadFrom { + MASTER = 0; + MASTER_PREFERRED = 1; + REPLICA = 2; + REPLICA_PREFERRED = 3; + } + ReadFrom read_from = 8; } message Subscription { @@ -183,8 +125,6 @@ message Store { // Configuration to connect to the store. Required. oneof config { RedisConfig redis_config = 11; - BigQueryConfig bigquery_config = 12; - CassandraConfig cassandra_config = 13; RedisClusterConfig redis_cluster_config = 14; } } diff --git a/protos/feast/serving/ServingService.proto b/protos/feast/serving/ServingService.proto index 9b98c071b80..5ed7c0c55d5 100644 --- a/protos/feast/serving/ServingService.proto +++ b/protos/feast/serving/ServingService.proto @@ -30,24 +30,8 @@ service ServingService { // Get information about this Feast serving. rpc GetFeastServingInfo (GetFeastServingInfoRequest) returns (GetFeastServingInfoResponse); - // Get online features synchronously. - rpc GetOnlineFeatures (GetOnlineFeaturesRequest) returns (GetOnlineFeaturesResponse); - // Get online features (v2) synchronously. rpc GetOnlineFeaturesV2 (GetOnlineFeaturesRequestV2) returns (GetOnlineFeaturesResponse); - - // Get batch features asynchronously. - // - // The client should check the status of the returned job periodically by - // calling ReloadJob to determine if the job has completed successfully - // or with an error. If the job completes successfully i.e. - // status = JOB_STATUS_DONE with no error, then the client can check - // the file_uris for the location to download feature values data. - // The client is assumed to have access to these file URIs. - rpc GetBatchFeatures (GetBatchFeaturesRequest) returns (GetBatchFeaturesResponse); - - // Get the latest job status for batch feature retrieval. - rpc GetJob (GetJobRequest) returns (GetJobResponse); } message GetFeastServingInfoRequest {} @@ -65,22 +49,6 @@ message GetFeastServingInfoResponse { string job_staging_location = 10; } -message FeatureReference { - // Project name. This field is optional, if unspecified will default to 'default'. - string project = 1; - - // Feature name - string name = 2; - - // Feature set name specifying the feature set of this referenced feature. - // This field is optional if the feature referenced is unique across the project - // in which case the feature set would be automatically infered - string feature_set = 5; - - // Feature version and max_age was removed in v0.5.0 - reserved 3, 4; -} - message FeatureReferenceV2 { // Name of the Feature Table to retrieve the feature from. string feature_table = 1; @@ -89,34 +57,6 @@ message FeatureReferenceV2 { string name = 2; } -message GetOnlineFeaturesRequest { - // List of features that are being retrieved - repeated FeatureReference features = 4; - - // List of entity rows, containing entity id and timestamp data. - // Used during retrieval of feature rows and for joining feature - // rows into a final dataset - repeated EntityRow entity_rows = 2; - - // Option to omit entities from the response. If true, only feature - // values will be returned. - bool omit_entities_in_response = 3; - - // Optional field to specify project name override. If specified, uses the - // given project for retrieval. Overrides the projects specified in - // Feature References if both are specified. - string project = 5; - - message EntityRow { - // Request timestamp of this row. This value will be used, - // together with maxAge, to determine feature staleness. - google.protobuf.Timestamp entity_timestamp = 1; - - // Map containing mapping of entity name to entity value. - map fields = 2; - } -} - message GetOnlineFeaturesRequestV2 { // List of features that are being retrieved repeated FeatureReferenceV2 features = 4; @@ -141,18 +81,6 @@ message GetOnlineFeaturesRequestV2 { } } -message GetBatchFeaturesRequest { - // List of features that are being retrieved - repeated FeatureReference features = 3; - - // Source of the entity dataset containing the timestamps and entity keys to retrieve - // features for. - DatasetSource dataset_source = 2; - - // Compute statistics for the dataset retrieved - bool compute_statistics = 4; -} - message GetOnlineFeaturesResponse { // Feature values retrieved from feast. repeated FieldValues field_values = 1; @@ -187,18 +115,6 @@ message GetOnlineFeaturesResponse { } } -message GetBatchFeaturesResponse { - Job job = 1; -} - -message GetJobRequest { - Job job = 1; -} - -message GetJobResponse { - Job job = 1; -} - enum FeastServingType { FEAST_SERVING_TYPE_INVALID = 0; // Online serving receives entity data directly and synchronously and will @@ -208,56 +124,3 @@ enum FeastServingType { // retrieval through a staging location. FEAST_SERVING_TYPE_BATCH = 2; } - -enum JobType { - JOB_TYPE_INVALID = 0; - JOB_TYPE_DOWNLOAD = 1; -} - -enum JobStatus { - JOB_STATUS_INVALID = 0; - JOB_STATUS_PENDING = 1; - JOB_STATUS_RUNNING = 2; - JOB_STATUS_DONE = 3; -} - -enum DataFormat { - DATA_FORMAT_INVALID = 0; - DATA_FORMAT_AVRO = 1; -} - -message Job { - string id = 1; - // Output only. The type of the job. - JobType type = 2; - // Output only. Current state of the job. - JobStatus status = 3; - // Output only. If not empty, the job has failed with this error message. - string error = 4; - // Output only. The list of URIs for the files to be downloaded or - // uploaded (depends on the job type) for this particular job. - repeated string file_uris = 5; - // Output only. The data format for all the files. - // For CSV format, the files contain both feature values and a column header. - DataFormat data_format = 6; - // Output only. The statistics computed over - // the retrieved dataset. Only available for BigQuery stores. - tensorflow.metadata.v0.DatasetFeatureStatisticsList dataset_feature_statistics_list = 7; -} - -message DatasetSource { - oneof dataset_source { - // File source to load the dataset from. - FileSource file_source = 1; - } - - message FileSource { - // URIs to retrieve the dataset from, e.g. gs://bucket/directory/object.csv. Wildcards are - // supported. This data must be compatible to be uploaded to the serving store, and also be - // accessible by this serving instance. - repeated string file_uris = 1; - - // Format of the data. Currently only avro is supported. - DataFormat data_format = 2; - } -} diff --git a/protos/feast/storage/Redis.proto b/protos/feast/storage/Redis.proto index fe7d4805094..a662e352f48 100644 --- a/protos/feast/storage/Redis.proto +++ b/protos/feast/storage/Redis.proto @@ -25,19 +25,6 @@ option java_outer_classname = "RedisProto"; option java_package = "feast.proto.storage"; option go_package = "github.com/feast-dev/feast/sdk/go/protos/feast/storage"; -message RedisKey { - // Field number 1 is reserved for a future distributing hash if needed - // (for when redis is clustered). - - // FeatureSet this row belongs to, this is defined as featureSetName. - string feature_set = 2; - - // List of fields containing entity names and their respective values - // contained within this feature row. The entities should be sorted - // by the entity name alphabetically in ascending order. - repeated feast.types.Field entities = 3; -} - message RedisKeyV2 { string project = 1; diff --git a/protos/feast/types/FeatureRow.proto b/protos/feast/types/FeatureRow.proto deleted file mode 100644 index fd8a561c7bc..00000000000 --- a/protos/feast/types/FeatureRow.proto +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Copyright 2018 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -syntax = "proto3"; - -import "google/protobuf/timestamp.proto"; -import "feast/types/Field.proto"; - -package feast.types; - -option java_package = "feast.proto.types"; -option java_outer_classname = "FeatureRowProto"; -option go_package = "github.com/feast-dev/feast/sdk/go/protos/feast/types"; - -message FeatureRow { - - // Fields in the feature row. - repeated Field fields = 2; - - // Timestamp of the feature row. While the actual definition of this timestamp may vary - // depending on the upstream feature creation pipelines, this is the timestamp that Feast - // will use to perform joins, determine latest values, and coalesce rows. - google.protobuf.Timestamp event_timestamp = 3; - - // Complete reference to the featureSet this featureRow belongs to, in the form of - // /. This value will be used by the feast ingestion job to filter - // rows, and write the values to the correct tables. - string feature_set = 6; - - // Identifier tying this feature row to a specific ingestion job. - string ingestion_id = 7; -} diff --git a/protos/feast/types/FeatureRowExtended.proto b/protos/feast/types/FeatureRowExtended.proto deleted file mode 100644 index f922fe66bfd..00000000000 --- a/protos/feast/types/FeatureRowExtended.proto +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Copyright 2018 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -syntax = "proto3"; - -import "google/protobuf/timestamp.proto"; -import "feast/types/FeatureRow.proto"; - -package feast.types; - -option java_package = "feast.proto.types"; -option java_outer_classname = "FeatureRowExtendedProto"; -option go_package = "github.com/feast-dev/feast/sdk/go/protos/feast/types"; - -message Error { - string cause = 1; // exception class name - string transform = 2; // name of transform where the error occurred - string message = 3; - string stack_trace = 4; -} - -message Attempt { - int32 attempts = 1; - Error error = 2; -} - -message FeatureRowExtended { - FeatureRow row = 1; - Attempt last_attempt = 2; - google.protobuf.Timestamp first_seen = 3; -} diff --git a/sdk/go/client.go b/sdk/go/client.go index 17d4dd350d2..4deb0a789cc 100644 --- a/sdk/go/client.go +++ b/sdk/go/client.go @@ -6,6 +6,7 @@ import ( "fmt" "github.com/feast-dev/feast/sdk/go/protos/feast/serving" + "github.com/opentracing-contrib/go-grpc" "github.com/opentracing/opentracing-go" "go.opencensus.io/plugin/ocgrpc" "google.golang.org/grpc" @@ -88,6 +89,11 @@ func NewSecureGrpcClientWithDialOptions(host string, port int, security Security options = append(options, grpc.WithPerRPCCredentials(security.Credential)) } + // Enable tracing if a global tracer is registered + tracingInterceptor := grpc.WithUnaryInterceptor( + otgrpc.OpenTracingClientInterceptor(opentracing.GlobalTracer())) + options = append(options, tracingInterceptor) + conn, err := grpc.Dial(adr, options...) if err != nil { return nil, err @@ -100,9 +106,6 @@ func NewSecureGrpcClientWithDialOptions(host string, port int, security Security // GetOnlineFeatures gets the latest values of the request features from the Feast serving instance provided. func (fc *GrpcClient) GetOnlineFeatures(ctx context.Context, req *OnlineFeaturesRequest) ( *OnlineFeaturesResponse, error) { - span, ctx := opentracing.StartSpanFromContext(ctx, "get_online_features") - defer span.Finish() - featuresRequest, err := req.buildRequest() if err != nil { return nil, err @@ -122,9 +125,6 @@ func (fc *GrpcClient) GetOnlineFeatures(ctx context.Context, req *OnlineFeatures // GetFeastServingInfo gets information about the feast serving instance this client is connected to. func (fc *GrpcClient) GetFeastServingInfo(ctx context.Context, in *serving.GetFeastServingInfoRequest) ( *serving.GetFeastServingInfoResponse, error) { - span, ctx := opentracing.StartSpanFromContext(ctx, "get_info") - defer span.Finish() - return fc.cli.GetFeastServingInfo(ctx, in) } diff --git a/sdk/go/client_test.go b/sdk/go/client_test.go index 914592c44b6..a94a577e84c 100644 --- a/sdk/go/client_test.go +++ b/sdk/go/client_test.go @@ -9,7 +9,6 @@ import ( "github.com/feast-dev/feast/sdk/go/protos/feast/types" "github.com/golang/mock/gomock" "github.com/google/go-cmp/cmp" - "github.com/opentracing/opentracing-go" ) func TestGetOnlineFeatures(t *testing.T) { @@ -59,10 +58,9 @@ func TestGetOnlineFeatures(t *testing.T) { defer ctrl.Finish() cli := mock_serving.NewMockServingServiceClient(ctrl) ctx := context.Background() - _, traceCtx := opentracing.StartSpanFromContext(ctx, "get_online_features") rawRequest, _ := tc.req.buildRequest() resp := tc.want.RawResponse - cli.EXPECT().GetOnlineFeaturesV2(traceCtx, rawRequest).Return(resp, nil).Times(1) + cli.EXPECT().GetOnlineFeaturesV2(ctx, rawRequest).Return(resp, nil).Times(1) client := &GrpcClient{ cli: cli, diff --git a/sdk/go/go.mod b/sdk/go/go.mod index 712241d356f..d3b454d55c3 100644 --- a/sdk/go/go.mod +++ b/sdk/go/go.mod @@ -6,6 +6,7 @@ require ( github.com/golang/mock v1.4.3 github.com/golang/protobuf v1.4.2 github.com/google/go-cmp v0.5.1 + github.com/opentracing-contrib/go-grpc v0.0.0-20200813121455-4a6760c71486 github.com/opentracing/opentracing-go v1.1.0 go.opencensus.io v0.22.4 golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d diff --git a/sdk/go/go.sum b/sdk/go/go.sum index dba3041f042..f9a22785a53 100644 --- a/sdk/go/go.sum +++ b/sdk/go/go.sum @@ -108,15 +108,20 @@ github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm4 github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5 h1:sjZBwGj9Jlw33ImPtvFviGYvseOtDM7hkSKB7+Tv3SM= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= +github.com/grpc-ecosystem/grpc-opentracing v0.0.0-20180507213350-8e809c8a8645/go.mod h1:6iZfnjpejD4L/4DwD7NryNaJyCQdzwWwH2MWhCA90Kw= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/opentracing-contrib/go-grpc v0.0.0-20200813121455-4a6760c71486 h1:K35HCWaOTJIPW6cDHK4yj3QfRY/NhE0pBbfoc0M2NMQ= +github.com/opentracing-contrib/go-grpc v0.0.0-20200813121455-4a6760c71486/go.mod h1:DYR5Eij8rJl8h7gblRrOZ8g0kW1umSpKqYIBTgeDtLo= github.com/opentracing/opentracing-go v1.1.0 h1:pWlfV3Bxv7k65HYwkikxat0+s3pV4bsqf19k25Ur8rU= github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= @@ -131,8 +136,6 @@ github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= -go.opencensus.io v0.22.1 h1:8dP3SGL7MPB94crU3bEPplMPe83FI4EouesJUeFHv50= -go.opencensus.io v0.22.1/go.mod h1:Ap50jQcDJrx6rB6VgeeFPtuPIf3wMRvRfrfYDO6+BmA= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.4 h1:LYy1Hy3MJdrCdMwwzxA/dRok4ejH+RwNGbuoD9fCjto= @@ -186,6 +189,7 @@ golang.org/x/net v0.0.0-20190620200207-3b0461eec859 h1:R/3boaszxrf1GEUWTVDzSKVwL golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190921015927-1a5e07d1ff72/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -320,6 +324,7 @@ google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7 google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.6 h1:lMO5rYAqUxkmaj76jAkRUvt5JZgFymx/+Q5Mzfivuhc= google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8 h1:Nw54tB0rB7hY/N0NQvRW8DG4Yk3Q6T9cu9RcFQDu1tc= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= @@ -357,6 +362,7 @@ google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZi google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.23.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= @@ -381,6 +387,7 @@ google.golang.org/protobuf v1.25.0 h1:Ejskq+SyPohKW+1uil0JJMtmHCgJPJ/qWTxr8qp+R4 google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= diff --git a/sdk/go/mocks/serving_mock.go b/sdk/go/mocks/serving_mock.go index de2020f6029..00d2e768ef8 100644 --- a/sdk/go/mocks/serving_mock.go +++ b/sdk/go/mocks/serving_mock.go @@ -36,26 +36,6 @@ func (m *MockServingServiceClient) EXPECT() *MockServingServiceClientMockRecorde return m.recorder } -// GetBatchFeatures mocks base method -func (m *MockServingServiceClient) GetBatchFeatures(arg0 context.Context, arg1 *serving.GetBatchFeaturesRequest, arg2 ...grpc.CallOption) (*serving.GetBatchFeaturesResponse, error) { - m.ctrl.T.Helper() - varargs := []interface{}{arg0, arg1} - for _, a := range arg2 { - varargs = append(varargs, a) - } - ret := m.ctrl.Call(m, "GetBatchFeatures", varargs...) - ret0, _ := ret[0].(*serving.GetBatchFeaturesResponse) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// GetBatchFeatures indicates an expected call of GetBatchFeatures -func (mr *MockServingServiceClientMockRecorder) GetBatchFeatures(arg0, arg1 interface{}, arg2 ...interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - varargs := append([]interface{}{arg0, arg1}, arg2...) - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetBatchFeatures", reflect.TypeOf((*MockServingServiceClient)(nil).GetBatchFeatures), varargs...) -} - // GetFeastServingInfo mocks base method func (m *MockServingServiceClient) GetFeastServingInfo(arg0 context.Context, arg1 *serving.GetFeastServingInfoRequest, arg2 ...grpc.CallOption) (*serving.GetFeastServingInfoResponse, error) { m.ctrl.T.Helper() @@ -76,46 +56,6 @@ func (mr *MockServingServiceClientMockRecorder) GetFeastServingInfo(arg0, arg1 i return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetFeastServingInfo", reflect.TypeOf((*MockServingServiceClient)(nil).GetFeastServingInfo), varargs...) } -// GetJob mocks base method -func (m *MockServingServiceClient) GetJob(arg0 context.Context, arg1 *serving.GetJobRequest, arg2 ...grpc.CallOption) (*serving.GetJobResponse, error) { - m.ctrl.T.Helper() - varargs := []interface{}{arg0, arg1} - for _, a := range arg2 { - varargs = append(varargs, a) - } - ret := m.ctrl.Call(m, "GetJob", varargs...) - ret0, _ := ret[0].(*serving.GetJobResponse) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// GetJob indicates an expected call of GetJob -func (mr *MockServingServiceClientMockRecorder) GetJob(arg0, arg1 interface{}, arg2 ...interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - varargs := append([]interface{}{arg0, arg1}, arg2...) - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetJob", reflect.TypeOf((*MockServingServiceClient)(nil).GetJob), varargs...) -} - -// GetOnlineFeatures mocks base method -func (m *MockServingServiceClient) GetOnlineFeatures(arg0 context.Context, arg1 *serving.GetOnlineFeaturesRequest, arg2 ...grpc.CallOption) (*serving.GetOnlineFeaturesResponse, error) { - m.ctrl.T.Helper() - varargs := []interface{}{arg0, arg1} - for _, a := range arg2 { - varargs = append(varargs, a) - } - ret := m.ctrl.Call(m, "GetOnlineFeatures", varargs...) - ret0, _ := ret[0].(*serving.GetOnlineFeaturesResponse) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// GetOnlineFeatures indicates an expected call of GetOnlineFeatures -func (mr *MockServingServiceClientMockRecorder) GetOnlineFeatures(arg0, arg1 interface{}, arg2 ...interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - varargs := append([]interface{}{arg0, arg1}, arg2...) - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetOnlineFeatures", reflect.TypeOf((*MockServingServiceClient)(nil).GetOnlineFeatures), varargs...) -} - // GetOnlineFeaturesV2 mocks base method func (m *MockServingServiceClient) GetOnlineFeaturesV2(arg0 context.Context, arg1 *serving.GetOnlineFeaturesRequestV2, arg2 ...grpc.CallOption) (*serving.GetOnlineFeaturesResponse, error) { m.ctrl.T.Helper() diff --git a/sdk/go/protos/feast/core/CoreService.pb.go b/sdk/go/protos/feast/core/CoreService.pb.go index c54a5bc3151..2d9b0a8e0ad 100644 --- a/sdk/go/protos/feast/core/CoreService.pb.go +++ b/sdk/go/protos/feast/core/CoreService.pb.go @@ -17,16 +17,16 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.25.0 -// protoc v3.10.0 +// protoc v3.12.4 // source: feast/core/CoreService.proto package core import ( context "context" - v0 "github.com/feast-dev/feast/sdk/go/protos/tensorflow_metadata/proto/v0" + _ "github.com/feast-dev/feast/sdk/go/protos/tensorflow_metadata/proto/v0" proto "github.com/golang/protobuf/proto" - timestamp "github.com/golang/protobuf/ptypes/timestamp" + _ "github.com/golang/protobuf/ptypes/timestamp" grpc "google.golang.org/grpc" codes "google.golang.org/grpc/codes" status "google.golang.org/grpc/status" @@ -47,63 +47,6 @@ const ( // of the legacy proto package is being used. const _ = proto.ProtoPackageIsVersion4 -// TODO: 0 should correspond to invalid rather than NO_CHANGE -type ApplyFeatureSetResponse_Status int32 - -const ( - // Latest feature set is consistent with provided feature set - ApplyFeatureSetResponse_NO_CHANGE ApplyFeatureSetResponse_Status = 0 - // New feature set created - ApplyFeatureSetResponse_CREATED ApplyFeatureSetResponse_Status = 1 - // Error occurred while trying to apply changes - ApplyFeatureSetResponse_ERROR ApplyFeatureSetResponse_Status = 2 - // Changes detected and updated successfully - ApplyFeatureSetResponse_UPDATED ApplyFeatureSetResponse_Status = 3 -) - -// Enum value maps for ApplyFeatureSetResponse_Status. -var ( - ApplyFeatureSetResponse_Status_name = map[int32]string{ - 0: "NO_CHANGE", - 1: "CREATED", - 2: "ERROR", - 3: "UPDATED", - } - ApplyFeatureSetResponse_Status_value = map[string]int32{ - "NO_CHANGE": 0, - "CREATED": 1, - "ERROR": 2, - "UPDATED": 3, - } -) - -func (x ApplyFeatureSetResponse_Status) Enum() *ApplyFeatureSetResponse_Status { - p := new(ApplyFeatureSetResponse_Status) - *p = x - return p -} - -func (x ApplyFeatureSetResponse_Status) String() string { - return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) -} - -func (ApplyFeatureSetResponse_Status) Descriptor() protoreflect.EnumDescriptor { - return file_feast_core_CoreService_proto_enumTypes[0].Descriptor() -} - -func (ApplyFeatureSetResponse_Status) Type() protoreflect.EnumType { - return &file_feast_core_CoreService_proto_enumTypes[0] -} - -func (x ApplyFeatureSetResponse_Status) Number() protoreflect.EnumNumber { - return protoreflect.EnumNumber(x) -} - -// Deprecated: Use ApplyFeatureSetResponse_Status.Descriptor instead. -func (ApplyFeatureSetResponse_Status) EnumDescriptor() ([]byte, []int) { - return file_feast_core_CoreService_proto_rawDescGZIP(), []int{15, 0} -} - type UpdateStoreResponse_Status int32 const ( @@ -136,11 +79,11 @@ func (x UpdateStoreResponse_Status) String() string { } func (UpdateStoreResponse_Status) Descriptor() protoreflect.EnumDescriptor { - return file_feast_core_CoreService_proto_enumTypes[1].Descriptor() + return file_feast_core_CoreService_proto_enumTypes[0].Descriptor() } func (UpdateStoreResponse_Status) Type() protoreflect.EnumType { - return &file_feast_core_CoreService_proto_enumTypes[1] + return &file_feast_core_CoreService_proto_enumTypes[0] } func (x UpdateStoreResponse_Status) Number() protoreflect.EnumNumber { @@ -149,208 +92,7 @@ func (x UpdateStoreResponse_Status) Number() protoreflect.EnumNumber { // Deprecated: Use UpdateStoreResponse_Status.Descriptor instead. func (UpdateStoreResponse_Status) EnumDescriptor() ([]byte, []int) { - return file_feast_core_CoreService_proto_rawDescGZIP(), []int{19, 0} -} - -// Request for a single feature set -type GetFeatureSetRequest struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - // Name of project the feature set belongs to. If omitted will default to 'default' project. - Project string `protobuf:"bytes,3,opt,name=project,proto3" json:"project,omitempty"` - // Name of feature set (required). - Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` -} - -func (x *GetFeatureSetRequest) Reset() { - *x = GetFeatureSetRequest{} - if protoimpl.UnsafeEnabled { - mi := &file_feast_core_CoreService_proto_msgTypes[0] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *GetFeatureSetRequest) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*GetFeatureSetRequest) ProtoMessage() {} - -func (x *GetFeatureSetRequest) ProtoReflect() protoreflect.Message { - mi := &file_feast_core_CoreService_proto_msgTypes[0] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use GetFeatureSetRequest.ProtoReflect.Descriptor instead. -func (*GetFeatureSetRequest) Descriptor() ([]byte, []int) { - return file_feast_core_CoreService_proto_rawDescGZIP(), []int{0} -} - -func (x *GetFeatureSetRequest) GetProject() string { - if x != nil { - return x.Project - } - return "" -} - -func (x *GetFeatureSetRequest) GetName() string { - if x != nil { - return x.Name - } - return "" -} - -// Response containing a single feature set -type GetFeatureSetResponse struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - FeatureSet *FeatureSet `protobuf:"bytes,1,opt,name=feature_set,json=featureSet,proto3" json:"feature_set,omitempty"` -} - -func (x *GetFeatureSetResponse) Reset() { - *x = GetFeatureSetResponse{} - if protoimpl.UnsafeEnabled { - mi := &file_feast_core_CoreService_proto_msgTypes[1] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *GetFeatureSetResponse) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*GetFeatureSetResponse) ProtoMessage() {} - -func (x *GetFeatureSetResponse) ProtoReflect() protoreflect.Message { - mi := &file_feast_core_CoreService_proto_msgTypes[1] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use GetFeatureSetResponse.ProtoReflect.Descriptor instead. -func (*GetFeatureSetResponse) Descriptor() ([]byte, []int) { - return file_feast_core_CoreService_proto_rawDescGZIP(), []int{1} -} - -func (x *GetFeatureSetResponse) GetFeatureSet() *FeatureSet { - if x != nil { - return x.FeatureSet - } - return nil -} - -// Retrieves details for all versions of a specific feature set -type ListFeatureSetsRequest struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - Filter *ListFeatureSetsRequest_Filter `protobuf:"bytes,1,opt,name=filter,proto3" json:"filter,omitempty"` -} - -func (x *ListFeatureSetsRequest) Reset() { - *x = ListFeatureSetsRequest{} - if protoimpl.UnsafeEnabled { - mi := &file_feast_core_CoreService_proto_msgTypes[2] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *ListFeatureSetsRequest) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*ListFeatureSetsRequest) ProtoMessage() {} - -func (x *ListFeatureSetsRequest) ProtoReflect() protoreflect.Message { - mi := &file_feast_core_CoreService_proto_msgTypes[2] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use ListFeatureSetsRequest.ProtoReflect.Descriptor instead. -func (*ListFeatureSetsRequest) Descriptor() ([]byte, []int) { - return file_feast_core_CoreService_proto_rawDescGZIP(), []int{2} -} - -func (x *ListFeatureSetsRequest) GetFilter() *ListFeatureSetsRequest_Filter { - if x != nil { - return x.Filter - } - return nil -} - -type ListFeatureSetsResponse struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - FeatureSets []*FeatureSet `protobuf:"bytes,1,rep,name=feature_sets,json=featureSets,proto3" json:"feature_sets,omitempty"` -} - -func (x *ListFeatureSetsResponse) Reset() { - *x = ListFeatureSetsResponse{} - if protoimpl.UnsafeEnabled { - mi := &file_feast_core_CoreService_proto_msgTypes[3] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *ListFeatureSetsResponse) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*ListFeatureSetsResponse) ProtoMessage() {} - -func (x *ListFeatureSetsResponse) ProtoReflect() protoreflect.Message { - mi := &file_feast_core_CoreService_proto_msgTypes[3] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use ListFeatureSetsResponse.ProtoReflect.Descriptor instead. -func (*ListFeatureSetsResponse) Descriptor() ([]byte, []int) { - return file_feast_core_CoreService_proto_rawDescGZIP(), []int{3} -} - -func (x *ListFeatureSetsResponse) GetFeatureSets() []*FeatureSet { - if x != nil { - return x.FeatureSets - } - return nil + return file_feast_core_CoreService_proto_rawDescGZIP(), []int{13, 0} } // Request for a single entity @@ -368,7 +110,7 @@ type GetEntityRequest struct { func (x *GetEntityRequest) Reset() { *x = GetEntityRequest{} if protoimpl.UnsafeEnabled { - mi := &file_feast_core_CoreService_proto_msgTypes[4] + mi := &file_feast_core_CoreService_proto_msgTypes[0] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -381,7 +123,7 @@ func (x *GetEntityRequest) String() string { func (*GetEntityRequest) ProtoMessage() {} func (x *GetEntityRequest) ProtoReflect() protoreflect.Message { - mi := &file_feast_core_CoreService_proto_msgTypes[4] + mi := &file_feast_core_CoreService_proto_msgTypes[0] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -394,7 +136,7 @@ func (x *GetEntityRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use GetEntityRequest.ProtoReflect.Descriptor instead. func (*GetEntityRequest) Descriptor() ([]byte, []int) { - return file_feast_core_CoreService_proto_rawDescGZIP(), []int{4} + return file_feast_core_CoreService_proto_rawDescGZIP(), []int{0} } func (x *GetEntityRequest) GetName() string { @@ -423,7 +165,7 @@ type GetEntityResponse struct { func (x *GetEntityResponse) Reset() { *x = GetEntityResponse{} if protoimpl.UnsafeEnabled { - mi := &file_feast_core_CoreService_proto_msgTypes[5] + mi := &file_feast_core_CoreService_proto_msgTypes[1] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -436,7 +178,7 @@ func (x *GetEntityResponse) String() string { func (*GetEntityResponse) ProtoMessage() {} func (x *GetEntityResponse) ProtoReflect() protoreflect.Message { - mi := &file_feast_core_CoreService_proto_msgTypes[5] + mi := &file_feast_core_CoreService_proto_msgTypes[1] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -449,7 +191,7 @@ func (x *GetEntityResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use GetEntityResponse.ProtoReflect.Descriptor instead. func (*GetEntityResponse) Descriptor() ([]byte, []int) { - return file_feast_core_CoreService_proto_rawDescGZIP(), []int{5} + return file_feast_core_CoreService_proto_rawDescGZIP(), []int{1} } func (x *GetEntityResponse) GetEntity() *Entity { @@ -471,7 +213,7 @@ type ListEntitiesRequest struct { func (x *ListEntitiesRequest) Reset() { *x = ListEntitiesRequest{} if protoimpl.UnsafeEnabled { - mi := &file_feast_core_CoreService_proto_msgTypes[6] + mi := &file_feast_core_CoreService_proto_msgTypes[2] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -484,7 +226,7 @@ func (x *ListEntitiesRequest) String() string { func (*ListEntitiesRequest) ProtoMessage() {} func (x *ListEntitiesRequest) ProtoReflect() protoreflect.Message { - mi := &file_feast_core_CoreService_proto_msgTypes[6] + mi := &file_feast_core_CoreService_proto_msgTypes[2] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -497,7 +239,7 @@ func (x *ListEntitiesRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use ListEntitiesRequest.ProtoReflect.Descriptor instead. func (*ListEntitiesRequest) Descriptor() ([]byte, []int) { - return file_feast_core_CoreService_proto_rawDescGZIP(), []int{6} + return file_feast_core_CoreService_proto_rawDescGZIP(), []int{2} } func (x *ListEntitiesRequest) GetFilter() *ListEntitiesRequest_Filter { @@ -518,7 +260,7 @@ type ListEntitiesResponse struct { func (x *ListEntitiesResponse) Reset() { *x = ListEntitiesResponse{} if protoimpl.UnsafeEnabled { - mi := &file_feast_core_CoreService_proto_msgTypes[7] + mi := &file_feast_core_CoreService_proto_msgTypes[3] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -531,7 +273,7 @@ func (x *ListEntitiesResponse) String() string { func (*ListEntitiesResponse) ProtoMessage() {} func (x *ListEntitiesResponse) ProtoReflect() protoreflect.Message { - mi := &file_feast_core_CoreService_proto_msgTypes[7] + mi := &file_feast_core_CoreService_proto_msgTypes[3] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -544,7 +286,7 @@ func (x *ListEntitiesResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use ListEntitiesResponse.ProtoReflect.Descriptor instead. func (*ListEntitiesResponse) Descriptor() ([]byte, []int) { - return file_feast_core_CoreService_proto_rawDescGZIP(), []int{7} + return file_feast_core_CoreService_proto_rawDescGZIP(), []int{3} } func (x *ListEntitiesResponse) GetEntities() []*Entity { @@ -565,7 +307,7 @@ type ListFeaturesRequest struct { func (x *ListFeaturesRequest) Reset() { *x = ListFeaturesRequest{} if protoimpl.UnsafeEnabled { - mi := &file_feast_core_CoreService_proto_msgTypes[8] + mi := &file_feast_core_CoreService_proto_msgTypes[4] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -578,7 +320,7 @@ func (x *ListFeaturesRequest) String() string { func (*ListFeaturesRequest) ProtoMessage() {} func (x *ListFeaturesRequest) ProtoReflect() protoreflect.Message { - mi := &file_feast_core_CoreService_proto_msgTypes[8] + mi := &file_feast_core_CoreService_proto_msgTypes[4] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -591,7 +333,7 @@ func (x *ListFeaturesRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use ListFeaturesRequest.ProtoReflect.Descriptor instead. func (*ListFeaturesRequest) Descriptor() ([]byte, []int) { - return file_feast_core_CoreService_proto_rawDescGZIP(), []int{8} + return file_feast_core_CoreService_proto_rawDescGZIP(), []int{4} } func (x *ListFeaturesRequest) GetFilter() *ListFeaturesRequest_Filter { @@ -606,13 +348,13 @@ type ListFeaturesResponse struct { sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - Features map[string]*FeatureSpec `protobuf:"bytes,1,rep,name=features,proto3" json:"features,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` + Features map[string]*FeatureSpecV2 `protobuf:"bytes,2,rep,name=features,proto3" json:"features,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` } func (x *ListFeaturesResponse) Reset() { *x = ListFeaturesResponse{} if protoimpl.UnsafeEnabled { - mi := &file_feast_core_CoreService_proto_msgTypes[9] + mi := &file_feast_core_CoreService_proto_msgTypes[5] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -625,7 +367,7 @@ func (x *ListFeaturesResponse) String() string { func (*ListFeaturesResponse) ProtoMessage() {} func (x *ListFeaturesResponse) ProtoReflect() protoreflect.Message { - mi := &file_feast_core_CoreService_proto_msgTypes[9] + mi := &file_feast_core_CoreService_proto_msgTypes[5] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -638,10 +380,10 @@ func (x *ListFeaturesResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use ListFeaturesResponse.ProtoReflect.Descriptor instead. func (*ListFeaturesResponse) Descriptor() ([]byte, []int) { - return file_feast_core_CoreService_proto_rawDescGZIP(), []int{9} + return file_feast_core_CoreService_proto_rawDescGZIP(), []int{5} } -func (x *ListFeaturesResponse) GetFeatures() map[string]*FeatureSpec { +func (x *ListFeaturesResponse) GetFeatures() map[string]*FeatureSpecV2 { if x != nil { return x.Features } @@ -659,7 +401,7 @@ type ListStoresRequest struct { func (x *ListStoresRequest) Reset() { *x = ListStoresRequest{} if protoimpl.UnsafeEnabled { - mi := &file_feast_core_CoreService_proto_msgTypes[10] + mi := &file_feast_core_CoreService_proto_msgTypes[6] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -672,7 +414,7 @@ func (x *ListStoresRequest) String() string { func (*ListStoresRequest) ProtoMessage() {} func (x *ListStoresRequest) ProtoReflect() protoreflect.Message { - mi := &file_feast_core_CoreService_proto_msgTypes[10] + mi := &file_feast_core_CoreService_proto_msgTypes[6] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -685,7 +427,7 @@ func (x *ListStoresRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use ListStoresRequest.ProtoReflect.Descriptor instead. func (*ListStoresRequest) Descriptor() ([]byte, []int) { - return file_feast_core_CoreService_proto_rawDescGZIP(), []int{10} + return file_feast_core_CoreService_proto_rawDescGZIP(), []int{6} } func (x *ListStoresRequest) GetFilter() *ListStoresRequest_Filter { @@ -706,7 +448,7 @@ type ListStoresResponse struct { func (x *ListStoresResponse) Reset() { *x = ListStoresResponse{} if protoimpl.UnsafeEnabled { - mi := &file_feast_core_CoreService_proto_msgTypes[11] + mi := &file_feast_core_CoreService_proto_msgTypes[7] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -719,7 +461,7 @@ func (x *ListStoresResponse) String() string { func (*ListStoresResponse) ProtoMessage() {} func (x *ListStoresResponse) ProtoReflect() protoreflect.Message { - mi := &file_feast_core_CoreService_proto_msgTypes[11] + mi := &file_feast_core_CoreService_proto_msgTypes[7] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -732,7 +474,7 @@ func (x *ListStoresResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use ListStoresResponse.ProtoReflect.Descriptor instead. func (*ListStoresResponse) Descriptor() ([]byte, []int) { - return file_feast_core_CoreService_proto_rawDescGZIP(), []int{11} + return file_feast_core_CoreService_proto_rawDescGZIP(), []int{7} } func (x *ListStoresResponse) GetStore() []*Store { @@ -757,7 +499,7 @@ type ApplyEntityRequest struct { func (x *ApplyEntityRequest) Reset() { *x = ApplyEntityRequest{} if protoimpl.UnsafeEnabled { - mi := &file_feast_core_CoreService_proto_msgTypes[12] + mi := &file_feast_core_CoreService_proto_msgTypes[8] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -770,7 +512,7 @@ func (x *ApplyEntityRequest) String() string { func (*ApplyEntityRequest) ProtoMessage() {} func (x *ApplyEntityRequest) ProtoReflect() protoreflect.Message { - mi := &file_feast_core_CoreService_proto_msgTypes[12] + mi := &file_feast_core_CoreService_proto_msgTypes[8] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -783,7 +525,7 @@ func (x *ApplyEntityRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use ApplyEntityRequest.ProtoReflect.Descriptor instead. func (*ApplyEntityRequest) Descriptor() ([]byte, []int) { - return file_feast_core_CoreService_proto_rawDescGZIP(), []int{12} + return file_feast_core_CoreService_proto_rawDescGZIP(), []int{8} } func (x *ApplyEntityRequest) GetSpec() *EntitySpecV2 { @@ -811,7 +553,7 @@ type ApplyEntityResponse struct { func (x *ApplyEntityResponse) Reset() { *x = ApplyEntityResponse{} if protoimpl.UnsafeEnabled { - mi := &file_feast_core_CoreService_proto_msgTypes[13] + mi := &file_feast_core_CoreService_proto_msgTypes[9] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -824,7 +566,7 @@ func (x *ApplyEntityResponse) String() string { func (*ApplyEntityResponse) ProtoMessage() {} func (x *ApplyEntityResponse) ProtoReflect() protoreflect.Message { - mi := &file_feast_core_CoreService_proto_msgTypes[13] + mi := &file_feast_core_CoreService_proto_msgTypes[9] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -837,7 +579,7 @@ func (x *ApplyEntityResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use ApplyEntityResponse.ProtoReflect.Descriptor instead. func (*ApplyEntityResponse) Descriptor() ([]byte, []int) { - return file_feast_core_CoreService_proto_rawDescGZIP(), []int{13} + return file_feast_core_CoreService_proto_rawDescGZIP(), []int{9} } func (x *ApplyEntityResponse) GetEntity() *Entity { @@ -847,134 +589,29 @@ func (x *ApplyEntityResponse) GetEntity() *Entity { return nil } -type ApplyFeatureSetRequest struct { +type GetFeastCoreVersionRequest struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - - // Feature set version - // If project is unspecified, will default to 'default' project. - // If project specified does not exist, the project would be automatically created. - FeatureSet *FeatureSet `protobuf:"bytes,1,opt,name=feature_set,json=featureSet,proto3" json:"feature_set,omitempty"` } -func (x *ApplyFeatureSetRequest) Reset() { - *x = ApplyFeatureSetRequest{} +func (x *GetFeastCoreVersionRequest) Reset() { + *x = GetFeastCoreVersionRequest{} if protoimpl.UnsafeEnabled { - mi := &file_feast_core_CoreService_proto_msgTypes[14] + mi := &file_feast_core_CoreService_proto_msgTypes[10] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } } -func (x *ApplyFeatureSetRequest) String() string { +func (x *GetFeastCoreVersionRequest) String() string { return protoimpl.X.MessageStringOf(x) } -func (*ApplyFeatureSetRequest) ProtoMessage() {} - -func (x *ApplyFeatureSetRequest) ProtoReflect() protoreflect.Message { - mi := &file_feast_core_CoreService_proto_msgTypes[14] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use ApplyFeatureSetRequest.ProtoReflect.Descriptor instead. -func (*ApplyFeatureSetRequest) Descriptor() ([]byte, []int) { - return file_feast_core_CoreService_proto_rawDescGZIP(), []int{14} -} - -func (x *ApplyFeatureSetRequest) GetFeatureSet() *FeatureSet { - if x != nil { - return x.FeatureSet - } - return nil -} - -type ApplyFeatureSetResponse struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - FeatureSet *FeatureSet `protobuf:"bytes,1,opt,name=feature_set,json=featureSet,proto3" json:"feature_set,omitempty"` - Status ApplyFeatureSetResponse_Status `protobuf:"varint,2,opt,name=status,proto3,enum=feast.core.ApplyFeatureSetResponse_Status" json:"status,omitempty"` -} - -func (x *ApplyFeatureSetResponse) Reset() { - *x = ApplyFeatureSetResponse{} - if protoimpl.UnsafeEnabled { - mi := &file_feast_core_CoreService_proto_msgTypes[15] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *ApplyFeatureSetResponse) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*ApplyFeatureSetResponse) ProtoMessage() {} - -func (x *ApplyFeatureSetResponse) ProtoReflect() protoreflect.Message { - mi := &file_feast_core_CoreService_proto_msgTypes[15] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use ApplyFeatureSetResponse.ProtoReflect.Descriptor instead. -func (*ApplyFeatureSetResponse) Descriptor() ([]byte, []int) { - return file_feast_core_CoreService_proto_rawDescGZIP(), []int{15} -} - -func (x *ApplyFeatureSetResponse) GetFeatureSet() *FeatureSet { - if x != nil { - return x.FeatureSet - } - return nil -} - -func (x *ApplyFeatureSetResponse) GetStatus() ApplyFeatureSetResponse_Status { - if x != nil { - return x.Status - } - return ApplyFeatureSetResponse_NO_CHANGE -} - -type GetFeastCoreVersionRequest struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields -} - -func (x *GetFeastCoreVersionRequest) Reset() { - *x = GetFeastCoreVersionRequest{} - if protoimpl.UnsafeEnabled { - mi := &file_feast_core_CoreService_proto_msgTypes[16] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *GetFeastCoreVersionRequest) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*GetFeastCoreVersionRequest) ProtoMessage() {} +func (*GetFeastCoreVersionRequest) ProtoMessage() {} func (x *GetFeastCoreVersionRequest) ProtoReflect() protoreflect.Message { - mi := &file_feast_core_CoreService_proto_msgTypes[16] + mi := &file_feast_core_CoreService_proto_msgTypes[10] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -987,7 +624,7 @@ func (x *GetFeastCoreVersionRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use GetFeastCoreVersionRequest.ProtoReflect.Descriptor instead. func (*GetFeastCoreVersionRequest) Descriptor() ([]byte, []int) { - return file_feast_core_CoreService_proto_rawDescGZIP(), []int{16} + return file_feast_core_CoreService_proto_rawDescGZIP(), []int{10} } type GetFeastCoreVersionResponse struct { @@ -1001,7 +638,7 @@ type GetFeastCoreVersionResponse struct { func (x *GetFeastCoreVersionResponse) Reset() { *x = GetFeastCoreVersionResponse{} if protoimpl.UnsafeEnabled { - mi := &file_feast_core_CoreService_proto_msgTypes[17] + mi := &file_feast_core_CoreService_proto_msgTypes[11] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1014,7 +651,7 @@ func (x *GetFeastCoreVersionResponse) String() string { func (*GetFeastCoreVersionResponse) ProtoMessage() {} func (x *GetFeastCoreVersionResponse) ProtoReflect() protoreflect.Message { - mi := &file_feast_core_CoreService_proto_msgTypes[17] + mi := &file_feast_core_CoreService_proto_msgTypes[11] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1027,7 +664,7 @@ func (x *GetFeastCoreVersionResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use GetFeastCoreVersionResponse.ProtoReflect.Descriptor instead. func (*GetFeastCoreVersionResponse) Descriptor() ([]byte, []int) { - return file_feast_core_CoreService_proto_rawDescGZIP(), []int{17} + return file_feast_core_CoreService_proto_rawDescGZIP(), []int{11} } func (x *GetFeastCoreVersionResponse) GetVersion() string { @@ -1048,7 +685,7 @@ type UpdateStoreRequest struct { func (x *UpdateStoreRequest) Reset() { *x = UpdateStoreRequest{} if protoimpl.UnsafeEnabled { - mi := &file_feast_core_CoreService_proto_msgTypes[18] + mi := &file_feast_core_CoreService_proto_msgTypes[12] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1061,7 +698,7 @@ func (x *UpdateStoreRequest) String() string { func (*UpdateStoreRequest) ProtoMessage() {} func (x *UpdateStoreRequest) ProtoReflect() protoreflect.Message { - mi := &file_feast_core_CoreService_proto_msgTypes[18] + mi := &file_feast_core_CoreService_proto_msgTypes[12] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1074,7 +711,7 @@ func (x *UpdateStoreRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use UpdateStoreRequest.ProtoReflect.Descriptor instead. func (*UpdateStoreRequest) Descriptor() ([]byte, []int) { - return file_feast_core_CoreService_proto_rawDescGZIP(), []int{18} + return file_feast_core_CoreService_proto_rawDescGZIP(), []int{12} } func (x *UpdateStoreRequest) GetStore() *Store { @@ -1096,7 +733,7 @@ type UpdateStoreResponse struct { func (x *UpdateStoreResponse) Reset() { *x = UpdateStoreResponse{} if protoimpl.UnsafeEnabled { - mi := &file_feast_core_CoreService_proto_msgTypes[19] + mi := &file_feast_core_CoreService_proto_msgTypes[13] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1109,7 +746,7 @@ func (x *UpdateStoreResponse) String() string { func (*UpdateStoreResponse) ProtoMessage() {} func (x *UpdateStoreResponse) ProtoReflect() protoreflect.Message { - mi := &file_feast_core_CoreService_proto_msgTypes[19] + mi := &file_feast_core_CoreService_proto_msgTypes[13] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1122,7 +759,7 @@ func (x *UpdateStoreResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use UpdateStoreResponse.ProtoReflect.Descriptor instead. func (*UpdateStoreResponse) Descriptor() ([]byte, []int) { - return file_feast_core_CoreService_proto_rawDescGZIP(), []int{19} + return file_feast_core_CoreService_proto_rawDescGZIP(), []int{13} } func (x *UpdateStoreResponse) GetStore() *Store { @@ -1152,7 +789,7 @@ type CreateProjectRequest struct { func (x *CreateProjectRequest) Reset() { *x = CreateProjectRequest{} if protoimpl.UnsafeEnabled { - mi := &file_feast_core_CoreService_proto_msgTypes[20] + mi := &file_feast_core_CoreService_proto_msgTypes[14] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1165,7 +802,7 @@ func (x *CreateProjectRequest) String() string { func (*CreateProjectRequest) ProtoMessage() {} func (x *CreateProjectRequest) ProtoReflect() protoreflect.Message { - mi := &file_feast_core_CoreService_proto_msgTypes[20] + mi := &file_feast_core_CoreService_proto_msgTypes[14] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1178,7 +815,7 @@ func (x *CreateProjectRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use CreateProjectRequest.ProtoReflect.Descriptor instead. func (*CreateProjectRequest) Descriptor() ([]byte, []int) { - return file_feast_core_CoreService_proto_rawDescGZIP(), []int{20} + return file_feast_core_CoreService_proto_rawDescGZIP(), []int{14} } func (x *CreateProjectRequest) GetName() string { @@ -1198,7 +835,7 @@ type CreateProjectResponse struct { func (x *CreateProjectResponse) Reset() { *x = CreateProjectResponse{} if protoimpl.UnsafeEnabled { - mi := &file_feast_core_CoreService_proto_msgTypes[21] + mi := &file_feast_core_CoreService_proto_msgTypes[15] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1211,7 +848,7 @@ func (x *CreateProjectResponse) String() string { func (*CreateProjectResponse) ProtoMessage() {} func (x *CreateProjectResponse) ProtoReflect() protoreflect.Message { - mi := &file_feast_core_CoreService_proto_msgTypes[21] + mi := &file_feast_core_CoreService_proto_msgTypes[15] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1224,7 +861,7 @@ func (x *CreateProjectResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use CreateProjectResponse.ProtoReflect.Descriptor instead. func (*CreateProjectResponse) Descriptor() ([]byte, []int) { - return file_feast_core_CoreService_proto_rawDescGZIP(), []int{21} + return file_feast_core_CoreService_proto_rawDescGZIP(), []int{15} } // Request for the archival of a project @@ -1240,7 +877,7 @@ type ArchiveProjectRequest struct { func (x *ArchiveProjectRequest) Reset() { *x = ArchiveProjectRequest{} if protoimpl.UnsafeEnabled { - mi := &file_feast_core_CoreService_proto_msgTypes[22] + mi := &file_feast_core_CoreService_proto_msgTypes[16] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1253,7 +890,7 @@ func (x *ArchiveProjectRequest) String() string { func (*ArchiveProjectRequest) ProtoMessage() {} func (x *ArchiveProjectRequest) ProtoReflect() protoreflect.Message { - mi := &file_feast_core_CoreService_proto_msgTypes[22] + mi := &file_feast_core_CoreService_proto_msgTypes[16] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1266,7 +903,7 @@ func (x *ArchiveProjectRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use ArchiveProjectRequest.ProtoReflect.Descriptor instead. func (*ArchiveProjectRequest) Descriptor() ([]byte, []int) { - return file_feast_core_CoreService_proto_rawDescGZIP(), []int{22} + return file_feast_core_CoreService_proto_rawDescGZIP(), []int{16} } func (x *ArchiveProjectRequest) GetName() string { @@ -1286,7 +923,7 @@ type ArchiveProjectResponse struct { func (x *ArchiveProjectResponse) Reset() { *x = ArchiveProjectResponse{} if protoimpl.UnsafeEnabled { - mi := &file_feast_core_CoreService_proto_msgTypes[23] + mi := &file_feast_core_CoreService_proto_msgTypes[17] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1299,7 +936,7 @@ func (x *ArchiveProjectResponse) String() string { func (*ArchiveProjectResponse) ProtoMessage() {} func (x *ArchiveProjectResponse) ProtoReflect() protoreflect.Message { - mi := &file_feast_core_CoreService_proto_msgTypes[23] + mi := &file_feast_core_CoreService_proto_msgTypes[17] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1312,7 +949,7 @@ func (x *ArchiveProjectResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use ArchiveProjectResponse.ProtoReflect.Descriptor instead. func (*ArchiveProjectResponse) Descriptor() ([]byte, []int) { - return file_feast_core_CoreService_proto_rawDescGZIP(), []int{23} + return file_feast_core_CoreService_proto_rawDescGZIP(), []int{17} } // Request for listing of projects @@ -1325,7 +962,7 @@ type ListProjectsRequest struct { func (x *ListProjectsRequest) Reset() { *x = ListProjectsRequest{} if protoimpl.UnsafeEnabled { - mi := &file_feast_core_CoreService_proto_msgTypes[24] + mi := &file_feast_core_CoreService_proto_msgTypes[18] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1338,7 +975,7 @@ func (x *ListProjectsRequest) String() string { func (*ListProjectsRequest) ProtoMessage() {} func (x *ListProjectsRequest) ProtoReflect() protoreflect.Message { - mi := &file_feast_core_CoreService_proto_msgTypes[24] + mi := &file_feast_core_CoreService_proto_msgTypes[18] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1351,7 +988,7 @@ func (x *ListProjectsRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use ListProjectsRequest.ProtoReflect.Descriptor instead. func (*ListProjectsRequest) Descriptor() ([]byte, []int) { - return file_feast_core_CoreService_proto_rawDescGZIP(), []int{24} + return file_feast_core_CoreService_proto_rawDescGZIP(), []int{18} } // Response for listing of projects @@ -1367,7 +1004,7 @@ type ListProjectsResponse struct { func (x *ListProjectsResponse) Reset() { *x = ListProjectsResponse{} if protoimpl.UnsafeEnabled { - mi := &file_feast_core_CoreService_proto_msgTypes[25] + mi := &file_feast_core_CoreService_proto_msgTypes[19] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1380,7 +1017,7 @@ func (x *ListProjectsResponse) String() string { func (*ListProjectsResponse) ProtoMessage() {} func (x *ListProjectsResponse) ProtoReflect() protoreflect.Message { - mi := &file_feast_core_CoreService_proto_msgTypes[25] + mi := &file_feast_core_CoreService_proto_msgTypes[19] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1393,7 +1030,7 @@ func (x *ListProjectsResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use ListProjectsResponse.ProtoReflect.Descriptor instead. func (*ListProjectsResponse) Descriptor() ([]byte, []int) { - return file_feast_core_CoreService_proto_rawDescGZIP(), []int{25} + return file_feast_core_CoreService_proto_rawDescGZIP(), []int{19} } func (x *ListProjectsResponse) GetProjects() []string { @@ -1403,32 +1040,29 @@ func (x *ListProjectsResponse) GetProjects() []string { return nil } -// Request for listing ingestion jobs -type ListIngestionJobsRequest struct { +type UpdateFeatureSetStatusResponse struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - - Filter *ListIngestionJobsRequest_Filter `protobuf:"bytes,1,opt,name=filter,proto3" json:"filter,omitempty"` } -func (x *ListIngestionJobsRequest) Reset() { - *x = ListIngestionJobsRequest{} +func (x *UpdateFeatureSetStatusResponse) Reset() { + *x = UpdateFeatureSetStatusResponse{} if protoimpl.UnsafeEnabled { - mi := &file_feast_core_CoreService_proto_msgTypes[26] + mi := &file_feast_core_CoreService_proto_msgTypes[20] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } } -func (x *ListIngestionJobsRequest) String() string { +func (x *UpdateFeatureSetStatusResponse) String() string { return protoimpl.X.MessageStringOf(x) } -func (*ListIngestionJobsRequest) ProtoMessage() {} +func (*UpdateFeatureSetStatusResponse) ProtoMessage() {} -func (x *ListIngestionJobsRequest) ProtoReflect() protoreflect.Message { - mi := &file_feast_core_CoreService_proto_msgTypes[26] +func (x *UpdateFeatureSetStatusResponse) ProtoReflect() protoreflect.Message { + mi := &file_feast_core_CoreService_proto_msgTypes[20] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1439,44 +1073,40 @@ func (x *ListIngestionJobsRequest) ProtoReflect() protoreflect.Message { return mi.MessageOf(x) } -// Deprecated: Use ListIngestionJobsRequest.ProtoReflect.Descriptor instead. -func (*ListIngestionJobsRequest) Descriptor() ([]byte, []int) { - return file_feast_core_CoreService_proto_rawDescGZIP(), []int{26} -} - -func (x *ListIngestionJobsRequest) GetFilter() *ListIngestionJobsRequest_Filter { - if x != nil { - return x.Filter - } - return nil +// Deprecated: Use UpdateFeatureSetStatusResponse.ProtoReflect.Descriptor instead. +func (*UpdateFeatureSetStatusResponse) Descriptor() ([]byte, []int) { + return file_feast_core_CoreService_proto_rawDescGZIP(), []int{20} } -// Response from listing ingestion jobs -type ListIngestionJobsResponse struct { +type ApplyFeatureTableRequest struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - Jobs []*IngestionJob `protobuf:"bytes,1,rep,name=jobs,proto3" json:"jobs,omitempty"` + // Optional. Name of the Project to apply the Feature Table to. + // If unspecified, will apply FeatureTable to the default project. + Project string `protobuf:"bytes,1,opt,name=project,proto3" json:"project,omitempty"` + // Feature Table specification to apply + TableSpec *FeatureTableSpec `protobuf:"bytes,2,opt,name=table_spec,json=tableSpec,proto3" json:"table_spec,omitempty"` } -func (x *ListIngestionJobsResponse) Reset() { - *x = ListIngestionJobsResponse{} +func (x *ApplyFeatureTableRequest) Reset() { + *x = ApplyFeatureTableRequest{} if protoimpl.UnsafeEnabled { - mi := &file_feast_core_CoreService_proto_msgTypes[27] + mi := &file_feast_core_CoreService_proto_msgTypes[21] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } } -func (x *ListIngestionJobsResponse) String() string { +func (x *ApplyFeatureTableRequest) String() string { return protoimpl.X.MessageStringOf(x) } -func (*ListIngestionJobsResponse) ProtoMessage() {} +func (*ApplyFeatureTableRequest) ProtoMessage() {} -func (x *ListIngestionJobsResponse) ProtoReflect() protoreflect.Message { - mi := &file_feast_core_CoreService_proto_msgTypes[27] +func (x *ApplyFeatureTableRequest) ProtoReflect() protoreflect.Message { + mi := &file_feast_core_CoreService_proto_msgTypes[21] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1487,45 +1117,50 @@ func (x *ListIngestionJobsResponse) ProtoReflect() protoreflect.Message { return mi.MessageOf(x) } -// Deprecated: Use ListIngestionJobsResponse.ProtoReflect.Descriptor instead. -func (*ListIngestionJobsResponse) Descriptor() ([]byte, []int) { - return file_feast_core_CoreService_proto_rawDescGZIP(), []int{27} +// Deprecated: Use ApplyFeatureTableRequest.ProtoReflect.Descriptor instead. +func (*ApplyFeatureTableRequest) Descriptor() ([]byte, []int) { + return file_feast_core_CoreService_proto_rawDescGZIP(), []int{21} +} + +func (x *ApplyFeatureTableRequest) GetProject() string { + if x != nil { + return x.Project + } + return "" } -func (x *ListIngestionJobsResponse) GetJobs() []*IngestionJob { +func (x *ApplyFeatureTableRequest) GetTableSpec() *FeatureTableSpec { if x != nil { - return x.Jobs + return x.TableSpec } return nil } -// Request to restart ingestion job -type RestartIngestionJobRequest struct { +type ApplyFeatureTableResponse struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - // Job ID assigned by Feast - Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"` + Table *FeatureTable `protobuf:"bytes,1,opt,name=table,proto3" json:"table,omitempty"` } -func (x *RestartIngestionJobRequest) Reset() { - *x = RestartIngestionJobRequest{} +func (x *ApplyFeatureTableResponse) Reset() { + *x = ApplyFeatureTableResponse{} if protoimpl.UnsafeEnabled { - mi := &file_feast_core_CoreService_proto_msgTypes[28] + mi := &file_feast_core_CoreService_proto_msgTypes[22] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } } -func (x *RestartIngestionJobRequest) String() string { +func (x *ApplyFeatureTableResponse) String() string { return protoimpl.X.MessageStringOf(x) } -func (*RestartIngestionJobRequest) ProtoMessage() {} +func (*ApplyFeatureTableResponse) ProtoMessage() {} -func (x *RestartIngestionJobRequest) ProtoReflect() protoreflect.Message { - mi := &file_feast_core_CoreService_proto_msgTypes[28] +func (x *ApplyFeatureTableResponse) ProtoReflect() protoreflect.Message { + mi := &file_feast_core_CoreService_proto_msgTypes[22] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1536,42 +1171,47 @@ func (x *RestartIngestionJobRequest) ProtoReflect() protoreflect.Message { return mi.MessageOf(x) } -// Deprecated: Use RestartIngestionJobRequest.ProtoReflect.Descriptor instead. -func (*RestartIngestionJobRequest) Descriptor() ([]byte, []int) { - return file_feast_core_CoreService_proto_rawDescGZIP(), []int{28} +// Deprecated: Use ApplyFeatureTableResponse.ProtoReflect.Descriptor instead. +func (*ApplyFeatureTableResponse) Descriptor() ([]byte, []int) { + return file_feast_core_CoreService_proto_rawDescGZIP(), []int{22} } -func (x *RestartIngestionJobRequest) GetId() string { +func (x *ApplyFeatureTableResponse) GetTable() *FeatureTable { if x != nil { - return x.Id + return x.Table } - return "" + return nil } -// Response from restartingan injestion job -type RestartIngestionJobResponse struct { +type GetFeatureTableRequest struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields + + // Optional. Name of the Project to retrieve the Feature Table from. + // If unspecified, will apply FeatureTable to the default project. + Project string `protobuf:"bytes,1,opt,name=project,proto3" json:"project,omitempty"` + // Name of the FeatureTable to retrieve. + Name string `protobuf:"bytes,2,opt,name=name,proto3" json:"name,omitempty"` } -func (x *RestartIngestionJobResponse) Reset() { - *x = RestartIngestionJobResponse{} +func (x *GetFeatureTableRequest) Reset() { + *x = GetFeatureTableRequest{} if protoimpl.UnsafeEnabled { - mi := &file_feast_core_CoreService_proto_msgTypes[29] + mi := &file_feast_core_CoreService_proto_msgTypes[23] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } } -func (x *RestartIngestionJobResponse) String() string { +func (x *GetFeatureTableRequest) String() string { return protoimpl.X.MessageStringOf(x) } -func (*RestartIngestionJobResponse) ProtoMessage() {} +func (*GetFeatureTableRequest) ProtoMessage() {} -func (x *RestartIngestionJobResponse) ProtoReflect() protoreflect.Message { - mi := &file_feast_core_CoreService_proto_msgTypes[29] +func (x *GetFeatureTableRequest) ProtoReflect() protoreflect.Message { + mi := &file_feast_core_CoreService_proto_msgTypes[23] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1582,38 +1222,51 @@ func (x *RestartIngestionJobResponse) ProtoReflect() protoreflect.Message { return mi.MessageOf(x) } -// Deprecated: Use RestartIngestionJobResponse.ProtoReflect.Descriptor instead. -func (*RestartIngestionJobResponse) Descriptor() ([]byte, []int) { - return file_feast_core_CoreService_proto_rawDescGZIP(), []int{29} +// Deprecated: Use GetFeatureTableRequest.ProtoReflect.Descriptor instead. +func (*GetFeatureTableRequest) Descriptor() ([]byte, []int) { + return file_feast_core_CoreService_proto_rawDescGZIP(), []int{23} +} + +func (x *GetFeatureTableRequest) GetProject() string { + if x != nil { + return x.Project + } + return "" +} + +func (x *GetFeatureTableRequest) GetName() string { + if x != nil { + return x.Name + } + return "" } -// Request to stop ingestion job -type StopIngestionJobRequest struct { +type GetFeatureTableResponse struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - // Job ID assigned by Feast - Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"` + // The Feature Table retrieved. + Table *FeatureTable `protobuf:"bytes,1,opt,name=table,proto3" json:"table,omitempty"` } -func (x *StopIngestionJobRequest) Reset() { - *x = StopIngestionJobRequest{} +func (x *GetFeatureTableResponse) Reset() { + *x = GetFeatureTableResponse{} if protoimpl.UnsafeEnabled { - mi := &file_feast_core_CoreService_proto_msgTypes[30] + mi := &file_feast_core_CoreService_proto_msgTypes[24] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } } -func (x *StopIngestionJobRequest) String() string { +func (x *GetFeatureTableResponse) String() string { return protoimpl.X.MessageStringOf(x) } -func (*StopIngestionJobRequest) ProtoMessage() {} +func (*GetFeatureTableResponse) ProtoMessage() {} -func (x *StopIngestionJobRequest) ProtoReflect() protoreflect.Message { - mi := &file_feast_core_CoreService_proto_msgTypes[30] +func (x *GetFeatureTableResponse) ProtoReflect() protoreflect.Message { + mi := &file_feast_core_CoreService_proto_msgTypes[24] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1624,523 +1277,16 @@ func (x *StopIngestionJobRequest) ProtoReflect() protoreflect.Message { return mi.MessageOf(x) } -// Deprecated: Use StopIngestionJobRequest.ProtoReflect.Descriptor instead. -func (*StopIngestionJobRequest) Descriptor() ([]byte, []int) { - return file_feast_core_CoreService_proto_rawDescGZIP(), []int{30} +// Deprecated: Use GetFeatureTableResponse.ProtoReflect.Descriptor instead. +func (*GetFeatureTableResponse) Descriptor() ([]byte, []int) { + return file_feast_core_CoreService_proto_rawDescGZIP(), []int{24} } -func (x *StopIngestionJobRequest) GetId() string { +func (x *GetFeatureTableResponse) GetTable() *FeatureTable { if x != nil { - return x.Id + return x.Table } - return "" -} - -// Request from stopping an ingestion job -type StopIngestionJobResponse struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields -} - -func (x *StopIngestionJobResponse) Reset() { - *x = StopIngestionJobResponse{} - if protoimpl.UnsafeEnabled { - mi := &file_feast_core_CoreService_proto_msgTypes[31] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *StopIngestionJobResponse) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*StopIngestionJobResponse) ProtoMessage() {} - -func (x *StopIngestionJobResponse) ProtoReflect() protoreflect.Message { - mi := &file_feast_core_CoreService_proto_msgTypes[31] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use StopIngestionJobResponse.ProtoReflect.Descriptor instead. -func (*StopIngestionJobResponse) Descriptor() ([]byte, []int) { - return file_feast_core_CoreService_proto_rawDescGZIP(), []int{31} -} - -type GetFeatureStatisticsRequest struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - // Feature set to retrieve the statistics for. A fully qualified feature set - // id in the format of project/feature_set must be provided. - FeatureSetId string `protobuf:"bytes,1,opt,name=feature_set_id,json=featureSetId,proto3" json:"feature_set_id,omitempty"` - // Optional filter which filters returned statistics by selected features. These - // features must be present in the data that is being processed. - Features []string `protobuf:"bytes,2,rep,name=features,proto3" json:"features,omitempty"` - // Optional filter to select store over which the statistics will retrieved. - // Only historical stores are allowed. - Store string `protobuf:"bytes,3,opt,name=store,proto3" json:"store,omitempty"` - // Optional start and end dates over which to filter statistical data - // Start date is inclusive, but end date is not. - // Only dates are supported, not times. - // Cannot be used with dataset_ids. - // If this period spans multiple days, unaggregatable statistics will be dropped. - StartDate *timestamp.Timestamp `protobuf:"bytes,4,opt,name=start_date,json=startDate,proto3" json:"start_date,omitempty"` - EndDate *timestamp.Timestamp `protobuf:"bytes,5,opt,name=end_date,json=endDate,proto3" json:"end_date,omitempty"` - // Optional list of ingestion Ids by which to filter data before - // retrieving statistics. - // Cannot be used with the date ranges - // If multiple dataset ids are provided, unaggregatable statistics will be dropped. - IngestionIds []string `protobuf:"bytes,6,rep,name=ingestion_ids,json=ingestionIds,proto3" json:"ingestion_ids,omitempty"` - // Setting this flag to true will force a recalculation of statistics and overwrite results currently in the - // cache, if any. - ForceRefresh bool `protobuf:"varint,7,opt,name=force_refresh,json=forceRefresh,proto3" json:"force_refresh,omitempty"` -} - -func (x *GetFeatureStatisticsRequest) Reset() { - *x = GetFeatureStatisticsRequest{} - if protoimpl.UnsafeEnabled { - mi := &file_feast_core_CoreService_proto_msgTypes[32] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *GetFeatureStatisticsRequest) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*GetFeatureStatisticsRequest) ProtoMessage() {} - -func (x *GetFeatureStatisticsRequest) ProtoReflect() protoreflect.Message { - mi := &file_feast_core_CoreService_proto_msgTypes[32] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use GetFeatureStatisticsRequest.ProtoReflect.Descriptor instead. -func (*GetFeatureStatisticsRequest) Descriptor() ([]byte, []int) { - return file_feast_core_CoreService_proto_rawDescGZIP(), []int{32} -} - -func (x *GetFeatureStatisticsRequest) GetFeatureSetId() string { - if x != nil { - return x.FeatureSetId - } - return "" -} - -func (x *GetFeatureStatisticsRequest) GetFeatures() []string { - if x != nil { - return x.Features - } - return nil -} - -func (x *GetFeatureStatisticsRequest) GetStore() string { - if x != nil { - return x.Store - } - return "" -} - -func (x *GetFeatureStatisticsRequest) GetStartDate() *timestamp.Timestamp { - if x != nil { - return x.StartDate - } - return nil -} - -func (x *GetFeatureStatisticsRequest) GetEndDate() *timestamp.Timestamp { - if x != nil { - return x.EndDate - } - return nil -} - -func (x *GetFeatureStatisticsRequest) GetIngestionIds() []string { - if x != nil { - return x.IngestionIds - } - return nil -} - -func (x *GetFeatureStatisticsRequest) GetForceRefresh() bool { - if x != nil { - return x.ForceRefresh - } - return false -} - -type GetFeatureStatisticsResponse struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - // Contains statistics for the requested data. - // Due to the limitations of TFDV and Facets, only a single dataset can be returned in, - // despite the message being of list type. - DatasetFeatureStatisticsList *v0.DatasetFeatureStatisticsList `protobuf:"bytes,1,opt,name=dataset_feature_statistics_list,json=datasetFeatureStatisticsList,proto3" json:"dataset_feature_statistics_list,omitempty"` -} - -func (x *GetFeatureStatisticsResponse) Reset() { - *x = GetFeatureStatisticsResponse{} - if protoimpl.UnsafeEnabled { - mi := &file_feast_core_CoreService_proto_msgTypes[33] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *GetFeatureStatisticsResponse) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*GetFeatureStatisticsResponse) ProtoMessage() {} - -func (x *GetFeatureStatisticsResponse) ProtoReflect() protoreflect.Message { - mi := &file_feast_core_CoreService_proto_msgTypes[33] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use GetFeatureStatisticsResponse.ProtoReflect.Descriptor instead. -func (*GetFeatureStatisticsResponse) Descriptor() ([]byte, []int) { - return file_feast_core_CoreService_proto_rawDescGZIP(), []int{33} -} - -func (x *GetFeatureStatisticsResponse) GetDatasetFeatureStatisticsList() *v0.DatasetFeatureStatisticsList { - if x != nil { - return x.DatasetFeatureStatisticsList - } - return nil -} - -type UpdateFeatureSetStatusRequest struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - // FeatureSetReference of FeatureSet to update - Reference *FeatureSetReference `protobuf:"bytes,1,opt,name=reference,proto3" json:"reference,omitempty"` - // Target status - Status FeatureSetStatus `protobuf:"varint,2,opt,name=status,proto3,enum=feast.core.FeatureSetStatus" json:"status,omitempty"` -} - -func (x *UpdateFeatureSetStatusRequest) Reset() { - *x = UpdateFeatureSetStatusRequest{} - if protoimpl.UnsafeEnabled { - mi := &file_feast_core_CoreService_proto_msgTypes[34] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *UpdateFeatureSetStatusRequest) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*UpdateFeatureSetStatusRequest) ProtoMessage() {} - -func (x *UpdateFeatureSetStatusRequest) ProtoReflect() protoreflect.Message { - mi := &file_feast_core_CoreService_proto_msgTypes[34] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use UpdateFeatureSetStatusRequest.ProtoReflect.Descriptor instead. -func (*UpdateFeatureSetStatusRequest) Descriptor() ([]byte, []int) { - return file_feast_core_CoreService_proto_rawDescGZIP(), []int{34} -} - -func (x *UpdateFeatureSetStatusRequest) GetReference() *FeatureSetReference { - if x != nil { - return x.Reference - } - return nil -} - -func (x *UpdateFeatureSetStatusRequest) GetStatus() FeatureSetStatus { - if x != nil { - return x.Status - } - return FeatureSetStatus_STATUS_INVALID -} - -type UpdateFeatureSetStatusResponse struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields -} - -func (x *UpdateFeatureSetStatusResponse) Reset() { - *x = UpdateFeatureSetStatusResponse{} - if protoimpl.UnsafeEnabled { - mi := &file_feast_core_CoreService_proto_msgTypes[35] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *UpdateFeatureSetStatusResponse) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*UpdateFeatureSetStatusResponse) ProtoMessage() {} - -func (x *UpdateFeatureSetStatusResponse) ProtoReflect() protoreflect.Message { - mi := &file_feast_core_CoreService_proto_msgTypes[35] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use UpdateFeatureSetStatusResponse.ProtoReflect.Descriptor instead. -func (*UpdateFeatureSetStatusResponse) Descriptor() ([]byte, []int) { - return file_feast_core_CoreService_proto_rawDescGZIP(), []int{35} -} - -type ApplyFeatureTableRequest struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - // Optional. Name of the Project to apply the Feature Table to. - // If unspecified, will apply FeatureTable to the default project. - Project string `protobuf:"bytes,1,opt,name=project,proto3" json:"project,omitempty"` - // Feature Table specification to apply - TableSpec *FeatureTableSpec `protobuf:"bytes,2,opt,name=table_spec,json=tableSpec,proto3" json:"table_spec,omitempty"` -} - -func (x *ApplyFeatureTableRequest) Reset() { - *x = ApplyFeatureTableRequest{} - if protoimpl.UnsafeEnabled { - mi := &file_feast_core_CoreService_proto_msgTypes[36] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *ApplyFeatureTableRequest) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*ApplyFeatureTableRequest) ProtoMessage() {} - -func (x *ApplyFeatureTableRequest) ProtoReflect() protoreflect.Message { - mi := &file_feast_core_CoreService_proto_msgTypes[36] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use ApplyFeatureTableRequest.ProtoReflect.Descriptor instead. -func (*ApplyFeatureTableRequest) Descriptor() ([]byte, []int) { - return file_feast_core_CoreService_proto_rawDescGZIP(), []int{36} -} - -func (x *ApplyFeatureTableRequest) GetProject() string { - if x != nil { - return x.Project - } - return "" -} - -func (x *ApplyFeatureTableRequest) GetTableSpec() *FeatureTableSpec { - if x != nil { - return x.TableSpec - } - return nil -} - -type ApplyFeatureTableResponse struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - Table *FeatureTable `protobuf:"bytes,1,opt,name=table,proto3" json:"table,omitempty"` -} - -func (x *ApplyFeatureTableResponse) Reset() { - *x = ApplyFeatureTableResponse{} - if protoimpl.UnsafeEnabled { - mi := &file_feast_core_CoreService_proto_msgTypes[37] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *ApplyFeatureTableResponse) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*ApplyFeatureTableResponse) ProtoMessage() {} - -func (x *ApplyFeatureTableResponse) ProtoReflect() protoreflect.Message { - mi := &file_feast_core_CoreService_proto_msgTypes[37] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use ApplyFeatureTableResponse.ProtoReflect.Descriptor instead. -func (*ApplyFeatureTableResponse) Descriptor() ([]byte, []int) { - return file_feast_core_CoreService_proto_rawDescGZIP(), []int{37} -} - -func (x *ApplyFeatureTableResponse) GetTable() *FeatureTable { - if x != nil { - return x.Table - } - return nil -} - -type GetFeatureTableRequest struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - // Optional. Name of the Project to retrieve the Feature Table from. - // If unspecified, will apply FeatureTable to the default project. - Project string `protobuf:"bytes,1,opt,name=project,proto3" json:"project,omitempty"` - // Name of the FeatureTable to retrieve. - Name string `protobuf:"bytes,2,opt,name=name,proto3" json:"name,omitempty"` -} - -func (x *GetFeatureTableRequest) Reset() { - *x = GetFeatureTableRequest{} - if protoimpl.UnsafeEnabled { - mi := &file_feast_core_CoreService_proto_msgTypes[38] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *GetFeatureTableRequest) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*GetFeatureTableRequest) ProtoMessage() {} - -func (x *GetFeatureTableRequest) ProtoReflect() protoreflect.Message { - mi := &file_feast_core_CoreService_proto_msgTypes[38] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use GetFeatureTableRequest.ProtoReflect.Descriptor instead. -func (*GetFeatureTableRequest) Descriptor() ([]byte, []int) { - return file_feast_core_CoreService_proto_rawDescGZIP(), []int{38} -} - -func (x *GetFeatureTableRequest) GetProject() string { - if x != nil { - return x.Project - } - return "" -} - -func (x *GetFeatureTableRequest) GetName() string { - if x != nil { - return x.Name - } - return "" -} - -type GetFeatureTableResponse struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - // The Feature Table retrieved. - Table *FeatureTable `protobuf:"bytes,1,opt,name=table,proto3" json:"table,omitempty"` -} - -func (x *GetFeatureTableResponse) Reset() { - *x = GetFeatureTableResponse{} - if protoimpl.UnsafeEnabled { - mi := &file_feast_core_CoreService_proto_msgTypes[39] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *GetFeatureTableResponse) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*GetFeatureTableResponse) ProtoMessage() {} - -func (x *GetFeatureTableResponse) ProtoReflect() protoreflect.Message { - mi := &file_feast_core_CoreService_proto_msgTypes[39] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use GetFeatureTableResponse.ProtoReflect.Descriptor instead. -func (*GetFeatureTableResponse) Descriptor() ([]byte, []int) { - return file_feast_core_CoreService_proto_rawDescGZIP(), []int{39} -} - -func (x *GetFeatureTableResponse) GetTable() *FeatureTable { - if x != nil { - return x.Table - } - return nil + return nil } type ListFeatureTablesRequest struct { @@ -2155,7 +1301,7 @@ type ListFeatureTablesRequest struct { func (x *ListFeatureTablesRequest) Reset() { *x = ListFeatureTablesRequest{} if protoimpl.UnsafeEnabled { - mi := &file_feast_core_CoreService_proto_msgTypes[40] + mi := &file_feast_core_CoreService_proto_msgTypes[25] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2168,7 +1314,7 @@ func (x *ListFeatureTablesRequest) String() string { func (*ListFeatureTablesRequest) ProtoMessage() {} func (x *ListFeatureTablesRequest) ProtoReflect() protoreflect.Message { - mi := &file_feast_core_CoreService_proto_msgTypes[40] + mi := &file_feast_core_CoreService_proto_msgTypes[25] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2181,7 +1327,7 @@ func (x *ListFeatureTablesRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use ListFeatureTablesRequest.ProtoReflect.Descriptor instead. func (*ListFeatureTablesRequest) Descriptor() ([]byte, []int) { - return file_feast_core_CoreService_proto_rawDescGZIP(), []int{40} + return file_feast_core_CoreService_proto_rawDescGZIP(), []int{25} } func (x *ListFeatureTablesRequest) GetFilter() *ListFeatureTablesRequest_Filter { @@ -2203,7 +1349,7 @@ type ListFeatureTablesResponse struct { func (x *ListFeatureTablesResponse) Reset() { *x = ListFeatureTablesResponse{} if protoimpl.UnsafeEnabled { - mi := &file_feast_core_CoreService_proto_msgTypes[41] + mi := &file_feast_core_CoreService_proto_msgTypes[26] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2216,7 +1362,7 @@ func (x *ListFeatureTablesResponse) String() string { func (*ListFeatureTablesResponse) ProtoMessage() {} func (x *ListFeatureTablesResponse) ProtoReflect() protoreflect.Message { - mi := &file_feast_core_CoreService_proto_msgTypes[41] + mi := &file_feast_core_CoreService_proto_msgTypes[26] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2229,7 +1375,7 @@ func (x *ListFeatureTablesResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use ListFeatureTablesResponse.ProtoReflect.Descriptor instead. func (*ListFeatureTablesResponse) Descriptor() ([]byte, []int) { - return file_feast_core_CoreService_proto_rawDescGZIP(), []int{41} + return file_feast_core_CoreService_proto_rawDescGZIP(), []int{26} } func (x *ListFeatureTablesResponse) GetTables() []*FeatureTable { @@ -2239,52 +1385,35 @@ func (x *ListFeatureTablesResponse) GetTables() []*FeatureTable { return nil } -type ListFeatureSetsRequest_Filter struct { +type DeleteFeatureTableRequest struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - // Name of project that the feature sets belongs to. This can be one of - // - [project_name] - // - * - // If an asterisk is provided, filtering on projects will be disabled. All projects will - // be matched. It is NOT possible to provide an asterisk with a string in order to do - // pattern matching. - // If unspecified this field will default to the default project 'default'. - Project string `protobuf:"bytes,3,opt,name=project,proto3" json:"project,omitempty"` - // Name of the desired feature set. Asterisks can be used as wildcards in the name. - // Matching on names is only permitted if a specific project is defined. It is disallowed - // If the project name is set to "*" - // e.g. - // - * can be used to match all feature sets - // - my-feature-set* can be used to match all features prefixed by "my-feature-set" - // - my-feature-set-6 can be used to select a single feature set - FeatureSetName string `protobuf:"bytes,1,opt,name=feature_set_name,json=featureSetName,proto3" json:"feature_set_name,omitempty"` - // User defined metadata for feature set. - // Feature sets with all matching labels will be returned. - Labels map[string]string `protobuf:"bytes,4,rep,name=labels,proto3" json:"labels,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` - // Filter by FeatureSet's current status - // Project and Feature Set name still must be specified (could be "*") - Status FeatureSetStatus `protobuf:"varint,5,opt,name=status,proto3,enum=feast.core.FeatureSetStatus" json:"status,omitempty"` + // Optional. Name of the Project to delete the Feature Table from. + // If unspecified, will delete FeatureTable from the default project. + Project string `protobuf:"bytes,1,opt,name=project,proto3" json:"project,omitempty"` + // Name of the FeatureTable to delete. + Name string `protobuf:"bytes,2,opt,name=name,proto3" json:"name,omitempty"` } -func (x *ListFeatureSetsRequest_Filter) Reset() { - *x = ListFeatureSetsRequest_Filter{} +func (x *DeleteFeatureTableRequest) Reset() { + *x = DeleteFeatureTableRequest{} if protoimpl.UnsafeEnabled { - mi := &file_feast_core_CoreService_proto_msgTypes[42] + mi := &file_feast_core_CoreService_proto_msgTypes[27] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } } -func (x *ListFeatureSetsRequest_Filter) String() string { +func (x *DeleteFeatureTableRequest) String() string { return protoimpl.X.MessageStringOf(x) } -func (*ListFeatureSetsRequest_Filter) ProtoMessage() {} +func (*DeleteFeatureTableRequest) ProtoMessage() {} -func (x *ListFeatureSetsRequest_Filter) ProtoReflect() protoreflect.Message { - mi := &file_feast_core_CoreService_proto_msgTypes[42] +func (x *DeleteFeatureTableRequest) ProtoReflect() protoreflect.Message { + mi := &file_feast_core_CoreService_proto_msgTypes[27] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2295,37 +1424,61 @@ func (x *ListFeatureSetsRequest_Filter) ProtoReflect() protoreflect.Message { return mi.MessageOf(x) } -// Deprecated: Use ListFeatureSetsRequest_Filter.ProtoReflect.Descriptor instead. -func (*ListFeatureSetsRequest_Filter) Descriptor() ([]byte, []int) { - return file_feast_core_CoreService_proto_rawDescGZIP(), []int{2, 0} +// Deprecated: Use DeleteFeatureTableRequest.ProtoReflect.Descriptor instead. +func (*DeleteFeatureTableRequest) Descriptor() ([]byte, []int) { + return file_feast_core_CoreService_proto_rawDescGZIP(), []int{27} } -func (x *ListFeatureSetsRequest_Filter) GetProject() string { +func (x *DeleteFeatureTableRequest) GetProject() string { if x != nil { return x.Project } return "" } -func (x *ListFeatureSetsRequest_Filter) GetFeatureSetName() string { +func (x *DeleteFeatureTableRequest) GetName() string { if x != nil { - return x.FeatureSetName + return x.Name } return "" } -func (x *ListFeatureSetsRequest_Filter) GetLabels() map[string]string { - if x != nil { - return x.Labels +type DeleteFeatureTableResponse struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields +} + +func (x *DeleteFeatureTableResponse) Reset() { + *x = DeleteFeatureTableResponse{} + if protoimpl.UnsafeEnabled { + mi := &file_feast_core_CoreService_proto_msgTypes[28] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } - return nil } -func (x *ListFeatureSetsRequest_Filter) GetStatus() FeatureSetStatus { - if x != nil { - return x.Status +func (x *DeleteFeatureTableResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*DeleteFeatureTableResponse) ProtoMessage() {} + +func (x *DeleteFeatureTableResponse) ProtoReflect() protoreflect.Message { + mi := &file_feast_core_CoreService_proto_msgTypes[28] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms } - return FeatureSetStatus_STATUS_INVALID + return mi.MessageOf(x) +} + +// Deprecated: Use DeleteFeatureTableResponse.ProtoReflect.Descriptor instead. +func (*DeleteFeatureTableResponse) Descriptor() ([]byte, []int) { + return file_feast_core_CoreService_proto_rawDescGZIP(), []int{28} } type ListEntitiesRequest_Filter struct { @@ -2345,7 +1498,7 @@ type ListEntitiesRequest_Filter struct { func (x *ListEntitiesRequest_Filter) Reset() { *x = ListEntitiesRequest_Filter{} if protoimpl.UnsafeEnabled { - mi := &file_feast_core_CoreService_proto_msgTypes[44] + mi := &file_feast_core_CoreService_proto_msgTypes[29] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2358,7 +1511,7 @@ func (x *ListEntitiesRequest_Filter) String() string { func (*ListEntitiesRequest_Filter) ProtoMessage() {} func (x *ListEntitiesRequest_Filter) ProtoReflect() protoreflect.Message { - mi := &file_feast_core_CoreService_proto_msgTypes[44] + mi := &file_feast_core_CoreService_proto_msgTypes[29] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2371,7 +1524,7 @@ func (x *ListEntitiesRequest_Filter) ProtoReflect() protoreflect.Message { // Deprecated: Use ListEntitiesRequest_Filter.ProtoReflect.Descriptor instead. func (*ListEntitiesRequest_Filter) Descriptor() ([]byte, []int) { - return file_feast_core_CoreService_proto_rawDescGZIP(), []int{6, 0} + return file_feast_core_CoreService_proto_rawDescGZIP(), []int{2, 0} } func (x *ListEntitiesRequest_Filter) GetProject() string { @@ -2397,9 +1550,9 @@ type ListFeaturesRequest_Filter struct { // Features with all matching labels will be returned. Labels map[string]string `protobuf:"bytes,1,rep,name=labels,proto3" json:"labels,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` // List of entities contained within the featureSet that the feature belongs to. - // Only feature sets with these entities will be searched for features. + // Only feature tables with these entities will be searched for features. Entities []string `protobuf:"bytes,2,rep,name=entities,proto3" json:"entities,omitempty"` - // Name of project that the feature sets belongs to. Filtering on projects is disabled. + // Name of project that the feature tables belongs to. Filtering on projects is disabled. // It is NOT possible to provide an asterisk with a string in order to do pattern matching. // If unspecified this field will default to the default project 'default'. Project string `protobuf:"bytes,3,opt,name=project,proto3" json:"project,omitempty"` @@ -2408,7 +1561,7 @@ type ListFeaturesRequest_Filter struct { func (x *ListFeaturesRequest_Filter) Reset() { *x = ListFeaturesRequest_Filter{} if protoimpl.UnsafeEnabled { - mi := &file_feast_core_CoreService_proto_msgTypes[46] + mi := &file_feast_core_CoreService_proto_msgTypes[31] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2421,7 +1574,7 @@ func (x *ListFeaturesRequest_Filter) String() string { func (*ListFeaturesRequest_Filter) ProtoMessage() {} func (x *ListFeaturesRequest_Filter) ProtoReflect() protoreflect.Message { - mi := &file_feast_core_CoreService_proto_msgTypes[46] + mi := &file_feast_core_CoreService_proto_msgTypes[31] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2434,108 +1587,56 @@ func (x *ListFeaturesRequest_Filter) ProtoReflect() protoreflect.Message { // Deprecated: Use ListFeaturesRequest_Filter.ProtoReflect.Descriptor instead. func (*ListFeaturesRequest_Filter) Descriptor() ([]byte, []int) { - return file_feast_core_CoreService_proto_rawDescGZIP(), []int{8, 0} + return file_feast_core_CoreService_proto_rawDescGZIP(), []int{4, 0} } func (x *ListFeaturesRequest_Filter) GetLabels() map[string]string { if x != nil { return x.Labels } - return nil -} - -func (x *ListFeaturesRequest_Filter) GetEntities() []string { - if x != nil { - return x.Entities - } - return nil -} - -func (x *ListFeaturesRequest_Filter) GetProject() string { - if x != nil { - return x.Project - } - return "" -} - -type ListStoresRequest_Filter struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - // Name of desired store. Regex is not supported in this query. - Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` -} - -func (x *ListStoresRequest_Filter) Reset() { - *x = ListStoresRequest_Filter{} - if protoimpl.UnsafeEnabled { - mi := &file_feast_core_CoreService_proto_msgTypes[49] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *ListStoresRequest_Filter) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*ListStoresRequest_Filter) ProtoMessage() {} - -func (x *ListStoresRequest_Filter) ProtoReflect() protoreflect.Message { - mi := &file_feast_core_CoreService_proto_msgTypes[49] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) + return nil } -// Deprecated: Use ListStoresRequest_Filter.ProtoReflect.Descriptor instead. -func (*ListStoresRequest_Filter) Descriptor() ([]byte, []int) { - return file_feast_core_CoreService_proto_rawDescGZIP(), []int{10, 0} +func (x *ListFeaturesRequest_Filter) GetEntities() []string { + if x != nil { + return x.Entities + } + return nil } -func (x *ListStoresRequest_Filter) GetName() string { +func (x *ListFeaturesRequest_Filter) GetProject() string { if x != nil { - return x.Name + return x.Project } return "" } -type ListIngestionJobsRequest_Filter struct { +type ListStoresRequest_Filter struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - // Filter by Job ID assigned by Feast - Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"` - // Filter by ingestion job target feature set. - FeatureSetReference *FeatureSetReference `protobuf:"bytes,2,opt,name=feature_set_reference,json=featureSetReference,proto3" json:"feature_set_reference,omitempty"` - // Filter by Name of store - StoreName string `protobuf:"bytes,3,opt,name=store_name,json=storeName,proto3" json:"store_name,omitempty"` + // Name of desired store. Regex is not supported in this query. + Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` } -func (x *ListIngestionJobsRequest_Filter) Reset() { - *x = ListIngestionJobsRequest_Filter{} +func (x *ListStoresRequest_Filter) Reset() { + *x = ListStoresRequest_Filter{} if protoimpl.UnsafeEnabled { - mi := &file_feast_core_CoreService_proto_msgTypes[50] + mi := &file_feast_core_CoreService_proto_msgTypes[34] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } } -func (x *ListIngestionJobsRequest_Filter) String() string { +func (x *ListStoresRequest_Filter) String() string { return protoimpl.X.MessageStringOf(x) } -func (*ListIngestionJobsRequest_Filter) ProtoMessage() {} +func (*ListStoresRequest_Filter) ProtoMessage() {} -func (x *ListIngestionJobsRequest_Filter) ProtoReflect() protoreflect.Message { - mi := &file_feast_core_CoreService_proto_msgTypes[50] +func (x *ListStoresRequest_Filter) ProtoReflect() protoreflect.Message { + mi := &file_feast_core_CoreService_proto_msgTypes[34] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2546,28 +1647,14 @@ func (x *ListIngestionJobsRequest_Filter) ProtoReflect() protoreflect.Message { return mi.MessageOf(x) } -// Deprecated: Use ListIngestionJobsRequest_Filter.ProtoReflect.Descriptor instead. -func (*ListIngestionJobsRequest_Filter) Descriptor() ([]byte, []int) { - return file_feast_core_CoreService_proto_rawDescGZIP(), []int{26, 0} -} - -func (x *ListIngestionJobsRequest_Filter) GetId() string { - if x != nil { - return x.Id - } - return "" -} - -func (x *ListIngestionJobsRequest_Filter) GetFeatureSetReference() *FeatureSetReference { - if x != nil { - return x.FeatureSetReference - } - return nil +// Deprecated: Use ListStoresRequest_Filter.ProtoReflect.Descriptor instead. +func (*ListStoresRequest_Filter) Descriptor() ([]byte, []int) { + return file_feast_core_CoreService_proto_rawDescGZIP(), []int{6, 0} } -func (x *ListIngestionJobsRequest_Filter) GetStoreName() string { +func (x *ListStoresRequest_Filter) GetName() string { if x != nil { - return x.StoreName + return x.Name } return "" } @@ -2588,7 +1675,7 @@ type ListFeatureTablesRequest_Filter struct { func (x *ListFeatureTablesRequest_Filter) Reset() { *x = ListFeatureTablesRequest_Filter{} if protoimpl.UnsafeEnabled { - mi := &file_feast_core_CoreService_proto_msgTypes[51] + mi := &file_feast_core_CoreService_proto_msgTypes[35] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2601,7 +1688,7 @@ func (x *ListFeatureTablesRequest_Filter) String() string { func (*ListFeatureTablesRequest_Filter) ProtoMessage() {} func (x *ListFeatureTablesRequest_Filter) ProtoReflect() protoreflect.Message { - mi := &file_feast_core_CoreService_proto_msgTypes[51] + mi := &file_feast_core_CoreService_proto_msgTypes[35] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2614,7 +1701,7 @@ func (x *ListFeatureTablesRequest_Filter) ProtoReflect() protoreflect.Message { // Deprecated: Use ListFeatureTablesRequest_Filter.ProtoReflect.Descriptor instead. func (*ListFeatureTablesRequest_Filter) Descriptor() ([]byte, []int) { - return file_feast_core_CoreService_proto_rawDescGZIP(), []int{40, 0} + return file_feast_core_CoreService_proto_rawDescGZIP(), []int{25, 0} } func (x *ListFeatureTablesRequest_Filter) GetProject() string { @@ -2643,427 +1730,261 @@ var file_feast_core_CoreService_proto_rawDesc = []byte{ 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x76, 0x30, 0x2f, 0x73, 0x74, 0x61, 0x74, 0x69, 0x73, 0x74, 0x69, 0x63, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x17, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2f, 0x63, 0x6f, 0x72, 0x65, 0x2f, 0x45, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x2e, 0x70, 0x72, - 0x6f, 0x74, 0x6f, 0x1a, 0x1b, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2f, 0x63, 0x6f, 0x72, 0x65, 0x2f, - 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x53, 0x65, 0x74, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, - 0x1a, 0x1d, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2f, 0x63, 0x6f, 0x72, 0x65, 0x2f, 0x46, 0x65, 0x61, - 0x74, 0x75, 0x72, 0x65, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, - 0x16, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2f, 0x63, 0x6f, 0x72, 0x65, 0x2f, 0x53, 0x74, 0x6f, 0x72, - 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x24, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2f, 0x63, - 0x6f, 0x72, 0x65, 0x2f, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x53, 0x65, 0x74, 0x52, 0x65, - 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x1d, 0x66, - 0x65, 0x61, 0x73, 0x74, 0x2f, 0x63, 0x6f, 0x72, 0x65, 0x2f, 0x49, 0x6e, 0x67, 0x65, 0x73, 0x74, - 0x69, 0x6f, 0x6e, 0x4a, 0x6f, 0x62, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0x44, 0x0a, 0x14, - 0x47, 0x65, 0x74, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x53, 0x65, 0x74, 0x52, 0x65, 0x71, - 0x75, 0x65, 0x73, 0x74, 0x12, 0x18, 0x0a, 0x07, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x18, - 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x12, 0x12, - 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, - 0x6d, 0x65, 0x22, 0x50, 0x0a, 0x15, 0x47, 0x65, 0x74, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, - 0x53, 0x65, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x37, 0x0a, 0x0b, 0x66, - 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x5f, 0x73, 0x65, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, - 0x32, 0x16, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x46, 0x65, - 0x61, 0x74, 0x75, 0x72, 0x65, 0x53, 0x65, 0x74, 0x52, 0x0a, 0x66, 0x65, 0x61, 0x74, 0x75, 0x72, - 0x65, 0x53, 0x65, 0x74, 0x22, 0xea, 0x02, 0x0a, 0x16, 0x4c, 0x69, 0x73, 0x74, 0x46, 0x65, 0x61, - 0x74, 0x75, 0x72, 0x65, 0x53, 0x65, 0x74, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, - 0x41, 0x0a, 0x06, 0x66, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, - 0x29, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x4c, 0x69, 0x73, - 0x74, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x53, 0x65, 0x74, 0x73, 0x52, 0x65, 0x71, 0x75, - 0x65, 0x73, 0x74, 0x2e, 0x46, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x52, 0x06, 0x66, 0x69, 0x6c, 0x74, - 0x65, 0x72, 0x1a, 0x8c, 0x02, 0x0a, 0x06, 0x46, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x12, 0x18, 0x0a, - 0x07, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, - 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x12, 0x28, 0x0a, 0x10, 0x66, 0x65, 0x61, 0x74, 0x75, - 0x72, 0x65, 0x5f, 0x73, 0x65, 0x74, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x0e, 0x66, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x53, 0x65, 0x74, 0x4e, 0x61, 0x6d, - 0x65, 0x12, 0x4d, 0x0a, 0x06, 0x6c, 0x61, 0x62, 0x65, 0x6c, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, - 0x0b, 0x32, 0x35, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x4c, - 0x69, 0x73, 0x74, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x53, 0x65, 0x74, 0x73, 0x52, 0x65, + 0x6f, 0x74, 0x6f, 0x1a, 0x18, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2f, 0x63, 0x6f, 0x72, 0x65, 0x2f, + 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x1d, 0x66, + 0x65, 0x61, 0x73, 0x74, 0x2f, 0x63, 0x6f, 0x72, 0x65, 0x2f, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, + 0x65, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x16, 0x66, 0x65, + 0x61, 0x73, 0x74, 0x2f, 0x63, 0x6f, 0x72, 0x65, 0x2f, 0x53, 0x74, 0x6f, 0x72, 0x65, 0x2e, 0x70, + 0x72, 0x6f, 0x74, 0x6f, 0x22, 0x40, 0x0a, 0x10, 0x47, 0x65, 0x74, 0x45, 0x6e, 0x74, 0x69, 0x74, + 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x18, 0x0a, 0x07, + 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x70, + 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x22, 0x3f, 0x0a, 0x11, 0x47, 0x65, 0x74, 0x45, 0x6e, 0x74, + 0x69, 0x74, 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x2a, 0x0a, 0x06, 0x65, + 0x6e, 0x74, 0x69, 0x74, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x66, 0x65, + 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x45, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x52, + 0x06, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x22, 0x81, 0x02, 0x0a, 0x13, 0x4c, 0x69, 0x73, 0x74, + 0x45, 0x6e, 0x74, 0x69, 0x74, 0x69, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, + 0x3e, 0x0a, 0x06, 0x66, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, + 0x26, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x4c, 0x69, 0x73, + 0x74, 0x45, 0x6e, 0x74, 0x69, 0x74, 0x69, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x2e, 0x46, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x52, 0x06, 0x66, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x1a, + 0xa9, 0x01, 0x0a, 0x06, 0x46, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x12, 0x18, 0x0a, 0x07, 0x70, 0x72, + 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x70, 0x72, 0x6f, + 0x6a, 0x65, 0x63, 0x74, 0x12, 0x4a, 0x0a, 0x06, 0x6c, 0x61, 0x62, 0x65, 0x6c, 0x73, 0x18, 0x04, + 0x20, 0x03, 0x28, 0x0b, 0x32, 0x32, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, + 0x65, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x45, 0x6e, 0x74, 0x69, 0x74, 0x69, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x46, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x2e, 0x4c, 0x61, 0x62, 0x65, 0x6c, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x06, 0x6c, 0x61, 0x62, 0x65, 0x6c, 0x73, - 0x12, 0x34, 0x0a, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0e, - 0x32, 0x1c, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x46, 0x65, - 0x61, 0x74, 0x75, 0x72, 0x65, 0x53, 0x65, 0x74, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x06, - 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x1a, 0x39, 0x0a, 0x0b, 0x4c, 0x61, 0x62, 0x65, 0x6c, 0x73, - 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, - 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, - 0x01, 0x22, 0x54, 0x0a, 0x17, 0x4c, 0x69, 0x73, 0x74, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, - 0x53, 0x65, 0x74, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x39, 0x0a, 0x0c, - 0x66, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x5f, 0x73, 0x65, 0x74, 0x73, 0x18, 0x01, 0x20, 0x03, - 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, - 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x53, 0x65, 0x74, 0x52, 0x0b, 0x66, 0x65, 0x61, 0x74, - 0x75, 0x72, 0x65, 0x53, 0x65, 0x74, 0x73, 0x22, 0x40, 0x0a, 0x10, 0x47, 0x65, 0x74, 0x45, 0x6e, - 0x74, 0x69, 0x74, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x6e, - 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, - 0x18, 0x0a, 0x07, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x07, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x22, 0x3f, 0x0a, 0x11, 0x47, 0x65, 0x74, - 0x45, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x2a, - 0x0a, 0x06, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, - 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x45, 0x6e, 0x74, 0x69, - 0x74, 0x79, 0x52, 0x06, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x22, 0x81, 0x02, 0x0a, 0x13, 0x4c, - 0x69, 0x73, 0x74, 0x45, 0x6e, 0x74, 0x69, 0x74, 0x69, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, - 0x73, 0x74, 0x12, 0x3e, 0x0a, 0x06, 0x66, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x0b, 0x32, 0x26, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, - 0x4c, 0x69, 0x73, 0x74, 0x45, 0x6e, 0x74, 0x69, 0x74, 0x69, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, - 0x65, 0x73, 0x74, 0x2e, 0x46, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x52, 0x06, 0x66, 0x69, 0x6c, 0x74, - 0x65, 0x72, 0x1a, 0xa9, 0x01, 0x0a, 0x06, 0x46, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x12, 0x18, 0x0a, - 0x07, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, - 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x12, 0x4a, 0x0a, 0x06, 0x6c, 0x61, 0x62, 0x65, 0x6c, - 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x32, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, - 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x45, 0x6e, 0x74, 0x69, 0x74, 0x69, 0x65, - 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x46, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x2e, - 0x4c, 0x61, 0x62, 0x65, 0x6c, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x06, 0x6c, 0x61, 0x62, - 0x65, 0x6c, 0x73, 0x1a, 0x39, 0x0a, 0x0b, 0x4c, 0x61, 0x62, 0x65, 0x6c, 0x73, 0x45, 0x6e, 0x74, - 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x46, - 0x0a, 0x14, 0x4c, 0x69, 0x73, 0x74, 0x45, 0x6e, 0x74, 0x69, 0x74, 0x69, 0x65, 0x73, 0x52, 0x65, - 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x2e, 0x0a, 0x08, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x69, - 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, - 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x45, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x52, 0x08, 0x65, 0x6e, - 0x74, 0x69, 0x74, 0x69, 0x65, 0x73, 0x22, 0x9d, 0x02, 0x0a, 0x13, 0x4c, 0x69, 0x73, 0x74, 0x46, - 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x3e, - 0x0a, 0x06, 0x66, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x26, + 0x1a, 0x39, 0x0a, 0x0b, 0x4c, 0x61, 0x62, 0x65, 0x6c, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, + 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, + 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x46, 0x0a, 0x14, 0x4c, + 0x69, 0x73, 0x74, 0x45, 0x6e, 0x74, 0x69, 0x74, 0x69, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x12, 0x2e, 0x0a, 0x08, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x69, 0x65, 0x73, 0x18, + 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, + 0x72, 0x65, 0x2e, 0x45, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x52, 0x08, 0x65, 0x6e, 0x74, 0x69, 0x74, + 0x69, 0x65, 0x73, 0x22, 0x9d, 0x02, 0x0a, 0x13, 0x4c, 0x69, 0x73, 0x74, 0x46, 0x65, 0x61, 0x74, + 0x75, 0x72, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x3e, 0x0a, 0x06, 0x66, + 0x69, 0x6c, 0x74, 0x65, 0x72, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x26, 0x2e, 0x66, 0x65, + 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x46, 0x65, 0x61, + 0x74, 0x75, 0x72, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x46, 0x69, 0x6c, + 0x74, 0x65, 0x72, 0x52, 0x06, 0x66, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x1a, 0xc5, 0x01, 0x0a, 0x06, + 0x46, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x12, 0x4a, 0x0a, 0x06, 0x6c, 0x61, 0x62, 0x65, 0x6c, 0x73, + 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x32, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, + 0x6f, 0x72, 0x65, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x46, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x2e, 0x4c, + 0x61, 0x62, 0x65, 0x6c, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x06, 0x6c, 0x61, 0x62, 0x65, + 0x6c, 0x73, 0x12, 0x1a, 0x0a, 0x08, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x69, 0x65, 0x73, 0x18, 0x02, + 0x20, 0x03, 0x28, 0x09, 0x52, 0x08, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x69, 0x65, 0x73, 0x12, 0x18, + 0x0a, 0x07, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x07, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x1a, 0x39, 0x0a, 0x0b, 0x4c, 0x61, 0x62, 0x65, + 0x6c, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, + 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, + 0x02, 0x38, 0x01, 0x22, 0xc0, 0x01, 0x0a, 0x14, 0x4c, 0x69, 0x73, 0x74, 0x46, 0x65, 0x61, 0x74, + 0x75, 0x72, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x4a, 0x0a, 0x08, + 0x66, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x2e, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x4c, 0x69, 0x73, 0x74, - 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, - 0x46, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x52, 0x06, 0x66, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x1a, 0xc5, - 0x01, 0x0a, 0x06, 0x46, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x12, 0x4a, 0x0a, 0x06, 0x6c, 0x61, 0x62, - 0x65, 0x6c, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x32, 0x2e, 0x66, 0x65, 0x61, 0x73, - 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x46, 0x65, 0x61, 0x74, 0x75, + 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, + 0x2e, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x08, + 0x66, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x1a, 0x56, 0x0a, 0x0d, 0x46, 0x65, 0x61, 0x74, + 0x75, 0x72, 0x65, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x2f, 0x0a, 0x05, 0x76, + 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x66, 0x65, 0x61, + 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x53, + 0x70, 0x65, 0x63, 0x56, 0x32, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, + 0x4a, 0x04, 0x08, 0x01, 0x10, 0x02, 0x22, 0x6f, 0x0a, 0x11, 0x4c, 0x69, 0x73, 0x74, 0x53, 0x74, + 0x6f, 0x72, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x3c, 0x0a, 0x06, 0x66, + 0x69, 0x6c, 0x74, 0x65, 0x72, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x24, 0x2e, 0x66, 0x65, + 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x53, 0x74, 0x6f, 0x72, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x46, 0x69, 0x6c, 0x74, 0x65, - 0x72, 0x2e, 0x4c, 0x61, 0x62, 0x65, 0x6c, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x06, 0x6c, - 0x61, 0x62, 0x65, 0x6c, 0x73, 0x12, 0x1a, 0x0a, 0x08, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x69, 0x65, - 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x09, 0x52, 0x08, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x69, 0x65, - 0x73, 0x12, 0x18, 0x0a, 0x07, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x18, 0x03, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x07, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x1a, 0x39, 0x0a, 0x0b, 0x4c, - 0x61, 0x62, 0x65, 0x6c, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, - 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, - 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, - 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0xb8, 0x01, 0x0a, 0x14, 0x4c, 0x69, 0x73, 0x74, 0x46, - 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, - 0x4a, 0x0a, 0x08, 0x66, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, - 0x0b, 0x32, 0x2e, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x4c, - 0x69, 0x73, 0x74, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, - 0x6e, 0x73, 0x65, 0x2e, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x45, 0x6e, 0x74, 0x72, - 0x79, 0x52, 0x08, 0x66, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x1a, 0x54, 0x0a, 0x0d, 0x46, - 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, - 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x2d, - 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x17, 0x2e, - 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x46, 0x65, 0x61, 0x74, 0x75, - 0x72, 0x65, 0x53, 0x70, 0x65, 0x63, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, - 0x01, 0x22, 0x6f, 0x0a, 0x11, 0x4c, 0x69, 0x73, 0x74, 0x53, 0x74, 0x6f, 0x72, 0x65, 0x73, 0x52, - 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x3c, 0x0a, 0x06, 0x66, 0x69, 0x6c, 0x74, 0x65, 0x72, - 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x24, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, - 0x6f, 0x72, 0x65, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x53, 0x74, 0x6f, 0x72, 0x65, 0x73, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x46, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x52, 0x06, 0x66, 0x69, - 0x6c, 0x74, 0x65, 0x72, 0x1a, 0x1c, 0x0a, 0x06, 0x46, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x12, 0x12, - 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, - 0x6d, 0x65, 0x22, 0x3d, 0x0a, 0x12, 0x4c, 0x69, 0x73, 0x74, 0x53, 0x74, 0x6f, 0x72, 0x65, 0x73, - 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x27, 0x0a, 0x05, 0x73, 0x74, 0x6f, 0x72, - 0x65, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x11, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, - 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x53, 0x74, 0x6f, 0x72, 0x65, 0x52, 0x05, 0x73, 0x74, 0x6f, 0x72, - 0x65, 0x22, 0x5c, 0x0a, 0x12, 0x41, 0x70, 0x70, 0x6c, 0x79, 0x45, 0x6e, 0x74, 0x69, 0x74, 0x79, - 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x2c, 0x0a, 0x04, 0x73, 0x70, 0x65, 0x63, 0x18, - 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x18, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, - 0x72, 0x65, 0x2e, 0x45, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x53, 0x70, 0x65, 0x63, 0x56, 0x32, 0x52, - 0x04, 0x73, 0x70, 0x65, 0x63, 0x12, 0x18, 0x0a, 0x07, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, - 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x22, - 0x41, 0x0a, 0x13, 0x41, 0x70, 0x70, 0x6c, 0x79, 0x45, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x52, 0x65, - 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x2a, 0x0a, 0x06, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, - 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, - 0x6f, 0x72, 0x65, 0x2e, 0x45, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x52, 0x06, 0x65, 0x6e, 0x74, 0x69, - 0x74, 0x79, 0x22, 0x51, 0x0a, 0x16, 0x41, 0x70, 0x70, 0x6c, 0x79, 0x46, 0x65, 0x61, 0x74, 0x75, - 0x72, 0x65, 0x53, 0x65, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x37, 0x0a, 0x0b, - 0x66, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x5f, 0x73, 0x65, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, - 0x0b, 0x32, 0x16, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x46, - 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x53, 0x65, 0x74, 0x52, 0x0a, 0x66, 0x65, 0x61, 0x74, 0x75, - 0x72, 0x65, 0x53, 0x65, 0x74, 0x22, 0xd4, 0x01, 0x0a, 0x17, 0x41, 0x70, 0x70, 0x6c, 0x79, 0x46, - 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x53, 0x65, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, - 0x65, 0x12, 0x37, 0x0a, 0x0b, 0x66, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x5f, 0x73, 0x65, 0x74, - 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, - 0x6f, 0x72, 0x65, 0x2e, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x53, 0x65, 0x74, 0x52, 0x0a, - 0x66, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x53, 0x65, 0x74, 0x12, 0x42, 0x0a, 0x06, 0x73, 0x74, - 0x61, 0x74, 0x75, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x2a, 0x2e, 0x66, 0x65, 0x61, - 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x41, 0x70, 0x70, 0x6c, 0x79, 0x46, 0x65, 0x61, - 0x74, 0x75, 0x72, 0x65, 0x53, 0x65, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, - 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x22, 0x3c, + 0x72, 0x52, 0x06, 0x66, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x1a, 0x1c, 0x0a, 0x06, 0x46, 0x69, 0x6c, + 0x74, 0x65, 0x72, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x22, 0x3d, 0x0a, 0x12, 0x4c, 0x69, 0x73, 0x74, 0x53, + 0x74, 0x6f, 0x72, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x27, 0x0a, + 0x05, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x11, 0x2e, 0x66, + 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x53, 0x74, 0x6f, 0x72, 0x65, 0x52, + 0x05, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x22, 0x5c, 0x0a, 0x12, 0x41, 0x70, 0x70, 0x6c, 0x79, 0x45, + 0x6e, 0x74, 0x69, 0x74, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x2c, 0x0a, 0x04, + 0x73, 0x70, 0x65, 0x63, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x18, 0x2e, 0x66, 0x65, 0x61, + 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x45, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x53, 0x70, + 0x65, 0x63, 0x56, 0x32, 0x52, 0x04, 0x73, 0x70, 0x65, 0x63, 0x12, 0x18, 0x0a, 0x07, 0x70, 0x72, + 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x70, 0x72, 0x6f, + 0x6a, 0x65, 0x63, 0x74, 0x22, 0x41, 0x0a, 0x13, 0x41, 0x70, 0x70, 0x6c, 0x79, 0x45, 0x6e, 0x74, + 0x69, 0x74, 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x2a, 0x0a, 0x06, 0x65, + 0x6e, 0x74, 0x69, 0x74, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x66, 0x65, + 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x45, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x52, + 0x06, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x22, 0x1c, 0x0a, 0x1a, 0x47, 0x65, 0x74, 0x46, 0x65, + 0x61, 0x73, 0x74, 0x43, 0x6f, 0x72, 0x65, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x65, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x22, 0x37, 0x0a, 0x1b, 0x47, 0x65, 0x74, 0x46, 0x65, 0x61, 0x73, + 0x74, 0x43, 0x6f, 0x72, 0x65, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x70, + 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x22, 0x3d, + 0x0a, 0x12, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x53, 0x74, 0x6f, 0x72, 0x65, 0x52, 0x65, 0x71, + 0x75, 0x65, 0x73, 0x74, 0x12, 0x27, 0x0a, 0x05, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x0b, 0x32, 0x11, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, + 0x2e, 0x53, 0x74, 0x6f, 0x72, 0x65, 0x52, 0x05, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x22, 0xa4, 0x01, + 0x0a, 0x13, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x53, 0x74, 0x6f, 0x72, 0x65, 0x52, 0x65, 0x73, + 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x27, 0x0a, 0x05, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x0b, 0x32, 0x11, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, + 0x65, 0x2e, 0x53, 0x74, 0x6f, 0x72, 0x65, 0x52, 0x05, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x12, 0x3e, + 0x0a, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x26, + 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x55, 0x70, 0x64, 0x61, + 0x74, 0x65, 0x53, 0x74, 0x6f, 0x72, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, + 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x22, 0x24, 0x0a, 0x06, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x0d, 0x0a, 0x09, 0x4e, 0x4f, 0x5f, 0x43, - 0x48, 0x41, 0x4e, 0x47, 0x45, 0x10, 0x00, 0x12, 0x0b, 0x0a, 0x07, 0x43, 0x52, 0x45, 0x41, 0x54, - 0x45, 0x44, 0x10, 0x01, 0x12, 0x09, 0x0a, 0x05, 0x45, 0x52, 0x52, 0x4f, 0x52, 0x10, 0x02, 0x12, - 0x0b, 0x0a, 0x07, 0x55, 0x50, 0x44, 0x41, 0x54, 0x45, 0x44, 0x10, 0x03, 0x22, 0x1c, 0x0a, 0x1a, - 0x47, 0x65, 0x74, 0x46, 0x65, 0x61, 0x73, 0x74, 0x43, 0x6f, 0x72, 0x65, 0x56, 0x65, 0x72, 0x73, - 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x22, 0x37, 0x0a, 0x1b, 0x47, 0x65, - 0x74, 0x46, 0x65, 0x61, 0x73, 0x74, 0x43, 0x6f, 0x72, 0x65, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, - 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x76, 0x65, 0x72, - 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x76, 0x65, 0x72, 0x73, - 0x69, 0x6f, 0x6e, 0x22, 0x3d, 0x0a, 0x12, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x53, 0x74, 0x6f, - 0x72, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x27, 0x0a, 0x05, 0x73, 0x74, 0x6f, - 0x72, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x11, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, - 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x53, 0x74, 0x6f, 0x72, 0x65, 0x52, 0x05, 0x73, 0x74, 0x6f, - 0x72, 0x65, 0x22, 0xa4, 0x01, 0x0a, 0x13, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x53, 0x74, 0x6f, - 0x72, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x27, 0x0a, 0x05, 0x73, 0x74, - 0x6f, 0x72, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x11, 0x2e, 0x66, 0x65, 0x61, 0x73, - 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x53, 0x74, 0x6f, 0x72, 0x65, 0x52, 0x05, 0x73, 0x74, - 0x6f, 0x72, 0x65, 0x12, 0x3e, 0x0a, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x18, 0x02, 0x20, - 0x01, 0x28, 0x0e, 0x32, 0x26, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, - 0x2e, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x53, 0x74, 0x6f, 0x72, 0x65, 0x52, 0x65, 0x73, 0x70, - 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x06, 0x73, 0x74, 0x61, - 0x74, 0x75, 0x73, 0x22, 0x24, 0x0a, 0x06, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x0d, 0x0a, - 0x09, 0x4e, 0x4f, 0x5f, 0x43, 0x48, 0x41, 0x4e, 0x47, 0x45, 0x10, 0x00, 0x12, 0x0b, 0x0a, 0x07, - 0x55, 0x50, 0x44, 0x41, 0x54, 0x45, 0x44, 0x10, 0x01, 0x22, 0x2a, 0x0a, 0x14, 0x43, 0x72, 0x65, - 0x61, 0x74, 0x65, 0x50, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, - 0x74, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x22, 0x17, 0x0a, 0x15, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x50, - 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x2b, - 0x0a, 0x15, 0x41, 0x72, 0x63, 0x68, 0x69, 0x76, 0x65, 0x50, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, - 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, - 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x22, 0x18, 0x0a, 0x16, 0x41, - 0x72, 0x63, 0x68, 0x69, 0x76, 0x65, 0x50, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x52, 0x65, 0x73, - 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x15, 0x0a, 0x13, 0x4c, 0x69, 0x73, 0x74, 0x50, 0x72, 0x6f, - 0x6a, 0x65, 0x63, 0x74, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x22, 0x32, 0x0a, 0x14, - 0x4c, 0x69, 0x73, 0x74, 0x50, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x52, 0x65, 0x73, 0x70, - 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x1a, 0x0a, 0x08, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x73, - 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x08, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x73, - 0x22, 0xee, 0x01, 0x0a, 0x18, 0x4c, 0x69, 0x73, 0x74, 0x49, 0x6e, 0x67, 0x65, 0x73, 0x74, 0x69, - 0x6f, 0x6e, 0x4a, 0x6f, 0x62, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x43, 0x0a, - 0x06, 0x66, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2b, 0x2e, - 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x49, - 0x6e, 0x67, 0x65, 0x73, 0x74, 0x69, 0x6f, 0x6e, 0x4a, 0x6f, 0x62, 0x73, 0x52, 0x65, 0x71, 0x75, - 0x65, 0x73, 0x74, 0x2e, 0x46, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x52, 0x06, 0x66, 0x69, 0x6c, 0x74, - 0x65, 0x72, 0x1a, 0x8c, 0x01, 0x0a, 0x06, 0x46, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x12, 0x0e, 0x0a, - 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x12, 0x53, 0x0a, - 0x15, 0x66, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x5f, 0x73, 0x65, 0x74, 0x5f, 0x72, 0x65, 0x66, - 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1f, 0x2e, 0x66, + 0x48, 0x41, 0x4e, 0x47, 0x45, 0x10, 0x00, 0x12, 0x0b, 0x0a, 0x07, 0x55, 0x50, 0x44, 0x41, 0x54, + 0x45, 0x44, 0x10, 0x01, 0x22, 0x2a, 0x0a, 0x14, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x50, 0x72, + 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x12, 0x0a, 0x04, + 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, + 0x22, 0x17, 0x0a, 0x15, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x50, 0x72, 0x6f, 0x6a, 0x65, 0x63, + 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x2b, 0x0a, 0x15, 0x41, 0x72, 0x63, + 0x68, 0x69, 0x76, 0x65, 0x50, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, + 0x73, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x22, 0x18, 0x0a, 0x16, 0x41, 0x72, 0x63, 0x68, 0x69, 0x76, + 0x65, 0x50, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, + 0x22, 0x15, 0x0a, 0x13, 0x4c, 0x69, 0x73, 0x74, 0x50, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x73, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x22, 0x32, 0x0a, 0x14, 0x4c, 0x69, 0x73, 0x74, 0x50, + 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, + 0x1a, 0x0a, 0x08, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, + 0x09, 0x52, 0x08, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x22, 0x20, 0x0a, 0x1e, 0x55, + 0x70, 0x64, 0x61, 0x74, 0x65, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x53, 0x65, 0x74, 0x53, + 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x71, 0x0a, + 0x18, 0x41, 0x70, 0x70, 0x6c, 0x79, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x54, 0x61, 0x62, + 0x6c, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x18, 0x0a, 0x07, 0x70, 0x72, 0x6f, + 0x6a, 0x65, 0x63, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x70, 0x72, 0x6f, 0x6a, + 0x65, 0x63, 0x74, 0x12, 0x3b, 0x0a, 0x0a, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x5f, 0x73, 0x70, 0x65, + 0x63, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, + 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x54, 0x61, 0x62, 0x6c, + 0x65, 0x53, 0x70, 0x65, 0x63, 0x52, 0x09, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x53, 0x70, 0x65, 0x63, + 0x22, 0x4b, 0x0a, 0x19, 0x41, 0x70, 0x70, 0x6c, 0x79, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, + 0x54, 0x61, 0x62, 0x6c, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x2e, 0x0a, + 0x05, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x18, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, - 0x65, 0x53, 0x65, 0x74, 0x52, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x52, 0x13, 0x66, - 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x53, 0x65, 0x74, 0x52, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, - 0x63, 0x65, 0x12, 0x1d, 0x0a, 0x0a, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x5f, 0x6e, 0x61, 0x6d, 0x65, - 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x4e, 0x61, 0x6d, - 0x65, 0x22, 0x49, 0x0a, 0x19, 0x4c, 0x69, 0x73, 0x74, 0x49, 0x6e, 0x67, 0x65, 0x73, 0x74, 0x69, - 0x6f, 0x6e, 0x4a, 0x6f, 0x62, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x2c, - 0x0a, 0x04, 0x6a, 0x6f, 0x62, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x18, 0x2e, 0x66, - 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x49, 0x6e, 0x67, 0x65, 0x73, 0x74, - 0x69, 0x6f, 0x6e, 0x4a, 0x6f, 0x62, 0x52, 0x04, 0x6a, 0x6f, 0x62, 0x73, 0x22, 0x2c, 0x0a, 0x1a, - 0x52, 0x65, 0x73, 0x74, 0x61, 0x72, 0x74, 0x49, 0x6e, 0x67, 0x65, 0x73, 0x74, 0x69, 0x6f, 0x6e, - 0x4a, 0x6f, 0x62, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, - 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x22, 0x1d, 0x0a, 0x1b, 0x52, 0x65, - 0x73, 0x74, 0x61, 0x72, 0x74, 0x49, 0x6e, 0x67, 0x65, 0x73, 0x74, 0x69, 0x6f, 0x6e, 0x4a, 0x6f, - 0x62, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x29, 0x0a, 0x17, 0x53, 0x74, 0x6f, - 0x70, 0x49, 0x6e, 0x67, 0x65, 0x73, 0x74, 0x69, 0x6f, 0x6e, 0x4a, 0x6f, 0x62, 0x52, 0x65, 0x71, - 0x75, 0x65, 0x73, 0x74, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x02, 0x69, 0x64, 0x22, 0x1a, 0x0a, 0x18, 0x53, 0x74, 0x6f, 0x70, 0x49, 0x6e, 0x67, 0x65, - 0x73, 0x74, 0x69, 0x6f, 0x6e, 0x4a, 0x6f, 0x62, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, - 0x22, 0xb1, 0x02, 0x0a, 0x1b, 0x47, 0x65, 0x74, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x53, - 0x74, 0x61, 0x74, 0x69, 0x73, 0x74, 0x69, 0x63, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, - 0x12, 0x24, 0x0a, 0x0e, 0x66, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x5f, 0x73, 0x65, 0x74, 0x5f, - 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x66, 0x65, 0x61, 0x74, 0x75, 0x72, - 0x65, 0x53, 0x65, 0x74, 0x49, 0x64, 0x12, 0x1a, 0x0a, 0x08, 0x66, 0x65, 0x61, 0x74, 0x75, 0x72, - 0x65, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x09, 0x52, 0x08, 0x66, 0x65, 0x61, 0x74, 0x75, 0x72, - 0x65, 0x73, 0x12, 0x14, 0x0a, 0x05, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x05, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x12, 0x39, 0x0a, 0x0a, 0x73, 0x74, 0x61, 0x72, - 0x74, 0x5f, 0x64, 0x61, 0x74, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, - 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, - 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x09, 0x73, 0x74, 0x61, 0x72, 0x74, 0x44, - 0x61, 0x74, 0x65, 0x12, 0x35, 0x0a, 0x08, 0x65, 0x6e, 0x64, 0x5f, 0x64, 0x61, 0x74, 0x65, 0x18, - 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, - 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, - 0x70, 0x52, 0x07, 0x65, 0x6e, 0x64, 0x44, 0x61, 0x74, 0x65, 0x12, 0x23, 0x0a, 0x0d, 0x69, 0x6e, - 0x67, 0x65, 0x73, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x69, 0x64, 0x73, 0x18, 0x06, 0x20, 0x03, 0x28, - 0x09, 0x52, 0x0c, 0x69, 0x6e, 0x67, 0x65, 0x73, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x64, 0x73, 0x12, - 0x23, 0x0a, 0x0d, 0x66, 0x6f, 0x72, 0x63, 0x65, 0x5f, 0x72, 0x65, 0x66, 0x72, 0x65, 0x73, 0x68, - 0x18, 0x07, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0c, 0x66, 0x6f, 0x72, 0x63, 0x65, 0x52, 0x65, 0x66, - 0x72, 0x65, 0x73, 0x68, 0x22, 0x9b, 0x01, 0x0a, 0x1c, 0x47, 0x65, 0x74, 0x46, 0x65, 0x61, 0x74, - 0x75, 0x72, 0x65, 0x53, 0x74, 0x61, 0x74, 0x69, 0x73, 0x74, 0x69, 0x63, 0x73, 0x52, 0x65, 0x73, - 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x7b, 0x0a, 0x1f, 0x64, 0x61, 0x74, 0x61, 0x73, 0x65, 0x74, - 0x5f, 0x66, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x5f, 0x73, 0x74, 0x61, 0x74, 0x69, 0x73, 0x74, - 0x69, 0x63, 0x73, 0x5f, 0x6c, 0x69, 0x73, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x34, - 0x2e, 0x74, 0x65, 0x6e, 0x73, 0x6f, 0x72, 0x66, 0x6c, 0x6f, 0x77, 0x2e, 0x6d, 0x65, 0x74, 0x61, - 0x64, 0x61, 0x74, 0x61, 0x2e, 0x76, 0x30, 0x2e, 0x44, 0x61, 0x74, 0x61, 0x73, 0x65, 0x74, 0x46, - 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x53, 0x74, 0x61, 0x74, 0x69, 0x73, 0x74, 0x69, 0x63, 0x73, - 0x4c, 0x69, 0x73, 0x74, 0x52, 0x1c, 0x64, 0x61, 0x74, 0x61, 0x73, 0x65, 0x74, 0x46, 0x65, 0x61, - 0x74, 0x75, 0x72, 0x65, 0x53, 0x74, 0x61, 0x74, 0x69, 0x73, 0x74, 0x69, 0x63, 0x73, 0x4c, 0x69, - 0x73, 0x74, 0x22, 0x94, 0x01, 0x0a, 0x1d, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x46, 0x65, 0x61, - 0x74, 0x75, 0x72, 0x65, 0x53, 0x65, 0x74, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x71, - 0x75, 0x65, 0x73, 0x74, 0x12, 0x3d, 0x0a, 0x09, 0x72, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, - 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1f, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, - 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x53, 0x65, 0x74, 0x52, - 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x52, 0x09, 0x72, 0x65, 0x66, 0x65, 0x72, 0x65, - 0x6e, 0x63, 0x65, 0x12, 0x34, 0x0a, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x18, 0x02, 0x20, - 0x01, 0x28, 0x0e, 0x32, 0x1c, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, - 0x2e, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x53, 0x65, 0x74, 0x53, 0x74, 0x61, 0x74, 0x75, - 0x73, 0x52, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x22, 0x20, 0x0a, 0x1e, 0x55, 0x70, 0x64, - 0x61, 0x74, 0x65, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x53, 0x65, 0x74, 0x53, 0x74, 0x61, - 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x71, 0x0a, 0x18, 0x41, - 0x70, 0x70, 0x6c, 0x79, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x54, 0x61, 0x62, 0x6c, 0x65, + 0x65, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x52, 0x05, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x22, 0x46, 0x0a, + 0x16, 0x47, 0x65, 0x74, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x18, 0x0a, 0x07, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, - 0x74, 0x12, 0x3b, 0x0a, 0x0a, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x5f, 0x73, 0x70, 0x65, 0x63, 0x18, - 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, - 0x72, 0x65, 0x2e, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x53, - 0x70, 0x65, 0x63, 0x52, 0x09, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x53, 0x70, 0x65, 0x63, 0x22, 0x4b, - 0x0a, 0x19, 0x41, 0x70, 0x70, 0x6c, 0x79, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x54, 0x61, - 0x62, 0x6c, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x2e, 0x0a, 0x05, 0x74, - 0x61, 0x62, 0x6c, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x18, 0x2e, 0x66, 0x65, 0x61, - 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x54, - 0x61, 0x62, 0x6c, 0x65, 0x52, 0x05, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x22, 0x46, 0x0a, 0x16, 0x47, - 0x65, 0x74, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x18, 0x0a, 0x07, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, - 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x12, - 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, - 0x61, 0x6d, 0x65, 0x22, 0x49, 0x0a, 0x17, 0x47, 0x65, 0x74, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, - 0x65, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x2e, - 0x0a, 0x05, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x18, 0x2e, - 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x46, 0x65, 0x61, 0x74, 0x75, - 0x72, 0x65, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x52, 0x05, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x22, 0x90, - 0x02, 0x0a, 0x18, 0x4c, 0x69, 0x73, 0x74, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x54, 0x61, - 0x62, 0x6c, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x43, 0x0a, 0x06, 0x66, - 0x69, 0x6c, 0x74, 0x65, 0x72, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2b, 0x2e, 0x66, 0x65, - 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x46, 0x65, 0x61, - 0x74, 0x75, 0x72, 0x65, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, - 0x74, 0x2e, 0x46, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x52, 0x06, 0x66, 0x69, 0x6c, 0x74, 0x65, 0x72, - 0x1a, 0xae, 0x01, 0x0a, 0x06, 0x46, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x12, 0x18, 0x0a, 0x07, 0x70, - 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x70, 0x72, - 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x12, 0x4f, 0x0a, 0x06, 0x6c, 0x61, 0x62, 0x65, 0x6c, 0x73, 0x18, - 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x37, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, - 0x72, 0x65, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x54, 0x61, - 0x62, 0x6c, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x46, 0x69, 0x6c, 0x74, - 0x65, 0x72, 0x2e, 0x4c, 0x61, 0x62, 0x65, 0x6c, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x06, - 0x6c, 0x61, 0x62, 0x65, 0x6c, 0x73, 0x1a, 0x39, 0x0a, 0x0b, 0x4c, 0x61, 0x62, 0x65, 0x6c, 0x73, - 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, - 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, - 0x01, 0x22, 0x4d, 0x0a, 0x19, 0x4c, 0x69, 0x73, 0x74, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, - 0x54, 0x61, 0x62, 0x6c, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x30, - 0x0a, 0x06, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x18, - 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x46, 0x65, 0x61, 0x74, - 0x75, 0x72, 0x65, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x52, 0x06, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x73, - 0x32, 0xde, 0x0c, 0x0a, 0x0b, 0x43, 0x6f, 0x72, 0x65, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, - 0x12, 0x66, 0x0a, 0x13, 0x47, 0x65, 0x74, 0x46, 0x65, 0x61, 0x73, 0x74, 0x43, 0x6f, 0x72, 0x65, - 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x26, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, - 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x47, 0x65, 0x74, 0x46, 0x65, 0x61, 0x73, 0x74, 0x43, 0x6f, 0x72, - 0x65, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, - 0x27, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x47, 0x65, 0x74, - 0x46, 0x65, 0x61, 0x73, 0x74, 0x43, 0x6f, 0x72, 0x65, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, - 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x54, 0x0a, 0x0d, 0x47, 0x65, 0x74, 0x46, - 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x53, 0x65, 0x74, 0x12, 0x20, 0x2e, 0x66, 0x65, 0x61, 0x73, - 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x47, 0x65, 0x74, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, - 0x65, 0x53, 0x65, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x21, 0x2e, 0x66, 0x65, - 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x47, 0x65, 0x74, 0x46, 0x65, 0x61, 0x74, - 0x75, 0x72, 0x65, 0x53, 0x65, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x48, - 0x0a, 0x09, 0x47, 0x65, 0x74, 0x45, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x12, 0x1c, 0x2e, 0x66, 0x65, - 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x47, 0x65, 0x74, 0x45, 0x6e, 0x74, 0x69, - 0x74, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1d, 0x2e, 0x66, 0x65, 0x61, 0x73, - 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x47, 0x65, 0x74, 0x45, 0x6e, 0x74, 0x69, 0x74, 0x79, - 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x5a, 0x0a, 0x0f, 0x4c, 0x69, 0x73, 0x74, - 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x53, 0x65, 0x74, 0x73, 0x12, 0x22, 0x2e, 0x66, 0x65, - 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x46, 0x65, 0x61, - 0x74, 0x75, 0x72, 0x65, 0x53, 0x65, 0x74, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, - 0x23, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x4c, 0x69, 0x73, - 0x74, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x53, 0x65, 0x74, 0x73, 0x52, 0x65, 0x73, 0x70, - 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x51, 0x0a, 0x0c, 0x4c, 0x69, 0x73, 0x74, 0x46, 0x65, 0x61, 0x74, - 0x75, 0x72, 0x65, 0x73, 0x12, 0x1f, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, - 0x65, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x20, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, - 0x72, 0x65, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x52, - 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x69, 0x0a, 0x14, 0x47, 0x65, 0x74, 0x46, 0x65, - 0x61, 0x74, 0x75, 0x72, 0x65, 0x53, 0x74, 0x61, 0x74, 0x69, 0x73, 0x74, 0x69, 0x63, 0x73, 0x12, - 0x27, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x47, 0x65, 0x74, - 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x53, 0x74, 0x61, 0x74, 0x69, 0x73, 0x74, 0x69, 0x63, - 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x28, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, - 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x47, 0x65, 0x74, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, - 0x53, 0x74, 0x61, 0x74, 0x69, 0x73, 0x74, 0x69, 0x63, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, - 0x73, 0x65, 0x12, 0x4b, 0x0a, 0x0a, 0x4c, 0x69, 0x73, 0x74, 0x53, 0x74, 0x6f, 0x72, 0x65, 0x73, - 0x12, 0x1d, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x4c, 0x69, - 0x73, 0x74, 0x53, 0x74, 0x6f, 0x72, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, - 0x1e, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x4c, 0x69, 0x73, - 0x74, 0x53, 0x74, 0x6f, 0x72, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, - 0x5a, 0x0a, 0x0f, 0x41, 0x70, 0x70, 0x6c, 0x79, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x53, - 0x65, 0x74, 0x12, 0x22, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, - 0x41, 0x70, 0x70, 0x6c, 0x79, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x53, 0x65, 0x74, 0x52, - 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x23, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, - 0x6f, 0x72, 0x65, 0x2e, 0x41, 0x70, 0x70, 0x6c, 0x79, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, - 0x53, 0x65, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x4e, 0x0a, 0x0b, 0x41, - 0x70, 0x70, 0x6c, 0x79, 0x45, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x12, 0x1e, 0x2e, 0x66, 0x65, 0x61, - 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x41, 0x70, 0x70, 0x6c, 0x79, 0x45, 0x6e, 0x74, - 0x69, 0x74, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1f, 0x2e, 0x66, 0x65, 0x61, - 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x41, 0x70, 0x70, 0x6c, 0x79, 0x45, 0x6e, 0x74, - 0x69, 0x74, 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x51, 0x0a, 0x0c, 0x4c, - 0x69, 0x73, 0x74, 0x45, 0x6e, 0x74, 0x69, 0x74, 0x69, 0x65, 0x73, 0x12, 0x1f, 0x2e, 0x66, 0x65, - 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x45, 0x6e, 0x74, - 0x69, 0x74, 0x69, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x20, 0x2e, 0x66, - 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x45, 0x6e, - 0x74, 0x69, 0x74, 0x69, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x4e, - 0x0a, 0x0b, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x53, 0x74, 0x6f, 0x72, 0x65, 0x12, 0x1e, 0x2e, - 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x55, 0x70, 0x64, 0x61, 0x74, - 0x65, 0x53, 0x74, 0x6f, 0x72, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1f, 0x2e, - 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x55, 0x70, 0x64, 0x61, 0x74, - 0x65, 0x53, 0x74, 0x6f, 0x72, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x54, - 0x0a, 0x0d, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x50, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x12, - 0x20, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x43, 0x72, 0x65, - 0x61, 0x74, 0x65, 0x50, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, - 0x74, 0x1a, 0x21, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x43, - 0x72, 0x65, 0x61, 0x74, 0x65, 0x50, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x52, 0x65, 0x73, 0x70, - 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x57, 0x0a, 0x0e, 0x41, 0x72, 0x63, 0x68, 0x69, 0x76, 0x65, 0x50, - 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x12, 0x21, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, - 0x6f, 0x72, 0x65, 0x2e, 0x41, 0x72, 0x63, 0x68, 0x69, 0x76, 0x65, 0x50, 0x72, 0x6f, 0x6a, 0x65, - 0x63, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x22, 0x2e, 0x66, 0x65, 0x61, 0x73, - 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x41, 0x72, 0x63, 0x68, 0x69, 0x76, 0x65, 0x50, 0x72, - 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x51, 0x0a, - 0x0c, 0x4c, 0x69, 0x73, 0x74, 0x50, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x12, 0x1f, 0x2e, - 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x50, - 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x20, + 0x74, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x22, 0x49, 0x0a, 0x17, 0x47, 0x65, 0x74, 0x46, 0x65, 0x61, 0x74, + 0x75, 0x72, 0x65, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, + 0x12, 0x2e, 0x0a, 0x05, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, + 0x18, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x46, 0x65, 0x61, + 0x74, 0x75, 0x72, 0x65, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x52, 0x05, 0x74, 0x61, 0x62, 0x6c, 0x65, + 0x22, 0x90, 0x02, 0x0a, 0x18, 0x4c, 0x69, 0x73, 0x74, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, + 0x54, 0x61, 0x62, 0x6c, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x43, 0x0a, + 0x06, 0x66, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2b, 0x2e, + 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x46, + 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, + 0x65, 0x73, 0x74, 0x2e, 0x46, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x52, 0x06, 0x66, 0x69, 0x6c, 0x74, + 0x65, 0x72, 0x1a, 0xae, 0x01, 0x0a, 0x06, 0x46, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x12, 0x18, 0x0a, + 0x07, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, + 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x12, 0x4f, 0x0a, 0x06, 0x6c, 0x61, 0x62, 0x65, 0x6c, + 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x37, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, + 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, + 0x54, 0x61, 0x62, 0x6c, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x46, 0x69, + 0x6c, 0x74, 0x65, 0x72, 0x2e, 0x4c, 0x61, 0x62, 0x65, 0x6c, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, + 0x52, 0x06, 0x6c, 0x61, 0x62, 0x65, 0x6c, 0x73, 0x1a, 0x39, 0x0a, 0x0b, 0x4c, 0x61, 0x62, 0x65, + 0x6c, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, + 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, + 0x02, 0x38, 0x01, 0x22, 0x4d, 0x0a, 0x19, 0x4c, 0x69, 0x73, 0x74, 0x46, 0x65, 0x61, 0x74, 0x75, + 0x72, 0x65, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, + 0x12, 0x30, 0x0a, 0x06, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, + 0x32, 0x18, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x46, 0x65, + 0x61, 0x74, 0x75, 0x72, 0x65, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x52, 0x06, 0x74, 0x61, 0x62, 0x6c, + 0x65, 0x73, 0x22, 0x49, 0x0a, 0x19, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x46, 0x65, 0x61, 0x74, + 0x75, 0x72, 0x65, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, + 0x18, 0x0a, 0x07, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x07, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, + 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x22, 0x1c, 0x0a, + 0x1a, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x54, 0x61, + 0x62, 0x6c, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x32, 0xd9, 0x09, 0x0a, 0x0b, + 0x43, 0x6f, 0x72, 0x65, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x66, 0x0a, 0x13, 0x47, + 0x65, 0x74, 0x46, 0x65, 0x61, 0x73, 0x74, 0x43, 0x6f, 0x72, 0x65, 0x56, 0x65, 0x72, 0x73, 0x69, + 0x6f, 0x6e, 0x12, 0x26, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, + 0x47, 0x65, 0x74, 0x46, 0x65, 0x61, 0x73, 0x74, 0x43, 0x6f, 0x72, 0x65, 0x56, 0x65, 0x72, 0x73, + 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x27, 0x2e, 0x66, 0x65, 0x61, + 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x47, 0x65, 0x74, 0x46, 0x65, 0x61, 0x73, 0x74, + 0x43, 0x6f, 0x72, 0x65, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x12, 0x48, 0x0a, 0x09, 0x47, 0x65, 0x74, 0x45, 0x6e, 0x74, 0x69, 0x74, 0x79, + 0x12, 0x1c, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x47, 0x65, + 0x74, 0x45, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1d, + 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x47, 0x65, 0x74, 0x45, + 0x6e, 0x74, 0x69, 0x74, 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x51, 0x0a, + 0x0c, 0x4c, 0x69, 0x73, 0x74, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x12, 0x1f, 0x2e, + 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x46, + 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x20, + 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x4c, 0x69, 0x73, 0x74, + 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, + 0x12, 0x4b, 0x0a, 0x0a, 0x4c, 0x69, 0x73, 0x74, 0x53, 0x74, 0x6f, 0x72, 0x65, 0x73, 0x12, 0x1d, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x4c, 0x69, 0x73, 0x74, - 0x50, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, - 0x12, 0x6f, 0x0a, 0x16, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, - 0x65, 0x53, 0x65, 0x74, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x29, 0x2e, 0x66, 0x65, 0x61, - 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x46, 0x65, - 0x61, 0x74, 0x75, 0x72, 0x65, 0x53, 0x65, 0x74, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x2a, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, - 0x72, 0x65, 0x2e, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, - 0x53, 0x65, 0x74, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, - 0x65, 0x12, 0x60, 0x0a, 0x11, 0x41, 0x70, 0x70, 0x6c, 0x79, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, - 0x65, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x12, 0x24, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, - 0x6f, 0x72, 0x65, 0x2e, 0x41, 0x70, 0x70, 0x6c, 0x79, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, - 0x54, 0x61, 0x62, 0x6c, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x25, 0x2e, 0x66, - 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x41, 0x70, 0x70, 0x6c, 0x79, 0x46, - 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, - 0x6e, 0x73, 0x65, 0x12, 0x60, 0x0a, 0x11, 0x4c, 0x69, 0x73, 0x74, 0x46, 0x65, 0x61, 0x74, 0x75, - 0x72, 0x65, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x73, 0x12, 0x24, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, - 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, - 0x65, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x25, + 0x53, 0x74, 0x6f, 0x72, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1e, 0x2e, + 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x53, + 0x74, 0x6f, 0x72, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x4e, 0x0a, + 0x0b, 0x41, 0x70, 0x70, 0x6c, 0x79, 0x45, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x12, 0x1e, 0x2e, 0x66, + 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x41, 0x70, 0x70, 0x6c, 0x79, 0x45, + 0x6e, 0x74, 0x69, 0x74, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1f, 0x2e, 0x66, + 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x41, 0x70, 0x70, 0x6c, 0x79, 0x45, + 0x6e, 0x74, 0x69, 0x74, 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x51, 0x0a, + 0x0c, 0x4c, 0x69, 0x73, 0x74, 0x45, 0x6e, 0x74, 0x69, 0x74, 0x69, 0x65, 0x73, 0x12, 0x1f, 0x2e, + 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x45, + 0x6e, 0x74, 0x69, 0x74, 0x69, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x20, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x4c, 0x69, 0x73, 0x74, - 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x73, 0x52, 0x65, 0x73, - 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x5a, 0x0a, 0x0f, 0x47, 0x65, 0x74, 0x46, 0x65, 0x61, 0x74, - 0x75, 0x72, 0x65, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x12, 0x22, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, - 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x47, 0x65, 0x74, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, - 0x54, 0x61, 0x62, 0x6c, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x23, 0x2e, 0x66, - 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x47, 0x65, 0x74, 0x46, 0x65, 0x61, - 0x74, 0x75, 0x72, 0x65, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, - 0x65, 0x32, 0xbf, 0x02, 0x0a, 0x14, 0x4a, 0x6f, 0x62, 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, - 0x6c, 0x65, 0x72, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x60, 0x0a, 0x11, 0x4c, 0x69, - 0x73, 0x74, 0x49, 0x6e, 0x67, 0x65, 0x73, 0x74, 0x69, 0x6f, 0x6e, 0x4a, 0x6f, 0x62, 0x73, 0x12, - 0x24, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x4c, 0x69, 0x73, - 0x74, 0x49, 0x6e, 0x67, 0x65, 0x73, 0x74, 0x69, 0x6f, 0x6e, 0x4a, 0x6f, 0x62, 0x73, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x25, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, - 0x72, 0x65, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x49, 0x6e, 0x67, 0x65, 0x73, 0x74, 0x69, 0x6f, 0x6e, - 0x4a, 0x6f, 0x62, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x66, 0x0a, 0x13, - 0x52, 0x65, 0x73, 0x74, 0x61, 0x72, 0x74, 0x49, 0x6e, 0x67, 0x65, 0x73, 0x74, 0x69, 0x6f, 0x6e, - 0x4a, 0x6f, 0x62, 0x12, 0x26, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, - 0x2e, 0x52, 0x65, 0x73, 0x74, 0x61, 0x72, 0x74, 0x49, 0x6e, 0x67, 0x65, 0x73, 0x74, 0x69, 0x6f, - 0x6e, 0x4a, 0x6f, 0x62, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x27, 0x2e, 0x66, 0x65, - 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x52, 0x65, 0x73, 0x74, 0x61, 0x72, 0x74, - 0x49, 0x6e, 0x67, 0x65, 0x73, 0x74, 0x69, 0x6f, 0x6e, 0x4a, 0x6f, 0x62, 0x52, 0x65, 0x73, 0x70, - 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x5d, 0x0a, 0x10, 0x53, 0x74, 0x6f, 0x70, 0x49, 0x6e, 0x67, 0x65, - 0x73, 0x74, 0x69, 0x6f, 0x6e, 0x4a, 0x6f, 0x62, 0x12, 0x23, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, - 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x53, 0x74, 0x6f, 0x70, 0x49, 0x6e, 0x67, 0x65, 0x73, 0x74, - 0x69, 0x6f, 0x6e, 0x4a, 0x6f, 0x62, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x24, 0x2e, - 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x53, 0x74, 0x6f, 0x70, 0x49, - 0x6e, 0x67, 0x65, 0x73, 0x74, 0x69, 0x6f, 0x6e, 0x4a, 0x6f, 0x62, 0x52, 0x65, 0x73, 0x70, 0x6f, - 0x6e, 0x73, 0x65, 0x42, 0x59, 0x0a, 0x10, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x70, 0x72, 0x6f, - 0x74, 0x6f, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x42, 0x10, 0x43, 0x6f, 0x72, 0x65, 0x53, 0x65, 0x72, - 0x76, 0x69, 0x63, 0x65, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x5a, 0x33, 0x67, 0x69, 0x74, 0x68, 0x75, - 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2d, 0x64, 0x65, 0x76, 0x2f, - 0x66, 0x65, 0x61, 0x73, 0x74, 0x2f, 0x73, 0x64, 0x6b, 0x2f, 0x67, 0x6f, 0x2f, 0x70, 0x72, 0x6f, - 0x74, 0x6f, 0x73, 0x2f, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2f, 0x63, 0x6f, 0x72, 0x65, 0x62, 0x06, - 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x45, 0x6e, 0x74, 0x69, 0x74, 0x69, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, + 0x12, 0x4e, 0x0a, 0x0b, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x53, 0x74, 0x6f, 0x72, 0x65, 0x12, + 0x1e, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x55, 0x70, 0x64, + 0x61, 0x74, 0x65, 0x53, 0x74, 0x6f, 0x72, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, + 0x1f, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x55, 0x70, 0x64, + 0x61, 0x74, 0x65, 0x53, 0x74, 0x6f, 0x72, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, + 0x12, 0x54, 0x0a, 0x0d, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x50, 0x72, 0x6f, 0x6a, 0x65, 0x63, + 0x74, 0x12, 0x20, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x43, + 0x72, 0x65, 0x61, 0x74, 0x65, 0x50, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x52, 0x65, 0x71, 0x75, + 0x65, 0x73, 0x74, 0x1a, 0x21, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, + 0x2e, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x50, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x52, 0x65, + 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x57, 0x0a, 0x0e, 0x41, 0x72, 0x63, 0x68, 0x69, 0x76, + 0x65, 0x50, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x12, 0x21, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, + 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x41, 0x72, 0x63, 0x68, 0x69, 0x76, 0x65, 0x50, 0x72, 0x6f, + 0x6a, 0x65, 0x63, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x22, 0x2e, 0x66, 0x65, + 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x41, 0x72, 0x63, 0x68, 0x69, 0x76, 0x65, + 0x50, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, + 0x51, 0x0a, 0x0c, 0x4c, 0x69, 0x73, 0x74, 0x50, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x12, + 0x1f, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x4c, 0x69, 0x73, + 0x74, 0x50, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x1a, 0x20, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x4c, 0x69, + 0x73, 0x74, 0x50, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, + 0x73, 0x65, 0x12, 0x60, 0x0a, 0x11, 0x41, 0x70, 0x70, 0x6c, 0x79, 0x46, 0x65, 0x61, 0x74, 0x75, + 0x72, 0x65, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x12, 0x24, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, + 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x41, 0x70, 0x70, 0x6c, 0x79, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, + 0x65, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x25, 0x2e, + 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x41, 0x70, 0x70, 0x6c, 0x79, + 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x52, 0x65, 0x73, 0x70, + 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x60, 0x0a, 0x11, 0x4c, 0x69, 0x73, 0x74, 0x46, 0x65, 0x61, 0x74, + 0x75, 0x72, 0x65, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x73, 0x12, 0x24, 0x2e, 0x66, 0x65, 0x61, 0x73, + 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x46, 0x65, 0x61, 0x74, 0x75, + 0x72, 0x65, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, + 0x25, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x4c, 0x69, 0x73, + 0x74, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x73, 0x52, 0x65, + 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x5a, 0x0a, 0x0f, 0x47, 0x65, 0x74, 0x46, 0x65, 0x61, + 0x74, 0x75, 0x72, 0x65, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x12, 0x22, 0x2e, 0x66, 0x65, 0x61, 0x73, + 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x47, 0x65, 0x74, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, + 0x65, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x23, 0x2e, + 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x47, 0x65, 0x74, 0x46, 0x65, + 0x61, 0x74, 0x75, 0x72, 0x65, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, + 0x73, 0x65, 0x12, 0x63, 0x0a, 0x12, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x46, 0x65, 0x61, 0x74, + 0x75, 0x72, 0x65, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x12, 0x25, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, + 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x46, 0x65, 0x61, 0x74, + 0x75, 0x72, 0x65, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, + 0x26, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x44, 0x65, 0x6c, + 0x65, 0x74, 0x65, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x52, + 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x42, 0x59, 0x0a, 0x10, 0x66, 0x65, 0x61, 0x73, 0x74, + 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x42, 0x10, 0x43, 0x6f, 0x72, + 0x65, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x5a, 0x33, 0x67, + 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2d, + 0x64, 0x65, 0x76, 0x2f, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2f, 0x73, 0x64, 0x6b, 0x2f, 0x67, 0x6f, + 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x73, 0x2f, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2f, 0x63, 0x6f, + 0x72, 0x65, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( @@ -3078,162 +1999,109 @@ func file_feast_core_CoreService_proto_rawDescGZIP() []byte { return file_feast_core_CoreService_proto_rawDescData } -var file_feast_core_CoreService_proto_enumTypes = make([]protoimpl.EnumInfo, 2) -var file_feast_core_CoreService_proto_msgTypes = make([]protoimpl.MessageInfo, 53) +var file_feast_core_CoreService_proto_enumTypes = make([]protoimpl.EnumInfo, 1) +var file_feast_core_CoreService_proto_msgTypes = make([]protoimpl.MessageInfo, 37) var file_feast_core_CoreService_proto_goTypes = []interface{}{ - (ApplyFeatureSetResponse_Status)(0), // 0: feast.core.ApplyFeatureSetResponse.Status - (UpdateStoreResponse_Status)(0), // 1: feast.core.UpdateStoreResponse.Status - (*GetFeatureSetRequest)(nil), // 2: feast.core.GetFeatureSetRequest - (*GetFeatureSetResponse)(nil), // 3: feast.core.GetFeatureSetResponse - (*ListFeatureSetsRequest)(nil), // 4: feast.core.ListFeatureSetsRequest - (*ListFeatureSetsResponse)(nil), // 5: feast.core.ListFeatureSetsResponse - (*GetEntityRequest)(nil), // 6: feast.core.GetEntityRequest - (*GetEntityResponse)(nil), // 7: feast.core.GetEntityResponse - (*ListEntitiesRequest)(nil), // 8: feast.core.ListEntitiesRequest - (*ListEntitiesResponse)(nil), // 9: feast.core.ListEntitiesResponse - (*ListFeaturesRequest)(nil), // 10: feast.core.ListFeaturesRequest - (*ListFeaturesResponse)(nil), // 11: feast.core.ListFeaturesResponse - (*ListStoresRequest)(nil), // 12: feast.core.ListStoresRequest - (*ListStoresResponse)(nil), // 13: feast.core.ListStoresResponse - (*ApplyEntityRequest)(nil), // 14: feast.core.ApplyEntityRequest - (*ApplyEntityResponse)(nil), // 15: feast.core.ApplyEntityResponse - (*ApplyFeatureSetRequest)(nil), // 16: feast.core.ApplyFeatureSetRequest - (*ApplyFeatureSetResponse)(nil), // 17: feast.core.ApplyFeatureSetResponse - (*GetFeastCoreVersionRequest)(nil), // 18: feast.core.GetFeastCoreVersionRequest - (*GetFeastCoreVersionResponse)(nil), // 19: feast.core.GetFeastCoreVersionResponse - (*UpdateStoreRequest)(nil), // 20: feast.core.UpdateStoreRequest - (*UpdateStoreResponse)(nil), // 21: feast.core.UpdateStoreResponse - (*CreateProjectRequest)(nil), // 22: feast.core.CreateProjectRequest - (*CreateProjectResponse)(nil), // 23: feast.core.CreateProjectResponse - (*ArchiveProjectRequest)(nil), // 24: feast.core.ArchiveProjectRequest - (*ArchiveProjectResponse)(nil), // 25: feast.core.ArchiveProjectResponse - (*ListProjectsRequest)(nil), // 26: feast.core.ListProjectsRequest - (*ListProjectsResponse)(nil), // 27: feast.core.ListProjectsResponse - (*ListIngestionJobsRequest)(nil), // 28: feast.core.ListIngestionJobsRequest - (*ListIngestionJobsResponse)(nil), // 29: feast.core.ListIngestionJobsResponse - (*RestartIngestionJobRequest)(nil), // 30: feast.core.RestartIngestionJobRequest - (*RestartIngestionJobResponse)(nil), // 31: feast.core.RestartIngestionJobResponse - (*StopIngestionJobRequest)(nil), // 32: feast.core.StopIngestionJobRequest - (*StopIngestionJobResponse)(nil), // 33: feast.core.StopIngestionJobResponse - (*GetFeatureStatisticsRequest)(nil), // 34: feast.core.GetFeatureStatisticsRequest - (*GetFeatureStatisticsResponse)(nil), // 35: feast.core.GetFeatureStatisticsResponse - (*UpdateFeatureSetStatusRequest)(nil), // 36: feast.core.UpdateFeatureSetStatusRequest - (*UpdateFeatureSetStatusResponse)(nil), // 37: feast.core.UpdateFeatureSetStatusResponse - (*ApplyFeatureTableRequest)(nil), // 38: feast.core.ApplyFeatureTableRequest - (*ApplyFeatureTableResponse)(nil), // 39: feast.core.ApplyFeatureTableResponse - (*GetFeatureTableRequest)(nil), // 40: feast.core.GetFeatureTableRequest - (*GetFeatureTableResponse)(nil), // 41: feast.core.GetFeatureTableResponse - (*ListFeatureTablesRequest)(nil), // 42: feast.core.ListFeatureTablesRequest - (*ListFeatureTablesResponse)(nil), // 43: feast.core.ListFeatureTablesResponse - (*ListFeatureSetsRequest_Filter)(nil), // 44: feast.core.ListFeatureSetsRequest.Filter - nil, // 45: feast.core.ListFeatureSetsRequest.Filter.LabelsEntry - (*ListEntitiesRequest_Filter)(nil), // 46: feast.core.ListEntitiesRequest.Filter - nil, // 47: feast.core.ListEntitiesRequest.Filter.LabelsEntry - (*ListFeaturesRequest_Filter)(nil), // 48: feast.core.ListFeaturesRequest.Filter - nil, // 49: feast.core.ListFeaturesRequest.Filter.LabelsEntry - nil, // 50: feast.core.ListFeaturesResponse.FeaturesEntry - (*ListStoresRequest_Filter)(nil), // 51: feast.core.ListStoresRequest.Filter - (*ListIngestionJobsRequest_Filter)(nil), // 52: feast.core.ListIngestionJobsRequest.Filter - (*ListFeatureTablesRequest_Filter)(nil), // 53: feast.core.ListFeatureTablesRequest.Filter - nil, // 54: feast.core.ListFeatureTablesRequest.Filter.LabelsEntry - (*FeatureSet)(nil), // 55: feast.core.FeatureSet - (*Entity)(nil), // 56: feast.core.Entity - (*Store)(nil), // 57: feast.core.Store - (*EntitySpecV2)(nil), // 58: feast.core.EntitySpecV2 - (*IngestionJob)(nil), // 59: feast.core.IngestionJob - (*timestamp.Timestamp)(nil), // 60: google.protobuf.Timestamp - (*v0.DatasetFeatureStatisticsList)(nil), // 61: tensorflow.metadata.v0.DatasetFeatureStatisticsList - (*FeatureSetReference)(nil), // 62: feast.core.FeatureSetReference - (FeatureSetStatus)(0), // 63: feast.core.FeatureSetStatus - (*FeatureTableSpec)(nil), // 64: feast.core.FeatureTableSpec - (*FeatureTable)(nil), // 65: feast.core.FeatureTable - (*FeatureSpec)(nil), // 66: feast.core.FeatureSpec + (UpdateStoreResponse_Status)(0), // 0: feast.core.UpdateStoreResponse.Status + (*GetEntityRequest)(nil), // 1: feast.core.GetEntityRequest + (*GetEntityResponse)(nil), // 2: feast.core.GetEntityResponse + (*ListEntitiesRequest)(nil), // 3: feast.core.ListEntitiesRequest + (*ListEntitiesResponse)(nil), // 4: feast.core.ListEntitiesResponse + (*ListFeaturesRequest)(nil), // 5: feast.core.ListFeaturesRequest + (*ListFeaturesResponse)(nil), // 6: feast.core.ListFeaturesResponse + (*ListStoresRequest)(nil), // 7: feast.core.ListStoresRequest + (*ListStoresResponse)(nil), // 8: feast.core.ListStoresResponse + (*ApplyEntityRequest)(nil), // 9: feast.core.ApplyEntityRequest + (*ApplyEntityResponse)(nil), // 10: feast.core.ApplyEntityResponse + (*GetFeastCoreVersionRequest)(nil), // 11: feast.core.GetFeastCoreVersionRequest + (*GetFeastCoreVersionResponse)(nil), // 12: feast.core.GetFeastCoreVersionResponse + (*UpdateStoreRequest)(nil), // 13: feast.core.UpdateStoreRequest + (*UpdateStoreResponse)(nil), // 14: feast.core.UpdateStoreResponse + (*CreateProjectRequest)(nil), // 15: feast.core.CreateProjectRequest + (*CreateProjectResponse)(nil), // 16: feast.core.CreateProjectResponse + (*ArchiveProjectRequest)(nil), // 17: feast.core.ArchiveProjectRequest + (*ArchiveProjectResponse)(nil), // 18: feast.core.ArchiveProjectResponse + (*ListProjectsRequest)(nil), // 19: feast.core.ListProjectsRequest + (*ListProjectsResponse)(nil), // 20: feast.core.ListProjectsResponse + (*UpdateFeatureSetStatusResponse)(nil), // 21: feast.core.UpdateFeatureSetStatusResponse + (*ApplyFeatureTableRequest)(nil), // 22: feast.core.ApplyFeatureTableRequest + (*ApplyFeatureTableResponse)(nil), // 23: feast.core.ApplyFeatureTableResponse + (*GetFeatureTableRequest)(nil), // 24: feast.core.GetFeatureTableRequest + (*GetFeatureTableResponse)(nil), // 25: feast.core.GetFeatureTableResponse + (*ListFeatureTablesRequest)(nil), // 26: feast.core.ListFeatureTablesRequest + (*ListFeatureTablesResponse)(nil), // 27: feast.core.ListFeatureTablesResponse + (*DeleteFeatureTableRequest)(nil), // 28: feast.core.DeleteFeatureTableRequest + (*DeleteFeatureTableResponse)(nil), // 29: feast.core.DeleteFeatureTableResponse + (*ListEntitiesRequest_Filter)(nil), // 30: feast.core.ListEntitiesRequest.Filter + nil, // 31: feast.core.ListEntitiesRequest.Filter.LabelsEntry + (*ListFeaturesRequest_Filter)(nil), // 32: feast.core.ListFeaturesRequest.Filter + nil, // 33: feast.core.ListFeaturesRequest.Filter.LabelsEntry + nil, // 34: feast.core.ListFeaturesResponse.FeaturesEntry + (*ListStoresRequest_Filter)(nil), // 35: feast.core.ListStoresRequest.Filter + (*ListFeatureTablesRequest_Filter)(nil), // 36: feast.core.ListFeatureTablesRequest.Filter + nil, // 37: feast.core.ListFeatureTablesRequest.Filter.LabelsEntry + (*Entity)(nil), // 38: feast.core.Entity + (*Store)(nil), // 39: feast.core.Store + (*EntitySpecV2)(nil), // 40: feast.core.EntitySpecV2 + (*FeatureTableSpec)(nil), // 41: feast.core.FeatureTableSpec + (*FeatureTable)(nil), // 42: feast.core.FeatureTable + (*FeatureSpecV2)(nil), // 43: feast.core.FeatureSpecV2 } var file_feast_core_CoreService_proto_depIdxs = []int32{ - 55, // 0: feast.core.GetFeatureSetResponse.feature_set:type_name -> feast.core.FeatureSet - 44, // 1: feast.core.ListFeatureSetsRequest.filter:type_name -> feast.core.ListFeatureSetsRequest.Filter - 55, // 2: feast.core.ListFeatureSetsResponse.feature_sets:type_name -> feast.core.FeatureSet - 56, // 3: feast.core.GetEntityResponse.entity:type_name -> feast.core.Entity - 46, // 4: feast.core.ListEntitiesRequest.filter:type_name -> feast.core.ListEntitiesRequest.Filter - 56, // 5: feast.core.ListEntitiesResponse.entities:type_name -> feast.core.Entity - 48, // 6: feast.core.ListFeaturesRequest.filter:type_name -> feast.core.ListFeaturesRequest.Filter - 50, // 7: feast.core.ListFeaturesResponse.features:type_name -> feast.core.ListFeaturesResponse.FeaturesEntry - 51, // 8: feast.core.ListStoresRequest.filter:type_name -> feast.core.ListStoresRequest.Filter - 57, // 9: feast.core.ListStoresResponse.store:type_name -> feast.core.Store - 58, // 10: feast.core.ApplyEntityRequest.spec:type_name -> feast.core.EntitySpecV2 - 56, // 11: feast.core.ApplyEntityResponse.entity:type_name -> feast.core.Entity - 55, // 12: feast.core.ApplyFeatureSetRequest.feature_set:type_name -> feast.core.FeatureSet - 55, // 13: feast.core.ApplyFeatureSetResponse.feature_set:type_name -> feast.core.FeatureSet - 0, // 14: feast.core.ApplyFeatureSetResponse.status:type_name -> feast.core.ApplyFeatureSetResponse.Status - 57, // 15: feast.core.UpdateStoreRequest.store:type_name -> feast.core.Store - 57, // 16: feast.core.UpdateStoreResponse.store:type_name -> feast.core.Store - 1, // 17: feast.core.UpdateStoreResponse.status:type_name -> feast.core.UpdateStoreResponse.Status - 52, // 18: feast.core.ListIngestionJobsRequest.filter:type_name -> feast.core.ListIngestionJobsRequest.Filter - 59, // 19: feast.core.ListIngestionJobsResponse.jobs:type_name -> feast.core.IngestionJob - 60, // 20: feast.core.GetFeatureStatisticsRequest.start_date:type_name -> google.protobuf.Timestamp - 60, // 21: feast.core.GetFeatureStatisticsRequest.end_date:type_name -> google.protobuf.Timestamp - 61, // 22: feast.core.GetFeatureStatisticsResponse.dataset_feature_statistics_list:type_name -> tensorflow.metadata.v0.DatasetFeatureStatisticsList - 62, // 23: feast.core.UpdateFeatureSetStatusRequest.reference:type_name -> feast.core.FeatureSetReference - 63, // 24: feast.core.UpdateFeatureSetStatusRequest.status:type_name -> feast.core.FeatureSetStatus - 64, // 25: feast.core.ApplyFeatureTableRequest.table_spec:type_name -> feast.core.FeatureTableSpec - 65, // 26: feast.core.ApplyFeatureTableResponse.table:type_name -> feast.core.FeatureTable - 65, // 27: feast.core.GetFeatureTableResponse.table:type_name -> feast.core.FeatureTable - 53, // 28: feast.core.ListFeatureTablesRequest.filter:type_name -> feast.core.ListFeatureTablesRequest.Filter - 65, // 29: feast.core.ListFeatureTablesResponse.tables:type_name -> feast.core.FeatureTable - 45, // 30: feast.core.ListFeatureSetsRequest.Filter.labels:type_name -> feast.core.ListFeatureSetsRequest.Filter.LabelsEntry - 63, // 31: feast.core.ListFeatureSetsRequest.Filter.status:type_name -> feast.core.FeatureSetStatus - 47, // 32: feast.core.ListEntitiesRequest.Filter.labels:type_name -> feast.core.ListEntitiesRequest.Filter.LabelsEntry - 49, // 33: feast.core.ListFeaturesRequest.Filter.labels:type_name -> feast.core.ListFeaturesRequest.Filter.LabelsEntry - 66, // 34: feast.core.ListFeaturesResponse.FeaturesEntry.value:type_name -> feast.core.FeatureSpec - 62, // 35: feast.core.ListIngestionJobsRequest.Filter.feature_set_reference:type_name -> feast.core.FeatureSetReference - 54, // 36: feast.core.ListFeatureTablesRequest.Filter.labels:type_name -> feast.core.ListFeatureTablesRequest.Filter.LabelsEntry - 18, // 37: feast.core.CoreService.GetFeastCoreVersion:input_type -> feast.core.GetFeastCoreVersionRequest - 2, // 38: feast.core.CoreService.GetFeatureSet:input_type -> feast.core.GetFeatureSetRequest - 6, // 39: feast.core.CoreService.GetEntity:input_type -> feast.core.GetEntityRequest - 4, // 40: feast.core.CoreService.ListFeatureSets:input_type -> feast.core.ListFeatureSetsRequest - 10, // 41: feast.core.CoreService.ListFeatures:input_type -> feast.core.ListFeaturesRequest - 34, // 42: feast.core.CoreService.GetFeatureStatistics:input_type -> feast.core.GetFeatureStatisticsRequest - 12, // 43: feast.core.CoreService.ListStores:input_type -> feast.core.ListStoresRequest - 16, // 44: feast.core.CoreService.ApplyFeatureSet:input_type -> feast.core.ApplyFeatureSetRequest - 14, // 45: feast.core.CoreService.ApplyEntity:input_type -> feast.core.ApplyEntityRequest - 8, // 46: feast.core.CoreService.ListEntities:input_type -> feast.core.ListEntitiesRequest - 20, // 47: feast.core.CoreService.UpdateStore:input_type -> feast.core.UpdateStoreRequest - 22, // 48: feast.core.CoreService.CreateProject:input_type -> feast.core.CreateProjectRequest - 24, // 49: feast.core.CoreService.ArchiveProject:input_type -> feast.core.ArchiveProjectRequest - 26, // 50: feast.core.CoreService.ListProjects:input_type -> feast.core.ListProjectsRequest - 36, // 51: feast.core.CoreService.UpdateFeatureSetStatus:input_type -> feast.core.UpdateFeatureSetStatusRequest - 38, // 52: feast.core.CoreService.ApplyFeatureTable:input_type -> feast.core.ApplyFeatureTableRequest - 42, // 53: feast.core.CoreService.ListFeatureTables:input_type -> feast.core.ListFeatureTablesRequest - 40, // 54: feast.core.CoreService.GetFeatureTable:input_type -> feast.core.GetFeatureTableRequest - 28, // 55: feast.core.JobControllerService.ListIngestionJobs:input_type -> feast.core.ListIngestionJobsRequest - 30, // 56: feast.core.JobControllerService.RestartIngestionJob:input_type -> feast.core.RestartIngestionJobRequest - 32, // 57: feast.core.JobControllerService.StopIngestionJob:input_type -> feast.core.StopIngestionJobRequest - 19, // 58: feast.core.CoreService.GetFeastCoreVersion:output_type -> feast.core.GetFeastCoreVersionResponse - 3, // 59: feast.core.CoreService.GetFeatureSet:output_type -> feast.core.GetFeatureSetResponse - 7, // 60: feast.core.CoreService.GetEntity:output_type -> feast.core.GetEntityResponse - 5, // 61: feast.core.CoreService.ListFeatureSets:output_type -> feast.core.ListFeatureSetsResponse - 11, // 62: feast.core.CoreService.ListFeatures:output_type -> feast.core.ListFeaturesResponse - 35, // 63: feast.core.CoreService.GetFeatureStatistics:output_type -> feast.core.GetFeatureStatisticsResponse - 13, // 64: feast.core.CoreService.ListStores:output_type -> feast.core.ListStoresResponse - 17, // 65: feast.core.CoreService.ApplyFeatureSet:output_type -> feast.core.ApplyFeatureSetResponse - 15, // 66: feast.core.CoreService.ApplyEntity:output_type -> feast.core.ApplyEntityResponse - 9, // 67: feast.core.CoreService.ListEntities:output_type -> feast.core.ListEntitiesResponse - 21, // 68: feast.core.CoreService.UpdateStore:output_type -> feast.core.UpdateStoreResponse - 23, // 69: feast.core.CoreService.CreateProject:output_type -> feast.core.CreateProjectResponse - 25, // 70: feast.core.CoreService.ArchiveProject:output_type -> feast.core.ArchiveProjectResponse - 27, // 71: feast.core.CoreService.ListProjects:output_type -> feast.core.ListProjectsResponse - 37, // 72: feast.core.CoreService.UpdateFeatureSetStatus:output_type -> feast.core.UpdateFeatureSetStatusResponse - 39, // 73: feast.core.CoreService.ApplyFeatureTable:output_type -> feast.core.ApplyFeatureTableResponse - 43, // 74: feast.core.CoreService.ListFeatureTables:output_type -> feast.core.ListFeatureTablesResponse - 41, // 75: feast.core.CoreService.GetFeatureTable:output_type -> feast.core.GetFeatureTableResponse - 29, // 76: feast.core.JobControllerService.ListIngestionJobs:output_type -> feast.core.ListIngestionJobsResponse - 31, // 77: feast.core.JobControllerService.RestartIngestionJob:output_type -> feast.core.RestartIngestionJobResponse - 33, // 78: feast.core.JobControllerService.StopIngestionJob:output_type -> feast.core.StopIngestionJobResponse - 58, // [58:79] is the sub-list for method output_type - 37, // [37:58] is the sub-list for method input_type - 37, // [37:37] is the sub-list for extension type_name - 37, // [37:37] is the sub-list for extension extendee - 0, // [0:37] is the sub-list for field type_name + 38, // 0: feast.core.GetEntityResponse.entity:type_name -> feast.core.Entity + 30, // 1: feast.core.ListEntitiesRequest.filter:type_name -> feast.core.ListEntitiesRequest.Filter + 38, // 2: feast.core.ListEntitiesResponse.entities:type_name -> feast.core.Entity + 32, // 3: feast.core.ListFeaturesRequest.filter:type_name -> feast.core.ListFeaturesRequest.Filter + 34, // 4: feast.core.ListFeaturesResponse.features:type_name -> feast.core.ListFeaturesResponse.FeaturesEntry + 35, // 5: feast.core.ListStoresRequest.filter:type_name -> feast.core.ListStoresRequest.Filter + 39, // 6: feast.core.ListStoresResponse.store:type_name -> feast.core.Store + 40, // 7: feast.core.ApplyEntityRequest.spec:type_name -> feast.core.EntitySpecV2 + 38, // 8: feast.core.ApplyEntityResponse.entity:type_name -> feast.core.Entity + 39, // 9: feast.core.UpdateStoreRequest.store:type_name -> feast.core.Store + 39, // 10: feast.core.UpdateStoreResponse.store:type_name -> feast.core.Store + 0, // 11: feast.core.UpdateStoreResponse.status:type_name -> feast.core.UpdateStoreResponse.Status + 41, // 12: feast.core.ApplyFeatureTableRequest.table_spec:type_name -> feast.core.FeatureTableSpec + 42, // 13: feast.core.ApplyFeatureTableResponse.table:type_name -> feast.core.FeatureTable + 42, // 14: feast.core.GetFeatureTableResponse.table:type_name -> feast.core.FeatureTable + 36, // 15: feast.core.ListFeatureTablesRequest.filter:type_name -> feast.core.ListFeatureTablesRequest.Filter + 42, // 16: feast.core.ListFeatureTablesResponse.tables:type_name -> feast.core.FeatureTable + 31, // 17: feast.core.ListEntitiesRequest.Filter.labels:type_name -> feast.core.ListEntitiesRequest.Filter.LabelsEntry + 33, // 18: feast.core.ListFeaturesRequest.Filter.labels:type_name -> feast.core.ListFeaturesRequest.Filter.LabelsEntry + 43, // 19: feast.core.ListFeaturesResponse.FeaturesEntry.value:type_name -> feast.core.FeatureSpecV2 + 37, // 20: feast.core.ListFeatureTablesRequest.Filter.labels:type_name -> feast.core.ListFeatureTablesRequest.Filter.LabelsEntry + 11, // 21: feast.core.CoreService.GetFeastCoreVersion:input_type -> feast.core.GetFeastCoreVersionRequest + 1, // 22: feast.core.CoreService.GetEntity:input_type -> feast.core.GetEntityRequest + 5, // 23: feast.core.CoreService.ListFeatures:input_type -> feast.core.ListFeaturesRequest + 7, // 24: feast.core.CoreService.ListStores:input_type -> feast.core.ListStoresRequest + 9, // 25: feast.core.CoreService.ApplyEntity:input_type -> feast.core.ApplyEntityRequest + 3, // 26: feast.core.CoreService.ListEntities:input_type -> feast.core.ListEntitiesRequest + 13, // 27: feast.core.CoreService.UpdateStore:input_type -> feast.core.UpdateStoreRequest + 15, // 28: feast.core.CoreService.CreateProject:input_type -> feast.core.CreateProjectRequest + 17, // 29: feast.core.CoreService.ArchiveProject:input_type -> feast.core.ArchiveProjectRequest + 19, // 30: feast.core.CoreService.ListProjects:input_type -> feast.core.ListProjectsRequest + 22, // 31: feast.core.CoreService.ApplyFeatureTable:input_type -> feast.core.ApplyFeatureTableRequest + 26, // 32: feast.core.CoreService.ListFeatureTables:input_type -> feast.core.ListFeatureTablesRequest + 24, // 33: feast.core.CoreService.GetFeatureTable:input_type -> feast.core.GetFeatureTableRequest + 28, // 34: feast.core.CoreService.DeleteFeatureTable:input_type -> feast.core.DeleteFeatureTableRequest + 12, // 35: feast.core.CoreService.GetFeastCoreVersion:output_type -> feast.core.GetFeastCoreVersionResponse + 2, // 36: feast.core.CoreService.GetEntity:output_type -> feast.core.GetEntityResponse + 6, // 37: feast.core.CoreService.ListFeatures:output_type -> feast.core.ListFeaturesResponse + 8, // 38: feast.core.CoreService.ListStores:output_type -> feast.core.ListStoresResponse + 10, // 39: feast.core.CoreService.ApplyEntity:output_type -> feast.core.ApplyEntityResponse + 4, // 40: feast.core.CoreService.ListEntities:output_type -> feast.core.ListEntitiesResponse + 14, // 41: feast.core.CoreService.UpdateStore:output_type -> feast.core.UpdateStoreResponse + 16, // 42: feast.core.CoreService.CreateProject:output_type -> feast.core.CreateProjectResponse + 18, // 43: feast.core.CoreService.ArchiveProject:output_type -> feast.core.ArchiveProjectResponse + 20, // 44: feast.core.CoreService.ListProjects:output_type -> feast.core.ListProjectsResponse + 23, // 45: feast.core.CoreService.ApplyFeatureTable:output_type -> feast.core.ApplyFeatureTableResponse + 27, // 46: feast.core.CoreService.ListFeatureTables:output_type -> feast.core.ListFeatureTablesResponse + 25, // 47: feast.core.CoreService.GetFeatureTable:output_type -> feast.core.GetFeatureTableResponse + 29, // 48: feast.core.CoreService.DeleteFeatureTable:output_type -> feast.core.DeleteFeatureTableResponse + 35, // [35:49] is the sub-list for method output_type + 21, // [21:35] is the sub-list for method input_type + 21, // [21:21] is the sub-list for extension type_name + 21, // [21:21] is the sub-list for extension extendee + 0, // [0:21] is the sub-list for field type_name } func init() { file_feast_core_CoreService_proto_init() } @@ -3242,61 +2110,11 @@ func file_feast_core_CoreService_proto_init() { return } file_feast_core_Entity_proto_init() - file_feast_core_FeatureSet_proto_init() + file_feast_core_Feature_proto_init() file_feast_core_FeatureTable_proto_init() file_feast_core_Store_proto_init() - file_feast_core_FeatureSetReference_proto_init() - file_feast_core_IngestionJob_proto_init() if !protoimpl.UnsafeEnabled { file_feast_core_CoreService_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*GetFeatureSetRequest); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_feast_core_CoreService_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*GetFeatureSetResponse); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_feast_core_CoreService_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ListFeatureSetsRequest); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_feast_core_CoreService_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ListFeatureSetsResponse); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_feast_core_CoreService_proto_msgTypes[4].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*GetEntityRequest); i { case 0: return &v.state @@ -3308,7 +2126,7 @@ func file_feast_core_CoreService_proto_init() { return nil } } - file_feast_core_CoreService_proto_msgTypes[5].Exporter = func(v interface{}, i int) interface{} { + file_feast_core_CoreService_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*GetEntityResponse); i { case 0: return &v.state @@ -3320,7 +2138,7 @@ func file_feast_core_CoreService_proto_init() { return nil } } - file_feast_core_CoreService_proto_msgTypes[6].Exporter = func(v interface{}, i int) interface{} { + file_feast_core_CoreService_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*ListEntitiesRequest); i { case 0: return &v.state @@ -3332,7 +2150,7 @@ func file_feast_core_CoreService_proto_init() { return nil } } - file_feast_core_CoreService_proto_msgTypes[7].Exporter = func(v interface{}, i int) interface{} { + file_feast_core_CoreService_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*ListEntitiesResponse); i { case 0: return &v.state @@ -3344,7 +2162,7 @@ func file_feast_core_CoreService_proto_init() { return nil } } - file_feast_core_CoreService_proto_msgTypes[8].Exporter = func(v interface{}, i int) interface{} { + file_feast_core_CoreService_proto_msgTypes[4].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*ListFeaturesRequest); i { case 0: return &v.state @@ -3356,7 +2174,7 @@ func file_feast_core_CoreService_proto_init() { return nil } } - file_feast_core_CoreService_proto_msgTypes[9].Exporter = func(v interface{}, i int) interface{} { + file_feast_core_CoreService_proto_msgTypes[5].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*ListFeaturesResponse); i { case 0: return &v.state @@ -3368,7 +2186,7 @@ func file_feast_core_CoreService_proto_init() { return nil } } - file_feast_core_CoreService_proto_msgTypes[10].Exporter = func(v interface{}, i int) interface{} { + file_feast_core_CoreService_proto_msgTypes[6].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*ListStoresRequest); i { case 0: return &v.state @@ -3380,7 +2198,7 @@ func file_feast_core_CoreService_proto_init() { return nil } } - file_feast_core_CoreService_proto_msgTypes[11].Exporter = func(v interface{}, i int) interface{} { + file_feast_core_CoreService_proto_msgTypes[7].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*ListStoresResponse); i { case 0: return &v.state @@ -3392,7 +2210,7 @@ func file_feast_core_CoreService_proto_init() { return nil } } - file_feast_core_CoreService_proto_msgTypes[12].Exporter = func(v interface{}, i int) interface{} { + file_feast_core_CoreService_proto_msgTypes[8].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*ApplyEntityRequest); i { case 0: return &v.state @@ -3404,7 +2222,7 @@ func file_feast_core_CoreService_proto_init() { return nil } } - file_feast_core_CoreService_proto_msgTypes[13].Exporter = func(v interface{}, i int) interface{} { + file_feast_core_CoreService_proto_msgTypes[9].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*ApplyEntityResponse); i { case 0: return &v.state @@ -3416,31 +2234,7 @@ func file_feast_core_CoreService_proto_init() { return nil } } - file_feast_core_CoreService_proto_msgTypes[14].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ApplyFeatureSetRequest); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_feast_core_CoreService_proto_msgTypes[15].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ApplyFeatureSetResponse); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_feast_core_CoreService_proto_msgTypes[16].Exporter = func(v interface{}, i int) interface{} { + file_feast_core_CoreService_proto_msgTypes[10].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*GetFeastCoreVersionRequest); i { case 0: return &v.state @@ -3452,7 +2246,7 @@ func file_feast_core_CoreService_proto_init() { return nil } } - file_feast_core_CoreService_proto_msgTypes[17].Exporter = func(v interface{}, i int) interface{} { + file_feast_core_CoreService_proto_msgTypes[11].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*GetFeastCoreVersionResponse); i { case 0: return &v.state @@ -3464,7 +2258,7 @@ func file_feast_core_CoreService_proto_init() { return nil } } - file_feast_core_CoreService_proto_msgTypes[18].Exporter = func(v interface{}, i int) interface{} { + file_feast_core_CoreService_proto_msgTypes[12].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*UpdateStoreRequest); i { case 0: return &v.state @@ -3476,7 +2270,7 @@ func file_feast_core_CoreService_proto_init() { return nil } } - file_feast_core_CoreService_proto_msgTypes[19].Exporter = func(v interface{}, i int) interface{} { + file_feast_core_CoreService_proto_msgTypes[13].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*UpdateStoreResponse); i { case 0: return &v.state @@ -3488,7 +2282,7 @@ func file_feast_core_CoreService_proto_init() { return nil } } - file_feast_core_CoreService_proto_msgTypes[20].Exporter = func(v interface{}, i int) interface{} { + file_feast_core_CoreService_proto_msgTypes[14].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*CreateProjectRequest); i { case 0: return &v.state @@ -3500,7 +2294,7 @@ func file_feast_core_CoreService_proto_init() { return nil } } - file_feast_core_CoreService_proto_msgTypes[21].Exporter = func(v interface{}, i int) interface{} { + file_feast_core_CoreService_proto_msgTypes[15].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*CreateProjectResponse); i { case 0: return &v.state @@ -3512,116 +2306,8 @@ func file_feast_core_CoreService_proto_init() { return nil } } - file_feast_core_CoreService_proto_msgTypes[22].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ArchiveProjectRequest); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_feast_core_CoreService_proto_msgTypes[23].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ArchiveProjectResponse); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_feast_core_CoreService_proto_msgTypes[24].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ListProjectsRequest); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_feast_core_CoreService_proto_msgTypes[25].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ListProjectsResponse); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_feast_core_CoreService_proto_msgTypes[26].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ListIngestionJobsRequest); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_feast_core_CoreService_proto_msgTypes[27].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ListIngestionJobsResponse); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_feast_core_CoreService_proto_msgTypes[28].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*RestartIngestionJobRequest); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_feast_core_CoreService_proto_msgTypes[29].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*RestartIngestionJobResponse); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_feast_core_CoreService_proto_msgTypes[30].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*StopIngestionJobRequest); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_feast_core_CoreService_proto_msgTypes[31].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*StopIngestionJobResponse); i { + file_feast_core_CoreService_proto_msgTypes[16].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*ArchiveProjectRequest); i { case 0: return &v.state case 1: @@ -3632,8 +2318,8 @@ func file_feast_core_CoreService_proto_init() { return nil } } - file_feast_core_CoreService_proto_msgTypes[32].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*GetFeatureStatisticsRequest); i { + file_feast_core_CoreService_proto_msgTypes[17].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*ArchiveProjectResponse); i { case 0: return &v.state case 1: @@ -3644,8 +2330,8 @@ func file_feast_core_CoreService_proto_init() { return nil } } - file_feast_core_CoreService_proto_msgTypes[33].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*GetFeatureStatisticsResponse); i { + file_feast_core_CoreService_proto_msgTypes[18].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*ListProjectsRequest); i { case 0: return &v.state case 1: @@ -3656,8 +2342,8 @@ func file_feast_core_CoreService_proto_init() { return nil } } - file_feast_core_CoreService_proto_msgTypes[34].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*UpdateFeatureSetStatusRequest); i { + file_feast_core_CoreService_proto_msgTypes[19].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*ListProjectsResponse); i { case 0: return &v.state case 1: @@ -3668,7 +2354,7 @@ func file_feast_core_CoreService_proto_init() { return nil } } - file_feast_core_CoreService_proto_msgTypes[35].Exporter = func(v interface{}, i int) interface{} { + file_feast_core_CoreService_proto_msgTypes[20].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*UpdateFeatureSetStatusResponse); i { case 0: return &v.state @@ -3680,7 +2366,7 @@ func file_feast_core_CoreService_proto_init() { return nil } } - file_feast_core_CoreService_proto_msgTypes[36].Exporter = func(v interface{}, i int) interface{} { + file_feast_core_CoreService_proto_msgTypes[21].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*ApplyFeatureTableRequest); i { case 0: return &v.state @@ -3692,7 +2378,7 @@ func file_feast_core_CoreService_proto_init() { return nil } } - file_feast_core_CoreService_proto_msgTypes[37].Exporter = func(v interface{}, i int) interface{} { + file_feast_core_CoreService_proto_msgTypes[22].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*ApplyFeatureTableResponse); i { case 0: return &v.state @@ -3704,7 +2390,7 @@ func file_feast_core_CoreService_proto_init() { return nil } } - file_feast_core_CoreService_proto_msgTypes[38].Exporter = func(v interface{}, i int) interface{} { + file_feast_core_CoreService_proto_msgTypes[23].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*GetFeatureTableRequest); i { case 0: return &v.state @@ -3716,7 +2402,7 @@ func file_feast_core_CoreService_proto_init() { return nil } } - file_feast_core_CoreService_proto_msgTypes[39].Exporter = func(v interface{}, i int) interface{} { + file_feast_core_CoreService_proto_msgTypes[24].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*GetFeatureTableResponse); i { case 0: return &v.state @@ -3728,7 +2414,7 @@ func file_feast_core_CoreService_proto_init() { return nil } } - file_feast_core_CoreService_proto_msgTypes[40].Exporter = func(v interface{}, i int) interface{} { + file_feast_core_CoreService_proto_msgTypes[25].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*ListFeatureTablesRequest); i { case 0: return &v.state @@ -3740,7 +2426,7 @@ func file_feast_core_CoreService_proto_init() { return nil } } - file_feast_core_CoreService_proto_msgTypes[41].Exporter = func(v interface{}, i int) interface{} { + file_feast_core_CoreService_proto_msgTypes[26].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*ListFeatureTablesResponse); i { case 0: return &v.state @@ -3752,8 +2438,8 @@ func file_feast_core_CoreService_proto_init() { return nil } } - file_feast_core_CoreService_proto_msgTypes[42].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ListFeatureSetsRequest_Filter); i { + file_feast_core_CoreService_proto_msgTypes[27].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*DeleteFeatureTableRequest); i { case 0: return &v.state case 1: @@ -3764,8 +2450,8 @@ func file_feast_core_CoreService_proto_init() { return nil } } - file_feast_core_CoreService_proto_msgTypes[44].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ListEntitiesRequest_Filter); i { + file_feast_core_CoreService_proto_msgTypes[28].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*DeleteFeatureTableResponse); i { case 0: return &v.state case 1: @@ -3776,8 +2462,8 @@ func file_feast_core_CoreService_proto_init() { return nil } } - file_feast_core_CoreService_proto_msgTypes[46].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ListFeaturesRequest_Filter); i { + file_feast_core_CoreService_proto_msgTypes[29].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*ListEntitiesRequest_Filter); i { case 0: return &v.state case 1: @@ -3788,8 +2474,8 @@ func file_feast_core_CoreService_proto_init() { return nil } } - file_feast_core_CoreService_proto_msgTypes[49].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ListStoresRequest_Filter); i { + file_feast_core_CoreService_proto_msgTypes[31].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*ListFeaturesRequest_Filter); i { case 0: return &v.state case 1: @@ -3800,8 +2486,8 @@ func file_feast_core_CoreService_proto_init() { return nil } } - file_feast_core_CoreService_proto_msgTypes[50].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ListIngestionJobsRequest_Filter); i { + file_feast_core_CoreService_proto_msgTypes[34].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*ListStoresRequest_Filter); i { case 0: return &v.state case 1: @@ -3812,7 +2498,7 @@ func file_feast_core_CoreService_proto_init() { return nil } } - file_feast_core_CoreService_proto_msgTypes[51].Exporter = func(v interface{}, i int) interface{} { + file_feast_core_CoreService_proto_msgTypes[35].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*ListFeatureTablesRequest_Filter); i { case 0: return &v.state @@ -3830,10 +2516,10 @@ func file_feast_core_CoreService_proto_init() { File: protoimpl.DescBuilder{ GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: file_feast_core_CoreService_proto_rawDesc, - NumEnums: 2, - NumMessages: 53, + NumEnums: 1, + NumMessages: 37, NumExtensions: 0, - NumServices: 2, + NumServices: 1, }, GoTypes: file_feast_core_CoreService_proto_goTypes, DependencyIndexes: file_feast_core_CoreService_proto_depIdxs, @@ -3860,41 +2546,19 @@ const _ = grpc.SupportPackageIsVersion6 type CoreServiceClient interface { // Retrieve version information about this Feast deployment GetFeastCoreVersion(ctx context.Context, in *GetFeastCoreVersionRequest, opts ...grpc.CallOption) (*GetFeastCoreVersionResponse, error) - // Returns a specific feature set - GetFeatureSet(ctx context.Context, in *GetFeatureSetRequest, opts ...grpc.CallOption) (*GetFeatureSetResponse, error) // Returns a specific entity GetEntity(ctx context.Context, in *GetEntityRequest, opts ...grpc.CallOption) (*GetEntityResponse, error) - // Retrieve feature set details given a filter. - // - // Returns all feature sets matching that filter. If none are found, - // an empty list will be returned. - // If no filter is provided in the request, the response will contain all the feature - // sets currently stored in the registry. - ListFeatureSets(ctx context.Context, in *ListFeatureSetsRequest, opts ...grpc.CallOption) (*ListFeatureSetsResponse, error) // Returns all feature references and respective features matching that filter. If none are found // an empty map will be returned // If no filter is provided in the request, the response will contain all the features // currently stored in the default project. ListFeatures(ctx context.Context, in *ListFeaturesRequest, opts ...grpc.CallOption) (*ListFeaturesResponse, error) - // Get feature statistics computed over the data in the batch stores. - // - // Returns a dataset containing TFDV statistics mapped to each valid historical store. - GetFeatureStatistics(ctx context.Context, in *GetFeatureStatisticsRequest, opts ...grpc.CallOption) (*GetFeatureStatisticsResponse, error) // Retrieve store details given a filter. // // Returns all stores matching that filter. If none are found, an empty list will be returned. // If no filter is provided in the request, the response will contain all the stores currently // stored in the registry. ListStores(ctx context.Context, in *ListStoresRequest, opts ...grpc.CallOption) (*ListStoresResponse, error) - // Create or update and existing feature set. - // - // This function is idempotent - it will not create a new feature set if schema does not change. - // Schema changes will update the feature set if the changes are valid. - // All changes except the following are valid: - // - Changes to feature set id (name, project) - // - Changes to entities - // - Changes to feature name and type - ApplyFeatureSet(ctx context.Context, in *ApplyFeatureSetRequest, opts ...grpc.CallOption) (*ApplyFeatureSetResponse, error) // Create or update and existing entity. // // This function is idempotent - it will not create a new entity if schema does not change. @@ -3914,8 +2578,8 @@ type CoreServiceClient interface { // start or update the necessary feature population jobs for the updated store. UpdateStore(ctx context.Context, in *UpdateStoreRequest, opts ...grpc.CallOption) (*UpdateStoreResponse, error) // Creates a project. Projects serve as namespaces within which resources like features will be - // created. Feature set names as must be unique within a project while field (Feature/Entity) names - // must be unique within a Feature Set. Project names themselves must be globally unique. + // created. Feature table names as must be unique within a project while field (Feature/Entity) names + // must be unique within a Feature Table. Project names themselves must be globally unique. CreateProject(ctx context.Context, in *CreateProjectRequest, opts ...grpc.CallOption) (*CreateProjectResponse, error) // Archives a project. Archived projects will continue to exist and function, but won't be visible // through the Core API. Any existing ingestion or serving requests will continue to function, @@ -3924,8 +2588,6 @@ type CoreServiceClient interface { ArchiveProject(ctx context.Context, in *ArchiveProjectRequest, opts ...grpc.CallOption) (*ArchiveProjectResponse, error) // Lists all projects active projects. ListProjects(ctx context.Context, in *ListProjectsRequest, opts ...grpc.CallOption) (*ListProjectsResponse, error) - // Internal API for Job Controller to update featureSet's status once responsible ingestion job is running - UpdateFeatureSetStatus(ctx context.Context, in *UpdateFeatureSetStatusRequest, opts ...grpc.CallOption) (*UpdateFeatureSetStatusResponse, error) // Create or update an existing feature table. // This function is idempotent - it will not create a new feature table if the schema does not change. // Schema changes will update the feature table if the changes are valid. @@ -3942,6 +2604,8 @@ type CoreServiceClient interface { ListFeatureTables(ctx context.Context, in *ListFeatureTablesRequest, opts ...grpc.CallOption) (*ListFeatureTablesResponse, error) // Returns a specific feature table GetFeatureTable(ctx context.Context, in *GetFeatureTableRequest, opts ...grpc.CallOption) (*GetFeatureTableResponse, error) + // Delete a specific feature table + DeleteFeatureTable(ctx context.Context, in *DeleteFeatureTableRequest, opts ...grpc.CallOption) (*DeleteFeatureTableResponse, error) } type coreServiceClient struct { @@ -3961,15 +2625,6 @@ func (c *coreServiceClient) GetFeastCoreVersion(ctx context.Context, in *GetFeas return out, nil } -func (c *coreServiceClient) GetFeatureSet(ctx context.Context, in *GetFeatureSetRequest, opts ...grpc.CallOption) (*GetFeatureSetResponse, error) { - out := new(GetFeatureSetResponse) - err := c.cc.Invoke(ctx, "/feast.core.CoreService/GetFeatureSet", in, out, opts...) - if err != nil { - return nil, err - } - return out, nil -} - func (c *coreServiceClient) GetEntity(ctx context.Context, in *GetEntityRequest, opts ...grpc.CallOption) (*GetEntityResponse, error) { out := new(GetEntityResponse) err := c.cc.Invoke(ctx, "/feast.core.CoreService/GetEntity", in, out, opts...) @@ -3979,15 +2634,6 @@ func (c *coreServiceClient) GetEntity(ctx context.Context, in *GetEntityRequest, return out, nil } -func (c *coreServiceClient) ListFeatureSets(ctx context.Context, in *ListFeatureSetsRequest, opts ...grpc.CallOption) (*ListFeatureSetsResponse, error) { - out := new(ListFeatureSetsResponse) - err := c.cc.Invoke(ctx, "/feast.core.CoreService/ListFeatureSets", in, out, opts...) - if err != nil { - return nil, err - } - return out, nil -} - func (c *coreServiceClient) ListFeatures(ctx context.Context, in *ListFeaturesRequest, opts ...grpc.CallOption) (*ListFeaturesResponse, error) { out := new(ListFeaturesResponse) err := c.cc.Invoke(ctx, "/feast.core.CoreService/ListFeatures", in, out, opts...) @@ -3997,15 +2643,6 @@ func (c *coreServiceClient) ListFeatures(ctx context.Context, in *ListFeaturesRe return out, nil } -func (c *coreServiceClient) GetFeatureStatistics(ctx context.Context, in *GetFeatureStatisticsRequest, opts ...grpc.CallOption) (*GetFeatureStatisticsResponse, error) { - out := new(GetFeatureStatisticsResponse) - err := c.cc.Invoke(ctx, "/feast.core.CoreService/GetFeatureStatistics", in, out, opts...) - if err != nil { - return nil, err - } - return out, nil -} - func (c *coreServiceClient) ListStores(ctx context.Context, in *ListStoresRequest, opts ...grpc.CallOption) (*ListStoresResponse, error) { out := new(ListStoresResponse) err := c.cc.Invoke(ctx, "/feast.core.CoreService/ListStores", in, out, opts...) @@ -4015,15 +2652,6 @@ func (c *coreServiceClient) ListStores(ctx context.Context, in *ListStoresReques return out, nil } -func (c *coreServiceClient) ApplyFeatureSet(ctx context.Context, in *ApplyFeatureSetRequest, opts ...grpc.CallOption) (*ApplyFeatureSetResponse, error) { - out := new(ApplyFeatureSetResponse) - err := c.cc.Invoke(ctx, "/feast.core.CoreService/ApplyFeatureSet", in, out, opts...) - if err != nil { - return nil, err - } - return out, nil -} - func (c *coreServiceClient) ApplyEntity(ctx context.Context, in *ApplyEntityRequest, opts ...grpc.CallOption) (*ApplyEntityResponse, error) { out := new(ApplyEntityResponse) err := c.cc.Invoke(ctx, "/feast.core.CoreService/ApplyEntity", in, out, opts...) @@ -4078,15 +2706,6 @@ func (c *coreServiceClient) ListProjects(ctx context.Context, in *ListProjectsRe return out, nil } -func (c *coreServiceClient) UpdateFeatureSetStatus(ctx context.Context, in *UpdateFeatureSetStatusRequest, opts ...grpc.CallOption) (*UpdateFeatureSetStatusResponse, error) { - out := new(UpdateFeatureSetStatusResponse) - err := c.cc.Invoke(ctx, "/feast.core.CoreService/UpdateFeatureSetStatus", in, out, opts...) - if err != nil { - return nil, err - } - return out, nil -} - func (c *coreServiceClient) ApplyFeatureTable(ctx context.Context, in *ApplyFeatureTableRequest, opts ...grpc.CallOption) (*ApplyFeatureTableResponse, error) { out := new(ApplyFeatureTableResponse) err := c.cc.Invoke(ctx, "/feast.core.CoreService/ApplyFeatureTable", in, out, opts...) @@ -4114,45 +2733,32 @@ func (c *coreServiceClient) GetFeatureTable(ctx context.Context, in *GetFeatureT return out, nil } +func (c *coreServiceClient) DeleteFeatureTable(ctx context.Context, in *DeleteFeatureTableRequest, opts ...grpc.CallOption) (*DeleteFeatureTableResponse, error) { + out := new(DeleteFeatureTableResponse) + err := c.cc.Invoke(ctx, "/feast.core.CoreService/DeleteFeatureTable", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + // CoreServiceServer is the server API for CoreService service. type CoreServiceServer interface { // Retrieve version information about this Feast deployment GetFeastCoreVersion(context.Context, *GetFeastCoreVersionRequest) (*GetFeastCoreVersionResponse, error) - // Returns a specific feature set - GetFeatureSet(context.Context, *GetFeatureSetRequest) (*GetFeatureSetResponse, error) // Returns a specific entity GetEntity(context.Context, *GetEntityRequest) (*GetEntityResponse, error) - // Retrieve feature set details given a filter. - // - // Returns all feature sets matching that filter. If none are found, - // an empty list will be returned. - // If no filter is provided in the request, the response will contain all the feature - // sets currently stored in the registry. - ListFeatureSets(context.Context, *ListFeatureSetsRequest) (*ListFeatureSetsResponse, error) // Returns all feature references and respective features matching that filter. If none are found // an empty map will be returned // If no filter is provided in the request, the response will contain all the features // currently stored in the default project. ListFeatures(context.Context, *ListFeaturesRequest) (*ListFeaturesResponse, error) - // Get feature statistics computed over the data in the batch stores. - // - // Returns a dataset containing TFDV statistics mapped to each valid historical store. - GetFeatureStatistics(context.Context, *GetFeatureStatisticsRequest) (*GetFeatureStatisticsResponse, error) // Retrieve store details given a filter. // // Returns all stores matching that filter. If none are found, an empty list will be returned. // If no filter is provided in the request, the response will contain all the stores currently // stored in the registry. ListStores(context.Context, *ListStoresRequest) (*ListStoresResponse, error) - // Create or update and existing feature set. - // - // This function is idempotent - it will not create a new feature set if schema does not change. - // Schema changes will update the feature set if the changes are valid. - // All changes except the following are valid: - // - Changes to feature set id (name, project) - // - Changes to entities - // - Changes to feature name and type - ApplyFeatureSet(context.Context, *ApplyFeatureSetRequest) (*ApplyFeatureSetResponse, error) // Create or update and existing entity. // // This function is idempotent - it will not create a new entity if schema does not change. @@ -4172,8 +2778,8 @@ type CoreServiceServer interface { // start or update the necessary feature population jobs for the updated store. UpdateStore(context.Context, *UpdateStoreRequest) (*UpdateStoreResponse, error) // Creates a project. Projects serve as namespaces within which resources like features will be - // created. Feature set names as must be unique within a project while field (Feature/Entity) names - // must be unique within a Feature Set. Project names themselves must be globally unique. + // created. Feature table names as must be unique within a project while field (Feature/Entity) names + // must be unique within a Feature Table. Project names themselves must be globally unique. CreateProject(context.Context, *CreateProjectRequest) (*CreateProjectResponse, error) // Archives a project. Archived projects will continue to exist and function, but won't be visible // through the Core API. Any existing ingestion or serving requests will continue to function, @@ -4182,8 +2788,6 @@ type CoreServiceServer interface { ArchiveProject(context.Context, *ArchiveProjectRequest) (*ArchiveProjectResponse, error) // Lists all projects active projects. ListProjects(context.Context, *ListProjectsRequest) (*ListProjectsResponse, error) - // Internal API for Job Controller to update featureSet's status once responsible ingestion job is running - UpdateFeatureSetStatus(context.Context, *UpdateFeatureSetStatusRequest) (*UpdateFeatureSetStatusResponse, error) // Create or update an existing feature table. // This function is idempotent - it will not create a new feature table if the schema does not change. // Schema changes will update the feature table if the changes are valid. @@ -4200,6 +2804,8 @@ type CoreServiceServer interface { ListFeatureTables(context.Context, *ListFeatureTablesRequest) (*ListFeatureTablesResponse, error) // Returns a specific feature table GetFeatureTable(context.Context, *GetFeatureTableRequest) (*GetFeatureTableResponse, error) + // Delete a specific feature table + DeleteFeatureTable(context.Context, *DeleteFeatureTableRequest) (*DeleteFeatureTableResponse, error) } // UnimplementedCoreServiceServer can be embedded to have forward compatible implementations. @@ -4209,27 +2815,15 @@ type UnimplementedCoreServiceServer struct { func (*UnimplementedCoreServiceServer) GetFeastCoreVersion(context.Context, *GetFeastCoreVersionRequest) (*GetFeastCoreVersionResponse, error) { return nil, status.Errorf(codes.Unimplemented, "method GetFeastCoreVersion not implemented") } -func (*UnimplementedCoreServiceServer) GetFeatureSet(context.Context, *GetFeatureSetRequest) (*GetFeatureSetResponse, error) { - return nil, status.Errorf(codes.Unimplemented, "method GetFeatureSet not implemented") -} func (*UnimplementedCoreServiceServer) GetEntity(context.Context, *GetEntityRequest) (*GetEntityResponse, error) { return nil, status.Errorf(codes.Unimplemented, "method GetEntity not implemented") } -func (*UnimplementedCoreServiceServer) ListFeatureSets(context.Context, *ListFeatureSetsRequest) (*ListFeatureSetsResponse, error) { - return nil, status.Errorf(codes.Unimplemented, "method ListFeatureSets not implemented") -} func (*UnimplementedCoreServiceServer) ListFeatures(context.Context, *ListFeaturesRequest) (*ListFeaturesResponse, error) { return nil, status.Errorf(codes.Unimplemented, "method ListFeatures not implemented") } -func (*UnimplementedCoreServiceServer) GetFeatureStatistics(context.Context, *GetFeatureStatisticsRequest) (*GetFeatureStatisticsResponse, error) { - return nil, status.Errorf(codes.Unimplemented, "method GetFeatureStatistics not implemented") -} func (*UnimplementedCoreServiceServer) ListStores(context.Context, *ListStoresRequest) (*ListStoresResponse, error) { return nil, status.Errorf(codes.Unimplemented, "method ListStores not implemented") } -func (*UnimplementedCoreServiceServer) ApplyFeatureSet(context.Context, *ApplyFeatureSetRequest) (*ApplyFeatureSetResponse, error) { - return nil, status.Errorf(codes.Unimplemented, "method ApplyFeatureSet not implemented") -} func (*UnimplementedCoreServiceServer) ApplyEntity(context.Context, *ApplyEntityRequest) (*ApplyEntityResponse, error) { return nil, status.Errorf(codes.Unimplemented, "method ApplyEntity not implemented") } @@ -4248,9 +2842,6 @@ func (*UnimplementedCoreServiceServer) ArchiveProject(context.Context, *ArchiveP func (*UnimplementedCoreServiceServer) ListProjects(context.Context, *ListProjectsRequest) (*ListProjectsResponse, error) { return nil, status.Errorf(codes.Unimplemented, "method ListProjects not implemented") } -func (*UnimplementedCoreServiceServer) UpdateFeatureSetStatus(context.Context, *UpdateFeatureSetStatusRequest) (*UpdateFeatureSetStatusResponse, error) { - return nil, status.Errorf(codes.Unimplemented, "method UpdateFeatureSetStatus not implemented") -} func (*UnimplementedCoreServiceServer) ApplyFeatureTable(context.Context, *ApplyFeatureTableRequest) (*ApplyFeatureTableResponse, error) { return nil, status.Errorf(codes.Unimplemented, "method ApplyFeatureTable not implemented") } @@ -4260,6 +2851,9 @@ func (*UnimplementedCoreServiceServer) ListFeatureTables(context.Context, *ListF func (*UnimplementedCoreServiceServer) GetFeatureTable(context.Context, *GetFeatureTableRequest) (*GetFeatureTableResponse, error) { return nil, status.Errorf(codes.Unimplemented, "method GetFeatureTable not implemented") } +func (*UnimplementedCoreServiceServer) DeleteFeatureTable(context.Context, *DeleteFeatureTableRequest) (*DeleteFeatureTableResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method DeleteFeatureTable not implemented") +} func RegisterCoreServiceServer(s *grpc.Server, srv CoreServiceServer) { s.RegisterService(&_CoreService_serviceDesc, srv) @@ -4283,24 +2877,6 @@ func _CoreService_GetFeastCoreVersion_Handler(srv interface{}, ctx context.Conte return interceptor(ctx, in, info, handler) } -func _CoreService_GetFeatureSet_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - in := new(GetFeatureSetRequest) - if err := dec(in); err != nil { - return nil, err - } - if interceptor == nil { - return srv.(CoreServiceServer).GetFeatureSet(ctx, in) - } - info := &grpc.UnaryServerInfo{ - Server: srv, - FullMethod: "/feast.core.CoreService/GetFeatureSet", - } - handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(CoreServiceServer).GetFeatureSet(ctx, req.(*GetFeatureSetRequest)) - } - return interceptor(ctx, in, info, handler) -} - func _CoreService_GetEntity_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { in := new(GetEntityRequest) if err := dec(in); err != nil { @@ -4319,24 +2895,6 @@ func _CoreService_GetEntity_Handler(srv interface{}, ctx context.Context, dec fu return interceptor(ctx, in, info, handler) } -func _CoreService_ListFeatureSets_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - in := new(ListFeatureSetsRequest) - if err := dec(in); err != nil { - return nil, err - } - if interceptor == nil { - return srv.(CoreServiceServer).ListFeatureSets(ctx, in) - } - info := &grpc.UnaryServerInfo{ - Server: srv, - FullMethod: "/feast.core.CoreService/ListFeatureSets", - } - handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(CoreServiceServer).ListFeatureSets(ctx, req.(*ListFeatureSetsRequest)) - } - return interceptor(ctx, in, info, handler) -} - func _CoreService_ListFeatures_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { in := new(ListFeaturesRequest) if err := dec(in); err != nil { @@ -4355,24 +2913,6 @@ func _CoreService_ListFeatures_Handler(srv interface{}, ctx context.Context, dec return interceptor(ctx, in, info, handler) } -func _CoreService_GetFeatureStatistics_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - in := new(GetFeatureStatisticsRequest) - if err := dec(in); err != nil { - return nil, err - } - if interceptor == nil { - return srv.(CoreServiceServer).GetFeatureStatistics(ctx, in) - } - info := &grpc.UnaryServerInfo{ - Server: srv, - FullMethod: "/feast.core.CoreService/GetFeatureStatistics", - } - handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(CoreServiceServer).GetFeatureStatistics(ctx, req.(*GetFeatureStatisticsRequest)) - } - return interceptor(ctx, in, info, handler) -} - func _CoreService_ListStores_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { in := new(ListStoresRequest) if err := dec(in); err != nil { @@ -4391,24 +2931,6 @@ func _CoreService_ListStores_Handler(srv interface{}, ctx context.Context, dec f return interceptor(ctx, in, info, handler) } -func _CoreService_ApplyFeatureSet_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - in := new(ApplyFeatureSetRequest) - if err := dec(in); err != nil { - return nil, err - } - if interceptor == nil { - return srv.(CoreServiceServer).ApplyFeatureSet(ctx, in) - } - info := &grpc.UnaryServerInfo{ - Server: srv, - FullMethod: "/feast.core.CoreService/ApplyFeatureSet", - } - handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(CoreServiceServer).ApplyFeatureSet(ctx, req.(*ApplyFeatureSetRequest)) - } - return interceptor(ctx, in, info, handler) -} - func _CoreService_ApplyEntity_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { in := new(ApplyEntityRequest) if err := dec(in); err != nil { @@ -4517,24 +3039,6 @@ func _CoreService_ListProjects_Handler(srv interface{}, ctx context.Context, dec return interceptor(ctx, in, info, handler) } -func _CoreService_UpdateFeatureSetStatus_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - in := new(UpdateFeatureSetStatusRequest) - if err := dec(in); err != nil { - return nil, err - } - if interceptor == nil { - return srv.(CoreServiceServer).UpdateFeatureSetStatus(ctx, in) - } - info := &grpc.UnaryServerInfo{ - Server: srv, - FullMethod: "/feast.core.CoreService/UpdateFeatureSetStatus", - } - handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(CoreServiceServer).UpdateFeatureSetStatus(ctx, req.(*UpdateFeatureSetStatusRequest)) - } - return interceptor(ctx, in, info, handler) -} - func _CoreService_ApplyFeatureTable_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { in := new(ApplyFeatureTableRequest) if err := dec(in); err != nil { @@ -4589,6 +3093,24 @@ func _CoreService_GetFeatureTable_Handler(srv interface{}, ctx context.Context, return interceptor(ctx, in, info, handler) } +func _CoreService_DeleteFeatureTable_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(DeleteFeatureTableRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(CoreServiceServer).DeleteFeatureTable(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/feast.core.CoreService/DeleteFeatureTable", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(CoreServiceServer).DeleteFeatureTable(ctx, req.(*DeleteFeatureTableRequest)) + } + return interceptor(ctx, in, info, handler) +} + var _CoreService_serviceDesc = grpc.ServiceDesc{ ServiceName: "feast.core.CoreService", HandlerType: (*CoreServiceServer)(nil), @@ -4597,34 +3119,18 @@ var _CoreService_serviceDesc = grpc.ServiceDesc{ MethodName: "GetFeastCoreVersion", Handler: _CoreService_GetFeastCoreVersion_Handler, }, - { - MethodName: "GetFeatureSet", - Handler: _CoreService_GetFeatureSet_Handler, - }, { MethodName: "GetEntity", Handler: _CoreService_GetEntity_Handler, }, - { - MethodName: "ListFeatureSets", - Handler: _CoreService_ListFeatureSets_Handler, - }, { MethodName: "ListFeatures", Handler: _CoreService_ListFeatures_Handler, }, - { - MethodName: "GetFeatureStatistics", - Handler: _CoreService_GetFeatureStatistics_Handler, - }, { MethodName: "ListStores", Handler: _CoreService_ListStores_Handler, }, - { - MethodName: "ApplyFeatureSet", - Handler: _CoreService_ApplyFeatureSet_Handler, - }, { MethodName: "ApplyEntity", Handler: _CoreService_ApplyEntity_Handler, @@ -4649,10 +3155,6 @@ var _CoreService_serviceDesc = grpc.ServiceDesc{ MethodName: "ListProjects", Handler: _CoreService_ListProjects_Handler, }, - { - MethodName: "UpdateFeatureSetStatus", - Handler: _CoreService_UpdateFeatureSetStatus_Handler, - }, { MethodName: "ApplyFeatureTable", Handler: _CoreService_ApplyFeatureTable_Handler, @@ -4665,171 +3167,9 @@ var _CoreService_serviceDesc = grpc.ServiceDesc{ MethodName: "GetFeatureTable", Handler: _CoreService_GetFeatureTable_Handler, }, - }, - Streams: []grpc.StreamDesc{}, - Metadata: "feast/core/CoreService.proto", -} - -// JobControllerServiceClient is the client API for JobControllerService service. -// -// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://godoc.org/google.golang.org/grpc#ClientConn.NewStream. -type JobControllerServiceClient interface { - // List Ingestion Jobs given an optional filter. - // Returns allow ingestions matching the given request filter. - // Returns all ingestion jobs if no filter is provided. - // Returns an empty list if no ingestion jobs match the filter. - ListIngestionJobs(ctx context.Context, in *ListIngestionJobsRequest, opts ...grpc.CallOption) (*ListIngestionJobsResponse, error) - // Restart an Ingestion Job. Restarts the ingestion job with the given job id. - // NOTE: Data might be lost during the restart for some job runners. - // Does not support stopping a job in a transitional (ie pending, suspending, aborting), - // terminal state (ie suspended or aborted) or unknown status - RestartIngestionJob(ctx context.Context, in *RestartIngestionJobRequest, opts ...grpc.CallOption) (*RestartIngestionJobResponse, error) - // Stop an Ingestion Job. Stop (Aborts) the ingestion job with the given job id. - // Does nothing if the target job if already in a terminal state (ie suspended or aborted). - // Does not support stopping a job in a transitional (ie pending, suspending, aborting) or unknown status - StopIngestionJob(ctx context.Context, in *StopIngestionJobRequest, opts ...grpc.CallOption) (*StopIngestionJobResponse, error) -} - -type jobControllerServiceClient struct { - cc grpc.ClientConnInterface -} - -func NewJobControllerServiceClient(cc grpc.ClientConnInterface) JobControllerServiceClient { - return &jobControllerServiceClient{cc} -} - -func (c *jobControllerServiceClient) ListIngestionJobs(ctx context.Context, in *ListIngestionJobsRequest, opts ...grpc.CallOption) (*ListIngestionJobsResponse, error) { - out := new(ListIngestionJobsResponse) - err := c.cc.Invoke(ctx, "/feast.core.JobControllerService/ListIngestionJobs", in, out, opts...) - if err != nil { - return nil, err - } - return out, nil -} - -func (c *jobControllerServiceClient) RestartIngestionJob(ctx context.Context, in *RestartIngestionJobRequest, opts ...grpc.CallOption) (*RestartIngestionJobResponse, error) { - out := new(RestartIngestionJobResponse) - err := c.cc.Invoke(ctx, "/feast.core.JobControllerService/RestartIngestionJob", in, out, opts...) - if err != nil { - return nil, err - } - return out, nil -} - -func (c *jobControllerServiceClient) StopIngestionJob(ctx context.Context, in *StopIngestionJobRequest, opts ...grpc.CallOption) (*StopIngestionJobResponse, error) { - out := new(StopIngestionJobResponse) - err := c.cc.Invoke(ctx, "/feast.core.JobControllerService/StopIngestionJob", in, out, opts...) - if err != nil { - return nil, err - } - return out, nil -} - -// JobControllerServiceServer is the server API for JobControllerService service. -type JobControllerServiceServer interface { - // List Ingestion Jobs given an optional filter. - // Returns allow ingestions matching the given request filter. - // Returns all ingestion jobs if no filter is provided. - // Returns an empty list if no ingestion jobs match the filter. - ListIngestionJobs(context.Context, *ListIngestionJobsRequest) (*ListIngestionJobsResponse, error) - // Restart an Ingestion Job. Restarts the ingestion job with the given job id. - // NOTE: Data might be lost during the restart for some job runners. - // Does not support stopping a job in a transitional (ie pending, suspending, aborting), - // terminal state (ie suspended or aborted) or unknown status - RestartIngestionJob(context.Context, *RestartIngestionJobRequest) (*RestartIngestionJobResponse, error) - // Stop an Ingestion Job. Stop (Aborts) the ingestion job with the given job id. - // Does nothing if the target job if already in a terminal state (ie suspended or aborted). - // Does not support stopping a job in a transitional (ie pending, suspending, aborting) or unknown status - StopIngestionJob(context.Context, *StopIngestionJobRequest) (*StopIngestionJobResponse, error) -} - -// UnimplementedJobControllerServiceServer can be embedded to have forward compatible implementations. -type UnimplementedJobControllerServiceServer struct { -} - -func (*UnimplementedJobControllerServiceServer) ListIngestionJobs(context.Context, *ListIngestionJobsRequest) (*ListIngestionJobsResponse, error) { - return nil, status.Errorf(codes.Unimplemented, "method ListIngestionJobs not implemented") -} -func (*UnimplementedJobControllerServiceServer) RestartIngestionJob(context.Context, *RestartIngestionJobRequest) (*RestartIngestionJobResponse, error) { - return nil, status.Errorf(codes.Unimplemented, "method RestartIngestionJob not implemented") -} -func (*UnimplementedJobControllerServiceServer) StopIngestionJob(context.Context, *StopIngestionJobRequest) (*StopIngestionJobResponse, error) { - return nil, status.Errorf(codes.Unimplemented, "method StopIngestionJob not implemented") -} - -func RegisterJobControllerServiceServer(s *grpc.Server, srv JobControllerServiceServer) { - s.RegisterService(&_JobControllerService_serviceDesc, srv) -} - -func _JobControllerService_ListIngestionJobs_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - in := new(ListIngestionJobsRequest) - if err := dec(in); err != nil { - return nil, err - } - if interceptor == nil { - return srv.(JobControllerServiceServer).ListIngestionJobs(ctx, in) - } - info := &grpc.UnaryServerInfo{ - Server: srv, - FullMethod: "/feast.core.JobControllerService/ListIngestionJobs", - } - handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(JobControllerServiceServer).ListIngestionJobs(ctx, req.(*ListIngestionJobsRequest)) - } - return interceptor(ctx, in, info, handler) -} - -func _JobControllerService_RestartIngestionJob_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - in := new(RestartIngestionJobRequest) - if err := dec(in); err != nil { - return nil, err - } - if interceptor == nil { - return srv.(JobControllerServiceServer).RestartIngestionJob(ctx, in) - } - info := &grpc.UnaryServerInfo{ - Server: srv, - FullMethod: "/feast.core.JobControllerService/RestartIngestionJob", - } - handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(JobControllerServiceServer).RestartIngestionJob(ctx, req.(*RestartIngestionJobRequest)) - } - return interceptor(ctx, in, info, handler) -} - -func _JobControllerService_StopIngestionJob_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - in := new(StopIngestionJobRequest) - if err := dec(in); err != nil { - return nil, err - } - if interceptor == nil { - return srv.(JobControllerServiceServer).StopIngestionJob(ctx, in) - } - info := &grpc.UnaryServerInfo{ - Server: srv, - FullMethod: "/feast.core.JobControllerService/StopIngestionJob", - } - handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(JobControllerServiceServer).StopIngestionJob(ctx, req.(*StopIngestionJobRequest)) - } - return interceptor(ctx, in, info, handler) -} - -var _JobControllerService_serviceDesc = grpc.ServiceDesc{ - ServiceName: "feast.core.JobControllerService", - HandlerType: (*JobControllerServiceServer)(nil), - Methods: []grpc.MethodDesc{ - { - MethodName: "ListIngestionJobs", - Handler: _JobControllerService_ListIngestionJobs_Handler, - }, - { - MethodName: "RestartIngestionJob", - Handler: _JobControllerService_RestartIngestionJob_Handler, - }, { - MethodName: "StopIngestionJob", - Handler: _JobControllerService_StopIngestionJob_Handler, + MethodName: "DeleteFeatureTable", + Handler: _CoreService_DeleteFeatureTable_Handler, }, }, Streams: []grpc.StreamDesc{}, diff --git a/sdk/go/protos/feast/core/DataFormat.pb.go b/sdk/go/protos/feast/core/DataFormat.pb.go index ac6d88f9c77..4a766cc8f9a 100644 --- a/sdk/go/protos/feast/core/DataFormat.pb.go +++ b/sdk/go/protos/feast/core/DataFormat.pb.go @@ -17,7 +17,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.25.0 -// protoc v3.10.0 +// protoc v3.12.4 // source: feast/core/DataFormat.proto package core diff --git a/sdk/go/protos/feast/core/DataSource.pb.go b/sdk/go/protos/feast/core/DataSource.pb.go index 4dd43d51960..1b43972d327 100644 --- a/sdk/go/protos/feast/core/DataSource.pb.go +++ b/sdk/go/protos/feast/core/DataSource.pb.go @@ -17,7 +17,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.25.0 -// protoc v3.10.0 +// protoc v3.12.4 // source: feast/core/DataSource.proto package core diff --git a/sdk/go/protos/feast/core/Entity.pb.go b/sdk/go/protos/feast/core/Entity.pb.go index f5a2c0d06af..0aed9133257 100644 --- a/sdk/go/protos/feast/core/Entity.pb.go +++ b/sdk/go/protos/feast/core/Entity.pb.go @@ -17,7 +17,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.25.0 -// protoc v3.10.0 +// protoc v3.12.4 // source: feast/core/Entity.proto package core diff --git a/sdk/go/protos/feast/core/Feature.pb.go b/sdk/go/protos/feast/core/Feature.pb.go index 9c2690e0f9f..1ad93ef8a1c 100644 --- a/sdk/go/protos/feast/core/Feature.pb.go +++ b/sdk/go/protos/feast/core/Feature.pb.go @@ -17,7 +17,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.25.0 -// protoc v3.10.0 +// protoc v3.12.4 // source: feast/core/Feature.proto package core diff --git a/sdk/go/protos/feast/core/FeatureTable.pb.go b/sdk/go/protos/feast/core/FeatureTable.pb.go index 48aef52950a..0c4fa1b1f0c 100644 --- a/sdk/go/protos/feast/core/FeatureTable.pb.go +++ b/sdk/go/protos/feast/core/FeatureTable.pb.go @@ -17,7 +17,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.25.0 -// protoc v3.10.0 +// protoc v3.12.4 // source: feast/core/FeatureTable.proto package core @@ -105,7 +105,7 @@ type FeatureTableSpec struct { sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - // Name of the feature set. Must be unique. Not updated. + // Name of the feature table. Must be unique. Not updated. Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` // List names of entities to associate with the Features defined in this // Feature Table. Not updatable. @@ -221,6 +221,9 @@ type FeatureTableMeta struct { LastUpdatedTimestamp *timestamp.Timestamp `protobuf:"bytes,2,opt,name=last_updated_timestamp,json=lastUpdatedTimestamp,proto3" json:"last_updated_timestamp,omitempty"` // Auto incrementing revision no. of this Feature Table Revision int64 `protobuf:"varint,3,opt,name=revision,proto3" json:"revision,omitempty"` + // Hash entities, features, batch_source and stream_source to inform JobService if + // jobs should be restarted should hash change + Hash string `protobuf:"bytes,4,opt,name=hash,proto3" json:"hash,omitempty"` } func (x *FeatureTableMeta) Reset() { @@ -276,6 +279,13 @@ func (x *FeatureTableMeta) GetRevision() int64 { return 0 } +func (x *FeatureTableMeta) GetHash() string { + if x != nil { + return x.Hash + } + return "" +} + var File_feast_core_FeatureTable_proto protoreflect.FileDescriptor var file_feast_core_FeatureTable_proto_rawDesc = []byte{ @@ -322,7 +332,7 @@ var file_feast_core_FeatureTable_proto_rawDesc = []byte{ 0x65, 0x1a, 0x39, 0x0a, 0x0b, 0x4c, 0x61, 0x62, 0x65, 0x6c, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0xc9, 0x01, 0x0a, + 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0xdd, 0x01, 0x0a, 0x10, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x4d, 0x65, 0x74, 0x61, 0x12, 0x47, 0x0a, 0x11, 0x63, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, @@ -335,13 +345,14 @@ var file_feast_core_FeatureTable_proto_rawDesc = []byte{ 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x14, 0x6c, 0x61, 0x73, 0x74, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x64, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x12, 0x1a, 0x0a, 0x08, 0x72, 0x65, 0x76, 0x69, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x03, 0x52, 0x08, - 0x72, 0x65, 0x76, 0x69, 0x73, 0x69, 0x6f, 0x6e, 0x42, 0x5a, 0x0a, 0x10, 0x66, 0x65, 0x61, 0x73, - 0x74, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x42, 0x11, 0x46, 0x65, - 0x61, 0x74, 0x75, 0x72, 0x65, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x5a, - 0x33, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x66, 0x65, 0x61, 0x73, - 0x74, 0x2d, 0x64, 0x65, 0x76, 0x2f, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2f, 0x73, 0x64, 0x6b, 0x2f, - 0x67, 0x6f, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x73, 0x2f, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2f, - 0x63, 0x6f, 0x72, 0x65, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x72, 0x65, 0x76, 0x69, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x12, 0x0a, 0x04, 0x68, 0x61, 0x73, 0x68, + 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x68, 0x61, 0x73, 0x68, 0x42, 0x5a, 0x0a, 0x10, + 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x63, 0x6f, 0x72, 0x65, + 0x42, 0x11, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x50, 0x72, + 0x6f, 0x74, 0x6f, 0x5a, 0x33, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, + 0x66, 0x65, 0x61, 0x73, 0x74, 0x2d, 0x64, 0x65, 0x76, 0x2f, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2f, + 0x73, 0x64, 0x6b, 0x2f, 0x67, 0x6f, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x73, 0x2f, 0x66, 0x65, + 0x61, 0x73, 0x74, 0x2f, 0x63, 0x6f, 0x72, 0x65, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( diff --git a/sdk/go/protos/feast/core/JobService.pb.go b/sdk/go/protos/feast/core/JobService.pb.go new file mode 100644 index 00000000000..221c530bb37 --- /dev/null +++ b/sdk/go/protos/feast/core/JobService.pb.go @@ -0,0 +1,1789 @@ +// +// Copyright 2018 The Feast Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.25.0 +// protoc v3.12.4 +// source: feast/core/JobService.proto + +package core + +import ( + context "context" + proto "github.com/golang/protobuf/proto" + timestamp "github.com/golang/protobuf/ptypes/timestamp" + grpc "google.golang.org/grpc" + codes "google.golang.org/grpc/codes" + status "google.golang.org/grpc/status" + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +// This is a compile-time assertion that a sufficiently up-to-date version +// of the legacy proto package is being used. +const _ = proto.ProtoPackageIsVersion4 + +type JobType int32 + +const ( + JobType_INVALID_JOB JobType = 0 + JobType_BATCH_INGESTION_JOB JobType = 1 + JobType_STREAM_INGESTION_JOB JobType = 2 + JobType_RETRIEVAL_JOB JobType = 4 +) + +// Enum value maps for JobType. +var ( + JobType_name = map[int32]string{ + 0: "INVALID_JOB", + 1: "BATCH_INGESTION_JOB", + 2: "STREAM_INGESTION_JOB", + 4: "RETRIEVAL_JOB", + } + JobType_value = map[string]int32{ + "INVALID_JOB": 0, + "BATCH_INGESTION_JOB": 1, + "STREAM_INGESTION_JOB": 2, + "RETRIEVAL_JOB": 4, + } +) + +func (x JobType) Enum() *JobType { + p := new(JobType) + *p = x + return p +} + +func (x JobType) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (JobType) Descriptor() protoreflect.EnumDescriptor { + return file_feast_core_JobService_proto_enumTypes[0].Descriptor() +} + +func (JobType) Type() protoreflect.EnumType { + return &file_feast_core_JobService_proto_enumTypes[0] +} + +func (x JobType) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Use JobType.Descriptor instead. +func (JobType) EnumDescriptor() ([]byte, []int) { + return file_feast_core_JobService_proto_rawDescGZIP(), []int{0} +} + +type JobStatus int32 + +const ( + JobStatus_JOB_STATUS_INVALID JobStatus = 0 + // The Job has be registered and waiting to get scheduled to run + JobStatus_JOB_STATUS_PENDING JobStatus = 1 + // The Job is currently processing its task + JobStatus_JOB_STATUS_RUNNING JobStatus = 2 + // The Job has successfully completed its task + JobStatus_JOB_STATUS_DONE JobStatus = 3 + // The Job has encountered an error while processing its task + JobStatus_JOB_STATUS_ERROR JobStatus = 4 +) + +// Enum value maps for JobStatus. +var ( + JobStatus_name = map[int32]string{ + 0: "JOB_STATUS_INVALID", + 1: "JOB_STATUS_PENDING", + 2: "JOB_STATUS_RUNNING", + 3: "JOB_STATUS_DONE", + 4: "JOB_STATUS_ERROR", + } + JobStatus_value = map[string]int32{ + "JOB_STATUS_INVALID": 0, + "JOB_STATUS_PENDING": 1, + "JOB_STATUS_RUNNING": 2, + "JOB_STATUS_DONE": 3, + "JOB_STATUS_ERROR": 4, + } +) + +func (x JobStatus) Enum() *JobStatus { + p := new(JobStatus) + *p = x + return p +} + +func (x JobStatus) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (JobStatus) Descriptor() protoreflect.EnumDescriptor { + return file_feast_core_JobService_proto_enumTypes[1].Descriptor() +} + +func (JobStatus) Type() protoreflect.EnumType { + return &file_feast_core_JobService_proto_enumTypes[1] +} + +func (x JobStatus) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Use JobStatus.Descriptor instead. +func (JobStatus) EnumDescriptor() ([]byte, []int) { + return file_feast_core_JobService_proto_rawDescGZIP(), []int{1} +} + +type Job struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // Identifier of the Job + Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"` + // Type of the Job + Type JobType `protobuf:"varint,2,opt,name=type,proto3,enum=feast.core.JobType" json:"type,omitempty"` + // Current job status + Status JobStatus `protobuf:"varint,3,opt,name=status,proto3,enum=feast.core.JobStatus" json:"status,omitempty"` + // Deterministic hash of the Job + Hash string `protobuf:"bytes,8,opt,name=hash,proto3" json:"hash,omitempty"` + // JobType specific metadata on the job + // + // Types that are assignable to Meta: + // *Job_Retrieval + // *Job_BatchIngestion + // *Job_StreamIngestion + Meta isJob_Meta `protobuf_oneof:"meta"` +} + +func (x *Job) Reset() { + *x = Job{} + if protoimpl.UnsafeEnabled { + mi := &file_feast_core_JobService_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *Job) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Job) ProtoMessage() {} + +func (x *Job) ProtoReflect() protoreflect.Message { + mi := &file_feast_core_JobService_proto_msgTypes[0] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Job.ProtoReflect.Descriptor instead. +func (*Job) Descriptor() ([]byte, []int) { + return file_feast_core_JobService_proto_rawDescGZIP(), []int{0} +} + +func (x *Job) GetId() string { + if x != nil { + return x.Id + } + return "" +} + +func (x *Job) GetType() JobType { + if x != nil { + return x.Type + } + return JobType_INVALID_JOB +} + +func (x *Job) GetStatus() JobStatus { + if x != nil { + return x.Status + } + return JobStatus_JOB_STATUS_INVALID +} + +func (x *Job) GetHash() string { + if x != nil { + return x.Hash + } + return "" +} + +func (m *Job) GetMeta() isJob_Meta { + if m != nil { + return m.Meta + } + return nil +} + +func (x *Job) GetRetrieval() *Job_RetrievalJobMeta { + if x, ok := x.GetMeta().(*Job_Retrieval); ok { + return x.Retrieval + } + return nil +} + +func (x *Job) GetBatchIngestion() *Job_OfflineToOnlineMeta { + if x, ok := x.GetMeta().(*Job_BatchIngestion); ok { + return x.BatchIngestion + } + return nil +} + +func (x *Job) GetStreamIngestion() *Job_StreamToOnlineMeta { + if x, ok := x.GetMeta().(*Job_StreamIngestion); ok { + return x.StreamIngestion + } + return nil +} + +type isJob_Meta interface { + isJob_Meta() +} + +type Job_Retrieval struct { + Retrieval *Job_RetrievalJobMeta `protobuf:"bytes,5,opt,name=retrieval,proto3,oneof"` +} + +type Job_BatchIngestion struct { + BatchIngestion *Job_OfflineToOnlineMeta `protobuf:"bytes,6,opt,name=batch_ingestion,json=batchIngestion,proto3,oneof"` +} + +type Job_StreamIngestion struct { + StreamIngestion *Job_StreamToOnlineMeta `protobuf:"bytes,7,opt,name=stream_ingestion,json=streamIngestion,proto3,oneof"` +} + +func (*Job_Retrieval) isJob_Meta() {} + +func (*Job_BatchIngestion) isJob_Meta() {} + +func (*Job_StreamIngestion) isJob_Meta() {} + +// Ingest data from offline store into online store +type StartOfflineToOnlineIngestionJobRequest struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // Feature table to ingest + Project string `protobuf:"bytes,1,opt,name=project,proto3" json:"project,omitempty"` + TableName string `protobuf:"bytes,2,opt,name=table_name,json=tableName,proto3" json:"table_name,omitempty"` + // Start of time range for source data from offline store + StartDate *timestamp.Timestamp `protobuf:"bytes,3,opt,name=start_date,json=startDate,proto3" json:"start_date,omitempty"` + // End of time range for source data from offline store + EndDate *timestamp.Timestamp `protobuf:"bytes,4,opt,name=end_date,json=endDate,proto3" json:"end_date,omitempty"` +} + +func (x *StartOfflineToOnlineIngestionJobRequest) Reset() { + *x = StartOfflineToOnlineIngestionJobRequest{} + if protoimpl.UnsafeEnabled { + mi := &file_feast_core_JobService_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *StartOfflineToOnlineIngestionJobRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*StartOfflineToOnlineIngestionJobRequest) ProtoMessage() {} + +func (x *StartOfflineToOnlineIngestionJobRequest) ProtoReflect() protoreflect.Message { + mi := &file_feast_core_JobService_proto_msgTypes[1] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use StartOfflineToOnlineIngestionJobRequest.ProtoReflect.Descriptor instead. +func (*StartOfflineToOnlineIngestionJobRequest) Descriptor() ([]byte, []int) { + return file_feast_core_JobService_proto_rawDescGZIP(), []int{1} +} + +func (x *StartOfflineToOnlineIngestionJobRequest) GetProject() string { + if x != nil { + return x.Project + } + return "" +} + +func (x *StartOfflineToOnlineIngestionJobRequest) GetTableName() string { + if x != nil { + return x.TableName + } + return "" +} + +func (x *StartOfflineToOnlineIngestionJobRequest) GetStartDate() *timestamp.Timestamp { + if x != nil { + return x.StartDate + } + return nil +} + +func (x *StartOfflineToOnlineIngestionJobRequest) GetEndDate() *timestamp.Timestamp { + if x != nil { + return x.EndDate + } + return nil +} + +type StartOfflineToOnlineIngestionJobResponse struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // Job ID assigned by Feast + Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"` +} + +func (x *StartOfflineToOnlineIngestionJobResponse) Reset() { + *x = StartOfflineToOnlineIngestionJobResponse{} + if protoimpl.UnsafeEnabled { + mi := &file_feast_core_JobService_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *StartOfflineToOnlineIngestionJobResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*StartOfflineToOnlineIngestionJobResponse) ProtoMessage() {} + +func (x *StartOfflineToOnlineIngestionJobResponse) ProtoReflect() protoreflect.Message { + mi := &file_feast_core_JobService_proto_msgTypes[2] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use StartOfflineToOnlineIngestionJobResponse.ProtoReflect.Descriptor instead. +func (*StartOfflineToOnlineIngestionJobResponse) Descriptor() ([]byte, []int) { + return file_feast_core_JobService_proto_rawDescGZIP(), []int{2} +} + +func (x *StartOfflineToOnlineIngestionJobResponse) GetId() string { + if x != nil { + return x.Id + } + return "" +} + +type GetHistoricalFeaturesRequest struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // List of feature references that are being retrieved + FeatureRefs []string `protobuf:"bytes,1,rep,name=feature_refs,json=featureRefs,proto3" json:"feature_refs,omitempty"` + // Batch DataSource that can be used to obtain entity values for historical retrieval. + // For each entity value, a feature value will be retrieved for that value/timestamp + // Only 'BATCH_*' source types are supported. + // Currently only BATCH_FILE source type is supported. + EntitySource *DataSource `protobuf:"bytes,2,opt,name=entity_source,json=entitySource,proto3" json:"entity_source,omitempty"` + // Optional field to specify project name override. If specified, uses the + // given project for retrieval. Overrides the projects specified in + // Feature References if both are specified. + Project string `protobuf:"bytes,3,opt,name=project,proto3" json:"project,omitempty"` + // Specifies the path in a bucket to write the exported feature data files + // Export to AWS S3 - s3://path/to/features + // Export to GCP GCS - gs://path/to/features + OutputLocation string `protobuf:"bytes,4,opt,name=output_location,json=outputLocation,proto3" json:"output_location,omitempty"` + // Specify format name for output, eg. parquet + OutputFormat string `protobuf:"bytes,5,opt,name=output_format,json=outputFormat,proto3" json:"output_format,omitempty"` +} + +func (x *GetHistoricalFeaturesRequest) Reset() { + *x = GetHistoricalFeaturesRequest{} + if protoimpl.UnsafeEnabled { + mi := &file_feast_core_JobService_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *GetHistoricalFeaturesRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*GetHistoricalFeaturesRequest) ProtoMessage() {} + +func (x *GetHistoricalFeaturesRequest) ProtoReflect() protoreflect.Message { + mi := &file_feast_core_JobService_proto_msgTypes[3] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use GetHistoricalFeaturesRequest.ProtoReflect.Descriptor instead. +func (*GetHistoricalFeaturesRequest) Descriptor() ([]byte, []int) { + return file_feast_core_JobService_proto_rawDescGZIP(), []int{3} +} + +func (x *GetHistoricalFeaturesRequest) GetFeatureRefs() []string { + if x != nil { + return x.FeatureRefs + } + return nil +} + +func (x *GetHistoricalFeaturesRequest) GetEntitySource() *DataSource { + if x != nil { + return x.EntitySource + } + return nil +} + +func (x *GetHistoricalFeaturesRequest) GetProject() string { + if x != nil { + return x.Project + } + return "" +} + +func (x *GetHistoricalFeaturesRequest) GetOutputLocation() string { + if x != nil { + return x.OutputLocation + } + return "" +} + +func (x *GetHistoricalFeaturesRequest) GetOutputFormat() string { + if x != nil { + return x.OutputFormat + } + return "" +} + +type GetHistoricalFeaturesResponse struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // Export Job with ID assigned by Feast + Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"` + OutputFileUri string `protobuf:"bytes,2,opt,name=output_file_uri,json=outputFileUri,proto3" json:"output_file_uri,omitempty"` +} + +func (x *GetHistoricalFeaturesResponse) Reset() { + *x = GetHistoricalFeaturesResponse{} + if protoimpl.UnsafeEnabled { + mi := &file_feast_core_JobService_proto_msgTypes[4] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *GetHistoricalFeaturesResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*GetHistoricalFeaturesResponse) ProtoMessage() {} + +func (x *GetHistoricalFeaturesResponse) ProtoReflect() protoreflect.Message { + mi := &file_feast_core_JobService_proto_msgTypes[4] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use GetHistoricalFeaturesResponse.ProtoReflect.Descriptor instead. +func (*GetHistoricalFeaturesResponse) Descriptor() ([]byte, []int) { + return file_feast_core_JobService_proto_rawDescGZIP(), []int{4} +} + +func (x *GetHistoricalFeaturesResponse) GetId() string { + if x != nil { + return x.Id + } + return "" +} + +func (x *GetHistoricalFeaturesResponse) GetOutputFileUri() string { + if x != nil { + return x.OutputFileUri + } + return "" +} + +type StartStreamToOnlineIngestionJobRequest struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // Feature table to ingest + Project string `protobuf:"bytes,1,opt,name=project,proto3" json:"project,omitempty"` + TableName string `protobuf:"bytes,2,opt,name=table_name,json=tableName,proto3" json:"table_name,omitempty"` +} + +func (x *StartStreamToOnlineIngestionJobRequest) Reset() { + *x = StartStreamToOnlineIngestionJobRequest{} + if protoimpl.UnsafeEnabled { + mi := &file_feast_core_JobService_proto_msgTypes[5] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *StartStreamToOnlineIngestionJobRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*StartStreamToOnlineIngestionJobRequest) ProtoMessage() {} + +func (x *StartStreamToOnlineIngestionJobRequest) ProtoReflect() protoreflect.Message { + mi := &file_feast_core_JobService_proto_msgTypes[5] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use StartStreamToOnlineIngestionJobRequest.ProtoReflect.Descriptor instead. +func (*StartStreamToOnlineIngestionJobRequest) Descriptor() ([]byte, []int) { + return file_feast_core_JobService_proto_rawDescGZIP(), []int{5} +} + +func (x *StartStreamToOnlineIngestionJobRequest) GetProject() string { + if x != nil { + return x.Project + } + return "" +} + +func (x *StartStreamToOnlineIngestionJobRequest) GetTableName() string { + if x != nil { + return x.TableName + } + return "" +} + +type StartStreamToOnlineIngestionJobResponse struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // Job ID assigned by Feast + Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"` +} + +func (x *StartStreamToOnlineIngestionJobResponse) Reset() { + *x = StartStreamToOnlineIngestionJobResponse{} + if protoimpl.UnsafeEnabled { + mi := &file_feast_core_JobService_proto_msgTypes[6] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *StartStreamToOnlineIngestionJobResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*StartStreamToOnlineIngestionJobResponse) ProtoMessage() {} + +func (x *StartStreamToOnlineIngestionJobResponse) ProtoReflect() protoreflect.Message { + mi := &file_feast_core_JobService_proto_msgTypes[6] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use StartStreamToOnlineIngestionJobResponse.ProtoReflect.Descriptor instead. +func (*StartStreamToOnlineIngestionJobResponse) Descriptor() ([]byte, []int) { + return file_feast_core_JobService_proto_rawDescGZIP(), []int{6} +} + +func (x *StartStreamToOnlineIngestionJobResponse) GetId() string { + if x != nil { + return x.Id + } + return "" +} + +type ListJobsRequest struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + IncludeTerminated bool `protobuf:"varint,1,opt,name=include_terminated,json=includeTerminated,proto3" json:"include_terminated,omitempty"` +} + +func (x *ListJobsRequest) Reset() { + *x = ListJobsRequest{} + if protoimpl.UnsafeEnabled { + mi := &file_feast_core_JobService_proto_msgTypes[7] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *ListJobsRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ListJobsRequest) ProtoMessage() {} + +func (x *ListJobsRequest) ProtoReflect() protoreflect.Message { + mi := &file_feast_core_JobService_proto_msgTypes[7] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ListJobsRequest.ProtoReflect.Descriptor instead. +func (*ListJobsRequest) Descriptor() ([]byte, []int) { + return file_feast_core_JobService_proto_rawDescGZIP(), []int{7} +} + +func (x *ListJobsRequest) GetIncludeTerminated() bool { + if x != nil { + return x.IncludeTerminated + } + return false +} + +type ListJobsResponse struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Jobs []*Job `protobuf:"bytes,1,rep,name=jobs,proto3" json:"jobs,omitempty"` +} + +func (x *ListJobsResponse) Reset() { + *x = ListJobsResponse{} + if protoimpl.UnsafeEnabled { + mi := &file_feast_core_JobService_proto_msgTypes[8] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *ListJobsResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ListJobsResponse) ProtoMessage() {} + +func (x *ListJobsResponse) ProtoReflect() protoreflect.Message { + mi := &file_feast_core_JobService_proto_msgTypes[8] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ListJobsResponse.ProtoReflect.Descriptor instead. +func (*ListJobsResponse) Descriptor() ([]byte, []int) { + return file_feast_core_JobService_proto_rawDescGZIP(), []int{8} +} + +func (x *ListJobsResponse) GetJobs() []*Job { + if x != nil { + return x.Jobs + } + return nil +} + +type GetJobRequest struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + JobId string `protobuf:"bytes,1,opt,name=job_id,json=jobId,proto3" json:"job_id,omitempty"` +} + +func (x *GetJobRequest) Reset() { + *x = GetJobRequest{} + if protoimpl.UnsafeEnabled { + mi := &file_feast_core_JobService_proto_msgTypes[9] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *GetJobRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*GetJobRequest) ProtoMessage() {} + +func (x *GetJobRequest) ProtoReflect() protoreflect.Message { + mi := &file_feast_core_JobService_proto_msgTypes[9] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use GetJobRequest.ProtoReflect.Descriptor instead. +func (*GetJobRequest) Descriptor() ([]byte, []int) { + return file_feast_core_JobService_proto_rawDescGZIP(), []int{9} +} + +func (x *GetJobRequest) GetJobId() string { + if x != nil { + return x.JobId + } + return "" +} + +type GetJobResponse struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Job *Job `protobuf:"bytes,1,opt,name=job,proto3" json:"job,omitempty"` +} + +func (x *GetJobResponse) Reset() { + *x = GetJobResponse{} + if protoimpl.UnsafeEnabled { + mi := &file_feast_core_JobService_proto_msgTypes[10] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *GetJobResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*GetJobResponse) ProtoMessage() {} + +func (x *GetJobResponse) ProtoReflect() protoreflect.Message { + mi := &file_feast_core_JobService_proto_msgTypes[10] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use GetJobResponse.ProtoReflect.Descriptor instead. +func (*GetJobResponse) Descriptor() ([]byte, []int) { + return file_feast_core_JobService_proto_rawDescGZIP(), []int{10} +} + +func (x *GetJobResponse) GetJob() *Job { + if x != nil { + return x.Job + } + return nil +} + +type CancelJobRequest struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + JobId string `protobuf:"bytes,1,opt,name=job_id,json=jobId,proto3" json:"job_id,omitempty"` +} + +func (x *CancelJobRequest) Reset() { + *x = CancelJobRequest{} + if protoimpl.UnsafeEnabled { + mi := &file_feast_core_JobService_proto_msgTypes[11] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *CancelJobRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*CancelJobRequest) ProtoMessage() {} + +func (x *CancelJobRequest) ProtoReflect() protoreflect.Message { + mi := &file_feast_core_JobService_proto_msgTypes[11] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use CancelJobRequest.ProtoReflect.Descriptor instead. +func (*CancelJobRequest) Descriptor() ([]byte, []int) { + return file_feast_core_JobService_proto_rawDescGZIP(), []int{11} +} + +func (x *CancelJobRequest) GetJobId() string { + if x != nil { + return x.JobId + } + return "" +} + +type CancelJobResponse struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields +} + +func (x *CancelJobResponse) Reset() { + *x = CancelJobResponse{} + if protoimpl.UnsafeEnabled { + mi := &file_feast_core_JobService_proto_msgTypes[12] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *CancelJobResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*CancelJobResponse) ProtoMessage() {} + +func (x *CancelJobResponse) ProtoReflect() protoreflect.Message { + mi := &file_feast_core_JobService_proto_msgTypes[12] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use CancelJobResponse.ProtoReflect.Descriptor instead. +func (*CancelJobResponse) Descriptor() ([]byte, []int) { + return file_feast_core_JobService_proto_rawDescGZIP(), []int{12} +} + +type Job_RetrievalJobMeta struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + OutputLocation string `protobuf:"bytes,4,opt,name=output_location,json=outputLocation,proto3" json:"output_location,omitempty"` +} + +func (x *Job_RetrievalJobMeta) Reset() { + *x = Job_RetrievalJobMeta{} + if protoimpl.UnsafeEnabled { + mi := &file_feast_core_JobService_proto_msgTypes[13] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *Job_RetrievalJobMeta) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Job_RetrievalJobMeta) ProtoMessage() {} + +func (x *Job_RetrievalJobMeta) ProtoReflect() protoreflect.Message { + mi := &file_feast_core_JobService_proto_msgTypes[13] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Job_RetrievalJobMeta.ProtoReflect.Descriptor instead. +func (*Job_RetrievalJobMeta) Descriptor() ([]byte, []int) { + return file_feast_core_JobService_proto_rawDescGZIP(), []int{0, 0} +} + +func (x *Job_RetrievalJobMeta) GetOutputLocation() string { + if x != nil { + return x.OutputLocation + } + return "" +} + +type Job_OfflineToOnlineMeta struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields +} + +func (x *Job_OfflineToOnlineMeta) Reset() { + *x = Job_OfflineToOnlineMeta{} + if protoimpl.UnsafeEnabled { + mi := &file_feast_core_JobService_proto_msgTypes[14] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *Job_OfflineToOnlineMeta) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Job_OfflineToOnlineMeta) ProtoMessage() {} + +func (x *Job_OfflineToOnlineMeta) ProtoReflect() protoreflect.Message { + mi := &file_feast_core_JobService_proto_msgTypes[14] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Job_OfflineToOnlineMeta.ProtoReflect.Descriptor instead. +func (*Job_OfflineToOnlineMeta) Descriptor() ([]byte, []int) { + return file_feast_core_JobService_proto_rawDescGZIP(), []int{0, 1} +} + +type Job_StreamToOnlineMeta struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields +} + +func (x *Job_StreamToOnlineMeta) Reset() { + *x = Job_StreamToOnlineMeta{} + if protoimpl.UnsafeEnabled { + mi := &file_feast_core_JobService_proto_msgTypes[15] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *Job_StreamToOnlineMeta) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Job_StreamToOnlineMeta) ProtoMessage() {} + +func (x *Job_StreamToOnlineMeta) ProtoReflect() protoreflect.Message { + mi := &file_feast_core_JobService_proto_msgTypes[15] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Job_StreamToOnlineMeta.ProtoReflect.Descriptor instead. +func (*Job_StreamToOnlineMeta) Descriptor() ([]byte, []int) { + return file_feast_core_JobService_proto_rawDescGZIP(), []int{0, 2} +} + +var File_feast_core_JobService_proto protoreflect.FileDescriptor + +var file_feast_core_JobService_proto_rawDesc = []byte{ + 0x0a, 0x1b, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2f, 0x63, 0x6f, 0x72, 0x65, 0x2f, 0x4a, 0x6f, 0x62, + 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x0a, 0x66, + 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x1a, 0x1f, 0x67, 0x6f, 0x6f, 0x67, 0x6c, + 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2f, 0x74, 0x69, 0x6d, 0x65, 0x73, + 0x74, 0x61, 0x6d, 0x70, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x1b, 0x66, 0x65, 0x61, 0x73, + 0x74, 0x2f, 0x63, 0x6f, 0x72, 0x65, 0x2f, 0x44, 0x61, 0x74, 0x61, 0x53, 0x6f, 0x75, 0x72, 0x63, + 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0xd6, 0x03, 0x0a, 0x03, 0x4a, 0x6f, 0x62, 0x12, + 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x12, + 0x27, 0x0a, 0x04, 0x74, 0x79, 0x70, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x13, 0x2e, + 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x4a, 0x6f, 0x62, 0x54, 0x79, + 0x70, 0x65, 0x52, 0x04, 0x74, 0x79, 0x70, 0x65, 0x12, 0x2d, 0x0a, 0x06, 0x73, 0x74, 0x61, 0x74, + 0x75, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x15, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, + 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x4a, 0x6f, 0x62, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, + 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x12, 0x0a, 0x04, 0x68, 0x61, 0x73, 0x68, 0x18, + 0x08, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x68, 0x61, 0x73, 0x68, 0x12, 0x40, 0x0a, 0x09, 0x72, + 0x65, 0x74, 0x72, 0x69, 0x65, 0x76, 0x61, 0x6c, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x20, + 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x4a, 0x6f, 0x62, 0x2e, + 0x52, 0x65, 0x74, 0x72, 0x69, 0x65, 0x76, 0x61, 0x6c, 0x4a, 0x6f, 0x62, 0x4d, 0x65, 0x74, 0x61, + 0x48, 0x00, 0x52, 0x09, 0x72, 0x65, 0x74, 0x72, 0x69, 0x65, 0x76, 0x61, 0x6c, 0x12, 0x4e, 0x0a, + 0x0f, 0x62, 0x61, 0x74, 0x63, 0x68, 0x5f, 0x69, 0x6e, 0x67, 0x65, 0x73, 0x74, 0x69, 0x6f, 0x6e, + 0x18, 0x06, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, + 0x6f, 0x72, 0x65, 0x2e, 0x4a, 0x6f, 0x62, 0x2e, 0x4f, 0x66, 0x66, 0x6c, 0x69, 0x6e, 0x65, 0x54, + 0x6f, 0x4f, 0x6e, 0x6c, 0x69, 0x6e, 0x65, 0x4d, 0x65, 0x74, 0x61, 0x48, 0x00, 0x52, 0x0e, 0x62, + 0x61, 0x74, 0x63, 0x68, 0x49, 0x6e, 0x67, 0x65, 0x73, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x4f, 0x0a, + 0x10, 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x5f, 0x69, 0x6e, 0x67, 0x65, 0x73, 0x74, 0x69, 0x6f, + 0x6e, 0x18, 0x07, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x22, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, + 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x4a, 0x6f, 0x62, 0x2e, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x54, + 0x6f, 0x4f, 0x6e, 0x6c, 0x69, 0x6e, 0x65, 0x4d, 0x65, 0x74, 0x61, 0x48, 0x00, 0x52, 0x0f, 0x73, + 0x74, 0x72, 0x65, 0x61, 0x6d, 0x49, 0x6e, 0x67, 0x65, 0x73, 0x74, 0x69, 0x6f, 0x6e, 0x1a, 0x3b, + 0x0a, 0x10, 0x52, 0x65, 0x74, 0x72, 0x69, 0x65, 0x76, 0x61, 0x6c, 0x4a, 0x6f, 0x62, 0x4d, 0x65, + 0x74, 0x61, 0x12, 0x27, 0x0a, 0x0f, 0x6f, 0x75, 0x74, 0x70, 0x75, 0x74, 0x5f, 0x6c, 0x6f, 0x63, + 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0e, 0x6f, 0x75, 0x74, + 0x70, 0x75, 0x74, 0x4c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x1a, 0x15, 0x0a, 0x13, 0x4f, + 0x66, 0x66, 0x6c, 0x69, 0x6e, 0x65, 0x54, 0x6f, 0x4f, 0x6e, 0x6c, 0x69, 0x6e, 0x65, 0x4d, 0x65, + 0x74, 0x61, 0x1a, 0x14, 0x0a, 0x12, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x54, 0x6f, 0x4f, 0x6e, + 0x6c, 0x69, 0x6e, 0x65, 0x4d, 0x65, 0x74, 0x61, 0x42, 0x06, 0x0a, 0x04, 0x6d, 0x65, 0x74, 0x61, + 0x22, 0xd4, 0x01, 0x0a, 0x27, 0x53, 0x74, 0x61, 0x72, 0x74, 0x4f, 0x66, 0x66, 0x6c, 0x69, 0x6e, + 0x65, 0x54, 0x6f, 0x4f, 0x6e, 0x6c, 0x69, 0x6e, 0x65, 0x49, 0x6e, 0x67, 0x65, 0x73, 0x74, 0x69, + 0x6f, 0x6e, 0x4a, 0x6f, 0x62, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x18, 0x0a, 0x07, + 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x70, + 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x12, 0x1d, 0x0a, 0x0a, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x5f, + 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x74, 0x61, 0x62, 0x6c, + 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x39, 0x0a, 0x0a, 0x73, 0x74, 0x61, 0x72, 0x74, 0x5f, 0x64, + 0x61, 0x74, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, + 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, + 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x09, 0x73, 0x74, 0x61, 0x72, 0x74, 0x44, 0x61, 0x74, 0x65, + 0x12, 0x35, 0x0a, 0x08, 0x65, 0x6e, 0x64, 0x5f, 0x64, 0x61, 0x74, 0x65, 0x18, 0x04, 0x20, 0x01, + 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, + 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x07, + 0x65, 0x6e, 0x64, 0x44, 0x61, 0x74, 0x65, 0x22, 0x3a, 0x0a, 0x28, 0x53, 0x74, 0x61, 0x72, 0x74, + 0x4f, 0x66, 0x66, 0x6c, 0x69, 0x6e, 0x65, 0x54, 0x6f, 0x4f, 0x6e, 0x6c, 0x69, 0x6e, 0x65, 0x49, + 0x6e, 0x67, 0x65, 0x73, 0x74, 0x69, 0x6f, 0x6e, 0x4a, 0x6f, 0x62, 0x52, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x02, 0x69, 0x64, 0x22, 0xe6, 0x01, 0x0a, 0x1c, 0x47, 0x65, 0x74, 0x48, 0x69, 0x73, 0x74, 0x6f, + 0x72, 0x69, 0x63, 0x61, 0x6c, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x52, 0x65, 0x71, + 0x75, 0x65, 0x73, 0x74, 0x12, 0x21, 0x0a, 0x0c, 0x66, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x5f, + 0x72, 0x65, 0x66, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0b, 0x66, 0x65, 0x61, 0x74, + 0x75, 0x72, 0x65, 0x52, 0x65, 0x66, 0x73, 0x12, 0x3b, 0x0a, 0x0d, 0x65, 0x6e, 0x74, 0x69, 0x74, + 0x79, 0x5f, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x16, + 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x44, 0x61, 0x74, 0x61, + 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x52, 0x0c, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x53, 0x6f, + 0x75, 0x72, 0x63, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x18, + 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x12, 0x27, + 0x0a, 0x0f, 0x6f, 0x75, 0x74, 0x70, 0x75, 0x74, 0x5f, 0x6c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, + 0x6e, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0e, 0x6f, 0x75, 0x74, 0x70, 0x75, 0x74, 0x4c, + 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x23, 0x0a, 0x0d, 0x6f, 0x75, 0x74, 0x70, 0x75, + 0x74, 0x5f, 0x66, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, + 0x6f, 0x75, 0x74, 0x70, 0x75, 0x74, 0x46, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x22, 0x57, 0x0a, 0x1d, + 0x47, 0x65, 0x74, 0x48, 0x69, 0x73, 0x74, 0x6f, 0x72, 0x69, 0x63, 0x61, 0x6c, 0x46, 0x65, 0x61, + 0x74, 0x75, 0x72, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x0e, 0x0a, + 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x12, 0x26, 0x0a, + 0x0f, 0x6f, 0x75, 0x74, 0x70, 0x75, 0x74, 0x5f, 0x66, 0x69, 0x6c, 0x65, 0x5f, 0x75, 0x72, 0x69, + 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, 0x6f, 0x75, 0x74, 0x70, 0x75, 0x74, 0x46, 0x69, + 0x6c, 0x65, 0x55, 0x72, 0x69, 0x22, 0x61, 0x0a, 0x26, 0x53, 0x74, 0x61, 0x72, 0x74, 0x53, 0x74, + 0x72, 0x65, 0x61, 0x6d, 0x54, 0x6f, 0x4f, 0x6e, 0x6c, 0x69, 0x6e, 0x65, 0x49, 0x6e, 0x67, 0x65, + 0x73, 0x74, 0x69, 0x6f, 0x6e, 0x4a, 0x6f, 0x62, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, + 0x18, 0x0a, 0x07, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x07, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x12, 0x1d, 0x0a, 0x0a, 0x74, 0x61, 0x62, + 0x6c, 0x65, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x74, + 0x61, 0x62, 0x6c, 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x22, 0x39, 0x0a, 0x27, 0x53, 0x74, 0x61, 0x72, + 0x74, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x54, 0x6f, 0x4f, 0x6e, 0x6c, 0x69, 0x6e, 0x65, 0x49, + 0x6e, 0x67, 0x65, 0x73, 0x74, 0x69, 0x6f, 0x6e, 0x4a, 0x6f, 0x62, 0x52, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x02, 0x69, 0x64, 0x22, 0x40, 0x0a, 0x0f, 0x4c, 0x69, 0x73, 0x74, 0x4a, 0x6f, 0x62, 0x73, 0x52, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x2d, 0x0a, 0x12, 0x69, 0x6e, 0x63, 0x6c, 0x75, 0x64, + 0x65, 0x5f, 0x74, 0x65, 0x72, 0x6d, 0x69, 0x6e, 0x61, 0x74, 0x65, 0x64, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x08, 0x52, 0x11, 0x69, 0x6e, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x54, 0x65, 0x72, 0x6d, 0x69, + 0x6e, 0x61, 0x74, 0x65, 0x64, 0x22, 0x37, 0x0a, 0x10, 0x4c, 0x69, 0x73, 0x74, 0x4a, 0x6f, 0x62, + 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x23, 0x0a, 0x04, 0x6a, 0x6f, 0x62, + 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x0f, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, + 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x4a, 0x6f, 0x62, 0x52, 0x04, 0x6a, 0x6f, 0x62, 0x73, 0x22, 0x26, + 0x0a, 0x0d, 0x47, 0x65, 0x74, 0x4a, 0x6f, 0x62, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, + 0x15, 0x0a, 0x06, 0x6a, 0x6f, 0x62, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x05, 0x6a, 0x6f, 0x62, 0x49, 0x64, 0x22, 0x33, 0x0a, 0x0e, 0x47, 0x65, 0x74, 0x4a, 0x6f, 0x62, + 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x21, 0x0a, 0x03, 0x6a, 0x6f, 0x62, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0f, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, + 0x72, 0x65, 0x2e, 0x4a, 0x6f, 0x62, 0x52, 0x03, 0x6a, 0x6f, 0x62, 0x22, 0x29, 0x0a, 0x10, 0x43, + 0x61, 0x6e, 0x63, 0x65, 0x6c, 0x4a, 0x6f, 0x62, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, + 0x15, 0x0a, 0x06, 0x6a, 0x6f, 0x62, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x05, 0x6a, 0x6f, 0x62, 0x49, 0x64, 0x22, 0x13, 0x0a, 0x11, 0x43, 0x61, 0x6e, 0x63, 0x65, 0x6c, + 0x4a, 0x6f, 0x62, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2a, 0x60, 0x0a, 0x07, 0x4a, + 0x6f, 0x62, 0x54, 0x79, 0x70, 0x65, 0x12, 0x0f, 0x0a, 0x0b, 0x49, 0x4e, 0x56, 0x41, 0x4c, 0x49, + 0x44, 0x5f, 0x4a, 0x4f, 0x42, 0x10, 0x00, 0x12, 0x17, 0x0a, 0x13, 0x42, 0x41, 0x54, 0x43, 0x48, + 0x5f, 0x49, 0x4e, 0x47, 0x45, 0x53, 0x54, 0x49, 0x4f, 0x4e, 0x5f, 0x4a, 0x4f, 0x42, 0x10, 0x01, + 0x12, 0x18, 0x0a, 0x14, 0x53, 0x54, 0x52, 0x45, 0x41, 0x4d, 0x5f, 0x49, 0x4e, 0x47, 0x45, 0x53, + 0x54, 0x49, 0x4f, 0x4e, 0x5f, 0x4a, 0x4f, 0x42, 0x10, 0x02, 0x12, 0x11, 0x0a, 0x0d, 0x52, 0x45, + 0x54, 0x52, 0x49, 0x45, 0x56, 0x41, 0x4c, 0x5f, 0x4a, 0x4f, 0x42, 0x10, 0x04, 0x2a, 0x7e, 0x0a, + 0x09, 0x4a, 0x6f, 0x62, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x16, 0x0a, 0x12, 0x4a, 0x4f, + 0x42, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, 0x5f, 0x49, 0x4e, 0x56, 0x41, 0x4c, 0x49, 0x44, + 0x10, 0x00, 0x12, 0x16, 0x0a, 0x12, 0x4a, 0x4f, 0x42, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, + 0x5f, 0x50, 0x45, 0x4e, 0x44, 0x49, 0x4e, 0x47, 0x10, 0x01, 0x12, 0x16, 0x0a, 0x12, 0x4a, 0x4f, + 0x42, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, 0x5f, 0x52, 0x55, 0x4e, 0x4e, 0x49, 0x4e, 0x47, + 0x10, 0x02, 0x12, 0x13, 0x0a, 0x0f, 0x4a, 0x4f, 0x42, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, + 0x5f, 0x44, 0x4f, 0x4e, 0x45, 0x10, 0x03, 0x12, 0x14, 0x0a, 0x10, 0x4a, 0x4f, 0x42, 0x5f, 0x53, + 0x54, 0x41, 0x54, 0x55, 0x53, 0x5f, 0x45, 0x52, 0x52, 0x4f, 0x52, 0x10, 0x04, 0x32, 0xe9, 0x04, + 0x0a, 0x0a, 0x4a, 0x6f, 0x62, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x8d, 0x01, 0x0a, + 0x20, 0x53, 0x74, 0x61, 0x72, 0x74, 0x4f, 0x66, 0x66, 0x6c, 0x69, 0x6e, 0x65, 0x54, 0x6f, 0x4f, + 0x6e, 0x6c, 0x69, 0x6e, 0x65, 0x49, 0x6e, 0x67, 0x65, 0x73, 0x74, 0x69, 0x6f, 0x6e, 0x4a, 0x6f, + 0x62, 0x12, 0x33, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x53, + 0x74, 0x61, 0x72, 0x74, 0x4f, 0x66, 0x66, 0x6c, 0x69, 0x6e, 0x65, 0x54, 0x6f, 0x4f, 0x6e, 0x6c, + 0x69, 0x6e, 0x65, 0x49, 0x6e, 0x67, 0x65, 0x73, 0x74, 0x69, 0x6f, 0x6e, 0x4a, 0x6f, 0x62, 0x52, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x34, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, + 0x6f, 0x72, 0x65, 0x2e, 0x53, 0x74, 0x61, 0x72, 0x74, 0x4f, 0x66, 0x66, 0x6c, 0x69, 0x6e, 0x65, + 0x54, 0x6f, 0x4f, 0x6e, 0x6c, 0x69, 0x6e, 0x65, 0x49, 0x6e, 0x67, 0x65, 0x73, 0x74, 0x69, 0x6f, + 0x6e, 0x4a, 0x6f, 0x62, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x6c, 0x0a, 0x15, + 0x47, 0x65, 0x74, 0x48, 0x69, 0x73, 0x74, 0x6f, 0x72, 0x69, 0x63, 0x61, 0x6c, 0x46, 0x65, 0x61, + 0x74, 0x75, 0x72, 0x65, 0x73, 0x12, 0x28, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, + 0x72, 0x65, 0x2e, 0x47, 0x65, 0x74, 0x48, 0x69, 0x73, 0x74, 0x6f, 0x72, 0x69, 0x63, 0x61, 0x6c, + 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, + 0x29, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x47, 0x65, 0x74, + 0x48, 0x69, 0x73, 0x74, 0x6f, 0x72, 0x69, 0x63, 0x61, 0x6c, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, + 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x8a, 0x01, 0x0a, 0x1f, 0x53, + 0x74, 0x61, 0x72, 0x74, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x54, 0x6f, 0x4f, 0x6e, 0x6c, 0x69, + 0x6e, 0x65, 0x49, 0x6e, 0x67, 0x65, 0x73, 0x74, 0x69, 0x6f, 0x6e, 0x4a, 0x6f, 0x62, 0x12, 0x32, + 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x53, 0x74, 0x61, 0x72, + 0x74, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x54, 0x6f, 0x4f, 0x6e, 0x6c, 0x69, 0x6e, 0x65, 0x49, + 0x6e, 0x67, 0x65, 0x73, 0x74, 0x69, 0x6f, 0x6e, 0x4a, 0x6f, 0x62, 0x52, 0x65, 0x71, 0x75, 0x65, + 0x73, 0x74, 0x1a, 0x33, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, + 0x53, 0x74, 0x61, 0x72, 0x74, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x54, 0x6f, 0x4f, 0x6e, 0x6c, + 0x69, 0x6e, 0x65, 0x49, 0x6e, 0x67, 0x65, 0x73, 0x74, 0x69, 0x6f, 0x6e, 0x4a, 0x6f, 0x62, 0x52, + 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x45, 0x0a, 0x08, 0x4c, 0x69, 0x73, 0x74, 0x4a, + 0x6f, 0x62, 0x73, 0x12, 0x1b, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, + 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x4a, 0x6f, 0x62, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x1a, 0x1c, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x4c, 0x69, + 0x73, 0x74, 0x4a, 0x6f, 0x62, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x48, + 0x0a, 0x09, 0x43, 0x61, 0x6e, 0x63, 0x65, 0x6c, 0x4a, 0x6f, 0x62, 0x12, 0x1c, 0x2e, 0x66, 0x65, + 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x43, 0x61, 0x6e, 0x63, 0x65, 0x6c, 0x4a, + 0x6f, 0x62, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1d, 0x2e, 0x66, 0x65, 0x61, 0x73, + 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x43, 0x61, 0x6e, 0x63, 0x65, 0x6c, 0x4a, 0x6f, 0x62, + 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x3f, 0x0a, 0x06, 0x47, 0x65, 0x74, 0x4a, + 0x6f, 0x62, 0x12, 0x19, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, + 0x47, 0x65, 0x74, 0x4a, 0x6f, 0x62, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1a, 0x2e, + 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x47, 0x65, 0x74, 0x4a, 0x6f, + 0x62, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x42, 0x58, 0x0a, 0x10, 0x66, 0x65, 0x61, + 0x73, 0x74, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x42, 0x0f, 0x4a, + 0x6f, 0x62, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x5a, 0x33, + 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x66, 0x65, 0x61, 0x73, 0x74, + 0x2d, 0x64, 0x65, 0x76, 0x2f, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2f, 0x73, 0x64, 0x6b, 0x2f, 0x67, + 0x6f, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x73, 0x2f, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2f, 0x63, + 0x6f, 0x72, 0x65, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, +} + +var ( + file_feast_core_JobService_proto_rawDescOnce sync.Once + file_feast_core_JobService_proto_rawDescData = file_feast_core_JobService_proto_rawDesc +) + +func file_feast_core_JobService_proto_rawDescGZIP() []byte { + file_feast_core_JobService_proto_rawDescOnce.Do(func() { + file_feast_core_JobService_proto_rawDescData = protoimpl.X.CompressGZIP(file_feast_core_JobService_proto_rawDescData) + }) + return file_feast_core_JobService_proto_rawDescData +} + +var file_feast_core_JobService_proto_enumTypes = make([]protoimpl.EnumInfo, 2) +var file_feast_core_JobService_proto_msgTypes = make([]protoimpl.MessageInfo, 16) +var file_feast_core_JobService_proto_goTypes = []interface{}{ + (JobType)(0), // 0: feast.core.JobType + (JobStatus)(0), // 1: feast.core.JobStatus + (*Job)(nil), // 2: feast.core.Job + (*StartOfflineToOnlineIngestionJobRequest)(nil), // 3: feast.core.StartOfflineToOnlineIngestionJobRequest + (*StartOfflineToOnlineIngestionJobResponse)(nil), // 4: feast.core.StartOfflineToOnlineIngestionJobResponse + (*GetHistoricalFeaturesRequest)(nil), // 5: feast.core.GetHistoricalFeaturesRequest + (*GetHistoricalFeaturesResponse)(nil), // 6: feast.core.GetHistoricalFeaturesResponse + (*StartStreamToOnlineIngestionJobRequest)(nil), // 7: feast.core.StartStreamToOnlineIngestionJobRequest + (*StartStreamToOnlineIngestionJobResponse)(nil), // 8: feast.core.StartStreamToOnlineIngestionJobResponse + (*ListJobsRequest)(nil), // 9: feast.core.ListJobsRequest + (*ListJobsResponse)(nil), // 10: feast.core.ListJobsResponse + (*GetJobRequest)(nil), // 11: feast.core.GetJobRequest + (*GetJobResponse)(nil), // 12: feast.core.GetJobResponse + (*CancelJobRequest)(nil), // 13: feast.core.CancelJobRequest + (*CancelJobResponse)(nil), // 14: feast.core.CancelJobResponse + (*Job_RetrievalJobMeta)(nil), // 15: feast.core.Job.RetrievalJobMeta + (*Job_OfflineToOnlineMeta)(nil), // 16: feast.core.Job.OfflineToOnlineMeta + (*Job_StreamToOnlineMeta)(nil), // 17: feast.core.Job.StreamToOnlineMeta + (*timestamp.Timestamp)(nil), // 18: google.protobuf.Timestamp + (*DataSource)(nil), // 19: feast.core.DataSource +} +var file_feast_core_JobService_proto_depIdxs = []int32{ + 0, // 0: feast.core.Job.type:type_name -> feast.core.JobType + 1, // 1: feast.core.Job.status:type_name -> feast.core.JobStatus + 15, // 2: feast.core.Job.retrieval:type_name -> feast.core.Job.RetrievalJobMeta + 16, // 3: feast.core.Job.batch_ingestion:type_name -> feast.core.Job.OfflineToOnlineMeta + 17, // 4: feast.core.Job.stream_ingestion:type_name -> feast.core.Job.StreamToOnlineMeta + 18, // 5: feast.core.StartOfflineToOnlineIngestionJobRequest.start_date:type_name -> google.protobuf.Timestamp + 18, // 6: feast.core.StartOfflineToOnlineIngestionJobRequest.end_date:type_name -> google.protobuf.Timestamp + 19, // 7: feast.core.GetHistoricalFeaturesRequest.entity_source:type_name -> feast.core.DataSource + 2, // 8: feast.core.ListJobsResponse.jobs:type_name -> feast.core.Job + 2, // 9: feast.core.GetJobResponse.job:type_name -> feast.core.Job + 3, // 10: feast.core.JobService.StartOfflineToOnlineIngestionJob:input_type -> feast.core.StartOfflineToOnlineIngestionJobRequest + 5, // 11: feast.core.JobService.GetHistoricalFeatures:input_type -> feast.core.GetHistoricalFeaturesRequest + 7, // 12: feast.core.JobService.StartStreamToOnlineIngestionJob:input_type -> feast.core.StartStreamToOnlineIngestionJobRequest + 9, // 13: feast.core.JobService.ListJobs:input_type -> feast.core.ListJobsRequest + 13, // 14: feast.core.JobService.CancelJob:input_type -> feast.core.CancelJobRequest + 11, // 15: feast.core.JobService.GetJob:input_type -> feast.core.GetJobRequest + 4, // 16: feast.core.JobService.StartOfflineToOnlineIngestionJob:output_type -> feast.core.StartOfflineToOnlineIngestionJobResponse + 6, // 17: feast.core.JobService.GetHistoricalFeatures:output_type -> feast.core.GetHistoricalFeaturesResponse + 8, // 18: feast.core.JobService.StartStreamToOnlineIngestionJob:output_type -> feast.core.StartStreamToOnlineIngestionJobResponse + 10, // 19: feast.core.JobService.ListJobs:output_type -> feast.core.ListJobsResponse + 14, // 20: feast.core.JobService.CancelJob:output_type -> feast.core.CancelJobResponse + 12, // 21: feast.core.JobService.GetJob:output_type -> feast.core.GetJobResponse + 16, // [16:22] is the sub-list for method output_type + 10, // [10:16] is the sub-list for method input_type + 10, // [10:10] is the sub-list for extension type_name + 10, // [10:10] is the sub-list for extension extendee + 0, // [0:10] is the sub-list for field type_name +} + +func init() { file_feast_core_JobService_proto_init() } +func file_feast_core_JobService_proto_init() { + if File_feast_core_JobService_proto != nil { + return + } + file_feast_core_DataSource_proto_init() + if !protoimpl.UnsafeEnabled { + file_feast_core_JobService_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*Job); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_feast_core_JobService_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*StartOfflineToOnlineIngestionJobRequest); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_feast_core_JobService_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*StartOfflineToOnlineIngestionJobResponse); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_feast_core_JobService_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*GetHistoricalFeaturesRequest); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_feast_core_JobService_proto_msgTypes[4].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*GetHistoricalFeaturesResponse); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_feast_core_JobService_proto_msgTypes[5].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*StartStreamToOnlineIngestionJobRequest); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_feast_core_JobService_proto_msgTypes[6].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*StartStreamToOnlineIngestionJobResponse); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_feast_core_JobService_proto_msgTypes[7].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*ListJobsRequest); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_feast_core_JobService_proto_msgTypes[8].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*ListJobsResponse); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_feast_core_JobService_proto_msgTypes[9].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*GetJobRequest); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_feast_core_JobService_proto_msgTypes[10].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*GetJobResponse); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_feast_core_JobService_proto_msgTypes[11].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*CancelJobRequest); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_feast_core_JobService_proto_msgTypes[12].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*CancelJobResponse); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_feast_core_JobService_proto_msgTypes[13].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*Job_RetrievalJobMeta); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_feast_core_JobService_proto_msgTypes[14].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*Job_OfflineToOnlineMeta); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_feast_core_JobService_proto_msgTypes[15].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*Job_StreamToOnlineMeta); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + } + file_feast_core_JobService_proto_msgTypes[0].OneofWrappers = []interface{}{ + (*Job_Retrieval)(nil), + (*Job_BatchIngestion)(nil), + (*Job_StreamIngestion)(nil), + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: file_feast_core_JobService_proto_rawDesc, + NumEnums: 2, + NumMessages: 16, + NumExtensions: 0, + NumServices: 1, + }, + GoTypes: file_feast_core_JobService_proto_goTypes, + DependencyIndexes: file_feast_core_JobService_proto_depIdxs, + EnumInfos: file_feast_core_JobService_proto_enumTypes, + MessageInfos: file_feast_core_JobService_proto_msgTypes, + }.Build() + File_feast_core_JobService_proto = out.File + file_feast_core_JobService_proto_rawDesc = nil + file_feast_core_JobService_proto_goTypes = nil + file_feast_core_JobService_proto_depIdxs = nil +} + +// Reference imports to suppress errors if they are not otherwise used. +var _ context.Context +var _ grpc.ClientConnInterface + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the grpc package it is being compiled against. +const _ = grpc.SupportPackageIsVersion6 + +// JobServiceClient is the client API for JobService service. +// +// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://godoc.org/google.golang.org/grpc#ClientConn.NewStream. +type JobServiceClient interface { + // Start job to ingest data from offline store into online store + StartOfflineToOnlineIngestionJob(ctx context.Context, in *StartOfflineToOnlineIngestionJobRequest, opts ...grpc.CallOption) (*StartOfflineToOnlineIngestionJobResponse, error) + // Produce a training dataset, return a job id that will provide a file reference + GetHistoricalFeatures(ctx context.Context, in *GetHistoricalFeaturesRequest, opts ...grpc.CallOption) (*GetHistoricalFeaturesResponse, error) + // Start job to ingest data from stream into online store + StartStreamToOnlineIngestionJob(ctx context.Context, in *StartStreamToOnlineIngestionJobRequest, opts ...grpc.CallOption) (*StartStreamToOnlineIngestionJobResponse, error) + // List all types of jobs + ListJobs(ctx context.Context, in *ListJobsRequest, opts ...grpc.CallOption) (*ListJobsResponse, error) + // Cancel a single job + CancelJob(ctx context.Context, in *CancelJobRequest, opts ...grpc.CallOption) (*CancelJobResponse, error) + // Get details of a single job + GetJob(ctx context.Context, in *GetJobRequest, opts ...grpc.CallOption) (*GetJobResponse, error) +} + +type jobServiceClient struct { + cc grpc.ClientConnInterface +} + +func NewJobServiceClient(cc grpc.ClientConnInterface) JobServiceClient { + return &jobServiceClient{cc} +} + +func (c *jobServiceClient) StartOfflineToOnlineIngestionJob(ctx context.Context, in *StartOfflineToOnlineIngestionJobRequest, opts ...grpc.CallOption) (*StartOfflineToOnlineIngestionJobResponse, error) { + out := new(StartOfflineToOnlineIngestionJobResponse) + err := c.cc.Invoke(ctx, "/feast.core.JobService/StartOfflineToOnlineIngestionJob", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *jobServiceClient) GetHistoricalFeatures(ctx context.Context, in *GetHistoricalFeaturesRequest, opts ...grpc.CallOption) (*GetHistoricalFeaturesResponse, error) { + out := new(GetHistoricalFeaturesResponse) + err := c.cc.Invoke(ctx, "/feast.core.JobService/GetHistoricalFeatures", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *jobServiceClient) StartStreamToOnlineIngestionJob(ctx context.Context, in *StartStreamToOnlineIngestionJobRequest, opts ...grpc.CallOption) (*StartStreamToOnlineIngestionJobResponse, error) { + out := new(StartStreamToOnlineIngestionJobResponse) + err := c.cc.Invoke(ctx, "/feast.core.JobService/StartStreamToOnlineIngestionJob", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *jobServiceClient) ListJobs(ctx context.Context, in *ListJobsRequest, opts ...grpc.CallOption) (*ListJobsResponse, error) { + out := new(ListJobsResponse) + err := c.cc.Invoke(ctx, "/feast.core.JobService/ListJobs", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *jobServiceClient) CancelJob(ctx context.Context, in *CancelJobRequest, opts ...grpc.CallOption) (*CancelJobResponse, error) { + out := new(CancelJobResponse) + err := c.cc.Invoke(ctx, "/feast.core.JobService/CancelJob", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *jobServiceClient) GetJob(ctx context.Context, in *GetJobRequest, opts ...grpc.CallOption) (*GetJobResponse, error) { + out := new(GetJobResponse) + err := c.cc.Invoke(ctx, "/feast.core.JobService/GetJob", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +// JobServiceServer is the server API for JobService service. +type JobServiceServer interface { + // Start job to ingest data from offline store into online store + StartOfflineToOnlineIngestionJob(context.Context, *StartOfflineToOnlineIngestionJobRequest) (*StartOfflineToOnlineIngestionJobResponse, error) + // Produce a training dataset, return a job id that will provide a file reference + GetHistoricalFeatures(context.Context, *GetHistoricalFeaturesRequest) (*GetHistoricalFeaturesResponse, error) + // Start job to ingest data from stream into online store + StartStreamToOnlineIngestionJob(context.Context, *StartStreamToOnlineIngestionJobRequest) (*StartStreamToOnlineIngestionJobResponse, error) + // List all types of jobs + ListJobs(context.Context, *ListJobsRequest) (*ListJobsResponse, error) + // Cancel a single job + CancelJob(context.Context, *CancelJobRequest) (*CancelJobResponse, error) + // Get details of a single job + GetJob(context.Context, *GetJobRequest) (*GetJobResponse, error) +} + +// UnimplementedJobServiceServer can be embedded to have forward compatible implementations. +type UnimplementedJobServiceServer struct { +} + +func (*UnimplementedJobServiceServer) StartOfflineToOnlineIngestionJob(context.Context, *StartOfflineToOnlineIngestionJobRequest) (*StartOfflineToOnlineIngestionJobResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method StartOfflineToOnlineIngestionJob not implemented") +} +func (*UnimplementedJobServiceServer) GetHistoricalFeatures(context.Context, *GetHistoricalFeaturesRequest) (*GetHistoricalFeaturesResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method GetHistoricalFeatures not implemented") +} +func (*UnimplementedJobServiceServer) StartStreamToOnlineIngestionJob(context.Context, *StartStreamToOnlineIngestionJobRequest) (*StartStreamToOnlineIngestionJobResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method StartStreamToOnlineIngestionJob not implemented") +} +func (*UnimplementedJobServiceServer) ListJobs(context.Context, *ListJobsRequest) (*ListJobsResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method ListJobs not implemented") +} +func (*UnimplementedJobServiceServer) CancelJob(context.Context, *CancelJobRequest) (*CancelJobResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method CancelJob not implemented") +} +func (*UnimplementedJobServiceServer) GetJob(context.Context, *GetJobRequest) (*GetJobResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method GetJob not implemented") +} + +func RegisterJobServiceServer(s *grpc.Server, srv JobServiceServer) { + s.RegisterService(&_JobService_serviceDesc, srv) +} + +func _JobService_StartOfflineToOnlineIngestionJob_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(StartOfflineToOnlineIngestionJobRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(JobServiceServer).StartOfflineToOnlineIngestionJob(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/feast.core.JobService/StartOfflineToOnlineIngestionJob", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(JobServiceServer).StartOfflineToOnlineIngestionJob(ctx, req.(*StartOfflineToOnlineIngestionJobRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _JobService_GetHistoricalFeatures_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(GetHistoricalFeaturesRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(JobServiceServer).GetHistoricalFeatures(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/feast.core.JobService/GetHistoricalFeatures", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(JobServiceServer).GetHistoricalFeatures(ctx, req.(*GetHistoricalFeaturesRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _JobService_StartStreamToOnlineIngestionJob_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(StartStreamToOnlineIngestionJobRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(JobServiceServer).StartStreamToOnlineIngestionJob(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/feast.core.JobService/StartStreamToOnlineIngestionJob", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(JobServiceServer).StartStreamToOnlineIngestionJob(ctx, req.(*StartStreamToOnlineIngestionJobRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _JobService_ListJobs_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(ListJobsRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(JobServiceServer).ListJobs(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/feast.core.JobService/ListJobs", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(JobServiceServer).ListJobs(ctx, req.(*ListJobsRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _JobService_CancelJob_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(CancelJobRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(JobServiceServer).CancelJob(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/feast.core.JobService/CancelJob", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(JobServiceServer).CancelJob(ctx, req.(*CancelJobRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _JobService_GetJob_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(GetJobRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(JobServiceServer).GetJob(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/feast.core.JobService/GetJob", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(JobServiceServer).GetJob(ctx, req.(*GetJobRequest)) + } + return interceptor(ctx, in, info, handler) +} + +var _JobService_serviceDesc = grpc.ServiceDesc{ + ServiceName: "feast.core.JobService", + HandlerType: (*JobServiceServer)(nil), + Methods: []grpc.MethodDesc{ + { + MethodName: "StartOfflineToOnlineIngestionJob", + Handler: _JobService_StartOfflineToOnlineIngestionJob_Handler, + }, + { + MethodName: "GetHistoricalFeatures", + Handler: _JobService_GetHistoricalFeatures_Handler, + }, + { + MethodName: "StartStreamToOnlineIngestionJob", + Handler: _JobService_StartStreamToOnlineIngestionJob_Handler, + }, + { + MethodName: "ListJobs", + Handler: _JobService_ListJobs_Handler, + }, + { + MethodName: "CancelJob", + Handler: _JobService_CancelJob_Handler, + }, + { + MethodName: "GetJob", + Handler: _JobService_GetJob_Handler, + }, + }, + Streams: []grpc.StreamDesc{}, + Metadata: "feast/core/JobService.proto", +} diff --git a/sdk/go/protos/feast/core/Store.pb.go b/sdk/go/protos/feast/core/Store.pb.go index b5ffae9326f..9498c6a40c7 100644 --- a/sdk/go/protos/feast/core/Store.pb.go +++ b/sdk/go/protos/feast/core/Store.pb.go @@ -17,7 +17,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.25.0 -// protoc v3.10.0 +// protoc v3.12.4 // source: feast/core/Store.proto package core @@ -52,59 +52,10 @@ const ( // - value: STRING // // Encodings: - // - key: byte array of RedisKey (refer to feast.storage.RedisKey) - // - value: byte array of FeatureRow (refer to feast.types.FeatureRow) + // - key: byte array of RedisKey (refer to feast.storage.RedisKeyV2) + // - value: Redis hashmap // - Store_REDIS Store_StoreType = 1 - // BigQuery stores a FeatureRow element as a row in a BigQuery table. - // - // Table name is derived is the same as the feature set name. - // - // The entities and features in a FeatureSetSpec corresponds to the - // fields in the BigQuery table (these make up the BigQuery schema). - // The name of the entity spec and feature spec corresponds to the column - // names, and the value_type of entity spec and feature spec corresponds - // to BigQuery standard SQL data type of the column. - // - // The following BigQuery fields are reserved for Feast internal use. - // Ingestion of entity or feature spec with names identical - // to the following field names will raise an exception during ingestion. - // - // column_name | column_data_type | description - // ====================|==================|================================ - // - event_timestamp | TIMESTAMP | event time of the FeatureRow - // - created_timestamp | TIMESTAMP | processing time of the ingestion of the FeatureRow - // - ingestion_id | STRING | unique id identifying groups of rows that have been ingested together - // - job_id | STRING | identifier for the job that writes the FeatureRow to the corresponding BigQuery table - // - // BigQuery table created will be partitioned by the field "event_timestamp" - // of the FeatureRow (https://cloud.google.com/bigquery/docs/partitioned-tables). - // - // The following table shows how ValueType in Feast is mapped to - // BigQuery Standard SQL data types - // (https://cloud.google.com/bigquery/docs/reference/standard-sql/data-types): - // - // BYTES : BYTES - // STRING : STRING - // INT32 : INT64 - // INT64 : IN64 - // DOUBLE : FLOAT64 - // FLOAT : FLOAT64 - // BOOL : BOOL - // BYTES_LIST : ARRAY - // STRING_LIST : ARRAY - // INT32_LIST : ARRAY - // INT64_LIST : ARRAY - // DOUBLE_LIST : ARRAY - // FLOAT_LIST : ARRAY - // BOOL_LIST : ARRAY - // - // The column mode in BigQuery is set to "Nullable" such that unset Value - // in a FeatureRow corresponds to NULL value in BigQuery. - // - Store_BIGQUERY Store_StoreType = 2 - // Unsupported in Feast 0.3 - Store_CASSANDRA Store_StoreType = 3 + Store_REDIS Store_StoreType = 1 Store_REDIS_CLUSTER Store_StoreType = 4 ) @@ -113,15 +64,11 @@ var ( Store_StoreType_name = map[int32]string{ 0: "INVALID", 1: "REDIS", - 2: "BIGQUERY", - 3: "CASSANDRA", 4: "REDIS_CLUSTER", } Store_StoreType_value = map[string]int32{ "INVALID": 0, "REDIS": 1, - "BIGQUERY": 2, - "CASSANDRA": 3, "REDIS_CLUSTER": 4, } ) @@ -158,9 +105,6 @@ func (Store_StoreType) EnumDescriptor() ([]byte, []int) { // The way FeatureRow is encoded and decoded when it is written to and read from // the Store depends on the type of the Store. // -// For example, a FeatureRow will materialize as a row in a table in -// BigQuery but it will materialize as a key, value pair element in Redis. -// type Store struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -176,8 +120,6 @@ type Store struct { // // Types that are assignable to Config: // *Store_RedisConfig_ - // *Store_BigqueryConfig - // *Store_CassandraConfig_ // *Store_RedisClusterConfig_ Config isStore_Config `protobuf_oneof:"config"` } @@ -249,20 +191,6 @@ func (x *Store) GetRedisConfig() *Store_RedisConfig { return nil } -func (x *Store) GetBigqueryConfig() *Store_BigQueryConfig { - if x, ok := x.GetConfig().(*Store_BigqueryConfig); ok { - return x.BigqueryConfig - } - return nil -} - -func (x *Store) GetCassandraConfig() *Store_CassandraConfig { - if x, ok := x.GetConfig().(*Store_CassandraConfig_); ok { - return x.CassandraConfig - } - return nil -} - func (x *Store) GetRedisClusterConfig() *Store_RedisClusterConfig { if x, ok := x.GetConfig().(*Store_RedisClusterConfig_); ok { return x.RedisClusterConfig @@ -278,24 +206,12 @@ type Store_RedisConfig_ struct { RedisConfig *Store_RedisConfig `protobuf:"bytes,11,opt,name=redis_config,json=redisConfig,proto3,oneof"` } -type Store_BigqueryConfig struct { - BigqueryConfig *Store_BigQueryConfig `protobuf:"bytes,12,opt,name=bigquery_config,json=bigqueryConfig,proto3,oneof"` -} - -type Store_CassandraConfig_ struct { - CassandraConfig *Store_CassandraConfig `protobuf:"bytes,13,opt,name=cassandra_config,json=cassandraConfig,proto3,oneof"` -} - type Store_RedisClusterConfig_ struct { RedisClusterConfig *Store_RedisClusterConfig `protobuf:"bytes,14,opt,name=redis_cluster_config,json=redisClusterConfig,proto3,oneof"` } func (*Store_RedisConfig_) isStore_Config() {} -func (*Store_BigqueryConfig) isStore_Config() {} - -func (*Store_CassandraConfig_) isStore_Config() {} - func (*Store_RedisClusterConfig_) isStore_Config() {} type Store_RedisConfig struct { @@ -312,6 +228,8 @@ type Store_RedisConfig struct { MaxRetries int32 `protobuf:"varint,4,opt,name=max_retries,json=maxRetries,proto3" json:"max_retries,omitempty"` // Optional. How often flush data to redis FlushFrequencySeconds int32 `protobuf:"varint,5,opt,name=flush_frequency_seconds,json=flushFrequencySeconds,proto3" json:"flush_frequency_seconds,omitempty"` + // Optional. Connect over SSL. + Ssl bool `protobuf:"varint,6,opt,name=ssl,proto3" json:"ssl,omitempty"` } func (x *Store_RedisConfig) Reset() { @@ -381,147 +299,11 @@ func (x *Store_RedisConfig) GetFlushFrequencySeconds() int32 { return 0 } -type Store_BigQueryConfig struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - ProjectId string `protobuf:"bytes,1,opt,name=project_id,json=projectId,proto3" json:"project_id,omitempty"` - DatasetId string `protobuf:"bytes,2,opt,name=dataset_id,json=datasetId,proto3" json:"dataset_id,omitempty"` - StagingLocation string `protobuf:"bytes,3,opt,name=staging_location,json=stagingLocation,proto3" json:"staging_location,omitempty"` - InitialRetryDelaySeconds int32 `protobuf:"varint,4,opt,name=initial_retry_delay_seconds,json=initialRetryDelaySeconds,proto3" json:"initial_retry_delay_seconds,omitempty"` - TotalTimeoutSeconds int32 `protobuf:"varint,5,opt,name=total_timeout_seconds,json=totalTimeoutSeconds,proto3" json:"total_timeout_seconds,omitempty"` - // Required. Frequency of running BQ load job and flushing all collected rows to BQ table - WriteTriggeringFrequencySeconds int32 `protobuf:"varint,6,opt,name=write_triggering_frequency_seconds,json=writeTriggeringFrequencySeconds,proto3" json:"write_triggering_frequency_seconds,omitempty"` -} - -func (x *Store_BigQueryConfig) Reset() { - *x = Store_BigQueryConfig{} - if protoimpl.UnsafeEnabled { - mi := &file_feast_core_Store_proto_msgTypes[2] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *Store_BigQueryConfig) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*Store_BigQueryConfig) ProtoMessage() {} - -func (x *Store_BigQueryConfig) ProtoReflect() protoreflect.Message { - mi := &file_feast_core_Store_proto_msgTypes[2] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use Store_BigQueryConfig.ProtoReflect.Descriptor instead. -func (*Store_BigQueryConfig) Descriptor() ([]byte, []int) { - return file_feast_core_Store_proto_rawDescGZIP(), []int{0, 1} -} - -func (x *Store_BigQueryConfig) GetProjectId() string { - if x != nil { - return x.ProjectId - } - return "" -} - -func (x *Store_BigQueryConfig) GetDatasetId() string { - if x != nil { - return x.DatasetId - } - return "" -} - -func (x *Store_BigQueryConfig) GetStagingLocation() string { - if x != nil { - return x.StagingLocation - } - return "" -} - -func (x *Store_BigQueryConfig) GetInitialRetryDelaySeconds() int32 { - if x != nil { - return x.InitialRetryDelaySeconds - } - return 0 -} - -func (x *Store_BigQueryConfig) GetTotalTimeoutSeconds() int32 { - if x != nil { - return x.TotalTimeoutSeconds - } - return 0 -} - -func (x *Store_BigQueryConfig) GetWriteTriggeringFrequencySeconds() int32 { - if x != nil { - return x.WriteTriggeringFrequencySeconds - } - return 0 -} - -type Store_CassandraConfig struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - Host string `protobuf:"bytes,1,opt,name=host,proto3" json:"host,omitempty"` - Port int32 `protobuf:"varint,2,opt,name=port,proto3" json:"port,omitempty"` -} - -func (x *Store_CassandraConfig) Reset() { - *x = Store_CassandraConfig{} - if protoimpl.UnsafeEnabled { - mi := &file_feast_core_Store_proto_msgTypes[3] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *Store_CassandraConfig) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*Store_CassandraConfig) ProtoMessage() {} - -func (x *Store_CassandraConfig) ProtoReflect() protoreflect.Message { - mi := &file_feast_core_Store_proto_msgTypes[3] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use Store_CassandraConfig.ProtoReflect.Descriptor instead. -func (*Store_CassandraConfig) Descriptor() ([]byte, []int) { - return file_feast_core_Store_proto_rawDescGZIP(), []int{0, 2} -} - -func (x *Store_CassandraConfig) GetHost() string { +func (x *Store_RedisConfig) GetSsl() bool { if x != nil { - return x.Host + return x.Ssl } - return "" -} - -func (x *Store_CassandraConfig) GetPort() int32 { - if x != nil { - return x.Port - } - return 0 + return false } type Store_RedisClusterConfig struct { @@ -547,7 +329,7 @@ type Store_RedisClusterConfig struct { func (x *Store_RedisClusterConfig) Reset() { *x = Store_RedisClusterConfig{} if protoimpl.UnsafeEnabled { - mi := &file_feast_core_Store_proto_msgTypes[4] + mi := &file_feast_core_Store_proto_msgTypes[2] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -560,7 +342,7 @@ func (x *Store_RedisClusterConfig) String() string { func (*Store_RedisClusterConfig) ProtoMessage() {} func (x *Store_RedisClusterConfig) ProtoReflect() protoreflect.Message { - mi := &file_feast_core_Store_proto_msgTypes[4] + mi := &file_feast_core_Store_proto_msgTypes[2] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -573,7 +355,7 @@ func (x *Store_RedisClusterConfig) ProtoReflect() protoreflect.Message { // Deprecated: Use Store_RedisClusterConfig.ProtoReflect.Descriptor instead. func (*Store_RedisClusterConfig) Descriptor() ([]byte, []int) { - return file_feast_core_Store_proto_rawDescGZIP(), []int{0, 3} + return file_feast_core_Store_proto_rawDescGZIP(), []int{0, 1} } func (x *Store_RedisClusterConfig) GetConnectionString() string { @@ -652,7 +434,7 @@ type Store_Subscription struct { func (x *Store_Subscription) Reset() { *x = Store_Subscription{} if protoimpl.UnsafeEnabled { - mi := &file_feast_core_Store_proto_msgTypes[5] + mi := &file_feast_core_Store_proto_msgTypes[3] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -665,7 +447,7 @@ func (x *Store_Subscription) String() string { func (*Store_Subscription) ProtoMessage() {} func (x *Store_Subscription) ProtoReflect() protoreflect.Message { - mi := &file_feast_core_Store_proto_msgTypes[5] + mi := &file_feast_core_Store_proto_msgTypes[3] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -678,7 +460,7 @@ func (x *Store_Subscription) ProtoReflect() protoreflect.Message { // Deprecated: Use Store_Subscription.ProtoReflect.Descriptor instead. func (*Store_Subscription) Descriptor() ([]byte, []int) { - return file_feast_core_Store_proto_rawDescGZIP(), []int{0, 4} + return file_feast_core_Store_proto_rawDescGZIP(), []int{0, 2} } func (x *Store_Subscription) GetProject() string { @@ -707,7 +489,7 @@ var File_feast_core_Store_proto protoreflect.FileDescriptor var file_feast_core_Store_proto_rawDesc = []byte{ 0x0a, 0x16, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2f, 0x63, 0x6f, 0x72, 0x65, 0x2f, 0x53, 0x74, 0x6f, 0x72, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x0a, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, - 0x63, 0x6f, 0x72, 0x65, 0x22, 0xfc, 0x0b, 0x0a, 0x05, 0x53, 0x74, 0x6f, 0x72, 0x65, 0x12, 0x12, + 0x63, 0x6f, 0x72, 0x65, 0x22, 0xf5, 0x07, 0x0a, 0x05, 0x53, 0x74, 0x6f, 0x72, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x2f, 0x0a, 0x04, 0x74, 0x79, 0x70, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x1b, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x53, 0x74, @@ -720,95 +502,63 @@ var file_feast_core_Store_proto_rawDesc = []byte{ 0x69, 0x73, 0x5f, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1d, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x53, 0x74, 0x6f, 0x72, 0x65, 0x2e, 0x52, 0x65, 0x64, 0x69, 0x73, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x48, 0x00, - 0x52, 0x0b, 0x72, 0x65, 0x64, 0x69, 0x73, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x4b, 0x0a, - 0x0f, 0x62, 0x69, 0x67, 0x71, 0x75, 0x65, 0x72, 0x79, 0x5f, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, - 0x18, 0x0c, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x20, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, - 0x6f, 0x72, 0x65, 0x2e, 0x53, 0x74, 0x6f, 0x72, 0x65, 0x2e, 0x42, 0x69, 0x67, 0x51, 0x75, 0x65, - 0x72, 0x79, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x48, 0x00, 0x52, 0x0e, 0x62, 0x69, 0x67, 0x71, - 0x75, 0x65, 0x72, 0x79, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x4e, 0x0a, 0x10, 0x63, 0x61, - 0x73, 0x73, 0x61, 0x6e, 0x64, 0x72, 0x61, 0x5f, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x18, 0x0d, - 0x20, 0x01, 0x28, 0x0b, 0x32, 0x21, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, - 0x65, 0x2e, 0x53, 0x74, 0x6f, 0x72, 0x65, 0x2e, 0x43, 0x61, 0x73, 0x73, 0x61, 0x6e, 0x64, 0x72, - 0x61, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x48, 0x00, 0x52, 0x0f, 0x63, 0x61, 0x73, 0x73, 0x61, - 0x6e, 0x64, 0x72, 0x61, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x58, 0x0a, 0x14, 0x72, 0x65, - 0x64, 0x69, 0x73, 0x5f, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x63, 0x6f, 0x6e, 0x66, - 0x69, 0x67, 0x18, 0x0e, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x24, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, - 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x53, 0x74, 0x6f, 0x72, 0x65, 0x2e, 0x52, 0x65, 0x64, 0x69, - 0x73, 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x48, 0x00, - 0x52, 0x12, 0x72, 0x65, 0x64, 0x69, 0x73, 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x43, 0x6f, - 0x6e, 0x66, 0x69, 0x67, 0x1a, 0xbc, 0x01, 0x0a, 0x0b, 0x52, 0x65, 0x64, 0x69, 0x73, 0x43, 0x6f, - 0x6e, 0x66, 0x69, 0x67, 0x12, 0x12, 0x0a, 0x04, 0x68, 0x6f, 0x73, 0x74, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x04, 0x68, 0x6f, 0x73, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x70, 0x6f, 0x72, 0x74, - 0x18, 0x02, 0x20, 0x01, 0x28, 0x05, 0x52, 0x04, 0x70, 0x6f, 0x72, 0x74, 0x12, 0x2c, 0x0a, 0x12, + 0x52, 0x0b, 0x72, 0x65, 0x64, 0x69, 0x73, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x58, 0x0a, + 0x14, 0x72, 0x65, 0x64, 0x69, 0x73, 0x5f, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x63, + 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x18, 0x0e, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x24, 0x2e, 0x66, 0x65, + 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x53, 0x74, 0x6f, 0x72, 0x65, 0x2e, 0x52, + 0x65, 0x64, 0x69, 0x73, 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, + 0x67, 0x48, 0x00, 0x52, 0x12, 0x72, 0x65, 0x64, 0x69, 0x73, 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, + 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x1a, 0xce, 0x01, 0x0a, 0x0b, 0x52, 0x65, 0x64, 0x69, + 0x73, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x12, 0x0a, 0x04, 0x68, 0x6f, 0x73, 0x74, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x68, 0x6f, 0x73, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x70, + 0x6f, 0x72, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x05, 0x52, 0x04, 0x70, 0x6f, 0x72, 0x74, 0x12, + 0x2c, 0x0a, 0x12, 0x69, 0x6e, 0x69, 0x74, 0x69, 0x61, 0x6c, 0x5f, 0x62, 0x61, 0x63, 0x6b, 0x6f, + 0x66, 0x66, 0x5f, 0x6d, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x05, 0x52, 0x10, 0x69, 0x6e, 0x69, + 0x74, 0x69, 0x61, 0x6c, 0x42, 0x61, 0x63, 0x6b, 0x6f, 0x66, 0x66, 0x4d, 0x73, 0x12, 0x1f, 0x0a, + 0x0b, 0x6d, 0x61, 0x78, 0x5f, 0x72, 0x65, 0x74, 0x72, 0x69, 0x65, 0x73, 0x18, 0x04, 0x20, 0x01, + 0x28, 0x05, 0x52, 0x0a, 0x6d, 0x61, 0x78, 0x52, 0x65, 0x74, 0x72, 0x69, 0x65, 0x73, 0x12, 0x36, + 0x0a, 0x17, 0x66, 0x6c, 0x75, 0x73, 0x68, 0x5f, 0x66, 0x72, 0x65, 0x71, 0x75, 0x65, 0x6e, 0x63, + 0x79, 0x5f, 0x73, 0x65, 0x63, 0x6f, 0x6e, 0x64, 0x73, 0x18, 0x05, 0x20, 0x01, 0x28, 0x05, 0x52, + 0x15, 0x66, 0x6c, 0x75, 0x73, 0x68, 0x46, 0x72, 0x65, 0x71, 0x75, 0x65, 0x6e, 0x63, 0x79, 0x53, + 0x65, 0x63, 0x6f, 0x6e, 0x64, 0x73, 0x12, 0x10, 0x0a, 0x03, 0x73, 0x73, 0x6c, 0x18, 0x06, 0x20, + 0x01, 0x28, 0x08, 0x52, 0x03, 0x73, 0x73, 0x6c, 0x1a, 0xb9, 0x02, 0x0a, 0x12, 0x52, 0x65, 0x64, + 0x69, 0x73, 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, + 0x2b, 0x0a, 0x11, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x73, 0x74, + 0x72, 0x69, 0x6e, 0x67, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x10, 0x63, 0x6f, 0x6e, 0x6e, + 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x12, 0x2c, 0x0a, 0x12, 0x69, 0x6e, 0x69, 0x74, 0x69, 0x61, 0x6c, 0x5f, 0x62, 0x61, 0x63, 0x6b, 0x6f, 0x66, 0x66, 0x5f, - 0x6d, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x05, 0x52, 0x10, 0x69, 0x6e, 0x69, 0x74, 0x69, 0x61, + 0x6d, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x05, 0x52, 0x10, 0x69, 0x6e, 0x69, 0x74, 0x69, 0x61, 0x6c, 0x42, 0x61, 0x63, 0x6b, 0x6f, 0x66, 0x66, 0x4d, 0x73, 0x12, 0x1f, 0x0a, 0x0b, 0x6d, 0x61, - 0x78, 0x5f, 0x72, 0x65, 0x74, 0x72, 0x69, 0x65, 0x73, 0x18, 0x04, 0x20, 0x01, 0x28, 0x05, 0x52, + 0x78, 0x5f, 0x72, 0x65, 0x74, 0x72, 0x69, 0x65, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0a, 0x6d, 0x61, 0x78, 0x52, 0x65, 0x74, 0x72, 0x69, 0x65, 0x73, 0x12, 0x36, 0x0a, 0x17, 0x66, 0x6c, 0x75, 0x73, 0x68, 0x5f, 0x66, 0x72, 0x65, 0x71, 0x75, 0x65, 0x6e, 0x63, 0x79, 0x5f, 0x73, - 0x65, 0x63, 0x6f, 0x6e, 0x64, 0x73, 0x18, 0x05, 0x20, 0x01, 0x28, 0x05, 0x52, 0x15, 0x66, 0x6c, + 0x65, 0x63, 0x6f, 0x6e, 0x64, 0x73, 0x18, 0x04, 0x20, 0x01, 0x28, 0x05, 0x52, 0x15, 0x66, 0x6c, 0x75, 0x73, 0x68, 0x46, 0x72, 0x65, 0x71, 0x75, 0x65, 0x6e, 0x63, 0x79, 0x53, 0x65, 0x63, 0x6f, - 0x6e, 0x64, 0x73, 0x1a, 0xb9, 0x02, 0x0a, 0x0e, 0x42, 0x69, 0x67, 0x51, 0x75, 0x65, 0x72, 0x79, - 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x1d, 0x0a, 0x0a, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, - 0x74, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x70, 0x72, 0x6f, 0x6a, - 0x65, 0x63, 0x74, 0x49, 0x64, 0x12, 0x1d, 0x0a, 0x0a, 0x64, 0x61, 0x74, 0x61, 0x73, 0x65, 0x74, - 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x64, 0x61, 0x74, 0x61, 0x73, - 0x65, 0x74, 0x49, 0x64, 0x12, 0x29, 0x0a, 0x10, 0x73, 0x74, 0x61, 0x67, 0x69, 0x6e, 0x67, 0x5f, - 0x6c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0f, - 0x73, 0x74, 0x61, 0x67, 0x69, 0x6e, 0x67, 0x4c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, - 0x3d, 0x0a, 0x1b, 0x69, 0x6e, 0x69, 0x74, 0x69, 0x61, 0x6c, 0x5f, 0x72, 0x65, 0x74, 0x72, 0x79, - 0x5f, 0x64, 0x65, 0x6c, 0x61, 0x79, 0x5f, 0x73, 0x65, 0x63, 0x6f, 0x6e, 0x64, 0x73, 0x18, 0x04, - 0x20, 0x01, 0x28, 0x05, 0x52, 0x18, 0x69, 0x6e, 0x69, 0x74, 0x69, 0x61, 0x6c, 0x52, 0x65, 0x74, - 0x72, 0x79, 0x44, 0x65, 0x6c, 0x61, 0x79, 0x53, 0x65, 0x63, 0x6f, 0x6e, 0x64, 0x73, 0x12, 0x32, - 0x0a, 0x15, 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x6f, 0x75, 0x74, 0x5f, - 0x73, 0x65, 0x63, 0x6f, 0x6e, 0x64, 0x73, 0x18, 0x05, 0x20, 0x01, 0x28, 0x05, 0x52, 0x13, 0x74, - 0x6f, 0x74, 0x61, 0x6c, 0x54, 0x69, 0x6d, 0x65, 0x6f, 0x75, 0x74, 0x53, 0x65, 0x63, 0x6f, 0x6e, - 0x64, 0x73, 0x12, 0x4b, 0x0a, 0x22, 0x77, 0x72, 0x69, 0x74, 0x65, 0x5f, 0x74, 0x72, 0x69, 0x67, - 0x67, 0x65, 0x72, 0x69, 0x6e, 0x67, 0x5f, 0x66, 0x72, 0x65, 0x71, 0x75, 0x65, 0x6e, 0x63, 0x79, - 0x5f, 0x73, 0x65, 0x63, 0x6f, 0x6e, 0x64, 0x73, 0x18, 0x06, 0x20, 0x01, 0x28, 0x05, 0x52, 0x1f, - 0x77, 0x72, 0x69, 0x74, 0x65, 0x54, 0x72, 0x69, 0x67, 0x67, 0x65, 0x72, 0x69, 0x6e, 0x67, 0x46, - 0x72, 0x65, 0x71, 0x75, 0x65, 0x6e, 0x63, 0x79, 0x53, 0x65, 0x63, 0x6f, 0x6e, 0x64, 0x73, 0x1a, - 0x39, 0x0a, 0x0f, 0x43, 0x61, 0x73, 0x73, 0x61, 0x6e, 0x64, 0x72, 0x61, 0x43, 0x6f, 0x6e, 0x66, - 0x69, 0x67, 0x12, 0x12, 0x0a, 0x04, 0x68, 0x6f, 0x73, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x04, 0x68, 0x6f, 0x73, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x70, 0x6f, 0x72, 0x74, 0x18, 0x02, - 0x20, 0x01, 0x28, 0x05, 0x52, 0x04, 0x70, 0x6f, 0x72, 0x74, 0x1a, 0xb9, 0x02, 0x0a, 0x12, 0x52, - 0x65, 0x64, 0x69, 0x73, 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, - 0x67, 0x12, 0x2b, 0x0a, 0x11, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x5f, - 0x73, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x10, 0x63, 0x6f, - 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x12, 0x2c, - 0x0a, 0x12, 0x69, 0x6e, 0x69, 0x74, 0x69, 0x61, 0x6c, 0x5f, 0x62, 0x61, 0x63, 0x6b, 0x6f, 0x66, - 0x66, 0x5f, 0x6d, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x05, 0x52, 0x10, 0x69, 0x6e, 0x69, 0x74, - 0x69, 0x61, 0x6c, 0x42, 0x61, 0x63, 0x6b, 0x6f, 0x66, 0x66, 0x4d, 0x73, 0x12, 0x1f, 0x0a, 0x0b, - 0x6d, 0x61, 0x78, 0x5f, 0x72, 0x65, 0x74, 0x72, 0x69, 0x65, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, - 0x05, 0x52, 0x0a, 0x6d, 0x61, 0x78, 0x52, 0x65, 0x74, 0x72, 0x69, 0x65, 0x73, 0x12, 0x36, 0x0a, - 0x17, 0x66, 0x6c, 0x75, 0x73, 0x68, 0x5f, 0x66, 0x72, 0x65, 0x71, 0x75, 0x65, 0x6e, 0x63, 0x79, - 0x5f, 0x73, 0x65, 0x63, 0x6f, 0x6e, 0x64, 0x73, 0x18, 0x04, 0x20, 0x01, 0x28, 0x05, 0x52, 0x15, - 0x66, 0x6c, 0x75, 0x73, 0x68, 0x46, 0x72, 0x65, 0x71, 0x75, 0x65, 0x6e, 0x63, 0x79, 0x53, 0x65, - 0x63, 0x6f, 0x6e, 0x64, 0x73, 0x12, 0x1d, 0x0a, 0x0a, 0x6b, 0x65, 0x79, 0x5f, 0x70, 0x72, 0x65, - 0x66, 0x69, 0x78, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x6b, 0x65, 0x79, 0x50, 0x72, - 0x65, 0x66, 0x69, 0x78, 0x12, 0x27, 0x0a, 0x0f, 0x65, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x5f, 0x66, - 0x61, 0x6c, 0x6c, 0x62, 0x61, 0x63, 0x6b, 0x18, 0x06, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0e, 0x65, - 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x46, 0x61, 0x6c, 0x6c, 0x62, 0x61, 0x63, 0x6b, 0x12, 0x27, 0x0a, - 0x0f, 0x66, 0x61, 0x6c, 0x6c, 0x62, 0x61, 0x63, 0x6b, 0x5f, 0x70, 0x72, 0x65, 0x66, 0x69, 0x78, - 0x18, 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0e, 0x66, 0x61, 0x6c, 0x6c, 0x62, 0x61, 0x63, 0x6b, - 0x50, 0x72, 0x65, 0x66, 0x69, 0x78, 0x1a, 0x5c, 0x0a, 0x0c, 0x53, 0x75, 0x62, 0x73, 0x63, 0x72, - 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x18, 0x0a, 0x07, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, - 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, - 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, - 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x65, 0x78, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x18, - 0x04, 0x20, 0x01, 0x28, 0x08, 0x52, 0x07, 0x65, 0x78, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x4a, 0x04, - 0x08, 0x02, 0x10, 0x03, 0x22, 0x53, 0x0a, 0x09, 0x53, 0x74, 0x6f, 0x72, 0x65, 0x54, 0x79, 0x70, - 0x65, 0x12, 0x0b, 0x0a, 0x07, 0x49, 0x4e, 0x56, 0x41, 0x4c, 0x49, 0x44, 0x10, 0x00, 0x12, 0x09, - 0x0a, 0x05, 0x52, 0x45, 0x44, 0x49, 0x53, 0x10, 0x01, 0x12, 0x0c, 0x0a, 0x08, 0x42, 0x49, 0x47, - 0x51, 0x55, 0x45, 0x52, 0x59, 0x10, 0x02, 0x12, 0x0d, 0x0a, 0x09, 0x43, 0x41, 0x53, 0x53, 0x41, - 0x4e, 0x44, 0x52, 0x41, 0x10, 0x03, 0x12, 0x11, 0x0a, 0x0d, 0x52, 0x45, 0x44, 0x49, 0x53, 0x5f, - 0x43, 0x4c, 0x55, 0x53, 0x54, 0x45, 0x52, 0x10, 0x04, 0x42, 0x08, 0x0a, 0x06, 0x63, 0x6f, 0x6e, - 0x66, 0x69, 0x67, 0x42, 0x53, 0x0a, 0x10, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x70, 0x72, 0x6f, - 0x74, 0x6f, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x42, 0x0a, 0x53, 0x74, 0x6f, 0x72, 0x65, 0x50, 0x72, - 0x6f, 0x74, 0x6f, 0x5a, 0x33, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, - 0x66, 0x65, 0x61, 0x73, 0x74, 0x2d, 0x64, 0x65, 0x76, 0x2f, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2f, - 0x73, 0x64, 0x6b, 0x2f, 0x67, 0x6f, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x73, 0x2f, 0x66, 0x65, - 0x61, 0x73, 0x74, 0x2f, 0x63, 0x6f, 0x72, 0x65, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x6e, 0x64, 0x73, 0x12, 0x1d, 0x0a, 0x0a, 0x6b, 0x65, 0x79, 0x5f, 0x70, 0x72, 0x65, 0x66, 0x69, + 0x78, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x6b, 0x65, 0x79, 0x50, 0x72, 0x65, 0x66, + 0x69, 0x78, 0x12, 0x27, 0x0a, 0x0f, 0x65, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x5f, 0x66, 0x61, 0x6c, + 0x6c, 0x62, 0x61, 0x63, 0x6b, 0x18, 0x06, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0e, 0x65, 0x6e, 0x61, + 0x62, 0x6c, 0x65, 0x46, 0x61, 0x6c, 0x6c, 0x62, 0x61, 0x63, 0x6b, 0x12, 0x27, 0x0a, 0x0f, 0x66, + 0x61, 0x6c, 0x6c, 0x62, 0x61, 0x63, 0x6b, 0x5f, 0x70, 0x72, 0x65, 0x66, 0x69, 0x78, 0x18, 0x07, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x0e, 0x66, 0x61, 0x6c, 0x6c, 0x62, 0x61, 0x63, 0x6b, 0x50, 0x72, + 0x65, 0x66, 0x69, 0x78, 0x1a, 0x5c, 0x0a, 0x0c, 0x53, 0x75, 0x62, 0x73, 0x63, 0x72, 0x69, 0x70, + 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x18, 0x0a, 0x07, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x18, + 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x12, 0x12, + 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, + 0x6d, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x65, 0x78, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x18, 0x04, 0x20, + 0x01, 0x28, 0x08, 0x52, 0x07, 0x65, 0x78, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x4a, 0x04, 0x08, 0x02, + 0x10, 0x03, 0x22, 0x4e, 0x0a, 0x09, 0x53, 0x74, 0x6f, 0x72, 0x65, 0x54, 0x79, 0x70, 0x65, 0x12, + 0x0b, 0x0a, 0x07, 0x49, 0x4e, 0x56, 0x41, 0x4c, 0x49, 0x44, 0x10, 0x00, 0x12, 0x09, 0x0a, 0x05, + 0x52, 0x45, 0x44, 0x49, 0x53, 0x10, 0x01, 0x12, 0x11, 0x0a, 0x0d, 0x52, 0x45, 0x44, 0x49, 0x53, + 0x5f, 0x43, 0x4c, 0x55, 0x53, 0x54, 0x45, 0x52, 0x10, 0x04, 0x22, 0x04, 0x08, 0x02, 0x10, 0x02, + 0x22, 0x04, 0x08, 0x03, 0x10, 0x03, 0x22, 0x04, 0x08, 0x0c, 0x10, 0x0c, 0x22, 0x04, 0x08, 0x0d, + 0x10, 0x0d, 0x42, 0x08, 0x0a, 0x06, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x42, 0x53, 0x0a, 0x10, + 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x63, 0x6f, 0x72, 0x65, + 0x42, 0x0a, 0x53, 0x74, 0x6f, 0x72, 0x65, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x5a, 0x33, 0x67, 0x69, + 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2d, 0x64, + 0x65, 0x76, 0x2f, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2f, 0x73, 0x64, 0x6b, 0x2f, 0x67, 0x6f, 0x2f, + 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x73, 0x2f, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2f, 0x63, 0x6f, 0x72, + 0x65, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( @@ -824,28 +574,24 @@ func file_feast_core_Store_proto_rawDescGZIP() []byte { } var file_feast_core_Store_proto_enumTypes = make([]protoimpl.EnumInfo, 1) -var file_feast_core_Store_proto_msgTypes = make([]protoimpl.MessageInfo, 6) +var file_feast_core_Store_proto_msgTypes = make([]protoimpl.MessageInfo, 4) var file_feast_core_Store_proto_goTypes = []interface{}{ (Store_StoreType)(0), // 0: feast.core.Store.StoreType (*Store)(nil), // 1: feast.core.Store (*Store_RedisConfig)(nil), // 2: feast.core.Store.RedisConfig - (*Store_BigQueryConfig)(nil), // 3: feast.core.Store.BigQueryConfig - (*Store_CassandraConfig)(nil), // 4: feast.core.Store.CassandraConfig - (*Store_RedisClusterConfig)(nil), // 5: feast.core.Store.RedisClusterConfig - (*Store_Subscription)(nil), // 6: feast.core.Store.Subscription + (*Store_RedisClusterConfig)(nil), // 3: feast.core.Store.RedisClusterConfig + (*Store_Subscription)(nil), // 4: feast.core.Store.Subscription } var file_feast_core_Store_proto_depIdxs = []int32{ 0, // 0: feast.core.Store.type:type_name -> feast.core.Store.StoreType - 6, // 1: feast.core.Store.subscriptions:type_name -> feast.core.Store.Subscription + 4, // 1: feast.core.Store.subscriptions:type_name -> feast.core.Store.Subscription 2, // 2: feast.core.Store.redis_config:type_name -> feast.core.Store.RedisConfig - 3, // 3: feast.core.Store.bigquery_config:type_name -> feast.core.Store.BigQueryConfig - 4, // 4: feast.core.Store.cassandra_config:type_name -> feast.core.Store.CassandraConfig - 5, // 5: feast.core.Store.redis_cluster_config:type_name -> feast.core.Store.RedisClusterConfig - 6, // [6:6] is the sub-list for method output_type - 6, // [6:6] is the sub-list for method input_type - 6, // [6:6] is the sub-list for extension type_name - 6, // [6:6] is the sub-list for extension extendee - 0, // [0:6] is the sub-list for field type_name + 3, // 3: feast.core.Store.redis_cluster_config:type_name -> feast.core.Store.RedisClusterConfig + 4, // [4:4] is the sub-list for method output_type + 4, // [4:4] is the sub-list for method input_type + 4, // [4:4] is the sub-list for extension type_name + 4, // [4:4] is the sub-list for extension extendee + 0, // [0:4] is the sub-list for field type_name } func init() { file_feast_core_Store_proto_init() } @@ -879,30 +625,6 @@ func file_feast_core_Store_proto_init() { } } file_feast_core_Store_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*Store_BigQueryConfig); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_feast_core_Store_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*Store_CassandraConfig); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_feast_core_Store_proto_msgTypes[4].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*Store_RedisClusterConfig); i { case 0: return &v.state @@ -914,7 +636,7 @@ func file_feast_core_Store_proto_init() { return nil } } - file_feast_core_Store_proto_msgTypes[5].Exporter = func(v interface{}, i int) interface{} { + file_feast_core_Store_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*Store_Subscription); i { case 0: return &v.state @@ -929,8 +651,6 @@ func file_feast_core_Store_proto_init() { } file_feast_core_Store_proto_msgTypes[0].OneofWrappers = []interface{}{ (*Store_RedisConfig_)(nil), - (*Store_BigqueryConfig)(nil), - (*Store_CassandraConfig_)(nil), (*Store_RedisClusterConfig_)(nil), } type x struct{} @@ -939,7 +659,7 @@ func file_feast_core_Store_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: file_feast_core_Store_proto_rawDesc, NumEnums: 1, - NumMessages: 6, + NumMessages: 4, NumExtensions: 0, NumServices: 0, }, diff --git a/sdk/go/protos/feast/serving/ServingService.pb.go b/sdk/go/protos/feast/serving/ServingService.pb.go index f77930562ed..7d40fa45d59 100644 --- a/sdk/go/protos/feast/serving/ServingService.pb.go +++ b/sdk/go/protos/feast/serving/ServingService.pb.go @@ -16,7 +16,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.25.0 -// protoc v3.10.0 +// protoc v3.12.4 // source: feast/serving/ServingService.proto package serving @@ -24,7 +24,7 @@ package serving import ( context "context" types "github.com/feast-dev/feast/sdk/go/protos/feast/types" - v0 "github.com/feast-dev/feast/sdk/go/protos/tensorflow_metadata/proto/v0" + _ "github.com/feast-dev/feast/sdk/go/protos/tensorflow_metadata/proto/v0" proto "github.com/golang/protobuf/proto" timestamp "github.com/golang/protobuf/ptypes/timestamp" grpc "google.golang.org/grpc" @@ -100,150 +100,6 @@ func (FeastServingType) EnumDescriptor() ([]byte, []int) { return file_feast_serving_ServingService_proto_rawDescGZIP(), []int{0} } -type JobType int32 - -const ( - JobType_JOB_TYPE_INVALID JobType = 0 - JobType_JOB_TYPE_DOWNLOAD JobType = 1 -) - -// Enum value maps for JobType. -var ( - JobType_name = map[int32]string{ - 0: "JOB_TYPE_INVALID", - 1: "JOB_TYPE_DOWNLOAD", - } - JobType_value = map[string]int32{ - "JOB_TYPE_INVALID": 0, - "JOB_TYPE_DOWNLOAD": 1, - } -) - -func (x JobType) Enum() *JobType { - p := new(JobType) - *p = x - return p -} - -func (x JobType) String() string { - return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) -} - -func (JobType) Descriptor() protoreflect.EnumDescriptor { - return file_feast_serving_ServingService_proto_enumTypes[1].Descriptor() -} - -func (JobType) Type() protoreflect.EnumType { - return &file_feast_serving_ServingService_proto_enumTypes[1] -} - -func (x JobType) Number() protoreflect.EnumNumber { - return protoreflect.EnumNumber(x) -} - -// Deprecated: Use JobType.Descriptor instead. -func (JobType) EnumDescriptor() ([]byte, []int) { - return file_feast_serving_ServingService_proto_rawDescGZIP(), []int{1} -} - -type JobStatus int32 - -const ( - JobStatus_JOB_STATUS_INVALID JobStatus = 0 - JobStatus_JOB_STATUS_PENDING JobStatus = 1 - JobStatus_JOB_STATUS_RUNNING JobStatus = 2 - JobStatus_JOB_STATUS_DONE JobStatus = 3 -) - -// Enum value maps for JobStatus. -var ( - JobStatus_name = map[int32]string{ - 0: "JOB_STATUS_INVALID", - 1: "JOB_STATUS_PENDING", - 2: "JOB_STATUS_RUNNING", - 3: "JOB_STATUS_DONE", - } - JobStatus_value = map[string]int32{ - "JOB_STATUS_INVALID": 0, - "JOB_STATUS_PENDING": 1, - "JOB_STATUS_RUNNING": 2, - "JOB_STATUS_DONE": 3, - } -) - -func (x JobStatus) Enum() *JobStatus { - p := new(JobStatus) - *p = x - return p -} - -func (x JobStatus) String() string { - return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) -} - -func (JobStatus) Descriptor() protoreflect.EnumDescriptor { - return file_feast_serving_ServingService_proto_enumTypes[2].Descriptor() -} - -func (JobStatus) Type() protoreflect.EnumType { - return &file_feast_serving_ServingService_proto_enumTypes[2] -} - -func (x JobStatus) Number() protoreflect.EnumNumber { - return protoreflect.EnumNumber(x) -} - -// Deprecated: Use JobStatus.Descriptor instead. -func (JobStatus) EnumDescriptor() ([]byte, []int) { - return file_feast_serving_ServingService_proto_rawDescGZIP(), []int{2} -} - -type DataFormat int32 - -const ( - DataFormat_DATA_FORMAT_INVALID DataFormat = 0 - DataFormat_DATA_FORMAT_AVRO DataFormat = 1 -) - -// Enum value maps for DataFormat. -var ( - DataFormat_name = map[int32]string{ - 0: "DATA_FORMAT_INVALID", - 1: "DATA_FORMAT_AVRO", - } - DataFormat_value = map[string]int32{ - "DATA_FORMAT_INVALID": 0, - "DATA_FORMAT_AVRO": 1, - } -) - -func (x DataFormat) Enum() *DataFormat { - p := new(DataFormat) - *p = x - return p -} - -func (x DataFormat) String() string { - return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) -} - -func (DataFormat) Descriptor() protoreflect.EnumDescriptor { - return file_feast_serving_ServingService_proto_enumTypes[3].Descriptor() -} - -func (DataFormat) Type() protoreflect.EnumType { - return &file_feast_serving_ServingService_proto_enumTypes[3] -} - -func (x DataFormat) Number() protoreflect.EnumNumber { - return protoreflect.EnumNumber(x) -} - -// Deprecated: Use DataFormat.Descriptor instead. -func (DataFormat) EnumDescriptor() ([]byte, []int) { - return file_feast_serving_ServingService_proto_rawDescGZIP(), []int{3} -} - type GetOnlineFeaturesResponse_FieldStatus int32 const ( @@ -292,11 +148,11 @@ func (x GetOnlineFeaturesResponse_FieldStatus) String() string { } func (GetOnlineFeaturesResponse_FieldStatus) Descriptor() protoreflect.EnumDescriptor { - return file_feast_serving_ServingService_proto_enumTypes[4].Descriptor() + return file_feast_serving_ServingService_proto_enumTypes[1].Descriptor() } func (GetOnlineFeaturesResponse_FieldStatus) Type() protoreflect.EnumType { - return &file_feast_serving_ServingService_proto_enumTypes[4] + return &file_feast_serving_ServingService_proto_enumTypes[1] } func (x GetOnlineFeaturesResponse_FieldStatus) Number() protoreflect.EnumNumber { @@ -305,7 +161,7 @@ func (x GetOnlineFeaturesResponse_FieldStatus) Number() protoreflect.EnumNumber // Deprecated: Use GetOnlineFeaturesResponse_FieldStatus.Descriptor instead. func (GetOnlineFeaturesResponse_FieldStatus) EnumDescriptor() ([]byte, []int) { - return file_feast_serving_ServingService_proto_rawDescGZIP(), []int{7, 0} + return file_feast_serving_ServingService_proto_rawDescGZIP(), []int{4, 0} } type GetFeastServingInfoRequest struct { @@ -414,74 +270,6 @@ func (x *GetFeastServingInfoResponse) GetJobStagingLocation() string { return "" } -type FeatureReference struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - // Project name. This field is optional, if unspecified will default to 'default'. - Project string `protobuf:"bytes,1,opt,name=project,proto3" json:"project,omitempty"` - // Feature name - Name string `protobuf:"bytes,2,opt,name=name,proto3" json:"name,omitempty"` - // Feature set name specifying the feature set of this referenced feature. - // This field is optional if the feature referenced is unique across the project - // in which case the feature set would be automatically infered - FeatureSet string `protobuf:"bytes,5,opt,name=feature_set,json=featureSet,proto3" json:"feature_set,omitempty"` -} - -func (x *FeatureReference) Reset() { - *x = FeatureReference{} - if protoimpl.UnsafeEnabled { - mi := &file_feast_serving_ServingService_proto_msgTypes[2] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *FeatureReference) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*FeatureReference) ProtoMessage() {} - -func (x *FeatureReference) ProtoReflect() protoreflect.Message { - mi := &file_feast_serving_ServingService_proto_msgTypes[2] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use FeatureReference.ProtoReflect.Descriptor instead. -func (*FeatureReference) Descriptor() ([]byte, []int) { - return file_feast_serving_ServingService_proto_rawDescGZIP(), []int{2} -} - -func (x *FeatureReference) GetProject() string { - if x != nil { - return x.Project - } - return "" -} - -func (x *FeatureReference) GetName() string { - if x != nil { - return x.Name - } - return "" -} - -func (x *FeatureReference) GetFeatureSet() string { - if x != nil { - return x.FeatureSet - } - return "" -} - type FeatureReferenceV2 struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -496,7 +284,7 @@ type FeatureReferenceV2 struct { func (x *FeatureReferenceV2) Reset() { *x = FeatureReferenceV2{} if protoimpl.UnsafeEnabled { - mi := &file_feast_serving_ServingService_proto_msgTypes[3] + mi := &file_feast_serving_ServingService_proto_msgTypes[2] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -509,7 +297,7 @@ func (x *FeatureReferenceV2) String() string { func (*FeatureReferenceV2) ProtoMessage() {} func (x *FeatureReferenceV2) ProtoReflect() protoreflect.Message { - mi := &file_feast_serving_ServingService_proto_msgTypes[3] + mi := &file_feast_serving_ServingService_proto_msgTypes[2] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -522,7 +310,7 @@ func (x *FeatureReferenceV2) ProtoReflect() protoreflect.Message { // Deprecated: Use FeatureReferenceV2.ProtoReflect.Descriptor instead. func (*FeatureReferenceV2) Descriptor() ([]byte, []int) { - return file_feast_serving_ServingService_proto_rawDescGZIP(), []int{3} + return file_feast_serving_ServingService_proto_rawDescGZIP(), []int{2} } func (x *FeatureReferenceV2) GetFeatureTable() string { @@ -539,86 +327,6 @@ func (x *FeatureReferenceV2) GetName() string { return "" } -type GetOnlineFeaturesRequest struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - // List of features that are being retrieved - Features []*FeatureReference `protobuf:"bytes,4,rep,name=features,proto3" json:"features,omitempty"` - // List of entity rows, containing entity id and timestamp data. - // Used during retrieval of feature rows and for joining feature - // rows into a final dataset - EntityRows []*GetOnlineFeaturesRequest_EntityRow `protobuf:"bytes,2,rep,name=entity_rows,json=entityRows,proto3" json:"entity_rows,omitempty"` - // Option to omit entities from the response. If true, only feature - // values will be returned. - OmitEntitiesInResponse bool `protobuf:"varint,3,opt,name=omit_entities_in_response,json=omitEntitiesInResponse,proto3" json:"omit_entities_in_response,omitempty"` - // Optional field to specify project name override. If specified, uses the - // given project for retrieval. Overrides the projects specified in - // Feature References if both are specified. - Project string `protobuf:"bytes,5,opt,name=project,proto3" json:"project,omitempty"` -} - -func (x *GetOnlineFeaturesRequest) Reset() { - *x = GetOnlineFeaturesRequest{} - if protoimpl.UnsafeEnabled { - mi := &file_feast_serving_ServingService_proto_msgTypes[4] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *GetOnlineFeaturesRequest) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*GetOnlineFeaturesRequest) ProtoMessage() {} - -func (x *GetOnlineFeaturesRequest) ProtoReflect() protoreflect.Message { - mi := &file_feast_serving_ServingService_proto_msgTypes[4] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use GetOnlineFeaturesRequest.ProtoReflect.Descriptor instead. -func (*GetOnlineFeaturesRequest) Descriptor() ([]byte, []int) { - return file_feast_serving_ServingService_proto_rawDescGZIP(), []int{4} -} - -func (x *GetOnlineFeaturesRequest) GetFeatures() []*FeatureReference { - if x != nil { - return x.Features - } - return nil -} - -func (x *GetOnlineFeaturesRequest) GetEntityRows() []*GetOnlineFeaturesRequest_EntityRow { - if x != nil { - return x.EntityRows - } - return nil -} - -func (x *GetOnlineFeaturesRequest) GetOmitEntitiesInResponse() bool { - if x != nil { - return x.OmitEntitiesInResponse - } - return false -} - -func (x *GetOnlineFeaturesRequest) GetProject() string { - if x != nil { - return x.Project - } - return "" -} - type GetOnlineFeaturesRequestV2 struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -639,7 +347,7 @@ type GetOnlineFeaturesRequestV2 struct { func (x *GetOnlineFeaturesRequestV2) Reset() { *x = GetOnlineFeaturesRequestV2{} if protoimpl.UnsafeEnabled { - mi := &file_feast_serving_ServingService_proto_msgTypes[5] + mi := &file_feast_serving_ServingService_proto_msgTypes[3] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -652,7 +360,7 @@ func (x *GetOnlineFeaturesRequestV2) String() string { func (*GetOnlineFeaturesRequestV2) ProtoMessage() {} func (x *GetOnlineFeaturesRequestV2) ProtoReflect() protoreflect.Message { - mi := &file_feast_serving_ServingService_proto_msgTypes[5] + mi := &file_feast_serving_ServingService_proto_msgTypes[3] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -665,7 +373,7 @@ func (x *GetOnlineFeaturesRequestV2) ProtoReflect() protoreflect.Message { // Deprecated: Use GetOnlineFeaturesRequestV2.ProtoReflect.Descriptor instead. func (*GetOnlineFeaturesRequestV2) Descriptor() ([]byte, []int) { - return file_feast_serving_ServingService_proto_rawDescGZIP(), []int{5} + return file_feast_serving_ServingService_proto_rawDescGZIP(), []int{3} } func (x *GetOnlineFeaturesRequestV2) GetFeatures() []*FeatureReferenceV2 { @@ -689,462 +397,32 @@ func (x *GetOnlineFeaturesRequestV2) GetProject() string { return "" } -type GetBatchFeaturesRequest struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - // List of features that are being retrieved - Features []*FeatureReference `protobuf:"bytes,3,rep,name=features,proto3" json:"features,omitempty"` - // Source of the entity dataset containing the timestamps and entity keys to retrieve - // features for. - DatasetSource *DatasetSource `protobuf:"bytes,2,opt,name=dataset_source,json=datasetSource,proto3" json:"dataset_source,omitempty"` - // Compute statistics for the dataset retrieved - ComputeStatistics bool `protobuf:"varint,4,opt,name=compute_statistics,json=computeStatistics,proto3" json:"compute_statistics,omitempty"` -} - -func (x *GetBatchFeaturesRequest) Reset() { - *x = GetBatchFeaturesRequest{} - if protoimpl.UnsafeEnabled { - mi := &file_feast_serving_ServingService_proto_msgTypes[6] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *GetBatchFeaturesRequest) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*GetBatchFeaturesRequest) ProtoMessage() {} - -func (x *GetBatchFeaturesRequest) ProtoReflect() protoreflect.Message { - mi := &file_feast_serving_ServingService_proto_msgTypes[6] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use GetBatchFeaturesRequest.ProtoReflect.Descriptor instead. -func (*GetBatchFeaturesRequest) Descriptor() ([]byte, []int) { - return file_feast_serving_ServingService_proto_rawDescGZIP(), []int{6} -} - -func (x *GetBatchFeaturesRequest) GetFeatures() []*FeatureReference { - if x != nil { - return x.Features - } - return nil -} - -func (x *GetBatchFeaturesRequest) GetDatasetSource() *DatasetSource { - if x != nil { - return x.DatasetSource - } - return nil -} - -func (x *GetBatchFeaturesRequest) GetComputeStatistics() bool { - if x != nil { - return x.ComputeStatistics - } - return false -} - type GetOnlineFeaturesResponse struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - // Feature values retrieved from feast. - FieldValues []*GetOnlineFeaturesResponse_FieldValues `protobuf:"bytes,1,rep,name=field_values,json=fieldValues,proto3" json:"field_values,omitempty"` -} - -func (x *GetOnlineFeaturesResponse) Reset() { - *x = GetOnlineFeaturesResponse{} - if protoimpl.UnsafeEnabled { - mi := &file_feast_serving_ServingService_proto_msgTypes[7] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *GetOnlineFeaturesResponse) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*GetOnlineFeaturesResponse) ProtoMessage() {} - -func (x *GetOnlineFeaturesResponse) ProtoReflect() protoreflect.Message { - mi := &file_feast_serving_ServingService_proto_msgTypes[7] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use GetOnlineFeaturesResponse.ProtoReflect.Descriptor instead. -func (*GetOnlineFeaturesResponse) Descriptor() ([]byte, []int) { - return file_feast_serving_ServingService_proto_rawDescGZIP(), []int{7} -} - -func (x *GetOnlineFeaturesResponse) GetFieldValues() []*GetOnlineFeaturesResponse_FieldValues { - if x != nil { - return x.FieldValues - } - return nil -} - -type GetBatchFeaturesResponse struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - Job *Job `protobuf:"bytes,1,opt,name=job,proto3" json:"job,omitempty"` -} - -func (x *GetBatchFeaturesResponse) Reset() { - *x = GetBatchFeaturesResponse{} - if protoimpl.UnsafeEnabled { - mi := &file_feast_serving_ServingService_proto_msgTypes[8] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *GetBatchFeaturesResponse) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*GetBatchFeaturesResponse) ProtoMessage() {} - -func (x *GetBatchFeaturesResponse) ProtoReflect() protoreflect.Message { - mi := &file_feast_serving_ServingService_proto_msgTypes[8] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use GetBatchFeaturesResponse.ProtoReflect.Descriptor instead. -func (*GetBatchFeaturesResponse) Descriptor() ([]byte, []int) { - return file_feast_serving_ServingService_proto_rawDescGZIP(), []int{8} -} - -func (x *GetBatchFeaturesResponse) GetJob() *Job { - if x != nil { - return x.Job - } - return nil -} - -type GetJobRequest struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - Job *Job `protobuf:"bytes,1,opt,name=job,proto3" json:"job,omitempty"` -} - -func (x *GetJobRequest) Reset() { - *x = GetJobRequest{} - if protoimpl.UnsafeEnabled { - mi := &file_feast_serving_ServingService_proto_msgTypes[9] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *GetJobRequest) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*GetJobRequest) ProtoMessage() {} - -func (x *GetJobRequest) ProtoReflect() protoreflect.Message { - mi := &file_feast_serving_ServingService_proto_msgTypes[9] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use GetJobRequest.ProtoReflect.Descriptor instead. -func (*GetJobRequest) Descriptor() ([]byte, []int) { - return file_feast_serving_ServingService_proto_rawDescGZIP(), []int{9} -} - -func (x *GetJobRequest) GetJob() *Job { - if x != nil { - return x.Job - } - return nil -} - -type GetJobResponse struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - Job *Job `protobuf:"bytes,1,opt,name=job,proto3" json:"job,omitempty"` -} - -func (x *GetJobResponse) Reset() { - *x = GetJobResponse{} - if protoimpl.UnsafeEnabled { - mi := &file_feast_serving_ServingService_proto_msgTypes[10] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *GetJobResponse) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*GetJobResponse) ProtoMessage() {} - -func (x *GetJobResponse) ProtoReflect() protoreflect.Message { - mi := &file_feast_serving_ServingService_proto_msgTypes[10] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use GetJobResponse.ProtoReflect.Descriptor instead. -func (*GetJobResponse) Descriptor() ([]byte, []int) { - return file_feast_serving_ServingService_proto_rawDescGZIP(), []int{10} -} - -func (x *GetJobResponse) GetJob() *Job { - if x != nil { - return x.Job - } - return nil -} - -type Job struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"` - // Output only. The type of the job. - Type JobType `protobuf:"varint,2,opt,name=type,proto3,enum=feast.serving.JobType" json:"type,omitempty"` - // Output only. Current state of the job. - Status JobStatus `protobuf:"varint,3,opt,name=status,proto3,enum=feast.serving.JobStatus" json:"status,omitempty"` - // Output only. If not empty, the job has failed with this error message. - Error string `protobuf:"bytes,4,opt,name=error,proto3" json:"error,omitempty"` - // Output only. The list of URIs for the files to be downloaded or - // uploaded (depends on the job type) for this particular job. - FileUris []string `protobuf:"bytes,5,rep,name=file_uris,json=fileUris,proto3" json:"file_uris,omitempty"` - // Output only. The data format for all the files. - // For CSV format, the files contain both feature values and a column header. - DataFormat DataFormat `protobuf:"varint,6,opt,name=data_format,json=dataFormat,proto3,enum=feast.serving.DataFormat" json:"data_format,omitempty"` - // Output only. The statistics computed over - // the retrieved dataset. Only available for BigQuery stores. - DatasetFeatureStatisticsList *v0.DatasetFeatureStatisticsList `protobuf:"bytes,7,opt,name=dataset_feature_statistics_list,json=datasetFeatureStatisticsList,proto3" json:"dataset_feature_statistics_list,omitempty"` -} - -func (x *Job) Reset() { - *x = Job{} - if protoimpl.UnsafeEnabled { - mi := &file_feast_serving_ServingService_proto_msgTypes[11] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *Job) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*Job) ProtoMessage() {} - -func (x *Job) ProtoReflect() protoreflect.Message { - mi := &file_feast_serving_ServingService_proto_msgTypes[11] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use Job.ProtoReflect.Descriptor instead. -func (*Job) Descriptor() ([]byte, []int) { - return file_feast_serving_ServingService_proto_rawDescGZIP(), []int{11} -} - -func (x *Job) GetId() string { - if x != nil { - return x.Id - } - return "" -} - -func (x *Job) GetType() JobType { - if x != nil { - return x.Type - } - return JobType_JOB_TYPE_INVALID -} - -func (x *Job) GetStatus() JobStatus { - if x != nil { - return x.Status - } - return JobStatus_JOB_STATUS_INVALID -} - -func (x *Job) GetError() string { - if x != nil { - return x.Error - } - return "" -} - -func (x *Job) GetFileUris() []string { - if x != nil { - return x.FileUris - } - return nil -} - -func (x *Job) GetDataFormat() DataFormat { - if x != nil { - return x.DataFormat - } - return DataFormat_DATA_FORMAT_INVALID -} - -func (x *Job) GetDatasetFeatureStatisticsList() *v0.DatasetFeatureStatisticsList { - if x != nil { - return x.DatasetFeatureStatisticsList - } - return nil -} - -type DatasetSource struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - // Types that are assignable to DatasetSource: - // *DatasetSource_FileSource_ - DatasetSource isDatasetSource_DatasetSource `protobuf_oneof:"dataset_source"` -} - -func (x *DatasetSource) Reset() { - *x = DatasetSource{} - if protoimpl.UnsafeEnabled { - mi := &file_feast_serving_ServingService_proto_msgTypes[12] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *DatasetSource) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*DatasetSource) ProtoMessage() {} - -func (x *DatasetSource) ProtoReflect() protoreflect.Message { - mi := &file_feast_serving_ServingService_proto_msgTypes[12] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use DatasetSource.ProtoReflect.Descriptor instead. -func (*DatasetSource) Descriptor() ([]byte, []int) { - return file_feast_serving_ServingService_proto_rawDescGZIP(), []int{12} -} - -func (m *DatasetSource) GetDatasetSource() isDatasetSource_DatasetSource { - if m != nil { - return m.DatasetSource - } - return nil -} - -func (x *DatasetSource) GetFileSource() *DatasetSource_FileSource { - if x, ok := x.GetDatasetSource().(*DatasetSource_FileSource_); ok { - return x.FileSource - } - return nil -} - -type isDatasetSource_DatasetSource interface { - isDatasetSource_DatasetSource() -} - -type DatasetSource_FileSource_ struct { - // File source to load the dataset from. - FileSource *DatasetSource_FileSource `protobuf:"bytes,1,opt,name=file_source,json=fileSource,proto3,oneof"` -} - -func (*DatasetSource_FileSource_) isDatasetSource_DatasetSource() {} - -type GetOnlineFeaturesRequest_EntityRow struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - // Request timestamp of this row. This value will be used, - // together with maxAge, to determine feature staleness. - EntityTimestamp *timestamp.Timestamp `protobuf:"bytes,1,opt,name=entity_timestamp,json=entityTimestamp,proto3" json:"entity_timestamp,omitempty"` - // Map containing mapping of entity name to entity value. - Fields map[string]*types.Value `protobuf:"bytes,2,rep,name=fields,proto3" json:"fields,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` + // Feature values retrieved from feast. + FieldValues []*GetOnlineFeaturesResponse_FieldValues `protobuf:"bytes,1,rep,name=field_values,json=fieldValues,proto3" json:"field_values,omitempty"` } -func (x *GetOnlineFeaturesRequest_EntityRow) Reset() { - *x = GetOnlineFeaturesRequest_EntityRow{} +func (x *GetOnlineFeaturesResponse) Reset() { + *x = GetOnlineFeaturesResponse{} if protoimpl.UnsafeEnabled { - mi := &file_feast_serving_ServingService_proto_msgTypes[13] + mi := &file_feast_serving_ServingService_proto_msgTypes[4] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } } -func (x *GetOnlineFeaturesRequest_EntityRow) String() string { +func (x *GetOnlineFeaturesResponse) String() string { return protoimpl.X.MessageStringOf(x) } -func (*GetOnlineFeaturesRequest_EntityRow) ProtoMessage() {} +func (*GetOnlineFeaturesResponse) ProtoMessage() {} -func (x *GetOnlineFeaturesRequest_EntityRow) ProtoReflect() protoreflect.Message { - mi := &file_feast_serving_ServingService_proto_msgTypes[13] +func (x *GetOnlineFeaturesResponse) ProtoReflect() protoreflect.Message { + mi := &file_feast_serving_ServingService_proto_msgTypes[4] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1155,21 +433,14 @@ func (x *GetOnlineFeaturesRequest_EntityRow) ProtoReflect() protoreflect.Message return mi.MessageOf(x) } -// Deprecated: Use GetOnlineFeaturesRequest_EntityRow.ProtoReflect.Descriptor instead. -func (*GetOnlineFeaturesRequest_EntityRow) Descriptor() ([]byte, []int) { - return file_feast_serving_ServingService_proto_rawDescGZIP(), []int{4, 0} -} - -func (x *GetOnlineFeaturesRequest_EntityRow) GetEntityTimestamp() *timestamp.Timestamp { - if x != nil { - return x.EntityTimestamp - } - return nil +// Deprecated: Use GetOnlineFeaturesResponse.ProtoReflect.Descriptor instead. +func (*GetOnlineFeaturesResponse) Descriptor() ([]byte, []int) { + return file_feast_serving_ServingService_proto_rawDescGZIP(), []int{4} } -func (x *GetOnlineFeaturesRequest_EntityRow) GetFields() map[string]*types.Value { +func (x *GetOnlineFeaturesResponse) GetFieldValues() []*GetOnlineFeaturesResponse_FieldValues { if x != nil { - return x.Fields + return x.FieldValues } return nil } @@ -1189,7 +460,7 @@ type GetOnlineFeaturesRequestV2_EntityRow struct { func (x *GetOnlineFeaturesRequestV2_EntityRow) Reset() { *x = GetOnlineFeaturesRequestV2_EntityRow{} if protoimpl.UnsafeEnabled { - mi := &file_feast_serving_ServingService_proto_msgTypes[15] + mi := &file_feast_serving_ServingService_proto_msgTypes[5] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1202,7 +473,7 @@ func (x *GetOnlineFeaturesRequestV2_EntityRow) String() string { func (*GetOnlineFeaturesRequestV2_EntityRow) ProtoMessage() {} func (x *GetOnlineFeaturesRequestV2_EntityRow) ProtoReflect() protoreflect.Message { - mi := &file_feast_serving_ServingService_proto_msgTypes[15] + mi := &file_feast_serving_ServingService_proto_msgTypes[5] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1215,7 +486,7 @@ func (x *GetOnlineFeaturesRequestV2_EntityRow) ProtoReflect() protoreflect.Messa // Deprecated: Use GetOnlineFeaturesRequestV2_EntityRow.ProtoReflect.Descriptor instead. func (*GetOnlineFeaturesRequestV2_EntityRow) Descriptor() ([]byte, []int) { - return file_feast_serving_ServingService_proto_rawDescGZIP(), []int{5, 0} + return file_feast_serving_ServingService_proto_rawDescGZIP(), []int{3, 0} } func (x *GetOnlineFeaturesRequestV2_EntityRow) GetTimestamp() *timestamp.Timestamp { @@ -1247,7 +518,7 @@ type GetOnlineFeaturesResponse_FieldValues struct { func (x *GetOnlineFeaturesResponse_FieldValues) Reset() { *x = GetOnlineFeaturesResponse_FieldValues{} if protoimpl.UnsafeEnabled { - mi := &file_feast_serving_ServingService_proto_msgTypes[17] + mi := &file_feast_serving_ServingService_proto_msgTypes[7] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1260,7 +531,7 @@ func (x *GetOnlineFeaturesResponse_FieldValues) String() string { func (*GetOnlineFeaturesResponse_FieldValues) ProtoMessage() {} func (x *GetOnlineFeaturesResponse_FieldValues) ProtoReflect() protoreflect.Message { - mi := &file_feast_serving_ServingService_proto_msgTypes[17] + mi := &file_feast_serving_ServingService_proto_msgTypes[7] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1273,7 +544,7 @@ func (x *GetOnlineFeaturesResponse_FieldValues) ProtoReflect() protoreflect.Mess // Deprecated: Use GetOnlineFeaturesResponse_FieldValues.ProtoReflect.Descriptor instead. func (*GetOnlineFeaturesResponse_FieldValues) Descriptor() ([]byte, []int) { - return file_feast_serving_ServingService_proto_rawDescGZIP(), []int{7, 0} + return file_feast_serving_ServingService_proto_rawDescGZIP(), []int{4, 0} } func (x *GetOnlineFeaturesResponse_FieldValues) GetFields() map[string]*types.Value { @@ -1290,65 +561,6 @@ func (x *GetOnlineFeaturesResponse_FieldValues) GetStatuses() map[string]GetOnli return nil } -type DatasetSource_FileSource struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - // URIs to retrieve the dataset from, e.g. gs://bucket/directory/object.csv. Wildcards are - // supported. This data must be compatible to be uploaded to the serving store, and also be - // accessible by this serving instance. - FileUris []string `protobuf:"bytes,1,rep,name=file_uris,json=fileUris,proto3" json:"file_uris,omitempty"` - // Format of the data. Currently only avro is supported. - DataFormat DataFormat `protobuf:"varint,2,opt,name=data_format,json=dataFormat,proto3,enum=feast.serving.DataFormat" json:"data_format,omitempty"` -} - -func (x *DatasetSource_FileSource) Reset() { - *x = DatasetSource_FileSource{} - if protoimpl.UnsafeEnabled { - mi := &file_feast_serving_ServingService_proto_msgTypes[20] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *DatasetSource_FileSource) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*DatasetSource_FileSource) ProtoMessage() {} - -func (x *DatasetSource_FileSource) ProtoReflect() protoreflect.Message { - mi := &file_feast_serving_ServingService_proto_msgTypes[20] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use DatasetSource_FileSource.ProtoReflect.Descriptor instead. -func (*DatasetSource_FileSource) Descriptor() ([]byte, []int) { - return file_feast_serving_ServingService_proto_rawDescGZIP(), []int{12, 0} -} - -func (x *DatasetSource_FileSource) GetFileUris() []string { - if x != nil { - return x.FileUris - } - return nil -} - -func (x *DatasetSource_FileSource) GetDataFormat() DataFormat { - if x != nil { - return x.DataFormat - } - return DataFormat_DATA_FORMAT_INVALID -} - var File_feast_serving_ServingService_proto protoreflect.FileDescriptor var file_feast_serving_ServingService_proto_rawDesc = []byte{ @@ -1373,236 +585,106 @@ var file_feast_serving_ServingService_proto_rawDesc = []byte{ 0x79, 0x70, 0x65, 0x52, 0x04, 0x74, 0x79, 0x70, 0x65, 0x12, 0x30, 0x0a, 0x14, 0x6a, 0x6f, 0x62, 0x5f, 0x73, 0x74, 0x61, 0x67, 0x69, 0x6e, 0x67, 0x5f, 0x6c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x09, 0x52, 0x12, 0x6a, 0x6f, 0x62, 0x53, 0x74, 0x61, 0x67, - 0x69, 0x6e, 0x67, 0x4c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x22, 0x6d, 0x0a, 0x10, 0x46, - 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x12, - 0x18, 0x0a, 0x07, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x07, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, - 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x1f, 0x0a, - 0x0b, 0x66, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x5f, 0x73, 0x65, 0x74, 0x18, 0x05, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x0a, 0x66, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x53, 0x65, 0x74, 0x4a, 0x04, - 0x08, 0x03, 0x10, 0x04, 0x4a, 0x04, 0x08, 0x04, 0x10, 0x05, 0x22, 0x4d, 0x0a, 0x12, 0x46, 0x65, - 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x56, 0x32, - 0x12, 0x23, 0x0a, 0x0d, 0x66, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x5f, 0x74, 0x61, 0x62, 0x6c, - 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x66, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, - 0x54, 0x61, 0x62, 0x6c, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x22, 0xfb, 0x03, 0x0a, 0x18, 0x47, 0x65, - 0x74, 0x4f, 0x6e, 0x6c, 0x69, 0x6e, 0x65, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x52, - 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x3b, 0x0a, 0x08, 0x66, 0x65, 0x61, 0x74, 0x75, 0x72, - 0x65, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1f, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, - 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, 0x2e, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, - 0x52, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x52, 0x08, 0x66, 0x65, 0x61, 0x74, 0x75, - 0x72, 0x65, 0x73, 0x12, 0x52, 0x0a, 0x0b, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x5f, 0x72, 0x6f, - 0x77, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x31, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, - 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, 0x2e, 0x47, 0x65, 0x74, 0x4f, 0x6e, 0x6c, 0x69, - 0x6e, 0x65, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, - 0x74, 0x2e, 0x45, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x52, 0x6f, 0x77, 0x52, 0x0a, 0x65, 0x6e, 0x74, - 0x69, 0x74, 0x79, 0x52, 0x6f, 0x77, 0x73, 0x12, 0x39, 0x0a, 0x19, 0x6f, 0x6d, 0x69, 0x74, 0x5f, - 0x65, 0x6e, 0x74, 0x69, 0x74, 0x69, 0x65, 0x73, 0x5f, 0x69, 0x6e, 0x5f, 0x72, 0x65, 0x73, 0x70, - 0x6f, 0x6e, 0x73, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x08, 0x52, 0x16, 0x6f, 0x6d, 0x69, 0x74, - 0x45, 0x6e, 0x74, 0x69, 0x74, 0x69, 0x65, 0x73, 0x49, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, - 0x73, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x18, 0x05, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x07, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x1a, 0xf8, 0x01, 0x0a, - 0x09, 0x45, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x52, 0x6f, 0x77, 0x12, 0x45, 0x0a, 0x10, 0x65, 0x6e, - 0x74, 0x69, 0x74, 0x79, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x18, 0x01, - 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, - 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, - 0x52, 0x0f, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, - 0x70, 0x12, 0x55, 0x0a, 0x06, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, - 0x0b, 0x32, 0x3d, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x6e, - 0x67, 0x2e, 0x47, 0x65, 0x74, 0x4f, 0x6e, 0x6c, 0x69, 0x6e, 0x65, 0x46, 0x65, 0x61, 0x74, 0x75, - 0x72, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x45, 0x6e, 0x74, 0x69, 0x74, - 0x79, 0x52, 0x6f, 0x77, 0x2e, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, - 0x52, 0x06, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x73, 0x1a, 0x4d, 0x0a, 0x0b, 0x46, 0x69, 0x65, 0x6c, - 0x64, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x28, 0x0a, 0x05, 0x76, 0x61, 0x6c, - 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, - 0x2e, 0x74, 0x79, 0x70, 0x65, 0x73, 0x2e, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x52, 0x05, 0x76, 0x61, - 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0xbb, 0x03, 0x0a, 0x1a, 0x47, 0x65, 0x74, 0x4f, - 0x6e, 0x6c, 0x69, 0x6e, 0x65, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x52, 0x65, 0x71, - 0x75, 0x65, 0x73, 0x74, 0x56, 0x32, 0x12, 0x3d, 0x0a, 0x08, 0x66, 0x65, 0x61, 0x74, 0x75, 0x72, - 0x65, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x21, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, - 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, 0x2e, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, - 0x52, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x56, 0x32, 0x52, 0x08, 0x66, 0x65, 0x61, - 0x74, 0x75, 0x72, 0x65, 0x73, 0x12, 0x54, 0x0a, 0x0b, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x5f, - 0x72, 0x6f, 0x77, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x33, 0x2e, 0x66, 0x65, 0x61, - 0x73, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, 0x2e, 0x47, 0x65, 0x74, 0x4f, 0x6e, - 0x6c, 0x69, 0x6e, 0x65, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, - 0x65, 0x73, 0x74, 0x56, 0x32, 0x2e, 0x45, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x52, 0x6f, 0x77, 0x52, - 0x0a, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x52, 0x6f, 0x77, 0x73, 0x12, 0x18, 0x0a, 0x07, 0x70, - 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x70, 0x72, - 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x1a, 0xed, 0x01, 0x0a, 0x09, 0x45, 0x6e, 0x74, 0x69, 0x74, 0x79, - 0x52, 0x6f, 0x77, 0x12, 0x38, 0x0a, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, - 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, - 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, - 0x6d, 0x70, 0x52, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x12, 0x57, 0x0a, - 0x06, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x3f, 0x2e, + 0x69, 0x6e, 0x67, 0x4c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x22, 0x4d, 0x0a, 0x12, 0x46, + 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x56, + 0x32, 0x12, 0x23, 0x0a, 0x0d, 0x66, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x5f, 0x74, 0x61, 0x62, + 0x6c, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x66, 0x65, 0x61, 0x74, 0x75, 0x72, + 0x65, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x22, 0xbb, 0x03, 0x0a, 0x1a, 0x47, + 0x65, 0x74, 0x4f, 0x6e, 0x6c, 0x69, 0x6e, 0x65, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x56, 0x32, 0x12, 0x3d, 0x0a, 0x08, 0x66, 0x65, 0x61, + 0x74, 0x75, 0x72, 0x65, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x21, 0x2e, 0x66, 0x65, + 0x61, 0x73, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, 0x2e, 0x46, 0x65, 0x61, 0x74, + 0x75, 0x72, 0x65, 0x52, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x56, 0x32, 0x52, 0x08, + 0x66, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x12, 0x54, 0x0a, 0x0b, 0x65, 0x6e, 0x74, 0x69, + 0x74, 0x79, 0x5f, 0x72, 0x6f, 0x77, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x33, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, 0x2e, 0x47, 0x65, 0x74, 0x4f, 0x6e, 0x6c, 0x69, 0x6e, 0x65, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x56, 0x32, 0x2e, 0x45, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x52, - 0x6f, 0x77, 0x2e, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x06, - 0x66, 0x69, 0x65, 0x6c, 0x64, 0x73, 0x1a, 0x4d, 0x0a, 0x0b, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x73, - 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x28, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, - 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x74, - 0x79, 0x70, 0x65, 0x73, 0x2e, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, - 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0xca, 0x01, 0x0a, 0x17, 0x47, 0x65, 0x74, 0x42, 0x61, 0x74, - 0x63, 0x68, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, - 0x74, 0x12, 0x3b, 0x0a, 0x08, 0x66, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x18, 0x03, 0x20, - 0x03, 0x28, 0x0b, 0x32, 0x1f, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, - 0x69, 0x6e, 0x67, 0x2e, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x65, 0x66, 0x65, 0x72, - 0x65, 0x6e, 0x63, 0x65, 0x52, 0x08, 0x66, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x12, 0x43, - 0x0a, 0x0e, 0x64, 0x61, 0x74, 0x61, 0x73, 0x65, 0x74, 0x5f, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, - 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x73, - 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, 0x2e, 0x44, 0x61, 0x74, 0x61, 0x73, 0x65, 0x74, 0x53, 0x6f, - 0x75, 0x72, 0x63, 0x65, 0x52, 0x0d, 0x64, 0x61, 0x74, 0x61, 0x73, 0x65, 0x74, 0x53, 0x6f, 0x75, - 0x72, 0x63, 0x65, 0x12, 0x2d, 0x0a, 0x12, 0x63, 0x6f, 0x6d, 0x70, 0x75, 0x74, 0x65, 0x5f, 0x73, - 0x74, 0x61, 0x74, 0x69, 0x73, 0x74, 0x69, 0x63, 0x73, 0x18, 0x04, 0x20, 0x01, 0x28, 0x08, 0x52, - 0x11, 0x63, 0x6f, 0x6d, 0x70, 0x75, 0x74, 0x65, 0x53, 0x74, 0x61, 0x74, 0x69, 0x73, 0x74, 0x69, - 0x63, 0x73, 0x22, 0xdd, 0x04, 0x0a, 0x19, 0x47, 0x65, 0x74, 0x4f, 0x6e, 0x6c, 0x69, 0x6e, 0x65, - 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, - 0x12, 0x57, 0x0a, 0x0c, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x5f, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x73, - 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x34, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x73, - 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, 0x2e, 0x47, 0x65, 0x74, 0x4f, 0x6e, 0x6c, 0x69, 0x6e, 0x65, - 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, - 0x2e, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x73, 0x52, 0x0b, 0x66, 0x69, - 0x65, 0x6c, 0x64, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x73, 0x1a, 0x89, 0x03, 0x0a, 0x0b, 0x46, 0x69, - 0x65, 0x6c, 0x64, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x73, 0x12, 0x58, 0x0a, 0x06, 0x66, 0x69, 0x65, - 0x6c, 0x64, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x40, 0x2e, 0x66, 0x65, 0x61, 0x73, - 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, 0x2e, 0x47, 0x65, 0x74, 0x4f, 0x6e, 0x6c, - 0x69, 0x6e, 0x65, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, - 0x6e, 0x73, 0x65, 0x2e, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x73, 0x2e, - 0x46, 0x69, 0x65, 0x6c, 0x64, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x06, 0x66, 0x69, 0x65, - 0x6c, 0x64, 0x73, 0x12, 0x5e, 0x0a, 0x08, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x65, 0x73, 0x18, - 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x42, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x73, 0x65, - 0x72, 0x76, 0x69, 0x6e, 0x67, 0x2e, 0x47, 0x65, 0x74, 0x4f, 0x6e, 0x6c, 0x69, 0x6e, 0x65, 0x46, - 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, - 0x46, 0x69, 0x65, 0x6c, 0x64, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x73, 0x2e, 0x53, 0x74, 0x61, 0x74, - 0x75, 0x73, 0x65, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x08, 0x73, 0x74, 0x61, 0x74, 0x75, - 0x73, 0x65, 0x73, 0x1a, 0x4d, 0x0a, 0x0b, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x73, 0x45, 0x6e, 0x74, - 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x03, 0x6b, 0x65, 0x79, 0x12, 0x28, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, - 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x74, 0x79, 0x70, 0x65, - 0x73, 0x2e, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, - 0x38, 0x01, 0x1a, 0x71, 0x0a, 0x0d, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x65, 0x73, 0x45, 0x6e, - 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x4a, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, - 0x20, 0x01, 0x28, 0x0e, 0x32, 0x34, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x73, 0x65, 0x72, - 0x76, 0x69, 0x6e, 0x67, 0x2e, 0x47, 0x65, 0x74, 0x4f, 0x6e, 0x6c, 0x69, 0x6e, 0x65, 0x46, 0x65, - 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x46, - 0x69, 0x65, 0x6c, 0x64, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, - 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x5b, 0x0a, 0x0b, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x53, 0x74, - 0x61, 0x74, 0x75, 0x73, 0x12, 0x0b, 0x0a, 0x07, 0x49, 0x4e, 0x56, 0x41, 0x4c, 0x49, 0x44, 0x10, - 0x00, 0x12, 0x0b, 0x0a, 0x07, 0x50, 0x52, 0x45, 0x53, 0x45, 0x4e, 0x54, 0x10, 0x01, 0x12, 0x0e, - 0x0a, 0x0a, 0x4e, 0x55, 0x4c, 0x4c, 0x5f, 0x56, 0x41, 0x4c, 0x55, 0x45, 0x10, 0x02, 0x12, 0x0d, - 0x0a, 0x09, 0x4e, 0x4f, 0x54, 0x5f, 0x46, 0x4f, 0x55, 0x4e, 0x44, 0x10, 0x03, 0x12, 0x13, 0x0a, - 0x0f, 0x4f, 0x55, 0x54, 0x53, 0x49, 0x44, 0x45, 0x5f, 0x4d, 0x41, 0x58, 0x5f, 0x41, 0x47, 0x45, - 0x10, 0x04, 0x22, 0x40, 0x0a, 0x18, 0x47, 0x65, 0x74, 0x42, 0x61, 0x74, 0x63, 0x68, 0x46, 0x65, - 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x24, - 0x0a, 0x03, 0x6a, 0x6f, 0x62, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x66, 0x65, - 0x61, 0x73, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, 0x2e, 0x4a, 0x6f, 0x62, 0x52, - 0x03, 0x6a, 0x6f, 0x62, 0x22, 0x35, 0x0a, 0x0d, 0x47, 0x65, 0x74, 0x4a, 0x6f, 0x62, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x24, 0x0a, 0x03, 0x6a, 0x6f, 0x62, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, - 0x6e, 0x67, 0x2e, 0x4a, 0x6f, 0x62, 0x52, 0x03, 0x6a, 0x6f, 0x62, 0x22, 0x36, 0x0a, 0x0e, 0x47, - 0x65, 0x74, 0x4a, 0x6f, 0x62, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x24, 0x0a, - 0x03, 0x6a, 0x6f, 0x62, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x66, 0x65, 0x61, - 0x73, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, 0x2e, 0x4a, 0x6f, 0x62, 0x52, 0x03, - 0x6a, 0x6f, 0x62, 0x22, 0xdf, 0x02, 0x0a, 0x03, 0x4a, 0x6f, 0x62, 0x12, 0x0e, 0x0a, 0x02, 0x69, - 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x12, 0x2a, 0x0a, 0x04, 0x74, - 0x79, 0x70, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x16, 0x2e, 0x66, 0x65, 0x61, 0x73, - 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, 0x2e, 0x4a, 0x6f, 0x62, 0x54, 0x79, 0x70, - 0x65, 0x52, 0x04, 0x74, 0x79, 0x70, 0x65, 0x12, 0x30, 0x0a, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, - 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x18, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, - 0x73, 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, 0x2e, 0x4a, 0x6f, 0x62, 0x53, 0x74, 0x61, 0x74, 0x75, - 0x73, 0x52, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x72, 0x72, - 0x6f, 0x72, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x12, - 0x1b, 0x0a, 0x09, 0x66, 0x69, 0x6c, 0x65, 0x5f, 0x75, 0x72, 0x69, 0x73, 0x18, 0x05, 0x20, 0x03, - 0x28, 0x09, 0x52, 0x08, 0x66, 0x69, 0x6c, 0x65, 0x55, 0x72, 0x69, 0x73, 0x12, 0x3a, 0x0a, 0x0b, - 0x64, 0x61, 0x74, 0x61, 0x5f, 0x66, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x18, 0x06, 0x20, 0x01, 0x28, - 0x0e, 0x32, 0x19, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x6e, - 0x67, 0x2e, 0x44, 0x61, 0x74, 0x61, 0x46, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x52, 0x0a, 0x64, 0x61, - 0x74, 0x61, 0x46, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x12, 0x7b, 0x0a, 0x1f, 0x64, 0x61, 0x74, 0x61, - 0x73, 0x65, 0x74, 0x5f, 0x66, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x5f, 0x73, 0x74, 0x61, 0x74, - 0x69, 0x73, 0x74, 0x69, 0x63, 0x73, 0x5f, 0x6c, 0x69, 0x73, 0x74, 0x18, 0x07, 0x20, 0x01, 0x28, - 0x0b, 0x32, 0x34, 0x2e, 0x74, 0x65, 0x6e, 0x73, 0x6f, 0x72, 0x66, 0x6c, 0x6f, 0x77, 0x2e, 0x6d, - 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x76, 0x30, 0x2e, 0x44, 0x61, 0x74, 0x61, 0x73, - 0x65, 0x74, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x53, 0x74, 0x61, 0x74, 0x69, 0x73, 0x74, - 0x69, 0x63, 0x73, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x1c, 0x64, 0x61, 0x74, 0x61, 0x73, 0x65, 0x74, - 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x53, 0x74, 0x61, 0x74, 0x69, 0x73, 0x74, 0x69, 0x63, - 0x73, 0x4c, 0x69, 0x73, 0x74, 0x22, 0xd4, 0x01, 0x0a, 0x0d, 0x44, 0x61, 0x74, 0x61, 0x73, 0x65, - 0x74, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x12, 0x4a, 0x0a, 0x0b, 0x66, 0x69, 0x6c, 0x65, 0x5f, - 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x27, 0x2e, 0x66, - 0x65, 0x61, 0x73, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, 0x2e, 0x44, 0x61, 0x74, - 0x61, 0x73, 0x65, 0x74, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x2e, 0x46, 0x69, 0x6c, 0x65, 0x53, - 0x6f, 0x75, 0x72, 0x63, 0x65, 0x48, 0x00, 0x52, 0x0a, 0x66, 0x69, 0x6c, 0x65, 0x53, 0x6f, 0x75, - 0x72, 0x63, 0x65, 0x1a, 0x65, 0x0a, 0x0a, 0x46, 0x69, 0x6c, 0x65, 0x53, 0x6f, 0x75, 0x72, 0x63, - 0x65, 0x12, 0x1b, 0x0a, 0x09, 0x66, 0x69, 0x6c, 0x65, 0x5f, 0x75, 0x72, 0x69, 0x73, 0x18, 0x01, - 0x20, 0x03, 0x28, 0x09, 0x52, 0x08, 0x66, 0x69, 0x6c, 0x65, 0x55, 0x72, 0x69, 0x73, 0x12, 0x3a, - 0x0a, 0x0b, 0x64, 0x61, 0x74, 0x61, 0x5f, 0x66, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x18, 0x02, 0x20, - 0x01, 0x28, 0x0e, 0x32, 0x19, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, - 0x69, 0x6e, 0x67, 0x2e, 0x44, 0x61, 0x74, 0x61, 0x46, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x52, 0x0a, - 0x64, 0x61, 0x74, 0x61, 0x46, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x42, 0x10, 0x0a, 0x0e, 0x64, 0x61, - 0x74, 0x61, 0x73, 0x65, 0x74, 0x5f, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x2a, 0x6f, 0x0a, 0x10, - 0x46, 0x65, 0x61, 0x73, 0x74, 0x53, 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, 0x54, 0x79, 0x70, 0x65, - 0x12, 0x1e, 0x0a, 0x1a, 0x46, 0x45, 0x41, 0x53, 0x54, 0x5f, 0x53, 0x45, 0x52, 0x56, 0x49, 0x4e, - 0x47, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x49, 0x4e, 0x56, 0x41, 0x4c, 0x49, 0x44, 0x10, 0x00, - 0x12, 0x1d, 0x0a, 0x19, 0x46, 0x45, 0x41, 0x53, 0x54, 0x5f, 0x53, 0x45, 0x52, 0x56, 0x49, 0x4e, - 0x47, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x4f, 0x4e, 0x4c, 0x49, 0x4e, 0x45, 0x10, 0x01, 0x12, - 0x1c, 0x0a, 0x18, 0x46, 0x45, 0x41, 0x53, 0x54, 0x5f, 0x53, 0x45, 0x52, 0x56, 0x49, 0x4e, 0x47, - 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x42, 0x41, 0x54, 0x43, 0x48, 0x10, 0x02, 0x2a, 0x36, 0x0a, - 0x07, 0x4a, 0x6f, 0x62, 0x54, 0x79, 0x70, 0x65, 0x12, 0x14, 0x0a, 0x10, 0x4a, 0x4f, 0x42, 0x5f, - 0x54, 0x59, 0x50, 0x45, 0x5f, 0x49, 0x4e, 0x56, 0x41, 0x4c, 0x49, 0x44, 0x10, 0x00, 0x12, 0x15, - 0x0a, 0x11, 0x4a, 0x4f, 0x42, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x44, 0x4f, 0x57, 0x4e, 0x4c, - 0x4f, 0x41, 0x44, 0x10, 0x01, 0x2a, 0x68, 0x0a, 0x09, 0x4a, 0x6f, 0x62, 0x53, 0x74, 0x61, 0x74, - 0x75, 0x73, 0x12, 0x16, 0x0a, 0x12, 0x4a, 0x4f, 0x42, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, - 0x5f, 0x49, 0x4e, 0x56, 0x41, 0x4c, 0x49, 0x44, 0x10, 0x00, 0x12, 0x16, 0x0a, 0x12, 0x4a, 0x4f, - 0x42, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, 0x5f, 0x50, 0x45, 0x4e, 0x44, 0x49, 0x4e, 0x47, - 0x10, 0x01, 0x12, 0x16, 0x0a, 0x12, 0x4a, 0x4f, 0x42, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, - 0x5f, 0x52, 0x55, 0x4e, 0x4e, 0x49, 0x4e, 0x47, 0x10, 0x02, 0x12, 0x13, 0x0a, 0x0f, 0x4a, 0x4f, - 0x42, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, 0x5f, 0x44, 0x4f, 0x4e, 0x45, 0x10, 0x03, 0x2a, - 0x3b, 0x0a, 0x0a, 0x44, 0x61, 0x74, 0x61, 0x46, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x12, 0x17, 0x0a, - 0x13, 0x44, 0x41, 0x54, 0x41, 0x5f, 0x46, 0x4f, 0x52, 0x4d, 0x41, 0x54, 0x5f, 0x49, 0x4e, 0x56, - 0x41, 0x4c, 0x49, 0x44, 0x10, 0x00, 0x12, 0x14, 0x0a, 0x10, 0x44, 0x41, 0x54, 0x41, 0x5f, 0x46, - 0x4f, 0x52, 0x4d, 0x41, 0x54, 0x5f, 0x41, 0x56, 0x52, 0x4f, 0x10, 0x01, 0x32, 0xfe, 0x03, 0x0a, - 0x0e, 0x53, 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, - 0x6c, 0x0a, 0x13, 0x47, 0x65, 0x74, 0x46, 0x65, 0x61, 0x73, 0x74, 0x53, 0x65, 0x72, 0x76, 0x69, - 0x6e, 0x67, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x29, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x73, - 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, 0x2e, 0x47, 0x65, 0x74, 0x46, 0x65, 0x61, 0x73, 0x74, 0x53, - 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, - 0x74, 0x1a, 0x2a, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x6e, - 0x67, 0x2e, 0x47, 0x65, 0x74, 0x46, 0x65, 0x61, 0x73, 0x74, 0x53, 0x65, 0x72, 0x76, 0x69, 0x6e, - 0x67, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x66, 0x0a, - 0x11, 0x47, 0x65, 0x74, 0x4f, 0x6e, 0x6c, 0x69, 0x6e, 0x65, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, - 0x65, 0x73, 0x12, 0x27, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, - 0x6e, 0x67, 0x2e, 0x47, 0x65, 0x74, 0x4f, 0x6e, 0x6c, 0x69, 0x6e, 0x65, 0x46, 0x65, 0x61, 0x74, - 0x75, 0x72, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x28, 0x2e, 0x66, 0x65, + 0x6f, 0x77, 0x52, 0x0a, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x52, 0x6f, 0x77, 0x73, 0x12, 0x18, + 0x0a, 0x07, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x07, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x1a, 0xed, 0x01, 0x0a, 0x09, 0x45, 0x6e, 0x74, + 0x69, 0x74, 0x79, 0x52, 0x6f, 0x77, 0x12, 0x38, 0x0a, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, + 0x61, 0x6d, 0x70, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, + 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, + 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, + 0x12, 0x57, 0x0a, 0x06, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, + 0x32, 0x3f, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, + 0x2e, 0x47, 0x65, 0x74, 0x4f, 0x6e, 0x6c, 0x69, 0x6e, 0x65, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, + 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x56, 0x32, 0x2e, 0x45, 0x6e, 0x74, 0x69, + 0x74, 0x79, 0x52, 0x6f, 0x77, 0x2e, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x73, 0x45, 0x6e, 0x74, 0x72, + 0x79, 0x52, 0x06, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x73, 0x1a, 0x4d, 0x0a, 0x0b, 0x46, 0x69, 0x65, + 0x6c, 0x64, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x28, 0x0a, 0x05, 0x76, 0x61, + 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x66, 0x65, 0x61, 0x73, + 0x74, 0x2e, 0x74, 0x79, 0x70, 0x65, 0x73, 0x2e, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x52, 0x05, 0x76, + 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0xdd, 0x04, 0x0a, 0x19, 0x47, 0x65, 0x74, + 0x4f, 0x6e, 0x6c, 0x69, 0x6e, 0x65, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x52, 0x65, + 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x57, 0x0a, 0x0c, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x5f, + 0x76, 0x61, 0x6c, 0x75, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x34, 0x2e, 0x66, + 0x65, 0x61, 0x73, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, 0x2e, 0x47, 0x65, 0x74, + 0x4f, 0x6e, 0x6c, 0x69, 0x6e, 0x65, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x52, 0x65, + 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x56, 0x61, 0x6c, 0x75, + 0x65, 0x73, 0x52, 0x0b, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x73, 0x1a, + 0x89, 0x03, 0x0a, 0x0b, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x73, 0x12, + 0x58, 0x0a, 0x06, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, + 0x40, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, 0x2e, + 0x47, 0x65, 0x74, 0x4f, 0x6e, 0x6c, 0x69, 0x6e, 0x65, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, + 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x56, + 0x61, 0x6c, 0x75, 0x65, 0x73, 0x2e, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x73, 0x45, 0x6e, 0x74, 0x72, + 0x79, 0x52, 0x06, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x73, 0x12, 0x5e, 0x0a, 0x08, 0x73, 0x74, 0x61, + 0x74, 0x75, 0x73, 0x65, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x42, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, 0x2e, 0x47, 0x65, 0x74, 0x4f, 0x6e, 0x6c, 0x69, 0x6e, 0x65, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x52, 0x65, 0x73, - 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x6a, 0x0a, 0x13, 0x47, 0x65, 0x74, 0x4f, 0x6e, 0x6c, 0x69, - 0x6e, 0x65, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x56, 0x32, 0x12, 0x29, 0x2e, 0x66, + 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x56, 0x61, 0x6c, 0x75, 0x65, + 0x73, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x65, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, + 0x08, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x65, 0x73, 0x1a, 0x4d, 0x0a, 0x0b, 0x46, 0x69, 0x65, + 0x6c, 0x64, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x28, 0x0a, 0x05, 0x76, 0x61, + 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x66, 0x65, 0x61, 0x73, + 0x74, 0x2e, 0x74, 0x79, 0x70, 0x65, 0x73, 0x2e, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x52, 0x05, 0x76, + 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x1a, 0x71, 0x0a, 0x0d, 0x53, 0x74, 0x61, 0x74, + 0x75, 0x73, 0x65, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x4a, 0x0a, 0x05, 0x76, + 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x34, 0x2e, 0x66, 0x65, 0x61, + 0x73, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, 0x2e, 0x47, 0x65, 0x74, 0x4f, 0x6e, + 0x6c, 0x69, 0x6e, 0x65, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, + 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, + 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x5b, 0x0a, 0x0b, 0x46, + 0x69, 0x65, 0x6c, 0x64, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x0b, 0x0a, 0x07, 0x49, 0x4e, + 0x56, 0x41, 0x4c, 0x49, 0x44, 0x10, 0x00, 0x12, 0x0b, 0x0a, 0x07, 0x50, 0x52, 0x45, 0x53, 0x45, + 0x4e, 0x54, 0x10, 0x01, 0x12, 0x0e, 0x0a, 0x0a, 0x4e, 0x55, 0x4c, 0x4c, 0x5f, 0x56, 0x41, 0x4c, + 0x55, 0x45, 0x10, 0x02, 0x12, 0x0d, 0x0a, 0x09, 0x4e, 0x4f, 0x54, 0x5f, 0x46, 0x4f, 0x55, 0x4e, + 0x44, 0x10, 0x03, 0x12, 0x13, 0x0a, 0x0f, 0x4f, 0x55, 0x54, 0x53, 0x49, 0x44, 0x45, 0x5f, 0x4d, + 0x41, 0x58, 0x5f, 0x41, 0x47, 0x45, 0x10, 0x04, 0x2a, 0x6f, 0x0a, 0x10, 0x46, 0x65, 0x61, 0x73, + 0x74, 0x53, 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, 0x54, 0x79, 0x70, 0x65, 0x12, 0x1e, 0x0a, 0x1a, + 0x46, 0x45, 0x41, 0x53, 0x54, 0x5f, 0x53, 0x45, 0x52, 0x56, 0x49, 0x4e, 0x47, 0x5f, 0x54, 0x59, + 0x50, 0x45, 0x5f, 0x49, 0x4e, 0x56, 0x41, 0x4c, 0x49, 0x44, 0x10, 0x00, 0x12, 0x1d, 0x0a, 0x19, + 0x46, 0x45, 0x41, 0x53, 0x54, 0x5f, 0x53, 0x45, 0x52, 0x56, 0x49, 0x4e, 0x47, 0x5f, 0x54, 0x59, + 0x50, 0x45, 0x5f, 0x4f, 0x4e, 0x4c, 0x49, 0x4e, 0x45, 0x10, 0x01, 0x12, 0x1c, 0x0a, 0x18, 0x46, + 0x45, 0x41, 0x53, 0x54, 0x5f, 0x53, 0x45, 0x52, 0x56, 0x49, 0x4e, 0x47, 0x5f, 0x54, 0x59, 0x50, + 0x45, 0x5f, 0x42, 0x41, 0x54, 0x43, 0x48, 0x10, 0x02, 0x32, 0xea, 0x01, 0x0a, 0x0e, 0x53, 0x65, + 0x72, 0x76, 0x69, 0x6e, 0x67, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x6c, 0x0a, 0x13, + 0x47, 0x65, 0x74, 0x46, 0x65, 0x61, 0x73, 0x74, 0x53, 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, 0x49, + 0x6e, 0x66, 0x6f, 0x12, 0x29, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, + 0x69, 0x6e, 0x67, 0x2e, 0x47, 0x65, 0x74, 0x46, 0x65, 0x61, 0x73, 0x74, 0x53, 0x65, 0x72, 0x76, + 0x69, 0x6e, 0x67, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x2a, + 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, 0x2e, 0x47, + 0x65, 0x74, 0x46, 0x65, 0x61, 0x73, 0x74, 0x53, 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, 0x49, 0x6e, + 0x66, 0x6f, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x6a, 0x0a, 0x13, 0x47, 0x65, + 0x74, 0x4f, 0x6e, 0x6c, 0x69, 0x6e, 0x65, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x56, + 0x32, 0x12, 0x29, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x6e, + 0x67, 0x2e, 0x47, 0x65, 0x74, 0x4f, 0x6e, 0x6c, 0x69, 0x6e, 0x65, 0x46, 0x65, 0x61, 0x74, 0x75, + 0x72, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x56, 0x32, 0x1a, 0x28, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, 0x2e, 0x47, 0x65, 0x74, 0x4f, 0x6e, 0x6c, 0x69, 0x6e, 0x65, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x56, 0x32, 0x1a, 0x28, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, - 0x73, 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, 0x2e, 0x47, 0x65, 0x74, 0x4f, 0x6e, 0x6c, 0x69, 0x6e, - 0x65, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, - 0x65, 0x12, 0x63, 0x0a, 0x10, 0x47, 0x65, 0x74, 0x42, 0x61, 0x74, 0x63, 0x68, 0x46, 0x65, 0x61, - 0x74, 0x75, 0x72, 0x65, 0x73, 0x12, 0x26, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x73, 0x65, - 0x72, 0x76, 0x69, 0x6e, 0x67, 0x2e, 0x47, 0x65, 0x74, 0x42, 0x61, 0x74, 0x63, 0x68, 0x46, 0x65, - 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x27, 0x2e, - 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, 0x2e, 0x47, 0x65, - 0x74, 0x42, 0x61, 0x74, 0x63, 0x68, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x52, 0x65, - 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x45, 0x0a, 0x06, 0x47, 0x65, 0x74, 0x4a, 0x6f, 0x62, - 0x12, 0x1c, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, - 0x2e, 0x47, 0x65, 0x74, 0x4a, 0x6f, 0x62, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1d, - 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, 0x2e, 0x47, - 0x65, 0x74, 0x4a, 0x6f, 0x62, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x42, 0x5e, 0x0a, - 0x13, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x73, 0x65, 0x72, - 0x76, 0x69, 0x6e, 0x67, 0x42, 0x0f, 0x53, 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, 0x41, 0x50, 0x49, - 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x5a, 0x36, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, - 0x6d, 0x2f, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2d, 0x64, 0x65, 0x76, 0x2f, 0x66, 0x65, 0x61, 0x73, - 0x74, 0x2f, 0x73, 0x64, 0x6b, 0x2f, 0x67, 0x6f, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x73, 0x2f, - 0x66, 0x65, 0x61, 0x73, 0x74, 0x2f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, 0x62, 0x06, 0x70, - 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x42, 0x5e, 0x0a, 0x13, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, + 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, 0x42, 0x0f, 0x53, + 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, 0x41, 0x50, 0x49, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x5a, 0x36, + 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x66, 0x65, 0x61, 0x73, 0x74, + 0x2d, 0x64, 0x65, 0x76, 0x2f, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2f, 0x73, 0x64, 0x6b, 0x2f, 0x67, + 0x6f, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x73, 0x2f, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2f, 0x73, + 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( @@ -1617,82 +699,45 @@ func file_feast_serving_ServingService_proto_rawDescGZIP() []byte { return file_feast_serving_ServingService_proto_rawDescData } -var file_feast_serving_ServingService_proto_enumTypes = make([]protoimpl.EnumInfo, 5) -var file_feast_serving_ServingService_proto_msgTypes = make([]protoimpl.MessageInfo, 21) +var file_feast_serving_ServingService_proto_enumTypes = make([]protoimpl.EnumInfo, 2) +var file_feast_serving_ServingService_proto_msgTypes = make([]protoimpl.MessageInfo, 10) var file_feast_serving_ServingService_proto_goTypes = []interface{}{ - (FeastServingType)(0), // 0: feast.serving.FeastServingType - (JobType)(0), // 1: feast.serving.JobType - (JobStatus)(0), // 2: feast.serving.JobStatus - (DataFormat)(0), // 3: feast.serving.DataFormat - (GetOnlineFeaturesResponse_FieldStatus)(0), // 4: feast.serving.GetOnlineFeaturesResponse.FieldStatus - (*GetFeastServingInfoRequest)(nil), // 5: feast.serving.GetFeastServingInfoRequest - (*GetFeastServingInfoResponse)(nil), // 6: feast.serving.GetFeastServingInfoResponse - (*FeatureReference)(nil), // 7: feast.serving.FeatureReference - (*FeatureReferenceV2)(nil), // 8: feast.serving.FeatureReferenceV2 - (*GetOnlineFeaturesRequest)(nil), // 9: feast.serving.GetOnlineFeaturesRequest - (*GetOnlineFeaturesRequestV2)(nil), // 10: feast.serving.GetOnlineFeaturesRequestV2 - (*GetBatchFeaturesRequest)(nil), // 11: feast.serving.GetBatchFeaturesRequest - (*GetOnlineFeaturesResponse)(nil), // 12: feast.serving.GetOnlineFeaturesResponse - (*GetBatchFeaturesResponse)(nil), // 13: feast.serving.GetBatchFeaturesResponse - (*GetJobRequest)(nil), // 14: feast.serving.GetJobRequest - (*GetJobResponse)(nil), // 15: feast.serving.GetJobResponse - (*Job)(nil), // 16: feast.serving.Job - (*DatasetSource)(nil), // 17: feast.serving.DatasetSource - (*GetOnlineFeaturesRequest_EntityRow)(nil), // 18: feast.serving.GetOnlineFeaturesRequest.EntityRow - nil, // 19: feast.serving.GetOnlineFeaturesRequest.EntityRow.FieldsEntry - (*GetOnlineFeaturesRequestV2_EntityRow)(nil), // 20: feast.serving.GetOnlineFeaturesRequestV2.EntityRow - nil, // 21: feast.serving.GetOnlineFeaturesRequestV2.EntityRow.FieldsEntry - (*GetOnlineFeaturesResponse_FieldValues)(nil), // 22: feast.serving.GetOnlineFeaturesResponse.FieldValues - nil, // 23: feast.serving.GetOnlineFeaturesResponse.FieldValues.FieldsEntry - nil, // 24: feast.serving.GetOnlineFeaturesResponse.FieldValues.StatusesEntry - (*DatasetSource_FileSource)(nil), // 25: feast.serving.DatasetSource.FileSource - (*v0.DatasetFeatureStatisticsList)(nil), // 26: tensorflow.metadata.v0.DatasetFeatureStatisticsList - (*timestamp.Timestamp)(nil), // 27: google.protobuf.Timestamp - (*types.Value)(nil), // 28: feast.types.Value + (FeastServingType)(0), // 0: feast.serving.FeastServingType + (GetOnlineFeaturesResponse_FieldStatus)(0), // 1: feast.serving.GetOnlineFeaturesResponse.FieldStatus + (*GetFeastServingInfoRequest)(nil), // 2: feast.serving.GetFeastServingInfoRequest + (*GetFeastServingInfoResponse)(nil), // 3: feast.serving.GetFeastServingInfoResponse + (*FeatureReferenceV2)(nil), // 4: feast.serving.FeatureReferenceV2 + (*GetOnlineFeaturesRequestV2)(nil), // 5: feast.serving.GetOnlineFeaturesRequestV2 + (*GetOnlineFeaturesResponse)(nil), // 6: feast.serving.GetOnlineFeaturesResponse + (*GetOnlineFeaturesRequestV2_EntityRow)(nil), // 7: feast.serving.GetOnlineFeaturesRequestV2.EntityRow + nil, // 8: feast.serving.GetOnlineFeaturesRequestV2.EntityRow.FieldsEntry + (*GetOnlineFeaturesResponse_FieldValues)(nil), // 9: feast.serving.GetOnlineFeaturesResponse.FieldValues + nil, // 10: feast.serving.GetOnlineFeaturesResponse.FieldValues.FieldsEntry + nil, // 11: feast.serving.GetOnlineFeaturesResponse.FieldValues.StatusesEntry + (*timestamp.Timestamp)(nil), // 12: google.protobuf.Timestamp + (*types.Value)(nil), // 13: feast.types.Value } var file_feast_serving_ServingService_proto_depIdxs = []int32{ 0, // 0: feast.serving.GetFeastServingInfoResponse.type:type_name -> feast.serving.FeastServingType - 7, // 1: feast.serving.GetOnlineFeaturesRequest.features:type_name -> feast.serving.FeatureReference - 18, // 2: feast.serving.GetOnlineFeaturesRequest.entity_rows:type_name -> feast.serving.GetOnlineFeaturesRequest.EntityRow - 8, // 3: feast.serving.GetOnlineFeaturesRequestV2.features:type_name -> feast.serving.FeatureReferenceV2 - 20, // 4: feast.serving.GetOnlineFeaturesRequestV2.entity_rows:type_name -> feast.serving.GetOnlineFeaturesRequestV2.EntityRow - 7, // 5: feast.serving.GetBatchFeaturesRequest.features:type_name -> feast.serving.FeatureReference - 17, // 6: feast.serving.GetBatchFeaturesRequest.dataset_source:type_name -> feast.serving.DatasetSource - 22, // 7: feast.serving.GetOnlineFeaturesResponse.field_values:type_name -> feast.serving.GetOnlineFeaturesResponse.FieldValues - 16, // 8: feast.serving.GetBatchFeaturesResponse.job:type_name -> feast.serving.Job - 16, // 9: feast.serving.GetJobRequest.job:type_name -> feast.serving.Job - 16, // 10: feast.serving.GetJobResponse.job:type_name -> feast.serving.Job - 1, // 11: feast.serving.Job.type:type_name -> feast.serving.JobType - 2, // 12: feast.serving.Job.status:type_name -> feast.serving.JobStatus - 3, // 13: feast.serving.Job.data_format:type_name -> feast.serving.DataFormat - 26, // 14: feast.serving.Job.dataset_feature_statistics_list:type_name -> tensorflow.metadata.v0.DatasetFeatureStatisticsList - 25, // 15: feast.serving.DatasetSource.file_source:type_name -> feast.serving.DatasetSource.FileSource - 27, // 16: feast.serving.GetOnlineFeaturesRequest.EntityRow.entity_timestamp:type_name -> google.protobuf.Timestamp - 19, // 17: feast.serving.GetOnlineFeaturesRequest.EntityRow.fields:type_name -> feast.serving.GetOnlineFeaturesRequest.EntityRow.FieldsEntry - 28, // 18: feast.serving.GetOnlineFeaturesRequest.EntityRow.FieldsEntry.value:type_name -> feast.types.Value - 27, // 19: feast.serving.GetOnlineFeaturesRequestV2.EntityRow.timestamp:type_name -> google.protobuf.Timestamp - 21, // 20: feast.serving.GetOnlineFeaturesRequestV2.EntityRow.fields:type_name -> feast.serving.GetOnlineFeaturesRequestV2.EntityRow.FieldsEntry - 28, // 21: feast.serving.GetOnlineFeaturesRequestV2.EntityRow.FieldsEntry.value:type_name -> feast.types.Value - 23, // 22: feast.serving.GetOnlineFeaturesResponse.FieldValues.fields:type_name -> feast.serving.GetOnlineFeaturesResponse.FieldValues.FieldsEntry - 24, // 23: feast.serving.GetOnlineFeaturesResponse.FieldValues.statuses:type_name -> feast.serving.GetOnlineFeaturesResponse.FieldValues.StatusesEntry - 28, // 24: feast.serving.GetOnlineFeaturesResponse.FieldValues.FieldsEntry.value:type_name -> feast.types.Value - 4, // 25: feast.serving.GetOnlineFeaturesResponse.FieldValues.StatusesEntry.value:type_name -> feast.serving.GetOnlineFeaturesResponse.FieldStatus - 3, // 26: feast.serving.DatasetSource.FileSource.data_format:type_name -> feast.serving.DataFormat - 5, // 27: feast.serving.ServingService.GetFeastServingInfo:input_type -> feast.serving.GetFeastServingInfoRequest - 9, // 28: feast.serving.ServingService.GetOnlineFeatures:input_type -> feast.serving.GetOnlineFeaturesRequest - 10, // 29: feast.serving.ServingService.GetOnlineFeaturesV2:input_type -> feast.serving.GetOnlineFeaturesRequestV2 - 11, // 30: feast.serving.ServingService.GetBatchFeatures:input_type -> feast.serving.GetBatchFeaturesRequest - 14, // 31: feast.serving.ServingService.GetJob:input_type -> feast.serving.GetJobRequest - 6, // 32: feast.serving.ServingService.GetFeastServingInfo:output_type -> feast.serving.GetFeastServingInfoResponse - 12, // 33: feast.serving.ServingService.GetOnlineFeatures:output_type -> feast.serving.GetOnlineFeaturesResponse - 12, // 34: feast.serving.ServingService.GetOnlineFeaturesV2:output_type -> feast.serving.GetOnlineFeaturesResponse - 13, // 35: feast.serving.ServingService.GetBatchFeatures:output_type -> feast.serving.GetBatchFeaturesResponse - 15, // 36: feast.serving.ServingService.GetJob:output_type -> feast.serving.GetJobResponse - 32, // [32:37] is the sub-list for method output_type - 27, // [27:32] is the sub-list for method input_type - 27, // [27:27] is the sub-list for extension type_name - 27, // [27:27] is the sub-list for extension extendee - 0, // [0:27] is the sub-list for field type_name + 4, // 1: feast.serving.GetOnlineFeaturesRequestV2.features:type_name -> feast.serving.FeatureReferenceV2 + 7, // 2: feast.serving.GetOnlineFeaturesRequestV2.entity_rows:type_name -> feast.serving.GetOnlineFeaturesRequestV2.EntityRow + 9, // 3: feast.serving.GetOnlineFeaturesResponse.field_values:type_name -> feast.serving.GetOnlineFeaturesResponse.FieldValues + 12, // 4: feast.serving.GetOnlineFeaturesRequestV2.EntityRow.timestamp:type_name -> google.protobuf.Timestamp + 8, // 5: feast.serving.GetOnlineFeaturesRequestV2.EntityRow.fields:type_name -> feast.serving.GetOnlineFeaturesRequestV2.EntityRow.FieldsEntry + 13, // 6: feast.serving.GetOnlineFeaturesRequestV2.EntityRow.FieldsEntry.value:type_name -> feast.types.Value + 10, // 7: feast.serving.GetOnlineFeaturesResponse.FieldValues.fields:type_name -> feast.serving.GetOnlineFeaturesResponse.FieldValues.FieldsEntry + 11, // 8: feast.serving.GetOnlineFeaturesResponse.FieldValues.statuses:type_name -> feast.serving.GetOnlineFeaturesResponse.FieldValues.StatusesEntry + 13, // 9: feast.serving.GetOnlineFeaturesResponse.FieldValues.FieldsEntry.value:type_name -> feast.types.Value + 1, // 10: feast.serving.GetOnlineFeaturesResponse.FieldValues.StatusesEntry.value:type_name -> feast.serving.GetOnlineFeaturesResponse.FieldStatus + 2, // 11: feast.serving.ServingService.GetFeastServingInfo:input_type -> feast.serving.GetFeastServingInfoRequest + 5, // 12: feast.serving.ServingService.GetOnlineFeaturesV2:input_type -> feast.serving.GetOnlineFeaturesRequestV2 + 3, // 13: feast.serving.ServingService.GetFeastServingInfo:output_type -> feast.serving.GetFeastServingInfoResponse + 6, // 14: feast.serving.ServingService.GetOnlineFeaturesV2:output_type -> feast.serving.GetOnlineFeaturesResponse + 13, // [13:15] is the sub-list for method output_type + 11, // [11:13] is the sub-list for method input_type + 11, // [11:11] is the sub-list for extension type_name + 11, // [11:11] is the sub-list for extension extendee + 0, // [0:11] is the sub-list for field type_name } func init() { file_feast_serving_ServingService_proto_init() } @@ -1726,18 +771,6 @@ func file_feast_serving_ServingService_proto_init() { } } file_feast_serving_ServingService_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*FeatureReference); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_feast_serving_ServingService_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*FeatureReferenceV2); i { case 0: return &v.state @@ -1749,19 +782,7 @@ func file_feast_serving_ServingService_proto_init() { return nil } } - file_feast_serving_ServingService_proto_msgTypes[4].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*GetOnlineFeaturesRequest); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_feast_serving_ServingService_proto_msgTypes[5].Exporter = func(v interface{}, i int) interface{} { + file_feast_serving_ServingService_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*GetOnlineFeaturesRequestV2); i { case 0: return &v.state @@ -1773,19 +794,7 @@ func file_feast_serving_ServingService_proto_init() { return nil } } - file_feast_serving_ServingService_proto_msgTypes[6].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*GetBatchFeaturesRequest); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_feast_serving_ServingService_proto_msgTypes[7].Exporter = func(v interface{}, i int) interface{} { + file_feast_serving_ServingService_proto_msgTypes[4].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*GetOnlineFeaturesResponse); i { case 0: return &v.state @@ -1797,79 +806,7 @@ func file_feast_serving_ServingService_proto_init() { return nil } } - file_feast_serving_ServingService_proto_msgTypes[8].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*GetBatchFeaturesResponse); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_feast_serving_ServingService_proto_msgTypes[9].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*GetJobRequest); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_feast_serving_ServingService_proto_msgTypes[10].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*GetJobResponse); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_feast_serving_ServingService_proto_msgTypes[11].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*Job); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_feast_serving_ServingService_proto_msgTypes[12].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*DatasetSource); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_feast_serving_ServingService_proto_msgTypes[13].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*GetOnlineFeaturesRequest_EntityRow); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_feast_serving_ServingService_proto_msgTypes[15].Exporter = func(v interface{}, i int) interface{} { + file_feast_serving_ServingService_proto_msgTypes[5].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*GetOnlineFeaturesRequestV2_EntityRow); i { case 0: return &v.state @@ -1881,7 +818,7 @@ func file_feast_serving_ServingService_proto_init() { return nil } } - file_feast_serving_ServingService_proto_msgTypes[17].Exporter = func(v interface{}, i int) interface{} { + file_feast_serving_ServingService_proto_msgTypes[7].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*GetOnlineFeaturesResponse_FieldValues); i { case 0: return &v.state @@ -1893,29 +830,14 @@ func file_feast_serving_ServingService_proto_init() { return nil } } - file_feast_serving_ServingService_proto_msgTypes[20].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*DatasetSource_FileSource); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - } - file_feast_serving_ServingService_proto_msgTypes[12].OneofWrappers = []interface{}{ - (*DatasetSource_FileSource_)(nil), } type x struct{} out := protoimpl.TypeBuilder{ File: protoimpl.DescBuilder{ GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: file_feast_serving_ServingService_proto_rawDesc, - NumEnums: 5, - NumMessages: 21, + NumEnums: 2, + NumMessages: 10, NumExtensions: 0, NumServices: 1, }, @@ -1944,21 +866,8 @@ const _ = grpc.SupportPackageIsVersion6 type ServingServiceClient interface { // Get information about this Feast serving. GetFeastServingInfo(ctx context.Context, in *GetFeastServingInfoRequest, opts ...grpc.CallOption) (*GetFeastServingInfoResponse, error) - // Get online features synchronously. - GetOnlineFeatures(ctx context.Context, in *GetOnlineFeaturesRequest, opts ...grpc.CallOption) (*GetOnlineFeaturesResponse, error) // Get online features (v2) synchronously. GetOnlineFeaturesV2(ctx context.Context, in *GetOnlineFeaturesRequestV2, opts ...grpc.CallOption) (*GetOnlineFeaturesResponse, error) - // Get batch features asynchronously. - // - // The client should check the status of the returned job periodically by - // calling ReloadJob to determine if the job has completed successfully - // or with an error. If the job completes successfully i.e. - // status = JOB_STATUS_DONE with no error, then the client can check - // the file_uris for the location to download feature values data. - // The client is assumed to have access to these file URIs. - GetBatchFeatures(ctx context.Context, in *GetBatchFeaturesRequest, opts ...grpc.CallOption) (*GetBatchFeaturesResponse, error) - // Get the latest job status for batch feature retrieval. - GetJob(ctx context.Context, in *GetJobRequest, opts ...grpc.CallOption) (*GetJobResponse, error) } type servingServiceClient struct { @@ -1978,15 +887,6 @@ func (c *servingServiceClient) GetFeastServingInfo(ctx context.Context, in *GetF return out, nil } -func (c *servingServiceClient) GetOnlineFeatures(ctx context.Context, in *GetOnlineFeaturesRequest, opts ...grpc.CallOption) (*GetOnlineFeaturesResponse, error) { - out := new(GetOnlineFeaturesResponse) - err := c.cc.Invoke(ctx, "/feast.serving.ServingService/GetOnlineFeatures", in, out, opts...) - if err != nil { - return nil, err - } - return out, nil -} - func (c *servingServiceClient) GetOnlineFeaturesV2(ctx context.Context, in *GetOnlineFeaturesRequestV2, opts ...grpc.CallOption) (*GetOnlineFeaturesResponse, error) { out := new(GetOnlineFeaturesResponse) err := c.cc.Invoke(ctx, "/feast.serving.ServingService/GetOnlineFeaturesV2", in, out, opts...) @@ -1996,43 +896,12 @@ func (c *servingServiceClient) GetOnlineFeaturesV2(ctx context.Context, in *GetO return out, nil } -func (c *servingServiceClient) GetBatchFeatures(ctx context.Context, in *GetBatchFeaturesRequest, opts ...grpc.CallOption) (*GetBatchFeaturesResponse, error) { - out := new(GetBatchFeaturesResponse) - err := c.cc.Invoke(ctx, "/feast.serving.ServingService/GetBatchFeatures", in, out, opts...) - if err != nil { - return nil, err - } - return out, nil -} - -func (c *servingServiceClient) GetJob(ctx context.Context, in *GetJobRequest, opts ...grpc.CallOption) (*GetJobResponse, error) { - out := new(GetJobResponse) - err := c.cc.Invoke(ctx, "/feast.serving.ServingService/GetJob", in, out, opts...) - if err != nil { - return nil, err - } - return out, nil -} - // ServingServiceServer is the server API for ServingService service. type ServingServiceServer interface { // Get information about this Feast serving. GetFeastServingInfo(context.Context, *GetFeastServingInfoRequest) (*GetFeastServingInfoResponse, error) - // Get online features synchronously. - GetOnlineFeatures(context.Context, *GetOnlineFeaturesRequest) (*GetOnlineFeaturesResponse, error) // Get online features (v2) synchronously. GetOnlineFeaturesV2(context.Context, *GetOnlineFeaturesRequestV2) (*GetOnlineFeaturesResponse, error) - // Get batch features asynchronously. - // - // The client should check the status of the returned job periodically by - // calling ReloadJob to determine if the job has completed successfully - // or with an error. If the job completes successfully i.e. - // status = JOB_STATUS_DONE with no error, then the client can check - // the file_uris for the location to download feature values data. - // The client is assumed to have access to these file URIs. - GetBatchFeatures(context.Context, *GetBatchFeaturesRequest) (*GetBatchFeaturesResponse, error) - // Get the latest job status for batch feature retrieval. - GetJob(context.Context, *GetJobRequest) (*GetJobResponse, error) } // UnimplementedServingServiceServer can be embedded to have forward compatible implementations. @@ -2042,18 +911,9 @@ type UnimplementedServingServiceServer struct { func (*UnimplementedServingServiceServer) GetFeastServingInfo(context.Context, *GetFeastServingInfoRequest) (*GetFeastServingInfoResponse, error) { return nil, status.Errorf(codes.Unimplemented, "method GetFeastServingInfo not implemented") } -func (*UnimplementedServingServiceServer) GetOnlineFeatures(context.Context, *GetOnlineFeaturesRequest) (*GetOnlineFeaturesResponse, error) { - return nil, status.Errorf(codes.Unimplemented, "method GetOnlineFeatures not implemented") -} func (*UnimplementedServingServiceServer) GetOnlineFeaturesV2(context.Context, *GetOnlineFeaturesRequestV2) (*GetOnlineFeaturesResponse, error) { return nil, status.Errorf(codes.Unimplemented, "method GetOnlineFeaturesV2 not implemented") } -func (*UnimplementedServingServiceServer) GetBatchFeatures(context.Context, *GetBatchFeaturesRequest) (*GetBatchFeaturesResponse, error) { - return nil, status.Errorf(codes.Unimplemented, "method GetBatchFeatures not implemented") -} -func (*UnimplementedServingServiceServer) GetJob(context.Context, *GetJobRequest) (*GetJobResponse, error) { - return nil, status.Errorf(codes.Unimplemented, "method GetJob not implemented") -} func RegisterServingServiceServer(s *grpc.Server, srv ServingServiceServer) { s.RegisterService(&_ServingService_serviceDesc, srv) @@ -2077,24 +937,6 @@ func _ServingService_GetFeastServingInfo_Handler(srv interface{}, ctx context.Co return interceptor(ctx, in, info, handler) } -func _ServingService_GetOnlineFeatures_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - in := new(GetOnlineFeaturesRequest) - if err := dec(in); err != nil { - return nil, err - } - if interceptor == nil { - return srv.(ServingServiceServer).GetOnlineFeatures(ctx, in) - } - info := &grpc.UnaryServerInfo{ - Server: srv, - FullMethod: "/feast.serving.ServingService/GetOnlineFeatures", - } - handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(ServingServiceServer).GetOnlineFeatures(ctx, req.(*GetOnlineFeaturesRequest)) - } - return interceptor(ctx, in, info, handler) -} - func _ServingService_GetOnlineFeaturesV2_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { in := new(GetOnlineFeaturesRequestV2) if err := dec(in); err != nil { @@ -2113,42 +955,6 @@ func _ServingService_GetOnlineFeaturesV2_Handler(srv interface{}, ctx context.Co return interceptor(ctx, in, info, handler) } -func _ServingService_GetBatchFeatures_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - in := new(GetBatchFeaturesRequest) - if err := dec(in); err != nil { - return nil, err - } - if interceptor == nil { - return srv.(ServingServiceServer).GetBatchFeatures(ctx, in) - } - info := &grpc.UnaryServerInfo{ - Server: srv, - FullMethod: "/feast.serving.ServingService/GetBatchFeatures", - } - handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(ServingServiceServer).GetBatchFeatures(ctx, req.(*GetBatchFeaturesRequest)) - } - return interceptor(ctx, in, info, handler) -} - -func _ServingService_GetJob_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - in := new(GetJobRequest) - if err := dec(in); err != nil { - return nil, err - } - if interceptor == nil { - return srv.(ServingServiceServer).GetJob(ctx, in) - } - info := &grpc.UnaryServerInfo{ - Server: srv, - FullMethod: "/feast.serving.ServingService/GetJob", - } - handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(ServingServiceServer).GetJob(ctx, req.(*GetJobRequest)) - } - return interceptor(ctx, in, info, handler) -} - var _ServingService_serviceDesc = grpc.ServiceDesc{ ServiceName: "feast.serving.ServingService", HandlerType: (*ServingServiceServer)(nil), @@ -2157,22 +963,10 @@ var _ServingService_serviceDesc = grpc.ServiceDesc{ MethodName: "GetFeastServingInfo", Handler: _ServingService_GetFeastServingInfo_Handler, }, - { - MethodName: "GetOnlineFeatures", - Handler: _ServingService_GetOnlineFeatures_Handler, - }, { MethodName: "GetOnlineFeaturesV2", Handler: _ServingService_GetOnlineFeaturesV2_Handler, }, - { - MethodName: "GetBatchFeatures", - Handler: _ServingService_GetBatchFeatures_Handler, - }, - { - MethodName: "GetJob", - Handler: _ServingService_GetJob_Handler, - }, }, Streams: []grpc.StreamDesc{}, Metadata: "feast/serving/ServingService.proto", diff --git a/sdk/go/protos/feast/storage/Redis.pb.go b/sdk/go/protos/feast/storage/Redis.pb.go index 7b3a3f6a774..761bf2bb6ea 100644 --- a/sdk/go/protos/feast/storage/Redis.pb.go +++ b/sdk/go/protos/feast/storage/Redis.pb.go @@ -16,7 +16,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.25.0 -// protoc v3.10.0 +// protoc v3.12.4 // source: feast/storage/Redis.proto package storage @@ -41,65 +41,6 @@ const ( // of the legacy proto package is being used. const _ = proto.ProtoPackageIsVersion4 -type RedisKey struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - // FeatureSet this row belongs to, this is defined as featureSetName. - FeatureSet string `protobuf:"bytes,2,opt,name=feature_set,json=featureSet,proto3" json:"feature_set,omitempty"` - // List of fields containing entity names and their respective values - // contained within this feature row. The entities should be sorted - // by the entity name alphabetically in ascending order. - Entities []*types.Field `protobuf:"bytes,3,rep,name=entities,proto3" json:"entities,omitempty"` -} - -func (x *RedisKey) Reset() { - *x = RedisKey{} - if protoimpl.UnsafeEnabled { - mi := &file_feast_storage_Redis_proto_msgTypes[0] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *RedisKey) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*RedisKey) ProtoMessage() {} - -func (x *RedisKey) ProtoReflect() protoreflect.Message { - mi := &file_feast_storage_Redis_proto_msgTypes[0] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use RedisKey.ProtoReflect.Descriptor instead. -func (*RedisKey) Descriptor() ([]byte, []int) { - return file_feast_storage_Redis_proto_rawDescGZIP(), []int{0} -} - -func (x *RedisKey) GetFeatureSet() string { - if x != nil { - return x.FeatureSet - } - return "" -} - -func (x *RedisKey) GetEntities() []*types.Field { - if x != nil { - return x.Entities - } - return nil -} - type RedisKeyV2 struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -113,7 +54,7 @@ type RedisKeyV2 struct { func (x *RedisKeyV2) Reset() { *x = RedisKeyV2{} if protoimpl.UnsafeEnabled { - mi := &file_feast_storage_Redis_proto_msgTypes[1] + mi := &file_feast_storage_Redis_proto_msgTypes[0] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -126,7 +67,7 @@ func (x *RedisKeyV2) String() string { func (*RedisKeyV2) ProtoMessage() {} func (x *RedisKeyV2) ProtoReflect() protoreflect.Message { - mi := &file_feast_storage_Redis_proto_msgTypes[1] + mi := &file_feast_storage_Redis_proto_msgTypes[0] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -139,7 +80,7 @@ func (x *RedisKeyV2) ProtoReflect() protoreflect.Message { // Deprecated: Use RedisKeyV2.ProtoReflect.Descriptor instead. func (*RedisKeyV2) Descriptor() ([]byte, []int) { - return file_feast_storage_Redis_proto_rawDescGZIP(), []int{1} + return file_feast_storage_Redis_proto_rawDescGZIP(), []int{0} } func (x *RedisKeyV2) GetProject() string { @@ -171,28 +112,22 @@ var file_feast_storage_Redis_proto_rawDesc = []byte{ 0x73, 0x74, 0x2e, 0x73, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x1a, 0x17, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2f, 0x74, 0x79, 0x70, 0x65, 0x73, 0x2f, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x17, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2f, 0x74, 0x79, 0x70, 0x65, 0x73, - 0x2f, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0x5b, 0x0a, 0x08, - 0x52, 0x65, 0x64, 0x69, 0x73, 0x4b, 0x65, 0x79, 0x12, 0x1f, 0x0a, 0x0b, 0x66, 0x65, 0x61, 0x74, - 0x75, 0x72, 0x65, 0x5f, 0x73, 0x65, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x66, - 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x53, 0x65, 0x74, 0x12, 0x2e, 0x0a, 0x08, 0x65, 0x6e, 0x74, - 0x69, 0x74, 0x69, 0x65, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x66, 0x65, - 0x61, 0x73, 0x74, 0x2e, 0x74, 0x79, 0x70, 0x65, 0x73, 0x2e, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x52, - 0x08, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x69, 0x65, 0x73, 0x22, 0x82, 0x01, 0x0a, 0x0a, 0x52, 0x65, - 0x64, 0x69, 0x73, 0x4b, 0x65, 0x79, 0x56, 0x32, 0x12, 0x18, 0x0a, 0x07, 0x70, 0x72, 0x6f, 0x6a, - 0x65, 0x63, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x70, 0x72, 0x6f, 0x6a, 0x65, - 0x63, 0x74, 0x12, 0x21, 0x0a, 0x0c, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x5f, 0x6e, 0x61, 0x6d, - 0x65, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0b, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, - 0x4e, 0x61, 0x6d, 0x65, 0x73, 0x12, 0x37, 0x0a, 0x0d, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x5f, - 0x76, 0x61, 0x6c, 0x75, 0x65, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x66, - 0x65, 0x61, 0x73, 0x74, 0x2e, 0x74, 0x79, 0x70, 0x65, 0x73, 0x2e, 0x56, 0x61, 0x6c, 0x75, 0x65, - 0x52, 0x0c, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x73, 0x42, 0x59, - 0x0a, 0x13, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x73, 0x74, - 0x6f, 0x72, 0x61, 0x67, 0x65, 0x42, 0x0a, 0x52, 0x65, 0x64, 0x69, 0x73, 0x50, 0x72, 0x6f, 0x74, - 0x6f, 0x5a, 0x36, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x66, 0x65, - 0x61, 0x73, 0x74, 0x2d, 0x64, 0x65, 0x76, 0x2f, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2f, 0x73, 0x64, - 0x6b, 0x2f, 0x67, 0x6f, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x73, 0x2f, 0x66, 0x65, 0x61, 0x73, - 0x74, 0x2f, 0x73, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, - 0x33, + 0x2f, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0x82, 0x01, 0x0a, + 0x0a, 0x52, 0x65, 0x64, 0x69, 0x73, 0x4b, 0x65, 0x79, 0x56, 0x32, 0x12, 0x18, 0x0a, 0x07, 0x70, + 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x70, 0x72, + 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x12, 0x21, 0x0a, 0x0c, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x5f, + 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0b, 0x65, 0x6e, 0x74, + 0x69, 0x74, 0x79, 0x4e, 0x61, 0x6d, 0x65, 0x73, 0x12, 0x37, 0x0a, 0x0d, 0x65, 0x6e, 0x74, 0x69, + 0x74, 0x79, 0x5f, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, + 0x12, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x74, 0x79, 0x70, 0x65, 0x73, 0x2e, 0x56, 0x61, + 0x6c, 0x75, 0x65, 0x52, 0x0c, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x56, 0x61, 0x6c, 0x75, 0x65, + 0x73, 0x42, 0x59, 0x0a, 0x13, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x2e, 0x73, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x42, 0x0a, 0x52, 0x65, 0x64, 0x69, 0x73, 0x50, + 0x72, 0x6f, 0x74, 0x6f, 0x5a, 0x36, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, + 0x2f, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2d, 0x64, 0x65, 0x76, 0x2f, 0x66, 0x65, 0x61, 0x73, 0x74, + 0x2f, 0x73, 0x64, 0x6b, 0x2f, 0x67, 0x6f, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x73, 0x2f, 0x66, + 0x65, 0x61, 0x73, 0x74, 0x2f, 0x73, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x62, 0x06, 0x70, 0x72, + 0x6f, 0x74, 0x6f, 0x33, } var ( @@ -207,21 +142,18 @@ func file_feast_storage_Redis_proto_rawDescGZIP() []byte { return file_feast_storage_Redis_proto_rawDescData } -var file_feast_storage_Redis_proto_msgTypes = make([]protoimpl.MessageInfo, 2) +var file_feast_storage_Redis_proto_msgTypes = make([]protoimpl.MessageInfo, 1) var file_feast_storage_Redis_proto_goTypes = []interface{}{ - (*RedisKey)(nil), // 0: feast.storage.RedisKey - (*RedisKeyV2)(nil), // 1: feast.storage.RedisKeyV2 - (*types.Field)(nil), // 2: feast.types.Field - (*types.Value)(nil), // 3: feast.types.Value + (*RedisKeyV2)(nil), // 0: feast.storage.RedisKeyV2 + (*types.Value)(nil), // 1: feast.types.Value } var file_feast_storage_Redis_proto_depIdxs = []int32{ - 2, // 0: feast.storage.RedisKey.entities:type_name -> feast.types.Field - 3, // 1: feast.storage.RedisKeyV2.entity_values:type_name -> feast.types.Value - 2, // [2:2] is the sub-list for method output_type - 2, // [2:2] is the sub-list for method input_type - 2, // [2:2] is the sub-list for extension type_name - 2, // [2:2] is the sub-list for extension extendee - 0, // [0:2] is the sub-list for field type_name + 1, // 0: feast.storage.RedisKeyV2.entity_values:type_name -> feast.types.Value + 1, // [1:1] is the sub-list for method output_type + 1, // [1:1] is the sub-list for method input_type + 1, // [1:1] is the sub-list for extension type_name + 1, // [1:1] is the sub-list for extension extendee + 0, // [0:1] is the sub-list for field type_name } func init() { file_feast_storage_Redis_proto_init() } @@ -231,18 +163,6 @@ func file_feast_storage_Redis_proto_init() { } if !protoimpl.UnsafeEnabled { file_feast_storage_Redis_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*RedisKey); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_feast_storage_Redis_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*RedisKeyV2); i { case 0: return &v.state @@ -261,7 +181,7 @@ func file_feast_storage_Redis_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: file_feast_storage_Redis_proto_rawDesc, NumEnums: 0, - NumMessages: 2, + NumMessages: 1, NumExtensions: 0, NumServices: 0, }, diff --git a/sdk/go/protos/feast/types/Field.pb.go b/sdk/go/protos/feast/types/Field.pb.go index f2562b72d95..9dad77cdb91 100644 --- a/sdk/go/protos/feast/types/Field.pb.go +++ b/sdk/go/protos/feast/types/Field.pb.go @@ -16,7 +16,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.25.0 -// protoc v3.10.0 +// protoc v3.12.4 // source: feast/types/Field.proto package types diff --git a/sdk/go/protos/feast/types/Value.pb.go b/sdk/go/protos/feast/types/Value.pb.go index 3625cef1a53..3b194356331 100644 --- a/sdk/go/protos/feast/types/Value.pb.go +++ b/sdk/go/protos/feast/types/Value.pb.go @@ -16,7 +16,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.25.0 -// protoc v3.10.0 +// protoc v3.12.4 // source: feast/types/Value.proto package types diff --git a/sdk/go/protos/tensorflow_metadata/proto/v0/path.pb.go b/sdk/go/protos/tensorflow_metadata/proto/v0/path.pb.go index a1e5137c727..1daa7687f94 100644 --- a/sdk/go/protos/tensorflow_metadata/proto/v0/path.pb.go +++ b/sdk/go/protos/tensorflow_metadata/proto/v0/path.pb.go @@ -16,7 +16,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.25.0 -// protoc v3.10.0 +// protoc v3.12.4 // source: tensorflow_metadata/proto/v0/path.proto package v0 diff --git a/sdk/go/protos/tensorflow_metadata/proto/v0/schema.pb.go b/sdk/go/protos/tensorflow_metadata/proto/v0/schema.pb.go index 25bf40bc7fe..940779a1917 100644 --- a/sdk/go/protos/tensorflow_metadata/proto/v0/schema.pb.go +++ b/sdk/go/protos/tensorflow_metadata/proto/v0/schema.pb.go @@ -16,7 +16,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.25.0 -// protoc v3.10.0 +// protoc v3.12.4 // source: tensorflow_metadata/proto/v0/schema.proto package v0 diff --git a/sdk/go/protos/tensorflow_metadata/proto/v0/statistics.pb.go b/sdk/go/protos/tensorflow_metadata/proto/v0/statistics.pb.go index 6a102a28afa..fbf6247a1d3 100644 --- a/sdk/go/protos/tensorflow_metadata/proto/v0/statistics.pb.go +++ b/sdk/go/protos/tensorflow_metadata/proto/v0/statistics.pb.go @@ -20,7 +20,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.25.0 -// protoc v3.10.0 +// protoc v3.12.4 // source: tensorflow_metadata/proto/v0/statistics.proto package v0 diff --git a/sdk/java/pom.xml b/sdk/java/pom.xml index fa1be41d086..6b2295b6b33 100644 --- a/sdk/java/pom.xml +++ b/sdk/java/pom.xml @@ -19,6 +19,7 @@ 5.5.2 2.28.2 + 0.33.0 @@ -60,6 +61,23 @@ protobuf-java + + + io.opentracing.contrib + opentracing-grpc + 0.2.3 + + + io.opentracing + opentracing-api + ${opentracing.version} + + + io.opentracing + opentracing-noop + ${opentracing.version} + + org.slf4j diff --git a/sdk/java/src/main/java/com/gojek/feast/FeastClient.java b/sdk/java/src/main/java/com/gojek/feast/FeastClient.java index a5ea279e822..94836d88aa4 100644 --- a/sdk/java/src/main/java/com/gojek/feast/FeastClient.java +++ b/sdk/java/src/main/java/com/gojek/feast/FeastClient.java @@ -29,6 +29,8 @@ import io.grpc.ManagedChannelBuilder; import io.grpc.netty.shaded.io.grpc.netty.GrpcSslContexts; import io.grpc.netty.shaded.io.grpc.netty.NettyChannelBuilder; +import io.opentracing.contrib.grpc.TracingClientInterceptor; +import io.opentracing.util.GlobalTracer; import java.io.File; import java.util.HashSet; import java.util.List; @@ -96,6 +98,7 @@ public static FeastClient createSecure(String host, int port, SecurityConfig sec // Disable TLS channel = ManagedChannelBuilder.forAddress(host, port).usePlaintext().build(); } + return new FeastClient(channel, securityConfig.getCredentials()); } @@ -187,10 +190,16 @@ public List getOnlineFeatures(List featureRefs, List rows, Str protected FeastClient(ManagedChannel channel, Optional credentials) { this.channel = channel; - ServingServiceBlockingStub servingStub = ServingServiceGrpc.newBlockingStub(channel); + TracingClientInterceptor tracingInterceptor = + TracingClientInterceptor.newBuilder().withTracer(GlobalTracer.get()).build(); + + ServingServiceBlockingStub servingStub = + ServingServiceGrpc.newBlockingStub(tracingInterceptor.intercept(channel)); + if (credentials.isPresent()) { servingStub = servingStub.withCallCredentials(credentials.get()); } + this.stub = servingStub; } diff --git a/sdk/python/docs/index.rst b/sdk/python/docs/index.rst index 57d3b5976ed..782ec9c83e2 100644 --- a/sdk/python/docs/index.rst +++ b/sdk/python/docs/index.rst @@ -7,19 +7,10 @@ Client .. automodule:: feast.client :members: - -Feature Set -================== - -.. automodule:: feast.feature_set - :members: - - -Feature +Data Source ================== -.. automodule:: feast.feature - :inherited-members: +.. automodule:: feast.data_source :members: @@ -31,24 +22,22 @@ Entity :members: -Value +Feature Table ================== -.. automodule:: feast.value_type +.. automodule:: feast.feature_table :members: - -Source +Feature ================== -.. automodule:: feast.source +.. automodule:: feast.feature + :inherited-members: :members: - -Job +Constants ================== -.. automodule:: feast.job +.. automodule:: feast.constants :members: - - + :exclude-members: AuthProvider, ConfigMeta diff --git a/sdk/python/feast/cli.py b/sdk/python/feast/cli.py index 9d79c405186..8b089ec081b 100644 --- a/sdk/python/feast/cli.py +++ b/sdk/python/feast/cli.py @@ -23,7 +23,7 @@ from feast.client import Client from feast.config import Config -from feast.constants import CONFIG_SPARK_LAUNCHER +from feast.constants import ConfigOptions as opt from feast.entity import Entity from feast.feature_table import FeatureTable from feast.job_service import start_job_service @@ -36,6 +36,7 @@ click.option("--serving-url", help="Set Feast serving URL to connect to"), click.option("--job-service-url", help="Set Feast job service URL to connect to"), ] +DATETIME_ISO = "%Y-%m-%dT%H:%M:%s" def common_options(func): @@ -145,7 +146,7 @@ def entity_create(filename, project): entities = [Entity.from_dict(entity_dict) for entity_dict in yaml_loader(filename)] feast_client = Client() # type: Client - feast_client.apply_entity(entities, project) + feast_client.apply(entities, project) @entity.command("describe") @@ -252,7 +253,7 @@ def feature_table_create(filename): FeatureTable.from_dict(ft_dict) for ft_dict in yaml_loader(filename) ] feast_client = Client() # type: Client - feast_client.apply_feature_table(feature_tables) + feast_client.apply(feature_tables) @feature_table.command("describe") @@ -381,7 +382,9 @@ def sync_offline_to_online(feature_table: str, start_time: str, end_time: str): client = Client() table = client.get_feature_table(feature_table) client.start_offline_to_online_ingestion( - table, datetime.fromisoformat(start_time), datetime.fromisoformat(end_time) + table, + datetime.strptime(start_time, DATETIME_ISO), + datetime.strptime(end_time, DATETIME_ISO), ) @@ -422,7 +425,7 @@ def stop_stream_to_online(feature_table: str): Stop stream to online sync job """ - spark_launcher = Config().get(CONFIG_SPARK_LAUNCHER) + spark_launcher = Config().get(opt.SPARK_LAUNCHER) if spark_launcher == "emr": import feast.pyspark.aws.jobs @@ -441,7 +444,7 @@ def list_jobs(): """ from tabulate import tabulate - spark_launcher = Config().get(CONFIG_SPARK_LAUNCHER) + spark_launcher = Config().get(opt.SPARK_LAUNCHER) if spark_launcher == "emr": import feast.pyspark.aws.jobs @@ -482,14 +485,18 @@ def get_historical_features( features: str, entity_df_path: str, entity_df_dtype: str, destination: str ): """ - Get historical features + Get historical features. This CLI command is mostly for testing/easy demos; use the + corresponding API method in production. + + The main reason why this command is unlikely to be more broadly useful is that we make quite a + few assumptions about the entity dataframe, namely: + * it has to have `event_timestamp` column + * it has to parse cleanly by `pandas.read_csv()` with no extra tuning of data types """ import pandas client = Client() - # TODO: clean this up - if entity_df_dtype: dtype = json.loads(entity_df_dtype) entity_df = pandas.read_csv( diff --git a/sdk/python/feast/client.py b/sdk/python/feast/client.py index 5e4b78b5ff1..31af5f56f2f 100644 --- a/sdk/python/feast/client.py +++ b/sdk/python/feast/client.py @@ -16,6 +16,7 @@ import os import shutil import uuid +import warnings from datetime import datetime from itertools import groupby from typing import Any, Dict, List, Optional, Union @@ -24,24 +25,7 @@ import pandas as pd from feast.config import Config -from feast.constants import ( - CONFIG_CORE_ENABLE_SSL_KEY, - CONFIG_CORE_SERVER_SSL_CERT_KEY, - CONFIG_CORE_URL_KEY, - CONFIG_ENABLE_AUTH_KEY, - CONFIG_GRPC_CONNECTION_TIMEOUT_DEFAULT_KEY, - CONFIG_JOB_SERVICE_ENABLE_SSL_KEY, - CONFIG_JOB_SERVICE_SERVER_SSL_CERT_KEY, - CONFIG_JOB_SERVICE_URL_KEY, - CONFIG_PROJECT_KEY, - CONFIG_SERVING_ENABLE_SSL_KEY, - CONFIG_SERVING_SERVER_SSL_CERT_KEY, - CONFIG_SERVING_URL_KEY, - CONFIG_SPARK_HISTORICAL_FEATURE_OUTPUT_FORMAT, - CONFIG_SPARK_HISTORICAL_FEATURE_OUTPUT_LOCATION, - CONFIG_SPARK_STAGING_LOCATION, - FEAST_DEFAULT_OPTIONS, -) +from feast.constants import ConfigOptions as opt from feast.core.CoreService_pb2 import ( ApplyEntityRequest, ApplyEntityResponse, @@ -59,6 +43,8 @@ GetFeatureTableResponse, ListEntitiesRequest, ListEntitiesResponse, + ListFeaturesRequest, + ListFeaturesResponse, ListFeatureTablesRequest, ListFeatureTablesResponse, ListProjectsRequest, @@ -76,12 +62,11 @@ from feast.data_format import ParquetFormat from feast.data_source import BigQuerySource, FileSource from feast.entity import Entity -from feast.feature import _build_feature_references +from feast.feature import Feature, FeatureRef, _build_feature_references from feast.feature_table import FeatureTable from feast.grpc import auth as feast_auth from feast.grpc.grpc import create_grpc_channel from feast.loaders.ingest import ( - BATCH_INGESTION_PRODUCTION_TIMEOUT, _check_field_mappings, _read_table_from_source, _upload_to_bq_source, @@ -121,6 +106,8 @@ CPU_COUNT: int = multiprocessing.cpu_count() +warnings.simplefilter("once", DeprecationWarning) + class Client: """ @@ -158,7 +145,7 @@ def __init__(self, options: Optional[Dict[str, str]] = None, **kwargs): self._auth_metadata: Optional[grpc.AuthMetadataPlugin] = None # Configure Auth Metadata Plugin if auth is enabled - if self._config.getboolean(CONFIG_ENABLE_AUTH_KEY): + if self._config.getboolean(opt.ENABLE_AUTH): self._auth_metadata = feast_auth.get_auth_metadata_plugin(self._config) @property @@ -170,12 +157,12 @@ def _core_service(self): """ if not self._core_service_stub: channel = create_grpc_channel( - url=self._config.get(CONFIG_CORE_URL_KEY), - enable_ssl=self._config.getboolean(CONFIG_CORE_ENABLE_SSL_KEY), - enable_auth=self._config.getboolean(CONFIG_ENABLE_AUTH_KEY), - ssl_server_cert_path=self._config.get(CONFIG_CORE_SERVER_SSL_CERT_KEY), + url=self._config.get(opt.CORE_URL), + enable_ssl=self._config.getboolean(opt.CORE_ENABLE_SSL), + enable_auth=self._config.getboolean(opt.ENABLE_AUTH), + ssl_server_cert_path=self._config.get(opt.CORE_SERVER_SSL_CERT), auth_metadata_plugin=self._auth_metadata, - timeout=self._config.getint(CONFIG_GRPC_CONNECTION_TIMEOUT_DEFAULT_KEY), + timeout=self._config.getint(opt.GRPC_CONNECTION_TIMEOUT), ) self._core_service_stub = CoreServiceStub(channel) return self._core_service_stub @@ -183,27 +170,38 @@ def _core_service(self): @property def _serving_service(self): """ - Creates or returns the gRPC Feast Serving Service Stub + Creates or returns the gRPC Feast Serving Service Stub. If both `opentracing` + and `grpcio-opentracing` are installed, an opentracing interceptor will be + instantiated based on the global tracer. Returns: ServingServiceStub """ if not self._serving_service_stub: channel = create_grpc_channel( - url=self._config.get(CONFIG_SERVING_URL_KEY), - enable_ssl=self._config.getboolean(CONFIG_SERVING_ENABLE_SSL_KEY), - enable_auth=self._config.getboolean(CONFIG_ENABLE_AUTH_KEY), - ssl_server_cert_path=self._config.get( - CONFIG_SERVING_SERVER_SSL_CERT_KEY - ), + url=self._config.get(opt.SERVING_URL), + enable_ssl=self._config.getboolean(opt.SERVING_ENABLE_SSL), + enable_auth=self._config.getboolean(opt.ENABLE_AUTH), + ssl_server_cert_path=self._config.get(opt.SERVING_SERVER_SSL_CERT), auth_metadata_plugin=self._auth_metadata, - timeout=self._config.getint(CONFIG_GRPC_CONNECTION_TIMEOUT_DEFAULT_KEY), + timeout=self._config.getint(opt.GRPC_CONNECTION_TIMEOUT), ) + try: + import opentracing + from grpc_opentracing import open_tracing_client_interceptor + from grpc_opentracing.grpcext import intercept_channel + + interceptor = open_tracing_client_interceptor( + opentracing.global_tracer() + ) + channel = intercept_channel(channel, interceptor) + except ImportError: + pass self._serving_service_stub = ServingServiceStub(channel) return self._serving_service_stub @property def _use_job_service(self) -> bool: - return self._config.exists(CONFIG_JOB_SERVICE_URL_KEY) + return self._config.exists(opt.JOB_SERVICE_URL) @property def _job_service(self): @@ -218,21 +216,19 @@ def _job_service(self): if not self._job_service_stub: channel = create_grpc_channel( - url=self._config.get(CONFIG_JOB_SERVICE_URL_KEY), - enable_ssl=self._config.getboolean(CONFIG_JOB_SERVICE_ENABLE_SSL_KEY), - enable_auth=self._config.getboolean(CONFIG_ENABLE_AUTH_KEY), - ssl_server_cert_path=self._config.get( - CONFIG_JOB_SERVICE_SERVER_SSL_CERT_KEY - ), + url=self._config.get(opt.JOB_SERVICE_URL), + enable_ssl=self._config.getboolean(opt.JOB_SERVICE_ENABLE_SSL), + enable_auth=self._config.getboolean(opt.ENABLE_AUTH), + ssl_server_cert_path=self._config.get(opt.JOB_SERVICE_SERVER_SSL_CERT), auth_metadata_plugin=self._auth_metadata, - timeout=self._config.getint(CONFIG_GRPC_CONNECTION_TIMEOUT_DEFAULT_KEY), + timeout=self._config.getint(opt.GRPC_CONNECTION_TIMEOUT), ) self._job_service_service_stub = JobServiceStub(channel) return self._job_service_service_stub def _extra_grpc_params(self) -> Dict[str, Any]: return dict( - timeout=self._config.getint(CONFIG_GRPC_CONNECTION_TIMEOUT_DEFAULT_KEY), + timeout=self._config.getint(opt.GRPC_CONNECTION_TIMEOUT), metadata=self._get_grpc_metadata(), ) @@ -244,7 +240,7 @@ def core_url(self) -> str: Returns: Feast Core URL string """ - return self._config.get(CONFIG_CORE_URL_KEY) + return self._config.get(opt.CORE_URL) @core_url.setter def core_url(self, value: str): @@ -254,7 +250,7 @@ def core_url(self, value: str): Args: value: Feast Core URL """ - self._config.set(CONFIG_CORE_URL_KEY, value) + self._config.set(opt.CORE_URL, value) @property def serving_url(self) -> str: @@ -264,7 +260,7 @@ def serving_url(self) -> str: Returns: Feast Serving URL string """ - return self._config.get(CONFIG_SERVING_URL_KEY) + return self._config.get(opt.SERVING_URL) @serving_url.setter def serving_url(self, value: str): @@ -274,7 +270,7 @@ def serving_url(self, value: str): Args: value: Feast Serving URL """ - self._config.set(CONFIG_SERVING_URL_KEY, value) + self._config.set(opt.SERVING_URL, value) @property def job_service_url(self) -> str: @@ -284,7 +280,7 @@ def job_service_url(self) -> str: Returns: Feast Job Service URL string """ - return self._config.get(CONFIG_JOB_SERVICE_URL_KEY) + return self._config.get(opt.JOB_SERVICE_URL) @job_service_url.setter def job_service_url(self, value: str): @@ -294,7 +290,7 @@ def job_service_url(self, value: str): Args: value: Feast Job Service URL """ - self._config.set(CONFIG_JOB_SERVICE_URL_KEY, value) + self._config.set(opt.JOB_SERVICE_URL, value) @property def core_secure(self) -> bool: @@ -304,7 +300,7 @@ def core_secure(self) -> bool: Returns: Whether client-side SSL/TLS is enabled """ - return self._config.getboolean(CONFIG_CORE_ENABLE_SSL_KEY) + return self._config.getboolean(opt.CORE_ENABLE_SSL) @core_secure.setter def core_secure(self, value: bool): @@ -314,7 +310,7 @@ def core_secure(self, value: bool): Args: value: True to enable client-side SSL/TLS """ - self._config.set(CONFIG_CORE_ENABLE_SSL_KEY, value) + self._config.set(opt.CORE_ENABLE_SSL, value) @property def serving_secure(self) -> bool: @@ -324,7 +320,7 @@ def serving_secure(self) -> bool: Returns: Whether client-side SSL/TLS is enabled """ - return self._config.getboolean(CONFIG_SERVING_ENABLE_SSL_KEY) + return self._config.getboolean(opt.SERVING_ENABLE_SSL) @serving_secure.setter def serving_secure(self, value: bool): @@ -334,7 +330,7 @@ def serving_secure(self, value: bool): Args: value: True to enable client-side SSL/TLS """ - self._config.set(CONFIG_SERVING_ENABLE_SSL_KEY, value) + self._config.set(opt.SERVING_ENABLE_SSL, value) @property def job_service_secure(self) -> bool: @@ -344,7 +340,7 @@ def job_service_secure(self) -> bool: Returns: Whether client-side SSL/TLS is enabled """ - return self._config.getboolean(CONFIG_JOB_SERVICE_ENABLE_SSL_KEY) + return self._config.getboolean(opt.JOB_SERVICE_ENABLE_SSL) @job_service_secure.setter def job_service_secure(self, value: bool): @@ -354,7 +350,7 @@ def job_service_secure(self, value: bool): Args: value: True to enable client-side SSL/TLS """ - self._config.set(CONFIG_JOB_SERVICE_ENABLE_SSL_KEY, value) + self._config.set(opt.JOB_SERVICE_ENABLE_SSL, value) def version(self): """ @@ -371,7 +367,7 @@ def version(self): if self.serving_url: serving_version = self._serving_service.GetFeastServingInfo( GetFeastServingInfoRequest(), - timeout=self._config.getint(CONFIG_GRPC_CONNECTION_TIMEOUT_DEFAULT_KEY), + timeout=self._config.getint(opt.GRPC_CONNECTION_TIMEOUT), metadata=self._get_grpc_metadata(), ).version result["serving"] = {"url": self.serving_url, "version": serving_version} @@ -379,7 +375,7 @@ def version(self): if self.core_url: core_version = self._core_service.GetFeastCoreVersion( GetFeastCoreVersionRequest(), - timeout=self._config.getint(CONFIG_GRPC_CONNECTION_TIMEOUT_DEFAULT_KEY), + timeout=self._config.getint(opt.GRPC_CONNECTION_TIMEOUT), metadata=self._get_grpc_metadata(), ).version result["core"] = {"url": self.core_url, "version": core_version} @@ -394,9 +390,9 @@ def project(self) -> str: Returns: Project name """ - if not self._config.get(CONFIG_PROJECT_KEY): + if not self._config.get(opt.PROJECT): raise ValueError("No project has been configured.") - return self._config.get(CONFIG_PROJECT_KEY) + return self._config.get(opt.PROJECT) def set_project(self, project: Optional[str] = None): """ @@ -406,8 +402,8 @@ def set_project(self, project: Optional[str] = None): project: Project to set as active. If unset, will reset to the default project. """ if project is None: - project = FEAST_DEFAULT_OPTIONS[CONFIG_PROJECT_KEY] - self._config.set(CONFIG_PROJECT_KEY, project) + project = opt().PROJECT + self._config.set(opt.PROJECT, project) def list_projects(self) -> List[str]: """ @@ -420,7 +416,7 @@ def list_projects(self) -> List[str]: response = self._core_service.ListProjects( ListProjectsRequest(), - timeout=self._config.getint(CONFIG_GRPC_CONNECTION_TIMEOUT_DEFAULT_KEY), + timeout=self._config.getint(opt.GRPC_CONNECTION_TIMEOUT), metadata=self._get_grpc_metadata(), ) # type: ListProjectsResponse return list(response.projects) @@ -435,7 +431,7 @@ def create_project(self, project: str): self._core_service.CreateProject( CreateProjectRequest(name=project), - timeout=self._config.getint(CONFIG_GRPC_CONNECTION_TIMEOUT_DEFAULT_KEY), + timeout=self._config.getint(opt.GRPC_CONNECTION_TIMEOUT), metadata=self._get_grpc_metadata(), ) # type: CreateProjectResponse @@ -452,7 +448,7 @@ def archive_project(self, project): try: self._core_service_stub.ArchiveProject( ArchiveProjectRequest(name=project), - timeout=self._config.getint(CONFIG_GRPC_CONNECTION_TIMEOUT_DEFAULT_KEY), + timeout=self._config.getint(opt.GRPC_CONNECTION_TIMEOUT), metadata=self._get_grpc_metadata(), ) # type: ArchiveProjectResponse except grpc.RpcError as e: @@ -460,15 +456,19 @@ def archive_project(self, project): # revert to the default project if self._project == project: - self._project = FEAST_DEFAULT_OPTIONS[CONFIG_PROJECT_KEY] + self._project = opt().PROJECT - def apply_entity(self, entities: Union[List[Entity], Entity], project: str = None): + def apply( + self, + objects: Union[List[Union[Entity, FeatureTable]], Entity, FeatureTable], + project: str = None, + ): """ - Idempotently registers entities with Feast Core. Either a single - entity or a list can be provided. + Idempotently registers entities and feature tables with Feast Core. Either a single + entity or feature table or a list can be provided. Args: - entities: List of entities that will be registered + objects: List of entities and/or feature tables that will be registered Examples: >>> from feast import Client @@ -484,8 +484,32 @@ def apply_entity(self, entities: Union[List[Entity], Entity], project: str = Non >>> "key": "val" >>> } >>> ) - >>> feast_client.apply_entity(entity) + >>> feast_client.apply(entity) + """ + + if project is None: + project = self.project + + if not isinstance(objects, list): + objects = [objects] + for obj in objects: + if isinstance(obj, Entity): + self._apply_entity(project, obj) # type: ignore + elif isinstance(obj, FeatureTable): + self._apply_feature_table(project, obj) # type: ignore + else: + raise ValueError( + f"Could not determine object type to apply {obj} with type {type(obj)}. Type must be Entity or FeatureTable." + ) + + def apply_entity(self, entities: Union[List[Entity], Entity], project: str = None): """ + Deprecated. Please see apply(). + """ + warnings.warn( + "The method apply_entity() is being deprecated. Please use apply() instead. Feast 0.10 and onwards will not support apply_entity().", + DeprecationWarning, + ) if project is None: project = self.project @@ -513,7 +537,7 @@ def _apply_entity(self, project: str, entity: Entity): try: apply_entity_response = self._core_service.ApplyEntity( ApplyEntityRequest(project=project, spec=entity_proto), # type: ignore - timeout=self._config.getint(CONFIG_GRPC_CONNECTION_TIMEOUT_DEFAULT_KEY), + timeout=self._config.getint(opt.GRPC_CONNECTION_TIMEOUT), metadata=self._get_grpc_metadata(), ) # type: ApplyEntityResponse except grpc.RpcError as e: @@ -590,12 +614,12 @@ def apply_feature_table( project: str = None, ): """ - Idempotently registers feature tables with Feast Core. Either a single - feature table or a list can be provided. - - Args: - feature_tables: List of feature tables that will be registered + Deprecated. Please see apply(). """ + warnings.warn( + "The method apply_feature_table() is being deprecated. Please use apply() instead. Feast 0.10 and onwards will not support apply_feature_table().", + DeprecationWarning, + ) if project is None: project = self.project @@ -625,7 +649,7 @@ def _apply_feature_table(self, project: str, feature_table: FeatureTable): try: apply_feature_table_response = self._core_service.ApplyFeatureTable( ApplyFeatureTableRequest(project=project, table_spec=feature_table_proto), # type: ignore - timeout=self._config.getint(CONFIG_GRPC_CONNECTION_TIMEOUT_DEFAULT_KEY), + timeout=self._config.getint(opt.GRPC_CONNECTION_TIMEOUT), metadata=self._get_grpc_metadata(), ) # type: ApplyFeatureTableResponse except grpc.RpcError as e: @@ -715,6 +739,51 @@ def delete_feature_table(self, name: str, project: str = None) -> None: except grpc.RpcError as e: raise grpc.RpcError(e.details()) + def list_features_by_ref( + self, + project: str = None, + entities: List[str] = list(), + labels: Dict[str, str] = dict(), + ) -> Dict[FeatureRef, Feature]: + """ + Retrieve a dictionary of feature reference to feature from Feast Core based on filters provided. + + Args: + project: Feast project that these features belongs to + entities: Feast entity that these features are associated with + labels: Feast labels that these features are associated with + + Returns: + Dictionary of + + Examples: + >>> from feast import Client + >>> + >>> feast_client = Client(core_url="localhost:6565") + >>> features = feast_client.list_features(project="test_project", entities=["driver_id"], labels={"key1":"val1","key2":"val2"}) + >>> print(features) + """ + + if project is None: + project = self.project + + filter = ListFeaturesRequest.Filter( + project=project, entities=entities, labels=labels + ) + + feature_protos = self._core_service.ListFeatures( + ListFeaturesRequest(filter=filter), metadata=self._get_grpc_metadata(), + ) # type: ListFeaturesResponse + + # Extract features and return + features_dict = {} + for ref_str, feature_proto in feature_protos.features.items(): + feature_ref = FeatureRef.from_str(ref_str) + feature = Feature.from_proto(feature_proto) + features_dict[feature_ref] = feature + + return features_dict + def ingest( self, feature_table: Union[str, FeatureTable], @@ -722,7 +791,7 @@ def ingest( project: str = None, chunk_size: int = 10000, max_workers: int = max(CPU_COUNT - 1, 1), - timeout: int = BATCH_INGESTION_PRODUCTION_TIMEOUT, + timeout: int = int(opt().BATCH_INGESTION_PRODUCTION_TIMEOUT), ) -> None: """ Batch load feature data into a FeatureTable. @@ -768,6 +837,8 @@ def ingest( if project is None: project = self.project + if isinstance(feature_table, str): + name = feature_table if isinstance(feature_table, FeatureTable): name = feature_table.name @@ -822,7 +893,9 @@ def ingest( try: if issubclass(type(feature_table.batch_source), FileSource): file_url = feature_table.batch_source.file_options.file_url.rstrip("*") - _upload_to_file_source(file_url, with_partitions, dest_path) + _upload_to_file_source( + file_url, with_partitions, dest_path, self._config + ) if issubclass(type(feature_table.batch_source), BigQuerySource): bq_table_ref = feature_table.batch_source.bigquery_options.table_ref feature_table_timestamp_column = ( @@ -847,7 +920,7 @@ def _get_grpc_metadata(self): Returns: Tuple of metadata to attach to each gRPC call """ - if self._config.getboolean(CONFIG_ENABLE_AUTH_KEY) and self._auth_metadata: + if self._config.getboolean(opt.ENABLE_AUTH) and self._auth_metadata: return self._auth_metadata.get_signed_meta() return () @@ -893,7 +966,7 @@ def get_online_features( entity_rows=_infer_online_entity_rows(entity_rows), project=project if project is not None else self.project, ), - timeout=self._config.getint(CONFIG_GRPC_CONNECTION_TIMEOUT_DEFAULT_KEY), + timeout=self._config.getint(opt.GRPC_CONNECTION_TIMEOUT), metadata=self._get_grpc_metadata(), ) except grpc.RpcError as e: @@ -950,12 +1023,18 @@ def get_historical_features( feature_refs, self.project ) + assert all(ft.batch_source.created_timestamp_column for ft in feature_tables), ( + "All BatchSources attached to retrieved FeatureTables " + "must have specified `created_timestamp_column` to be used in " + "historical dataset generation." + ) + if output_location is None: output_location = os.path.join( - self._config.get(CONFIG_SPARK_HISTORICAL_FEATURE_OUTPUT_LOCATION), + self._config.get(opt.HISTORICAL_FEATURE_OUTPUT_LOCATION), str(uuid.uuid4()), ) - output_format = self._config.get(CONFIG_SPARK_HISTORICAL_FEATURE_OUTPUT_FORMAT) + output_format = self._config.get(opt.HISTORICAL_FEATURE_OUTPUT_FORMAT) feature_sources = [ feature_table.batch_source for feature_table in feature_tables ] @@ -976,7 +1055,8 @@ def get_historical_features( else: entity_source = stage_entities_to_fs( entity_source, - staging_location=self._config.get(CONFIG_SPARK_STAGING_LOCATION), + staging_location=self._config.get(opt.SPARK_STAGING_LOCATION), + config=self._config, ) if self._use_job_service: @@ -995,6 +1075,8 @@ def get_historical_features( self._extra_grpc_params, response.id, output_file_uri=response.output_file_uri, + start_time=response.job_start_time.ToDatetime(), + log_uri=response.log_uri, ) else: return start_historical_feature_retrieval_job( @@ -1094,16 +1176,24 @@ def start_offline_to_online_ingestion( request.end_date.FromDatetime(end) response = self._job_service.StartOfflineToOnlineIngestionJob(request) return RemoteBatchIngestionJob( - self._job_service, self._extra_grpc_params, response.id, + self._job_service, + self._extra_grpc_params, + response.id, + feature_table.name, + response.job_start_time.ToDatetime(), + response.log_uri, ) def start_stream_to_online_ingestion( - self, feature_table: FeatureTable, extra_jars: Optional[List[str]] = None, + self, + feature_table: FeatureTable, + extra_jars: Optional[List[str]] = None, + project: str = None, ) -> SparkJob: if not self._use_job_service: return start_stream_to_online_ingestion( client=self, - project=self.project, + project=project or self.project, feature_table=feature_table, extra_jars=extra_jars or [], ) @@ -1113,14 +1203,23 @@ def start_stream_to_online_ingestion( ) response = self._job_service.StartStreamToOnlineIngestionJob(request) return RemoteStreamIngestionJob( - self._job_service, self._extra_grpc_params, response.id, + self._job_service, + self._extra_grpc_params, + response.id, + feature_table.name, + response.job_start_time, + response.log_uri, ) - def list_jobs(self, include_terminated: bool) -> List[SparkJob]: + def list_jobs( + self, include_terminated: bool, table_name: Optional[str] = None + ) -> List[SparkJob]: if not self._use_job_service: - return list_jobs(include_terminated, self) + return list_jobs(include_terminated, self, table_name) else: - request = ListJobsRequest(include_terminated=include_terminated) + request = ListJobsRequest( + include_terminated=include_terminated, table_name=table_name + ) response = self._job_service.ListJobs(request) return [ get_remote_job_from_proto( @@ -1142,4 +1241,4 @@ def get_job_by_id(self, job_id: str) -> SparkJob: def stage_dataframe( self, df: pd.DataFrame, event_timestamp_column: str, ) -> FileSource: - return stage_dataframe(df, event_timestamp_column, self) + return stage_dataframe(df, event_timestamp_column, self._config) diff --git a/sdk/python/feast/config.py b/sdk/python/feast/config.py index 1bbab4edbcf..788d257b9f3 100644 --- a/sdk/python/feast/config.py +++ b/sdk/python/feast/config.py @@ -24,11 +24,12 @@ CONFIG_FILE_DEFAULT_DIRECTORY, CONFIG_FILE_NAME, CONFIG_FILE_SECTION, - FEAST_CONFIG_FILE_ENV_KEY, + FEAST_CONFIG_FILE_ENV, ) -from feast.constants import FEAST_DEFAULT_OPTIONS as DEFAULTS +from feast.constants import ConfigOptions as opt _logger = logging.getLogger(__name__) +_UNSET = object() def _init_config(path: str): @@ -50,7 +51,7 @@ def _init_config(path: str): os.makedirs(os.path.dirname(config_dir), exist_ok=True) # Create the configuration file itself - config = ConfigParser(defaults=DEFAULTS) + config = ConfigParser(defaults=opt().defaults(), allow_no_value=True) if os.path.exists(path): config.read(path) @@ -58,9 +59,6 @@ def _init_config(path: str): if not config.has_section(CONFIG_FILE_SECTION): config.add_section(CONFIG_FILE_SECTION) - # Save the current configuration - config.write(open(path, "w")) - return config @@ -104,9 +102,7 @@ def __init__( if not path: path = join( expanduser("~"), - os.environ.get( - FEAST_CONFIG_FILE_ENV_KEY, CONFIG_FILE_DEFAULT_DIRECTORY, - ), + os.environ.get(FEAST_CONFIG_FILE_ENV, CONFIG_FILE_DEFAULT_DIRECTORY,), CONFIG_FILE_NAME, ) @@ -119,69 +115,66 @@ def __init__( self._config = config # type: ConfigParser self._path = path # type: str - def get(self, option): + def _get(self, option, default, get_method): + fallback = {} if default is _UNSET else {"fallback": default} + return get_method( + CONFIG_FILE_SECTION, + option, + vars={**_get_feast_env_vars(), **self._options}, + **fallback, + ) + + def get(self, option, default=_UNSET): """ Returns a single configuration option as a string Args: option: Name of the option + default: Default value to return if option is not found Returns: String option that is returned """ - return self._config.get( - CONFIG_FILE_SECTION, - option, - vars={**_get_feast_env_vars(), **self._options}, - ) + return self._get(option, default, self._config.get) - def getboolean(self, option): + def getboolean(self, option, default=_UNSET): """ Returns a single configuration option as a boolean Args: option: Name of the option + default: Default value to return if option is not found Returns: Boolean option value that is returned """ - return self._config.getboolean( - CONFIG_FILE_SECTION, - option, - vars={**_get_feast_env_vars(), **self._options}, - ) + return self._get(option, default, self._config.getboolean) - def getint(self, option): + def getint(self, option, default=_UNSET): """ Returns a single configuration option as an integer Args: option: Name of the option + default: Default value to return if option is not found Returns: Integer option value that is returned """ - return self._config.getint( - CONFIG_FILE_SECTION, - option, - vars={**_get_feast_env_vars(), **self._options}, - ) + return self._get(option, default, self._config.getint) - def getfloat(self, option): + def getfloat(self, option, default=_UNSET): """ Returns a single configuration option as an integer Args: option: Name of the option + default: Default value to return if option is not found Returns: Float option value that is returned """ - return self._config.getfloat( - CONFIG_FILE_SECTION, - option, - vars={**_get_feast_env_vars(), **self._options}, - ) + return self._get(option, default, self._config.getfloat) def set(self, option, value): """ @@ -213,7 +206,12 @@ def save(self): Save the current configuration to disk. This does not include environmental variables or initialized options """ - self._config.write(open(self._path, "w")) + defaults = self._config.defaults() + try: + self._config._defaults = {} + self._config.write(open(self._path, "w")) + finally: + self._config._defaults = defaults def __str__(self): result = "" diff --git a/sdk/python/feast/constants.py b/sdk/python/feast/constants.py index 0a4b4f52e85..1510f417c18 100644 --- a/sdk/python/feast/constants.py +++ b/sdk/python/feast/constants.py @@ -14,6 +14,7 @@ # limitations under the License. # from enum import Enum +from typing import Optional class AuthProvider(Enum): @@ -21,132 +22,246 @@ class AuthProvider(Enum): OAUTH = "oauth" -DATETIME_COLUMN = "datetime" - -# Environmental variable to specify Feast configuration file location -FEAST_CONFIG_FILE_ENV_KEY = "FEAST_CONFIG" - -# Default prefix to Feast environmental variables -CONFIG_FEAST_ENV_VAR_PREFIX = "FEAST_" - -# Default directory to Feast configuration file -CONFIG_FILE_DEFAULT_DIRECTORY = ".feast" - -# Default Feast configuration file name -CONFIG_FILE_NAME = "config" - -# Default section in Feast configuration file to specify options -CONFIG_FILE_SECTION = "general" - -# Feast Configuration Options -CONFIG_PROJECT_KEY = "project" -CONFIG_CORE_URL_KEY = "core_url" -CONFIG_CORE_ENABLE_SSL_KEY = "core_enable_ssl" -CONFIG_ENABLE_AUTH_KEY = "enable_auth" -CONFIG_ENABLE_AUTH_TOKEN_KEY = "auth_token" -CONFIG_CORE_SERVER_SSL_CERT_KEY = "core_server_ssl_cert" -CONFIG_JOB_CONTROLLER_SERVER_KEY = "jobcontroller_url" -CONFIG_SERVING_URL_KEY = "serving_url" -CONFIG_SERVING_ENABLE_SSL_KEY = "serving_enable_ssl" -CONFIG_SERVING_SERVER_SSL_CERT_KEY = "serving_server_ssl_cert" -CONFIG_JOB_SERVICE_URL_KEY = "job_service_url" -CONFIG_JOB_SERVICE_ENABLE_SSL_KEY = "job_service_enable_ssl" -CONFIG_JOB_SERVICE_SERVER_SSL_CERT_KEY = "job_service_server_ssl_cert" -CONFIG_GRPC_CONNECTION_TIMEOUT_DEFAULT_KEY = "grpc_connection_timeout_default" -CONFIG_GRPC_CONNECTION_TIMEOUT_APPLY_KEY = "grpc_connection_timeout_apply" -CONFIG_BATCH_FEATURE_REQUEST_WAIT_TIME_SECONDS_KEY = ( - "batch_feature_request_wait_time_seconds" -) -CONFIG_OAUTH_GRANT_TYPE_KEY = "oauth_grant_type" -CONFIG_OAUTH_CLIENT_ID_KEY = "oauth_client_id" -CONFIG_OAUTH_CLIENT_SECRET_KEY = "oauth_client_secret" -CONFIG_OAUTH_AUDIENCE_KEY = "oauth_audience" -CONFIG_OAUTH_TOKEN_REQUEST_URL_KEY = "oauth_token_request_url" -CONFIG_AUTH_PROVIDER = "auth_provider" - -CONFIG_TIMEOUT_KEY = "timeout" -CONFIG_MAX_WAIT_INTERVAL_KEY = "max_wait_interval" - -# Spark Job Config -CONFIG_SPARK_LAUNCHER = "spark_launcher" # standalone, dataproc, emr - -CONFIG_SPARK_STAGING_LOCATION = "spark_staging_location" - -CONFIG_SPARK_INGESTION_JOB_JAR = "spark_ingestion_jar" - -CONFIG_SPARK_STANDALONE_MASTER = "spark_standalone_master" -CONFIG_SPARK_HOME = "spark_home" - -CONFIG_SPARK_DATAPROC_CLUSTER_NAME = "dataproc_cluster_name" -CONFIG_SPARK_DATAPROC_PROJECT = "dataproc_project" -CONFIG_SPARK_DATAPROC_REGION = "dataproc_region" - -CONFIG_SPARK_HISTORICAL_FEATURE_OUTPUT_FORMAT = "historical_feature_output_format" -CONFIG_SPARK_HISTORICAL_FEATURE_OUTPUT_LOCATION = "historical_feature_output_location" - -CONFIG_REDIS_HOST = "redis_host" -CONFIG_REDIS_PORT = "redis_port" -CONFIG_REDIS_SSL = "redis_ssl" - -CONFIG_STATSD_ENABLED = "statsd_enabled" -CONFIG_STATSD_HOST = "statsd_host" -CONFIG_STATSD_PORT = "statsd_port" - -CONFIG_DEADLETTER_PATH = "deadletter_path" -CONFIG_STENCIL_URL = "stencil_url" - -CONFIG_SPARK_EMR_REGION = "emr_region" -CONFIG_SPARK_EMR_CLUSTER_ID = "emr_cluster_id" -CONFIG_SPARK_EMR_CLUSTER_TEMPLATE_PATH = "emr_cluster_template_path" -CONFIG_SPARK_EMR_LOG_LOCATION = "emr_log_location" - -# Configuration option default values -FEAST_DEFAULT_OPTIONS = { - # Default Feast project to use - CONFIG_PROJECT_KEY: "default", - # Default Feast Core URL - CONFIG_CORE_URL_KEY: "localhost:6565", - # Enable or disable TLS/SSL to Feast Core - CONFIG_CORE_ENABLE_SSL_KEY: "False", - # Enable user authentication to Feast Core - CONFIG_ENABLE_AUTH_KEY: "False", - # Path to certificate(s) to secure connection to Feast Core - CONFIG_CORE_SERVER_SSL_CERT_KEY: "", - # Default Feast Job Controller URL - CONFIG_JOB_CONTROLLER_SERVER_KEY: "localhost:6570", - # Default Feast Serving URL - CONFIG_SERVING_URL_KEY: "localhost:6565", - # Enable or disable TLS/SSL to Feast Serving - CONFIG_SERVING_ENABLE_SSL_KEY: "False", - # Path to certificate(s) to secure connection to Feast Serving - CONFIG_SERVING_SERVER_SSL_CERT_KEY: "", - # Default connection timeout to Feast Serving, Feast Core, and Feast Job Service (in seconds) - CONFIG_GRPC_CONNECTION_TIMEOUT_DEFAULT_KEY: "10", - # Default gRPC connection timeout when sending an ApplyFeatureSet command to - # Feast Core (in seconds) - CONFIG_GRPC_CONNECTION_TIMEOUT_APPLY_KEY: "600", - # Time to wait for batch feature requests before timing out. - CONFIG_BATCH_FEATURE_REQUEST_WAIT_TIME_SECONDS_KEY: "600", - CONFIG_TIMEOUT_KEY: "21600", - CONFIG_MAX_WAIT_INTERVAL_KEY: "60", - # Authentication Provider - Google OpenID/OAuth - CONFIG_AUTH_PROVIDER: "google", - CONFIG_SPARK_LAUNCHER: "dataproc", - CONFIG_SPARK_INGESTION_JOB_JAR: "https://storage.googleapis.com/feast-jobs/spark/" - "ingestion/feast-ingestion-spark-develop.jar", - CONFIG_SPARK_STANDALONE_MASTER: "local[*]", - CONFIG_REDIS_HOST: "localhost", - CONFIG_REDIS_PORT: "6379", - CONFIG_REDIS_SSL: "False", - CONFIG_SPARK_HISTORICAL_FEATURE_OUTPUT_FORMAT: "parquet", - # Enable or disable TLS/SSL to Feast Service - CONFIG_JOB_SERVICE_ENABLE_SSL_KEY: "False", - # Path to certificate(s) to secure connection to Feast Job Service - CONFIG_JOB_SERVICE_SERVER_SSL_CERT_KEY: "", - CONFIG_STATSD_ENABLED: "False", - # IngestionJob DeadLetter Destination - CONFIG_DEADLETTER_PATH: "", - # ProtoRegistry Address (currently only Stencil Server is supported as registry) - # https://github.com/gojekfarm/stencil - CONFIG_STENCIL_URL: "", -} +class Option: + def __init__(self, name, default): + self._name = name + self._default = default + + def __get__(self, instance, owner): + if instance is None: + return self._name.lower() + + return self._default + + +class ConfigMeta(type): + """ + Class factory which customizes ConfigOptions class instantiation. + Specifically, setting configuration option's name to lowercase of capitalized variable. + """ + + def __new__(cls, name, bases, attrs): + keys = [ + k for k, v in attrs.items() if not k.startswith("_") and not callable(v) + ] + attrs["__config_keys__"] = keys + attrs.update({k: Option(k, attrs[k]) for k in keys}) + return super().__new__(cls, name, bases, attrs) + + +#: Default datetime column name for point-in-time join +DATETIME_COLUMN: str = "datetime" + +#: Environmental variable to specify Feast configuration file location +FEAST_CONFIG_FILE_ENV: str = "FEAST_CONFIG" + +#: Default prefix to Feast environmental variables +CONFIG_FEAST_ENV_VAR_PREFIX: str = "FEAST_" + +#: Default directory to Feast configuration file +CONFIG_FILE_DEFAULT_DIRECTORY: str = ".feast" + +#: Default Feast configuration file name +CONFIG_FILE_NAME: str = "config" + +#: Default section in Feast configuration file to specify options +CONFIG_FILE_SECTION: str = "general" + +# Maximum interval(secs) to wait between retries for retry function +MAX_WAIT_INTERVAL: str = "60" + + +class ConfigOptions(metaclass=ConfigMeta): + """ Feast Configuration Options """ + + #: Feast project namespace to use + PROJECT: str = "default" + + #: Default Feast Core URL + CORE_URL: str = "localhost:6565" + + #: Enable or disable TLS/SSL to Feast Core + CORE_ENABLE_SSL: str = "False" + + #: Enable user authentication to Feast Core + ENABLE_AUTH: str = "False" + + #: JWT Auth token for user authentication to Feast + AUTH_TOKEN: Optional[str] = None + + #: Path to certificate(s) to secure connection to Feast Core + CORE_SERVER_SSL_CERT: str = "" + + #: Default Feast Serving URL + SERVING_URL: str = "localhost:6566" + + #: Enable or disable TLS/SSL to Feast Serving + SERVING_ENABLE_SSL: str = "False" + + #: Path to certificate(s) to secure connection to Feast Serving + SERVING_SERVER_SSL_CERT: str = "" + + #: Default Feast Job Service URL + JOB_SERVICE_URL: Optional[str] = None + + #: Enable or disable TLS/SSL to Feast Job Service + JOB_SERVICE_ENABLE_SSL: str = "False" + + #: Path to certificate(s) to secure connection to Feast Job Service + JOB_SERVICE_SERVER_SSL_CERT: str = "" + + #: Enable or disable control loop for Feast Job Service + JOB_SERVICE_ENABLE_CONTROL_LOOP: str = "False" + + #: Default connection timeout to Feast Serving, Feast Core, and Feast Job Service (in seconds) + GRPC_CONNECTION_TIMEOUT: str = "10" + + #: Default gRPC connection timeout when sending an ApplyFeatureTable command to Feast Core (in seconds) + GRPC_CONNECTION_TIMEOUT_APPLY: str = "600" + + #: Default timeout when running batch ingestion + BATCH_INGESTION_PRODUCTION_TIMEOUT: str = "120" + + #: Time to wait for historical feature requests before timing out. + BATCH_FEATURE_REQUEST_WAIT_TIME_SECONDS: str = "600" + + #: Endpoint URL for S3 storage_client + S3_ENDPOINT_URL: Optional[str] = None + + #: Account name for Azure blob storage_client + AZURE_BLOB_ACCOUNT_NAME: Optional[str] = None + + #: Account access key for Azure blob storage_client + AZURE_BLOB_ACCOUNT_ACCESS_KEY: Optional[str] = None + + #: Authentication Provider - Google OpenID/OAuth + #: + #: Options: "google" / "oauth" + AUTH_PROVIDER: str = "google" + + #: Spark Job launcher. The choice of storage is connected to the choice of SPARK_LAUNCHER. + #: + #: Options: "standalone", "dataproc", "emr" + SPARK_LAUNCHER: Optional[str] = None + + #: Feast Spark Job ingestion jobs staging location. The choice of storage is connected to the choice of SPARK_LAUNCHER. + #: + #: Eg. gs://some-bucket/output/, s3://some-bucket/output/, file:///data/subfolder/ + SPARK_STAGING_LOCATION: Optional[str] = None + + #: Feast Spark Job ingestion jar file. The choice of storage is connected to the choice of SPARK_LAUNCHER. + #: + #: Eg. "dataproc" (http and gs), "emr" (http and s3), "standalone" (http and file) + SPARK_INGESTION_JAR: str = "https://storage.googleapis.com/feast-jobs/spark/ingestion/feast-ingestion-spark-develop.jar" + + #: Spark resource manager master url + SPARK_STANDALONE_MASTER: str = "local[*]" + + #: Directory where Spark is installed + SPARK_HOME: Optional[str] = None + + #: The project id where the materialized view of BigQuerySource is going to be created + #: by default, use the same project where view is located + SPARK_BQ_MATERIALIZATION_PROJECT: Optional[str] = None + + #: The dataset id where the materialized view of BigQuerySource is going to be created + #: by default, use the same dataset where view is located + SPARK_BQ_MATERIALIZATION_DATASET: Optional[str] = None + + #: Dataproc cluster to run Feast Spark Jobs in + DATAPROC_CLUSTER_NAME: Optional[str] = None + + #: Project of Dataproc cluster + DATAPROC_PROJECT: Optional[str] = None + + #: Region of Dataproc cluster + DATAPROC_REGION: Optional[str] = None + + #: No. of executor instances for Dataproc cluster + DATAPROC_EXECUTOR_INSTANCES = "2" + + #: No. of executor cores for Dataproc cluster + DATAPROC_EXECUTOR_CORES = "2" + + #: No. of executor memory for Dataproc cluster + DATAPROC_EXECUTOR_MEMORY = "2g" + + # namespace to use for Spark jobs launched using k8s spark operator + SPARK_K8S_NAMESPACE = "default" + + # expect k8s spark operator to be running in the same cluster as Feast + SPARK_K8S_USE_INCLUSTER_CONFIG = "True" + + # SparkApplication resource template + SPARK_K8S_JOB_TEMPLATE_PATH = None + + #: File format of historical retrieval features + HISTORICAL_FEATURE_OUTPUT_FORMAT: str = "parquet" + + #: File location of historical retrieval features + HISTORICAL_FEATURE_OUTPUT_LOCATION: Optional[str] = None + + #: Default Redis host + REDIS_HOST: str = "localhost" + + #: Default Redis port + REDIS_PORT: str = "6379" + + #: Enable or disable TLS/SSL to Redis + REDIS_SSL: str = "False" + + #: Enable or disable StatsD + STATSD_ENABLED: str = "False" + + #: Default StatsD port + STATSD_HOST: Optional[str] = None + + #: Default StatsD port + STATSD_PORT: Optional[str] = None + + #: Ingestion Job DeadLetter Destination. The choice of storage is connected to the choice of SPARK_LAUNCHER. + #: + #: Eg. gs://some-bucket/output/, s3://some-bucket/output/, file:///data/subfolder/ + DEADLETTER_PATH: str = "" + + #: ProtoRegistry Address (currently only Stencil Server is supported as registry) + #: https://github.com/gojekfarm/stencil + STENCIL_URL: str = "" + + #: If set to true rows that do not pass custom validation (see feast.contrib.validation) + #: won't be saved to Online Storage + INGESTION_DROP_INVALID_ROWS = "False" + + #: EMR cluster to run Feast Spark Jobs in + EMR_CLUSTER_ID: Optional[str] = None + + #: Region of EMR cluster + EMR_REGION: Optional[str] = None + + #: Template path of EMR cluster + EMR_CLUSTER_TEMPLATE_PATH: Optional[str] = None + + #: Log path of EMR cluster + EMR_LOG_LOCATION: Optional[str] = None + + #: Oauth grant type + OAUTH_GRANT_TYPE: Optional[str] = None + + #: Oauth client ID + OAUTH_CLIENT_ID: Optional[str] = None + + #: Oauth client secret + OAUTH_CLIENT_SECRET: Optional[str] = None + + #: Oauth intended recipients + OAUTH_AUDIENCE: Optional[str] = None + + #: Oauth token request url + OAUTH_TOKEN_REQUEST_URL: Optional[str] = None + + def defaults(self): + return { + k: getattr(self, k) + for k in self.__config_keys__ + if getattr(self, k) is not None + } diff --git a/sdk/python/feast/pyspark/aws/__init__.py b/sdk/python/feast/contrib/__init__.py similarity index 100% rename from sdk/python/feast/pyspark/aws/__init__.py rename to sdk/python/feast/contrib/__init__.py diff --git a/infra/docker/jobcontroller/Dockerfile.debug b/sdk/python/feast/contrib/validation/__init__.py similarity index 100% rename from infra/docker/jobcontroller/Dockerfile.debug rename to sdk/python/feast/contrib/validation/__init__.py diff --git a/sdk/python/feast/contrib/validation/base.py b/sdk/python/feast/contrib/validation/base.py new file mode 100644 index 00000000000..3e956f34098 --- /dev/null +++ b/sdk/python/feast/contrib/validation/base.py @@ -0,0 +1,13 @@ +import io + +try: + from pyspark import cloudpickle +except ImportError: + raise ImportError("pyspark must be installed to enable validation functionality") + + +def serialize_udf(fun, return_type) -> bytes: + buffer = io.BytesIO() + command = (fun, return_type) + cloudpickle.dump(command, buffer) + return buffer.getvalue() diff --git a/sdk/python/feast/contrib/validation/ge.py b/sdk/python/feast/contrib/validation/ge.py new file mode 100644 index 00000000000..1315d4a59e2 --- /dev/null +++ b/sdk/python/feast/contrib/validation/ge.py @@ -0,0 +1,201 @@ +import io +import json +import os +from typing import TYPE_CHECKING +from urllib.parse import urlparse + +import pandas as pd + +from feast.constants import ConfigOptions +from feast.contrib.validation.base import serialize_udf +from feast.staging.storage_client import get_staging_client + +try: + from great_expectations.core import ExpectationConfiguration, ExpectationSuite + from great_expectations.dataset import PandasDataset +except ImportError: + raise ImportError( + "great_expectations must be installed to enable validation functionality. " + "Please install feast[validation]" + ) + +try: + from pyspark.sql.types import BooleanType +except ImportError: + raise ImportError( + "pyspark must be installed to enable validation functionality. " + "Please install feast[validation]" + ) + + +if TYPE_CHECKING: + from feast import Client, FeatureTable + + +GE_PACKED_ARCHIVE = "https://storage.googleapis.com/feast-jobs/spark/validation/pylibs-ge-%(platform)s.tar.gz" +_UNSET = object() + + +class ValidationUDF: + def __init__(self, name: str, pickled_code: bytes): + self.name = name + self.pickled_code = pickled_code + + +def drop_feature_table_prefix( + expectation_configuration: ExpectationConfiguration, prefix +): + kwargs = expectation_configuration.kwargs + for arg_name in ("column", "column_A", "column_B"): + if arg_name not in kwargs: + continue + + if kwargs[arg_name].startswith(prefix): + kwargs[arg_name] = kwargs[arg_name][len(prefix) :] + + +def prepare_expectations(suite: ExpectationSuite, feature_table: "FeatureTable"): + for expectation in suite.expectations: + drop_feature_table_prefix(expectation, f"{feature_table.name}__") + + return suite + + +def create_validation_udf( + name: str, expectations: ExpectationSuite, feature_table: "FeatureTable", +) -> ValidationUDF: + """ + Wraps your expectations into Spark UDF. + + Expectations should be generated & validated using training dataset: + >>> from great_expectations.dataset import PandasDataset + >>> ds = PandasDataset.from_dataset(you_training_df) + >>> ds.expect_column_values_to_be_between('column', 0, 100) + + >>> expectations = ds.get_expectation_suite() + + Important: you expectations should pass on training dataset, only successful checks + will be converted and stored in ExpectationSuite. + + Now you can create UDF that will validate data during ingestion: + >>> create_validation_udf("myValidation", expectations) + + :param name + :param expectations: collection of expectation gathered on training dataset + :param feature_table + :return: ValidationUDF with serialized code + """ + + expectations = prepare_expectations(expectations, feature_table) + + def udf(df: pd.DataFrame) -> pd.Series: + from datadog.dogstatsd import DogStatsd + + reporter = ( + DogStatsd( + host=os.environ["STATSD_HOST"], + port=int(os.environ["STATSD_PORT"]), + telemetry_min_flush_interval=0, + ) + if os.getenv("STATSD_HOST") and os.getenv("STATSD_PORT") + else DogStatsd() + ) + + ds = PandasDataset.from_dataset(df) + result = ds.validate(expectations, result_format="COMPLETE") + valid_rows = pd.Series([True] * df.shape[0]) + + for check in result.results: + if check.exception_info["raised_exception"]: + # ToDo: probably we should mark all rows as invalid + continue + + check_kwargs = check.expectation_config.kwargs + check_kwargs.pop("result_format", None) + check_name = "_".join( + [check.expectation_config.expectation_type] + + [ + str(v) + for v in check_kwargs.values() + if isinstance(v, (str, int, float)) + ] + ) + + if ( + "unexpected_count" in check.result + and check.result["unexpected_count"] > 0 + ): + reporter.increment( + "feast_feature_validation_check_failed", + value=check.result["unexpected_count"], + tags=[ + f"feature_table:{os.getenv('FEAST_INGESTION_FEATURE_TABLE', 'unknown')}", + f"project:{os.getenv('FEAST_INGESTION_PROJECT_NAME', 'default')}", + f"check:{check_name}", + ], + ) + + valid_rows.iloc[check.result["unexpected_index_list"]] = False + + elif "observed_value" in check.result and check.result["observed_value"]: + reporter.gauge( + "feast_feature_validation_observed_value", + value=int( + check.result["observed_value"] + * 100 # storing as decimal with precision 2 + ) + if not check.success + else 0, # nullify everything below threshold + tags=[ + f"feature_table:{os.getenv('FEAST_INGESTION_FEATURE_TABLE', 'unknown')}", + f"project:{os.getenv('FEAST_INGESTION_PROJECT_NAME', 'default')}", + f"check:{check_name}", + ], + ) + + return valid_rows + + pickled_code = serialize_udf(udf, BooleanType()) + return ValidationUDF(name, pickled_code) + + +def apply_validation( + client: "Client", + feature_table: "FeatureTable", + udf: ValidationUDF, + validation_window_secs: int, + include_py_libs=_UNSET, +): + """ + Uploads validation udf code to staging location & + stores path to udf code and required python libraries as FeatureTable labels. + """ + include_py_libs = ( + include_py_libs if include_py_libs is not _UNSET else GE_PACKED_ARCHIVE + ) + + staging_location = client._config.get(ConfigOptions.SPARK_STAGING_LOCATION).rstrip( + "/" + ) + staging_scheme = urlparse(staging_location).scheme + staging_client = get_staging_client(staging_scheme, client._config) + + pickled_code_fp = io.BytesIO(udf.pickled_code) + remote_path = f"{staging_location}/udfs/{feature_table.name}/{udf.name}.pickle" + staging_client.upload_fileobj( + pickled_code_fp, f"{udf.name}.pickle", remote_uri=urlparse(remote_path) + ) + + feature_table.labels.update( + { + "_validation": json.dumps( + dict( + name=udf.name, + pickled_code_path=remote_path, + include_archive_path=include_py_libs, + ) + ), + "_streaming_trigger_secs": str(validation_window_secs), + } + ) + client.apply_feature_table(feature_table) diff --git a/sdk/python/feast/feature.py b/sdk/python/feast/feature.py index 1d0e525a89b..16c8ca57eea 100644 --- a/sdk/python/feast/feature.py +++ b/sdk/python/feast/feature.py @@ -148,6 +148,15 @@ def to_proto(self) -> FeatureRefProto: return self.proto + def __repr__(self): + # return string representation of the reference + ref_str = self.proto.feature_table + ":" + self.proto.name + return ref_str + + def __str__(self): + # readable string of the reference + return f"FeatureRef<{self.__repr__()}>" + def _build_feature_references(feature_ref_strs: List[str]) -> List[FeatureRefProto]: """ diff --git a/sdk/python/feast/feature_table.py b/sdk/python/feast/feature_table.py index 5d32dcc114a..530b44fa5f6 100644 --- a/sdk/python/feast/feature_table.py +++ b/sdk/python/feast/feature_table.py @@ -406,3 +406,6 @@ def _update_from_feature_table(self, feature_table): self.stream_source = feature_table.stream_source self._created_timestamp = feature_table.created_timestamp self._last_updated_timestamp = feature_table.last_updated_timestamp + + def __repr__(self): + return f"FeatureTable <{self.name}>" diff --git a/sdk/python/feast/field.py b/sdk/python/feast/field.py deleted file mode 100644 index 2f54e82d6fe..00000000000 --- a/sdk/python/feast/field.py +++ /dev/null @@ -1,468 +0,0 @@ -# Copyright 2019 The Feast Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -from collections import OrderedDict -from typing import MutableMapping, Optional, Union - -from feast.core.FeatureSet_pb2 import FeatureSpec -from feast.value_type import ValueType -from tensorflow_metadata.proto.v0 import schema_pb2 - - -class Field: - """ - High level field type. This is the parent type to both entities and - features. - """ - - def __init__( - self, - name: str, - dtype: ValueType, - labels: Optional[MutableMapping[str, str]] = None, - ): - self._name = name - if not isinstance(dtype, ValueType): - raise ValueError("dtype is not a valid ValueType") - self._dtype = dtype - if labels is None: - self._labels = OrderedDict() # type: MutableMapping - else: - self._labels = labels - self._presence: Optional[schema_pb2.FeaturePresence] = None - self._group_presence: Optional[schema_pb2.FeaturePresenceWithinGroup] = None - self._shape: Optional[schema_pb2.FixedShape] = None - self._value_count: Optional[schema_pb2.ValueCount] = None - self._domain: Optional[str] = None - self._int_domain: Optional[schema_pb2.IntDomain] = None - self._float_domain: Optional[schema_pb2.FloatDomain] = None - self._string_domain: Optional[schema_pb2.StringDomain] = None - self._bool_domain: Optional[schema_pb2.BoolDomain] = None - self._struct_domain: Optional[schema_pb2.StructDomain] = None - self._natural_language_domain: Optional[schema_pb2.NaturalLanguageDomain] = None - self._image_domain: Optional[schema_pb2.ImageDomain] = None - self._mid_domain: Optional[schema_pb2.MIDDomain] = None - self._url_domain: Optional[schema_pb2.URLDomain] = None - self._time_domain: Optional[schema_pb2.TimeDomain] = None - self._time_of_day_domain: Optional[schema_pb2.TimeOfDayDomain] = None - - def __eq__(self, other): - if ( - self.name != other.name - or self.dtype != other.dtype - or self.labels != other.labels - ): - return False - return True - - @property - def name(self): - """ - Getter for name of this field - """ - return self._name - - @property - def dtype(self) -> ValueType: - """ - Getter for data type of this field - """ - return self._dtype - - @property - def labels(self) -> MutableMapping[str, str]: - """ - Getter for labels of this field - """ - return self._labels - - @property - def presence(self) -> Optional[schema_pb2.FeaturePresence]: - """ - Getter for presence of this field - """ - return self._presence - - @presence.setter - def presence(self, presence: schema_pb2.FeaturePresence): - """ - Setter for presence of this field - """ - if not isinstance(presence, schema_pb2.FeaturePresence): - raise TypeError("presence must be of FeaturePresence type") - self._clear_presence_constraints() - self._presence = presence - - @property - def group_presence(self) -> Optional[schema_pb2.FeaturePresenceWithinGroup]: - """ - Getter for group_presence of this field - """ - return self._group_presence - - @group_presence.setter - def group_presence(self, group_presence: schema_pb2.FeaturePresenceWithinGroup): - """ - Setter for group_presence of this field - """ - if not isinstance(group_presence, schema_pb2.FeaturePresenceWithinGroup): - raise TypeError("group_presence must be of FeaturePresenceWithinGroup type") - self._clear_presence_constraints() - self._group_presence = group_presence - - @property - def shape(self) -> Optional[schema_pb2.FixedShape]: - """ - Getter for shape of this field - """ - return self._shape - - @shape.setter - def shape(self, shape: schema_pb2.FixedShape): - """ - Setter for shape of this field - """ - if not isinstance(shape, schema_pb2.FixedShape): - raise TypeError("shape must be of FixedShape type") - self._clear_shape_type() - self._shape = shape - - @property - def value_count(self) -> Optional[schema_pb2.ValueCount]: - """ - Getter for value_count of this field - """ - return self._value_count - - @value_count.setter - def value_count(self, value_count: schema_pb2.ValueCount): - """ - Setter for value_count of this field - """ - if not isinstance(value_count, schema_pb2.ValueCount): - raise TypeError("value_count must be of ValueCount type") - self._clear_shape_type() - self._value_count = value_count - - @property - def domain(self) -> Optional[str]: - """ - Getter for domain of this field - """ - return self._domain - - @domain.setter - def domain(self, domain: str): - """ - Setter for domain of this field - """ - if not isinstance(domain, str): - raise TypeError("domain must be of str type") - self._clear_domain_info() - self._domain = domain - - @property - def int_domain(self) -> Optional[schema_pb2.IntDomain]: - """ - Getter for int_domain of this field - """ - return self._int_domain - - @int_domain.setter - def int_domain(self, int_domain: schema_pb2.IntDomain): - """ - Setter for int_domain of this field - """ - if not isinstance(int_domain, schema_pb2.IntDomain): - raise TypeError("int_domain must be of IntDomain type") - self._clear_domain_info() - self._int_domain = int_domain - - @property - def float_domain(self) -> Optional[schema_pb2.FloatDomain]: - """ - Getter for float_domain of this field - """ - return self._float_domain - - @float_domain.setter - def float_domain(self, float_domain: schema_pb2.FloatDomain): - """ - Setter for float_domain of this field - """ - if not isinstance(float_domain, schema_pb2.FloatDomain): - raise TypeError("float_domain must be of FloatDomain type") - self._clear_domain_info() - self._float_domain = float_domain - - @property - def string_domain(self) -> Optional[schema_pb2.StringDomain]: - """ - Getter for string_domain of this field - """ - return self._string_domain - - @string_domain.setter - def string_domain(self, string_domain: schema_pb2.StringDomain): - """ - Setter for string_domain of this field - """ - if not isinstance(string_domain, schema_pb2.StringDomain): - raise TypeError("string_domain must be of StringDomain type") - self._clear_domain_info() - self._string_domain = string_domain - - @property - def bool_domain(self) -> Optional[schema_pb2.BoolDomain]: - """ - Getter for bool_domain of this field - """ - return self._bool_domain - - @bool_domain.setter - def bool_domain(self, bool_domain: schema_pb2.BoolDomain): - """ - Setter for bool_domain of this field - """ - if not isinstance(bool_domain, schema_pb2.BoolDomain): - raise TypeError("bool_domain must be of BoolDomain type") - self._clear_domain_info() - self._bool_domain = bool_domain - - @property - def struct_domain(self) -> Optional[schema_pb2.StructDomain]: - """ - Getter for struct_domain of this field - """ - return self._struct_domain - - @struct_domain.setter - def struct_domain(self, struct_domain: schema_pb2.StructDomain): - """ - Setter for struct_domain of this field - """ - if not isinstance(struct_domain, schema_pb2.StructDomain): - raise TypeError("struct_domain must be of StructDomain type") - self._clear_domain_info() - self._struct_domain = struct_domain - - @property - def natural_language_domain(self) -> Optional[schema_pb2.NaturalLanguageDomain]: - """ - Getter for natural_language_domain of this field - """ - return self._natural_language_domain - - @natural_language_domain.setter - def natural_language_domain( - self, natural_language_domain: schema_pb2.NaturalLanguageDomain - ): - """ - Setter for natural_language_domin of this field - """ - if not isinstance(natural_language_domain, schema_pb2.NaturalLanguageDomain): - raise TypeError( - "natural_language_domain must be of NaturalLanguageDomain type" - ) - self._clear_domain_info() - self._natural_language_domain = natural_language_domain - - @property - def image_domain(self) -> Optional[schema_pb2.ImageDomain]: - """ - Getter for image_domain of this field - """ - return self._image_domain - - @image_domain.setter - def image_domain(self, image_domain: schema_pb2.ImageDomain): - """ - Setter for image_domain of this field - """ - if not isinstance(image_domain, schema_pb2.ImageDomain): - raise TypeError("image_domain must be of ImageDomain type") - self._clear_domain_info() - self._image_domain = image_domain - - @property - def mid_domain(self) -> Optional[schema_pb2.MIDDomain]: - """ - Getter for mid_domain of this field - """ - return self._mid_domain - - @mid_domain.setter - def mid_domain(self, mid_domain: schema_pb2.MIDDomain): - """ - Setter for mid_domain of this field - """ - if not isinstance(mid_domain, schema_pb2.MIDDomain): - raise TypeError("mid_domain must be of MIDDomain type") - self._clear_domain_info() - self._mid_domain = mid_domain - - @property - def url_domain(self) -> Optional[schema_pb2.URLDomain]: - """ - Getter for url_domain of this field - """ - return self._url_domain - - @url_domain.setter - def url_domain(self, url_domain: schema_pb2.URLDomain): - """ - Setter for url_domain of this field - """ - if not isinstance(url_domain, schema_pb2.URLDomain): - raise TypeError("url_domain must be of URLDomain type") - self._clear_domain_info() - self.url_domain = url_domain - - @property - def time_domain(self) -> Optional[schema_pb2.TimeDomain]: - """ - Getter for time_domain of this field - """ - return self._time_domain - - @time_domain.setter - def time_domain(self, time_domain: schema_pb2.TimeDomain): - """ - Setter for time_domain of this field - """ - if not isinstance(time_domain, schema_pb2.TimeDomain): - raise TypeError("time_domain must be of TimeDomain type") - self._clear_domain_info() - self._time_domain = time_domain - - @property - def time_of_day_domain(self) -> Optional[schema_pb2.TimeOfDayDomain]: - """ - Getter for time_of_day_domain of this field - """ - return self._time_of_day_domain - - @time_of_day_domain.setter - def time_of_day_domain(self, time_of_day_domain): - """ - Setter for time_of_day_domain of this field - """ - if not isinstance(time_of_day_domain, schema_pb2.TimeOfDayDomain): - raise TypeError("time_of_day_domain must be of TimeOfDayDomain type") - self._clear_domain_info() - self._time_of_day_domain = time_of_day_domain - - def update_presence_constraints( - self, feature: Union[schema_pb2.Feature, FeatureSpec] - ) -> None: - """ - Update the presence constraints in this field from Tensorflow Feature or - Feast FeatureSpec - - Args: - feature: Tensorflow Feature or Feast FeatureSpec - - Returns: None - """ - presence_constraints_case = feature.WhichOneof("presence_constraints") - if presence_constraints_case == "presence": - self.presence = feature.presence - elif presence_constraints_case == "group_presence": - self.group_presence = feature.group_presence - - def update_shape_type( - self, feature: Union[schema_pb2.Feature, FeatureSpec] - ) -> None: - """ - Update the shape type in this field from Tensorflow Feature or - Feast FeatureSpec - - Args: - feature: Tensorflow Feature or Feast FeatureSpec - - Returns: None - """ - shape_type_case = feature.WhichOneof("shape_type") - if shape_type_case == "shape": - self.shape = feature.shape - elif shape_type_case == "value_count": - self.value_count = feature.value_count - - def update_domain_info( - self, feature: Union[schema_pb2.Feature, FeatureSpec] - ) -> None: - """ - Update the domain info in this field from Tensorflow Feature or Feast FeatureSpec - - Args: - feature: Tensorflow Feature or Feast FeatureSpec - - Returns: None - """ - domain_info_case = feature.WhichOneof("domain_info") - if domain_info_case == "int_domain": - self.int_domain = feature.int_domain - elif domain_info_case == "float_domain": - self.float_domain = feature.float_domain - elif domain_info_case == "string_domain": - self.string_domain = feature.string_domain - elif domain_info_case == "bool_domain": - self.bool_domain = feature.bool_domain - elif domain_info_case == "struct_domain": - self.struct_domain = feature.struct_domain - elif domain_info_case == "natural_language_domain": - self.natural_language_domain = feature.natural_language_domain - elif domain_info_case == "image_domain": - self.image_domain = feature.image_domain - elif domain_info_case == "mid_domain": - self.mid_domain = feature.mid_domain - elif domain_info_case == "url_domain": - self.url_domain = feature.url_domain - elif domain_info_case == "time_domain": - self.time_domain = feature.time_domain - elif domain_info_case == "time_of_day_domain": - self.time_of_day_domain = feature.time_of_day_domain - - def to_proto(self): - """ - Unimplemented to_proto method for a field. This should be extended. - """ - pass - - def from_proto(self, proto): - """ - Unimplemented from_proto method for a field. This should be extended. - """ - pass - - def _clear_presence_constraints(self): - self._presence = None - self._group_presence = None - - def _clear_shape_type(self): - self._shape = None - self._value_count = None - - def _clear_domain_info(self): - self._domain = None - self._int_domain = None - self._float_domain = None - self._string_domain = None - self._bool_domain = None - self._struct_domain = None - self._natural_language_domain = None - self._image_domain = None - self._mid_domain = None - self._url_domain = None - self._time_domain = None - self._time_of_day_domain = None diff --git a/sdk/python/feast/grpc/auth.py b/sdk/python/feast/grpc/auth.py index 9680607b8e3..8614015f456 100644 --- a/sdk/python/feast/grpc/auth.py +++ b/sdk/python/feast/grpc/auth.py @@ -18,16 +18,8 @@ from google.auth.exceptions import DefaultCredentialsError from feast.config import Config -from feast.constants import ( - CONFIG_AUTH_PROVIDER, - CONFIG_ENABLE_AUTH_TOKEN_KEY, - CONFIG_OAUTH_AUDIENCE_KEY, - CONFIG_OAUTH_CLIENT_ID_KEY, - CONFIG_OAUTH_CLIENT_SECRET_KEY, - CONFIG_OAUTH_GRANT_TYPE_KEY, - CONFIG_OAUTH_TOKEN_REQUEST_URL_KEY, - AuthProvider, -) +from feast.constants import AuthProvider +from feast.constants import ConfigOptions as opt def get_auth_metadata_plugin(config: Config) -> grpc.AuthMetadataPlugin: @@ -44,9 +36,9 @@ def get_auth_metadata_plugin(config: Config) -> grpc.AuthMetadataPlugin: Args: config: Feast Configuration object """ - if AuthProvider(config.get(CONFIG_AUTH_PROVIDER)) == AuthProvider.GOOGLE: + if AuthProvider(config.get(opt.AUTH_PROVIDER)) == AuthProvider.GOOGLE: return GoogleOpenIDAuthMetadataPlugin(config) - elif AuthProvider(config.get(CONFIG_AUTH_PROVIDER)) == AuthProvider.OAUTH: + elif AuthProvider(config.get(opt.AUTH_PROVIDER)) == AuthProvider.OAUTH: return OAuthMetadataPlugin(config) else: raise RuntimeError( @@ -75,15 +67,15 @@ def __init__(self, config: Config): self._token = None # If provided, set a static token - if config.exists(CONFIG_ENABLE_AUTH_TOKEN_KEY): - self._static_token = config.get(CONFIG_ENABLE_AUTH_TOKEN_KEY) + if config.exists(opt.AUTH_TOKEN): + self._static_token = config.get(opt.AUTH_TOKEN) self._refresh_token(config) elif ( - config.exists(CONFIG_OAUTH_GRANT_TYPE_KEY) - and config.exists(CONFIG_OAUTH_CLIENT_ID_KEY) - and config.exists(CONFIG_OAUTH_CLIENT_SECRET_KEY) - and config.exists(CONFIG_OAUTH_AUDIENCE_KEY) - and config.exists(CONFIG_OAUTH_TOKEN_REQUEST_URL_KEY) + config.exists(opt.OAUTH_GRANT_TYPE) + and config.exists(opt.OAUTH_CLIENT_ID) + and config.exists(opt.OAUTH_CLIENT_SECRET) + and config.exists(opt.OAUTH_AUDIENCE) + and config.exists(opt.OAUTH_TOKEN_REQUEST_URL) ): self._refresh_token(config) else: @@ -112,14 +104,14 @@ def _refresh_token(self, config: Config): headers_token = {"content-type": "application/json"} data_token = { - "grant_type": config.get(CONFIG_OAUTH_GRANT_TYPE_KEY), - "client_id": config.get(CONFIG_OAUTH_CLIENT_ID_KEY), - "client_secret": config.get(CONFIG_OAUTH_CLIENT_SECRET_KEY), - "audience": config.get(CONFIG_OAUTH_AUDIENCE_KEY), + "grant_type": config.get(opt.OAUTH_GRANT_TYPE), + "client_id": config.get(opt.OAUTH_CLIENT_ID), + "client_secret": config.get(opt.OAUTH_CLIENT_SECRET), + "audience": config.get(opt.OAUTH_AUDIENCE), } data_token = json.dumps(data_token) response_token = requests.post( - config.get(CONFIG_OAUTH_TOKEN_REQUEST_URL_KEY), + config.get(opt.OAUTH_TOKEN_REQUEST_URL), headers=headers_token, data=data_token, ) @@ -171,8 +163,8 @@ def __init__(self, config: Config): self._token = None # If provided, set a static token - if config.exists(CONFIG_ENABLE_AUTH_TOKEN_KEY): - self._static_token = config.get(CONFIG_ENABLE_AUTH_TOKEN_KEY) + if config.exists(opt.AUTH_TOKEN): + self._static_token = config.get(opt.AUTH_TOKEN) self._request = requests.Request() self._refresh_token() diff --git a/sdk/python/feast/job_service.py b/sdk/python/feast/job_service.py index 62f8f3bed99..bf9f67df584 100644 --- a/sdk/python/feast/job_service.py +++ b/sdk/python/feast/job_service.py @@ -1,9 +1,18 @@ import logging +import os +import signal +import threading +import time +import traceback from concurrent.futures import ThreadPoolExecutor +from typing import Dict, List, Tuple, cast import grpc +from google.api_core.exceptions import FailedPrecondition +from google.protobuf.timestamp_pb2 import Timestamp import feast +from feast.constants import ConfigOptions as opt from feast.core import JobService_pb2_grpc from feast.core.JobService_pb2 import ( CancelJobResponse, @@ -31,6 +40,7 @@ ) from feast.pyspark.launcher import ( get_job_by_id, + get_stream_to_online_ingestion_params, list_jobs, start_historical_feature_retrieval_job, start_offline_to_online_ingestion, @@ -43,36 +53,42 @@ ) +def _job_to_proto(spark_job: SparkJob) -> JobProto: + job = JobProto() + job.id = spark_job.get_id() + job.log_uri = cast(str, spark_job.get_log_uri() or "") + status = spark_job.get_status() + if status == SparkJobStatus.COMPLETED: + job.status = JobStatus.JOB_STATUS_DONE + elif status == SparkJobStatus.IN_PROGRESS: + job.status = JobStatus.JOB_STATUS_RUNNING + elif status == SparkJobStatus.FAILED: + job.status = JobStatus.JOB_STATUS_ERROR + elif status == SparkJobStatus.STARTING: + job.status = JobStatus.JOB_STATUS_PENDING + else: + raise ValueError(f"Invalid job status {status}") + + if isinstance(spark_job, RetrievalJob): + job.type = JobType.RETRIEVAL_JOB + job.retrieval.output_location = spark_job.get_output_file_uri(block=False) + elif isinstance(spark_job, BatchIngestionJob): + job.type = JobType.BATCH_INGESTION_JOB + job.batch_ingestion.table_name = spark_job.get_feature_table() + elif isinstance(spark_job, StreamIngestionJob): + job.type = JobType.STREAM_INGESTION_JOB + job.stream_ingestion.table_name = spark_job.get_feature_table() + else: + raise ValueError(f"Invalid job type {job}") + + job.start_time.FromDatetime(spark_job.get_start_time()) + + return job + + class JobServiceServicer(JobService_pb2_grpc.JobServiceServicer): - def __init__(self): - self.client = feast.Client() - - def _job_to_proto(self, spark_job: SparkJob) -> JobProto: - job = JobProto() - job.id = spark_job.get_id() - status = spark_job.get_status() - if status == SparkJobStatus.COMPLETED: - job.status = JobStatus.JOB_STATUS_DONE - elif status == SparkJobStatus.IN_PROGRESS: - job.status = JobStatus.JOB_STATUS_RUNNING - elif status == SparkJobStatus.FAILED: - job.status = JobStatus.JOB_STATUS_ERROR - elif status == SparkJobStatus.STARTING: - job.status = JobStatus.JOB_STATUS_PENDING - else: - raise ValueError(f"Invalid job status {status}") - - if isinstance(spark_job, RetrievalJob): - job.type = JobType.RETRIEVAL_JOB - job.retrieval.output_location = spark_job.get_output_file_uri(block=False) - elif isinstance(spark_job, BatchIngestionJob): - job.type = JobType.BATCH_INGESTION_JOB - elif isinstance(spark_job, StreamIngestionJob): - job.type = JobType.STREAM_INGESTION_JOB - else: - raise ValueError(f"Invalid job type {job}") - - return job + def __init__(self, client): + self.client = client def StartOfflineToOnlineIngestionJob( self, request: StartOfflineToOnlineIngestionJobRequest, context @@ -88,7 +104,16 @@ def StartOfflineToOnlineIngestionJob( start=request.start_date.ToDatetime(), end=request.end_date.ToDatetime(), ) - return StartOfflineToOnlineIngestionJobResponse(id=job.get_id()) + + job_start_timestamp = Timestamp() + job_start_timestamp.FromDatetime(job.get_start_time()) + + return StartOfflineToOnlineIngestionJobResponse( + id=job.get_id(), + job_start_time=job_start_timestamp, + table_name=request.table_name, + log_uri=job.get_log_uri(), + ) def GetHistoricalFeatures(self, request: GetHistoricalFeaturesRequest, context): """Produce a training dataset, return a job id that will provide a file reference""" @@ -105,8 +130,13 @@ def GetHistoricalFeatures(self, request: GetHistoricalFeaturesRequest, context): output_file_uri = job.get_output_file_uri(block=False) + job_start_timestamp = Timestamp() + job_start_timestamp.FromDatetime(job.get_start_time()) + return GetHistoricalFeaturesResponse( - id=job.get_id(), output_file_uri=output_file_uri + id=job.get_id(), + output_file_uri=output_file_uri, + job_start_time=job_start_timestamp, ) def StartStreamToOnlineIngestionJob( @@ -117,6 +147,27 @@ def StartStreamToOnlineIngestionJob( feature_table = self.client.get_feature_table( request.table_name, request.project ) + + if self.client._config.getboolean(opt.JOB_SERVICE_ENABLE_CONTROL_LOOP): + # If the control loop is enabled, return existing stream ingestion job id instead of starting a new one + params = get_stream_to_online_ingestion_params( + self.client, request.project, feature_table, [] + ) + job_hash = params.get_job_hash() + for job in list_jobs(include_terminated=True, client=self.client): + if isinstance(job, StreamIngestionJob) and job.get_hash() == job_hash: + job_start_timestamp = Timestamp() + job_start_timestamp.FromDatetime(job.get_start_time()) + return StartStreamToOnlineIngestionJobResponse( + id=job.get_id(), + job_start_time=job_start_timestamp, + table_name=job.get_feature_table(), + log_uri=job.get_log_uri(), + ) + raise RuntimeError( + "Feast Job Service has control loop enabled, but couldn't find the existing stream ingestion job for the given FeatureTable" + ) + # TODO: add extra_jars to request job = start_stream_to_online_ingestion( client=self.client, @@ -124,14 +175,24 @@ def StartStreamToOnlineIngestionJob( feature_table=feature_table, extra_jars=[], ) - return StartStreamToOnlineIngestionJobResponse(id=job.get_id()) + + job_start_timestamp = Timestamp() + job_start_timestamp.FromDatetime(job.get_start_time()) + return StartStreamToOnlineIngestionJobResponse( + id=job.get_id(), + job_start_time=job_start_timestamp, + table_name=request.table_name, + log_uri=job.get_log_uri(), + ) def ListJobs(self, request, context): """List all types of jobs""" jobs = list_jobs( - include_terminated=request.include_terminated, client=self.client + include_terminated=request.include_terminated, + table_name=request.table_name, + client=self.client, ) - return ListJobsResponse(jobs=[self._job_to_proto(job) for job in jobs]) + return ListJobsResponse(jobs=[_job_to_proto(job) for job in jobs]) def CancelJob(self, request, context): """Stop a single job""" @@ -142,7 +203,30 @@ def CancelJob(self, request, context): def GetJob(self, request, context): """Get details of a single job""" job = get_job_by_id(request.job_id, client=self.client) - return GetJobResponse(job=self._job_to_proto(job)) + return GetJobResponse(job=_job_to_proto(job)) + + +def start_control_loop() -> None: + """Starts control loop that continuously ensures that correct jobs are being run. + + Currently this affects only the stream ingestion jobs. Please refer to + ensure_stream_ingestion_jobs for full documentation on how the check works. + + """ + logging.info( + "Feast Job Service is starting a control loop in a background thread, " + "which will ensure that stream ingestion jobs are successfully running." + ) + try: + client = feast.Client() + while True: + ensure_stream_ingestion_jobs(client, all_projects=True) + time.sleep(1) + except Exception: + traceback.print_exc() + finally: + # Send interrupt signal to the main thread to kill the server if control loop fails + os.kill(os.getpid(), signal.SIGINT) class HealthServicer(HealthService_pb2_grpc.HealthServicer): @@ -156,7 +240,7 @@ def intercept_service(self, continuation, handler_call_details): return continuation(handler_call_details) -def start_job_service(): +def start_job_service() -> None: """ Start Feast Job Service """ @@ -164,10 +248,105 @@ def start_job_service(): log_fmt = "%(asctime)s %(levelname)s %(message)s" logging.basicConfig(level=logging.INFO, format=log_fmt) + client = feast.Client() + + if client._config.getboolean(opt.JOB_SERVICE_ENABLE_CONTROL_LOOP): + # Start the control loop thread only if it's enabled from configs + thread = threading.Thread(target=start_control_loop, daemon=True) + thread.start() + server = grpc.server(ThreadPoolExecutor(), interceptors=(LoggingInterceptor(),)) - JobService_pb2_grpc.add_JobServiceServicer_to_server(JobServiceServicer(), server) + JobService_pb2_grpc.add_JobServiceServicer_to_server( + JobServiceServicer(client), server + ) HealthService_pb2_grpc.add_HealthServicer_to_server(HealthServicer(), server) server.add_insecure_port("[::]:6568") server.start() - print("Feast job server listening on port :6568") + logging.info("Feast Job Service is listening on port :6568") server.wait_for_termination() + + +def _get_expected_job_hash_to_table_refs( + client: feast.Client, projects: List[str] +) -> Dict[str, Tuple[str, str]]: + """ + Checks all feature tables for the requires project(s) and determines all required stream + ingestion jobs from them. Outputs a map of the expected job_hash to a tuple of (project, table_name). + + Args: + all_projects (bool): If true, runs the check for all project. + Otherwise only checks the current project. + + Returns: + Dict[str, Tuple[str, str]]: Map of job_hash -> (project, table_name) for expected stream ingestion jobs + """ + job_hash_to_table_refs = {} + + for project in projects: + feature_tables = client.list_feature_tables(project) + for feature_table in feature_tables: + if feature_table.stream_source is not None: + params = get_stream_to_online_ingestion_params( + client, project, feature_table, [] + ) + job_hash = params.get_job_hash() + job_hash_to_table_refs[job_hash] = (project, feature_table.name) + + return job_hash_to_table_refs + + +def ensure_stream_ingestion_jobs(client: feast.Client, all_projects: bool): + """Ensures all required stream ingestion jobs are running and cleans up the unnecessary jobs. + + More concretely, it will determine + - which stream ingestion jobs are running + - which stream ingestion jobs should be running + And it'll do 2 kinds of operations + - Cancel all running jobs that should not be running + - Start all non-existent jobs that should be running + + Args: + all_projects (bool): If true, runs the check for all project. + Otherwise only checks the client's current project. + """ + + projects = client.list_projects() if all_projects else [client.project] + + expected_job_hash_to_table_refs = _get_expected_job_hash_to_table_refs( + client, projects + ) + + expected_job_hashes = set(expected_job_hash_to_table_refs.keys()) + + jobs_by_hash: Dict[str, StreamIngestionJob] = {} + for job in client.list_jobs(include_terminated=False): + if isinstance(job, StreamIngestionJob): + jobs_by_hash[job.get_hash()] = job + + existing_job_hashes = set(jobs_by_hash.keys()) + + job_hashes_to_cancel = existing_job_hashes - expected_job_hashes + job_hashes_to_start = expected_job_hashes - existing_job_hashes + + logging.debug( + f"existing_job_hashes = {sorted(list(existing_job_hashes))} expected_job_hashes = {sorted(list(expected_job_hashes))}" + ) + + for job_hash in job_hashes_to_cancel: + job = jobs_by_hash[job_hash] + logging.info( + f"Cancelling a stream ingestion job with job_hash={job_hash} job_id={job.get_id()} status={job.get_status()}" + ) + try: + job.cancel() + except FailedPrecondition as exc: + logging.warning(f"Job canceling failed with exception {exc}") + + for job_hash in job_hashes_to_start: + # Any job that we wish to start should be among expected table refs map + project, table_name = expected_job_hash_to_table_refs[job_hash] + logging.info( + f"Starting a stream ingestion job for project={project}, table_name={table_name} with job_hash={job_hash}" + ) + feature_table = client.get_feature_table(name=table_name, project=project) + client.start_stream_to_online_ingestion(feature_table, [], project=project) diff --git a/sdk/python/feast/loaders/file.py b/sdk/python/feast/loaders/file.py index b0692457eed..e64a26d6063 100644 --- a/sdk/python/feast/loaders/file.py +++ b/sdk/python/feast/loaders/file.py @@ -40,11 +40,12 @@ def export_source_to_staging_location( Source of data to be staged. Can be a pandas DataFrame or a file path. - Only three types of source are allowed: + Only four types of source are allowed: * Pandas DataFrame * Local Avro file * GCS Avro file * S3 Avro file + * Azure Blob storage Avro file staging_location_uri (str): @@ -52,6 +53,7 @@ def export_source_to_staging_location( Examples: * gs://bucket/path/ * s3://bucket/path/ + * wasbs://bucket@account_name.blob.core.windows.net/path/ * file:///data/subfolder/ Returns: @@ -76,10 +78,9 @@ def export_source_to_staging_location( os.path.join(source_uri.netloc, source_uri.path) ) else: - # gs, s3 file provided as a source. - return get_staging_client(source_uri.scheme).list_files( - bucket=source_uri.hostname, path=source_uri.path - ) + # gs, s3, azure blob file provided as a source. + assert source_uri.hostname is not None + return get_staging_client(source_uri.scheme).list_files(uri=source_uri) else: raise Exception( f"Only string and DataFrame types are allowed as a " @@ -87,9 +88,12 @@ def export_source_to_staging_location( ) # Push data to required staging location - get_staging_client(uri.scheme).upload_file( - source_path, uri.hostname, str(uri.path).strip("/") + "/" + file_name, - ) + with open(source_path, "rb") as f: + get_staging_client(uri.scheme).upload_fileobj( + f, + source_path, + remote_uri=uri._replace(path=str(uri.path).strip("/") + "/" + file_name), + ) # Clean up, remove local staging file if dir_path and isinstance(source, pd.DataFrame) and len(dir_path) > 4: diff --git a/sdk/python/feast/loaders/ingest.py b/sdk/python/feast/loaders/ingest.py index dc87d5b32e5..820d53317a3 100644 --- a/sdk/python/feast/loaders/ingest.py +++ b/sdk/python/feast/loaders/ingest.py @@ -9,15 +9,9 @@ import pyarrow as pa from pyarrow import parquet as pq +from feast.config import Config from feast.staging.storage_client import get_staging_client -GRPC_CONNECTION_TIMEOUT_DEFAULT = 3 # type: int -GRPC_CONNECTION_TIMEOUT_APPLY = 300 # type: int -FEAST_SERVING_URL_ENV_KEY = "FEAST_SERVING_URL" # type: str -FEAST_CORE_URL_ENV_KEY = "FEAST_CORE_URL" # type: str -BATCH_FEATURE_REQUEST_WAIT_TIME_SECONDS = 300 -BATCH_INGESTION_PRODUCTION_TIMEOUT = 120 # type: int - def _check_field_mappings( column_names: List[str], @@ -173,33 +167,48 @@ def _read_table_from_source( def _upload_to_file_source( - file_url: str, with_partitions: bool, dest_path: str + file_url: str, with_partitions: bool, dest_path: str, config: Config ) -> None: """ Uploads data into a FileSource. Currently supports GCS, S3 and Local FS. Args: file_url: file url of FileSource defined for FeatureTable + with_partitions: whether to treat dest_path as dir with partitioned table + dest_path: path to file or dir to be uploaded + config: Config instance to configure FileSource """ from urllib.parse import urlparse uri = urlparse(file_url) - staging_client = get_staging_client(uri.scheme) + staging_client = get_staging_client(uri.scheme, config) if with_partitions: for path in glob.glob(os.path.join(dest_path, "**/*")): file_name = path.split("/")[-1] partition_col = path.split("/")[-2] - staging_client.upload_file( - path, - uri.hostname, - str(uri.path).strip("/") + "/" + partition_col + "/" + file_name, - ) + with open(path, "rb") as f: + staging_client.upload_fileobj( + f, + path, + remote_uri=uri._replace( + path=str(uri.path).rstrip("/") + + "/" + + partition_col + + "/" + + file_name + ), + ) else: file_name = dest_path.split("/")[-1] - staging_client.upload_file( - dest_path, uri.hostname, str(uri.path).strip("/") + "/" + file_name, - ) + with open(dest_path, "rb") as f: + staging_client.upload_fileobj( + f, + dest_path, + remote_uri=uri._replace( + path=str(uri.path).rstrip("/") + "/" + file_name + ), + ) def _upload_to_bq_source( diff --git a/sdk/python/feast/pyspark/abc.py b/sdk/python/feast/pyspark/abc.py index d3935ff65b8..5fc90e0ec43 100644 --- a/sdk/python/feast/pyspark/abc.py +++ b/sdk/python/feast/pyspark/abc.py @@ -1,14 +1,12 @@ import abc +import hashlib import json import os +from base64 import b64encode from datetime import datetime from enum import Enum from typing import Dict, List, Optional -import pandas - -from feast.data_source import FileSource - class SparkJobFailure(Exception): """ @@ -18,6 +16,9 @@ class SparkJobFailure(Exception): pass +BQ_SPARK_PACKAGE = "com.google.cloud.spark:spark-bigquery-with-dependencies_2.12:0.18.0" + + class SparkJobStatus(Enum): STARTING = 0 IN_PROGRESS = 1 @@ -66,6 +67,18 @@ def cancel(self): """ raise NotImplementedError + @abc.abstractmethod + def get_start_time(self) -> datetime: + """ + Get job start time. + """ + + def get_log_uri(self) -> Optional[str]: + """ + Get path to Spark job log, if applicable. + """ + return None + class SparkJobParameters(abc.ABC): @abc.abstractmethod @@ -103,6 +116,15 @@ def get_class_name(self) -> Optional[str]: """ return None + def get_extra_packages(self) -> List[str]: + """ + Getter for extra maven packages to be included on driver and executor + classpath if applicable. + Returns: + List[str]: List of maven packages + """ + return [] + @abc.abstractmethod def get_arguments(self) -> List[str]: """ @@ -121,6 +143,7 @@ def __init__( feature_tables_sources: List[Dict], entity_source: Dict, destination: Dict, + extra_packages: Optional[List[str]] = None, ): """ Args: @@ -129,6 +152,8 @@ def __init__( feature_tables (List[Dict]): List of feature table specification. The order of the feature table must correspond to that of feature_tables_sources. destination (Dict): Retrieval job output destination. + extra_packages (Optional[List[str]): Extra maven packages to be included on Spark driver + and executors classpath. Examples: >>> # Entity source from file @@ -232,6 +257,7 @@ def __init__( self._feature_tables_sources = feature_tables_sources self._entity_source = entity_source self._destination = destination + self._extra_packages = extra_packages if extra_packages else [] def get_name(self) -> str: all_feature_tables_names = [ft["name"] for ft in self._feature_tables] @@ -245,16 +271,22 @@ def get_main_file_path(self) -> str: os.path.dirname(__file__), "historical_feature_retrieval_job.py" ) + def get_extra_packages(self) -> List[str]: + return self._extra_packages + def get_arguments(self) -> List[str]: + def json_b64_encode(obj) -> str: + return b64encode(json.dumps(obj).encode("utf8")).decode("ascii") + return [ "--feature-tables", - json.dumps(self._feature_tables), + json_b64_encode(self._feature_tables), "--feature-tables-sources", - json.dumps(self._feature_tables_sources), + json_b64_encode(self._feature_tables_sources), "--entity-source", - json.dumps(self._entity_source), + json_b64_encode(self._entity_source), "--destination", - json.dumps(self._destination), + json_b64_encode(self._destination), ] def get_destination_path(self) -> str: @@ -303,6 +335,7 @@ def __init__( statsd_port: Optional[int] = None, deadletter_path: Optional[str] = None, stencil_url: Optional[str] = None, + drop_invalid_rows: bool = False, ): self._feature_table = feature_table self._source = source @@ -314,6 +347,7 @@ def __init__( self._statsd_port = statsd_port self._deadletter_path = deadletter_path self._stencil_url = stencil_url + self._drop_invalid_rows = drop_invalid_rows def _get_redis_config(self): return dict(host=self._redis_host, port=self._redis_port, ssl=self._redis_ssl) @@ -358,6 +392,9 @@ def get_arguments(self) -> List[str]: if self._stencil_url: args.extend(["--stencil-url", self._stencil_url]) + if self._drop_invalid_rows: + args.extend(["--drop-invalid"]) + return args @@ -426,6 +463,7 @@ def __init__( statsd_port: Optional[int] = None, deadletter_path: Optional[str] = None, stencil_url: Optional[str] = None, + drop_invalid_rows: bool = False, ): super().__init__( feature_table, @@ -438,6 +476,7 @@ def __init__( statsd_port, deadletter_path, stencil_url, + drop_invalid_rows, ) self._extra_jars = extra_jars @@ -456,18 +495,60 @@ def get_arguments(self) -> List[str]: "online", ] + def get_job_hash(self) -> str: + job_json = json.dumps( + {"source": self._source, "feature_table": self._feature_table}, + sort_keys=True, + ) + return hashlib.md5(job_json.encode()).hexdigest() + class BatchIngestionJob(SparkJob): """ Container for the ingestion job result """ + @abc.abstractmethod + def get_feature_table(self) -> str: + """ + Get the feature table name associated with this job. Return empty string if unable to + determine the feature table, such as when the job is created by the earlier + version of Feast. + + Returns: + str: Feature table name + """ + raise NotImplementedError + class StreamIngestionJob(SparkJob): """ Container for the streaming ingestion job result """ + def get_hash(self) -> str: + """Gets the consistent hash of this stream ingestion job. + + The hash needs to be persisted at the data processing layer, so that we can get the same + hash when retrieving the job from Spark. + + Returns: + str: The hash for this streaming ingestion job + """ + raise NotImplementedError + + @abc.abstractmethod + def get_feature_table(self) -> str: + """ + Get the feature table name associated with this job. Return `None` if unable to + determine the feature table, such as when the job is created by the earlier + version of Feast. + + Returns: + str: Feature table name + """ + raise NotImplementedError + class JobLauncher(abc.ABC): """ @@ -522,22 +603,12 @@ def start_stream_to_online_ingestion( """ raise NotImplementedError - @abc.abstractmethod - def stage_dataframe( - self, df: pandas.DataFrame, event_timestamp_column: str, - ) -> FileSource: - """ - Upload a pandas dataframe so it is available to the Spark cluster. - - Returns: - FileSource: representing the uploaded dataframe. - """ - raise NotImplementedError - @abc.abstractmethod def get_job_by_id(self, job_id: str) -> SparkJob: raise NotImplementedError @abc.abstractmethod - def list_jobs(self, include_terminated: bool) -> List[SparkJob]: + def list_jobs( + self, include_terminated: bool, table_name: Optional[str] + ) -> List[SparkJob]: raise NotImplementedError diff --git a/sdk/python/feast/pyspark/aws/jobs.py b/sdk/python/feast/pyspark/aws/jobs.py deleted file mode 100644 index 5579a95281b..00000000000 --- a/sdk/python/feast/pyspark/aws/jobs.py +++ /dev/null @@ -1,518 +0,0 @@ -import hashlib -import json -import logging -import os -import random -import string -import time -from typing import Any, Dict, List, NamedTuple, Optional, Tuple - -import boto3 -import botocore -import yaml - -from feast.client import Client -from feast.feature_table import FeatureTable -from feast.value_type import ValueType - -log = logging.getLogger("aws") - -# Config example: -# -# aws: -# logS3Prefix: "..a prefix for logs.." -# artifactS3Prefix: "..a prefix for jars.." -# existingClusterId: "..." # You need to set either existingClusterId -# runJobFlowTemplate: # or runJobFlowTemplate -# Name: "feast-ingestion-test" -# ReleaseLabel: emr-6.0.0 -# Instances: -# InstanceFleets: -# - InstanceFleetType: MASTER -# TargetOnDemandCapacity: 0 -# TargetSpotCapacity: 1 -# LaunchSpecifications: -# SpotSpecification: -# TimeoutDurationMinutes: 60 -# TimeoutAction: TERMINATE_CLUSTER -# InstanceTypeConfigs: -# - WeightedCapacity: 1 -# EbsConfiguration: -# EbsBlockDeviceConfigs: -# - VolumeSpecification: -# SizeInGB: 32 -# VolumeType: gp2 -# VolumesPerInstance: 2 -# BidPriceAsPercentageOfOnDemandPrice: 100 -# InstanceType: m4.xlarge -# - InstanceFleetType: CORE -# TargetOnDemandCapacity: 0 -# TargetSpotCapacity: 2 -# LaunchSpecifications: -# SpotSpecification: -# TimeoutDurationMinutes: 60 -# TimeoutAction: TERMINATE_CLUSTER -# InstanceTypeConfigs: -# - WeightedCapacity: 1 -# EbsConfiguration: -# EbsBlockDeviceConfigs: -# - VolumeSpecification: -# SizeInGB: 32 -# VolumeType: gp2 -# VolumesPerInstance: 2 -# BidPriceAsPercentageOfOnDemandPrice: 100 -# InstanceType: m4.xlarge -# Ec2SubnetIds: -# - "..a subnet id within a VPC with a route to redis..." -# AdditionalMasterSecurityGroups: -# - "..a security group that allows access to redis..." -# AdditionalSlaveSecurityGroups: -# - "..a security group that allows access to redis..." -# KeepJobFlowAliveWhenNoSteps: false -# BootstrapActions: -# - Name: "s3://aws-bigdata-blog/artifacts/resize_storage/resize_storage.sh" -# ScriptBootstrapAction: -# Path: "s3://aws-bigdata-blog/artifacts/resize_storage/resize_storage.sh" -# Args: -# - "--scaling-factor" -# - "1.5" -# Applications: -# - Name: Hadoop -# - Name: Hive -# - Name: Spark -# - Name: Livy -# JobFlowRole: my-spark-node -# ServiceRole: my-worker-node -# ScaleDownBehavior: TERMINATE_AT_TASK_COMPLETION -# redisConfig: -# host: my.redis.com -# port: 6379 -# ssl: true - -SUPPORTED_EMR_VERSION = "emr-6.0.0" -STREAM_TO_ONLINE_JOB_TYPE = "STREAM_TO_ONLINE_JOB" -OFFLINE_TO_ONLINE_JOB_TYPE = "OFFLINE_TO_ONLINE_JOB" - - -# EMR Step states considered "active", i.e. not terminated -ACTIVE_STEP_STATES = ["PENDING", "CANCEL_PENDING", "RUNNING"] -TERMINAL_STEP_STATES = ["COMPLETED", "CANCELLED", "FAILED", "INTERRUPTED"] - - -def _sanity_check_config(config, config_path: str): - """ - Sanity check the config. We don't really have to do this here but if the spark job fails - you'll only find out much later and this is annoying. Those are not exhaustive, just - some checks to help debugging common configuration issues. - """ - aws_config = config.get("aws", {}) - - if ("runJobFlowTemplate" not in aws_config) and ( - "existingClusterId" not in aws_config - ): - log.error("{config_path}: either clusterId or runJobFlowTemplate should be set") - elif "runJobFlowTemplate" in aws_config: - runJobFlowTemplate = aws_config["runJobFlowTemplate"] - releaseLabel = runJobFlowTemplate.get("ReleaseLabel") - if releaseLabel != SUPPORTED_EMR_VERSION: - log.warn( - f"{config_path}: ReleaseLabel is set to {releaseLabel}. Recommended: {SUPPORTED_EMR_VERSION}" - ) - - if "redisConfig" not in config: - log.error("{config_path}: redisConfig is not set") - - -def _get_config_path() -> str: - return os.environ["JOB_SERVICE_CONFIG_PATH"] - - -def _load_job_service_config(config_path: str): - with open(config_path) as f: - config = yaml.safe_load(f) - _sanity_check_config(config, config_path) - return config - - -def _random_string(length) -> str: - return "".join(random.choice(string.ascii_letters) for _ in range(length)) - - -def _batch_source_to_json(batch_source): - return { - "file": { - "path": batch_source.file_options.file_url, - "field_mapping": dict(batch_source.field_mapping), - "event_timestamp_column": batch_source.event_timestamp_column, - "created_timestamp_column": batch_source.created_timestamp_column, - "date_partition_column": batch_source.date_partition_column, - } - } - - -def _stream_source_to_json(stream_source): - return { - "kafka": { - "bootstrapServers": stream_source.kafka_options.bootstrap_servers, - "mapping": dict(stream_source.field_mapping), - "topic": stream_source.kafka_options.topic, - "timestampColumn": stream_source.timestamp_column, - "classpath": stream_source.kafka_options.class_path, - } - } - - -def _feature_table_to_json(client: Client, feature_table): - return { - "features": [ - {"name": f.name, "type": ValueType(f.dtype).name} - for f in feature_table.features - ], - "project": "default", - "name": feature_table.name, - "entities": [ - {"name": n, "type": client.get_entity(n).value_type} - for n in feature_table.entities - ], - } - - -def _s3_split_path(path: str) -> Tuple[str, str]: - """ Convert s3:// url to (bucket, key) """ - assert path.startswith("s3://") - _, _, bucket, key = path.split("/", 3) - return bucket, key - - -def _hash_file(local_path: str) -> str: - """ Compute sha256 hash of a file """ - h = hashlib.sha256() - with open(local_path, "rb") as f: - for block in iter(lambda: f.read(2 ** 20), b""): - h.update(block) - return h.hexdigest() - - -def _s3_upload(local_path: str, remote_path: str) -> str: - """ - Upload a local file to S3. We store the file sha256 sum in S3 metadata and skip the upload - if the file hasn't changed. - """ - bucket, key = _s3_split_path(remote_path) - client = boto3.client("s3") - - sha256sum = _hash_file(local_path) - - try: - head_response = client.head_object(Bucket=bucket, Key=key) - if head_response["Metadata"]["sha256sum"] == sha256sum: - # File already exists - return remote_path - else: - log.info("Uploading {local_path} to {remote_path}") - client.upload_file( - local_path, - bucket, - key, - ExtraArgs={"Metadata": {"sha256sum": sha256sum}}, - ) - return remote_path - except botocore.exceptions.ClientError as e: - if e.response["Error"]["Code"] == "404": - log.info("Uploading {local_path} to {remote_path}") - client.upload_file( - local_path, - bucket, - key, - ExtraArgs={"Metadata": {"sha256sum": sha256sum}}, - ) - return remote_path - else: - raise - - -def _upload_jar(jar_s3_prefix: str, local_path: str) -> str: - return _s3_upload( - local_path, os.path.join(jar_s3_prefix, os.path.basename(local_path)) - ) - - -def _get_ingestion_jar_s3_path(config) -> str: - """ - Extract job jar path from the configuration, upload it to S3 if necessary and return S3 path. - """ - jar_path = os.environ.get("INGESTION_JOB_JAR_PATH") - if jar_path is None: - raise ValueError("INGESTION_JOB_JAR_PATH not set") - elif jar_path.startswith("s3://"): - return jar_path - else: - artifactS3Prefix = config.get("aws").get("artifactS3Prefix") - if artifactS3Prefix: - return _upload_jar(artifactS3Prefix, jar_path) - else: - raise ValueError("artifactS3Prefix must be set") - - -def _sync_offline_to_online_step( - client: Client, config, feature_table, start_ts: str, end_ts: str -) -> Dict[str, Any]: - feature_table_json = _feature_table_to_json(client, feature_table) - source_json = _batch_source_to_json(feature_table.batch_source) - - return { - "Name": "Feast Ingestion", - "HadoopJarStep": { - # TODO: generate those from proto - "Properties": [ - { - "Key": "feast.step_metadata.job_type", - "Value": OFFLINE_TO_ONLINE_JOB_TYPE, - }, - { - "Key": "feast.step_metadata.offline_to_online.table_name", - "Value": feature_table.name, - }, - { - "Key": "feast.step_metadata.offline_to_online.start_ts", - "Value": start_ts, - }, - { - "Key": "feast.step_metadata.offline_to_online.end_ts", - "Value": end_ts, - }, - ], - "Args": [ - "spark-submit", - "--class", - "feast.ingestion.IngestionJob", - "--packages", - "com.google.cloud.spark:spark-bigquery-with-dependencies_2.12:0.17.2", - _get_ingestion_jar_s3_path(config), - "--mode", - "offline", - "--feature-table", - json.dumps(feature_table_json), - "--source", - json.dumps(source_json), - "--redis", - json.dumps(config["redisConfig"]), - "--start", - start_ts, - "--end", - end_ts, - ], - "Jar": "command-runner.jar", - }, - } - - -def _submit_emr_job(step: Dict[str, Any], config: Dict[str, Any]): - aws_config = config.get("aws", {}) - - emr = boto3.client("emr", region_name=aws_config.get("region")) - - if "existingClusterId" in aws_config: - step["ActionOnFailure"] = "CONTINUE" - step_ids = emr.add_job_flow_steps( - JobFlowId=aws_config["existingClusterId"], Steps=[step], - ) - print(step_ids) - else: - jobTemplate = aws_config["runJobFlowTemplate"] - step["ActionOnFailure"] = "TERMINATE_CLUSTER" - - jobTemplate["Steps"] = [step] - - if aws_config.get("logS3Prefix"): - jobTemplate["LogUri"] = os.path.join( - aws_config["logS3Prefix"], _random_string(5) - ) - - job = emr.run_job_flow(**jobTemplate) - print(job) - - -def sync_offline_to_online( - client: Client, feature_table: FeatureTable, start_ts: str, end_ts: str -): - config = _load_job_service_config(_get_config_path()) - step = _sync_offline_to_online_step(client, config, feature_table, start_ts, end_ts) - _submit_emr_job(step, config) - - -def _stream_ingestion_step( - client: Client, config, feature_table, jars: List[str] -) -> Dict[str, Any]: - feature_table_json = _feature_table_to_json(client, feature_table) - source_json = _stream_source_to_json(feature_table.stream_source) - - if jars: - jars_args = ["--jars", ",".join(jars)] - else: - jars_args = [] - - return { - "Name": "Feast Streaming Ingestion", - "HadoopJarStep": { - "Properties": [ - { - "Key": "feast.step_metadata.job_type", - "Value": STREAM_TO_ONLINE_JOB_TYPE, - }, - { - "Key": "feast.step_metadata.stream_to_online.table_name", - "Value": feature_table.name, - }, - ], - "Args": ["spark-submit", "--class", "feast.ingestion.IngestionJob"] - + jars_args - + [ - "--packages", - "com.google.cloud.spark:spark-bigquery-with-dependencies_2.12:0.17.2", - _get_ingestion_jar_s3_path(config), - "--mode", - "online", - "--feature-table", - json.dumps(feature_table_json), - "--source", - json.dumps(source_json), - "--redis", - json.dumps(config["redisConfig"]), - ], - "Jar": "command-runner.jar", - }, - } - - -def start_stream_to_online( - client: Client, feature_table: FeatureTable, jars: List[str] -): - if _get_stream_to_online_job(client, feature_table): - raise Exception("Job already running") - - config = _load_job_service_config(_get_config_path()) - step = _stream_ingestion_step(client, config, feature_table, jars) - _submit_emr_job(step, config) - - -class JobInfo(NamedTuple): - job_type: str - cluster_id: str - step_id: str - table_name: str - state: str - - -def list_jobs( - job_type: Optional[str], table_name: Optional[str], active_only=True -) -> List[JobInfo]: - """ - List Feast EMR jobs. - - Args: - job_type: optional filter by job type - table_name: optional filter by table name - active_only: filter only for "active" jobs, that is the ones that are running or pending, not terminated - - Returns: - A list of jobs. - """ - config = _load_job_service_config(_get_config_path()) - aws_config = config.get("aws", {}) - emr = boto3.client("emr", region_name=aws_config.get("region")) - paginator = emr.get_paginator("list_clusters") - res: List[JobInfo] = [] - for page in paginator.paginate( - ClusterStates=["STARTING", "BOOTSTRAPPING", "RUNNING", "WAITING", "TERMINATING"] - ): - for cluster in page["Clusters"]: - cluster_id = cluster["Id"] - step_paginator = emr.get_paginator("list_steps") - - list_steps_params = dict(ClusterId=cluster_id) - if active_only: - list_steps_params["StepStates"] = ACTIVE_STEP_STATES - - for step_page in step_paginator.paginate(**list_steps_params): - for step in step_page["Steps"]: - props = step["Config"]["Properties"] - if "feast.step_metadata.job_type" not in props: - continue - - step_table_name = props.get( - "feast.step_metadata.stream_to_online.table_name" - ) or props.get("feast.step_metadata.offline_to_online.table_name") - step_job_type = props["feast.step_metadata.job_type"] - - if table_name and step_table_name != table_name: - continue - - if job_type and step_job_type != job_type: - continue - - res.append( - JobInfo( - job_type=step_job_type, - cluster_id=cluster_id, - step_id=step["Id"], - state=step["Status"]["State"], - table_name=step_table_name, - ) - ) - return res - - -def _get_stream_to_online_job( - client: Client, feature_table: FeatureTable -) -> List[JobInfo]: - return list_jobs( - job_type=STREAM_TO_ONLINE_JOB_TYPE, - table_name=feature_table.name, - active_only=True, - ) - - -def _wait_for_job_state( - emr_client, job: JobInfo, desired_states: List[str], timeout_seconds=90 -): - """ - Wait up to timeout seconds for job to go into one of the desired states. - """ - start_time = time.time() - while time.time() - start_time < timeout_seconds: - response = emr_client.describe_step( - ClusterId=job.cluster_id, StepId=job.step_id - ) - state = response["Step"]["Status"]["State"] - if state in desired_states: - return - else: - time.sleep(0.5) - else: - raise TimeoutError( - f'Timeout waiting for job state to become {"|".join(desired_states)}' - ) - - -def _cancel_job(job_type, table_name): - """ - Cancel a EMR job. - """ - jobs = list_jobs(job_type=job_type, table_name=table_name, active_only=True) - config = _load_job_service_config(_get_config_path()) - aws_config = config.get("aws", {}) - - emr = boto3.client("emr", region_name=aws_config.get("region")) - for job in jobs: - emr.cancel_steps(ClusterId=job.cluster_id, StepIds=[job.step_id]) - - for job in jobs: - _wait_for_job_state(emr, job, TERMINAL_STEP_STATES) - - -def stop_stream_to_online(table_name: str): - """ - Stop offline-to-online ingestion job for the table. - """ - _cancel_job(STREAM_TO_ONLINE_JOB_TYPE, table_name) diff --git a/sdk/python/feast/pyspark/historical_feature_retrieval_job.py b/sdk/python/feast/pyspark/historical_feature_retrieval_job.py index a27ac3406b6..60b792b134a 100644 --- a/sdk/python/feast/pyspark/historical_feature_retrieval_job.py +++ b/sdk/python/feast/pyspark/historical_feature_retrieval_job.py @@ -1,11 +1,16 @@ import abc import argparse import json +from base64 import b64decode from datetime import timedelta from typing import Any, Dict, List, NamedTuple, Optional from pyspark.sql import DataFrame, SparkSession, Window from pyspark.sql.functions import col, expr, monotonically_increasing_id, row_number +from pyspark.sql.types import LongType + +EVENT_TIMESTAMP_ALIAS = "event_timestamp" +CREATED_TIMESTAMP_ALIAS = "created_timestamp" class Source(abc.ABC): @@ -123,6 +128,8 @@ class BigQuerySource(Source): event_timestamp_column (str): Column representing the event timestamp. created_timestamp_column (str): Column representing the creation timestamp. Required only if the source corresponds to a feature table. + materialization (Dict[str, str]): Optional. Destination for materialized view, + e.g. dict(project="...", dataset="...). """ def __init__( @@ -133,6 +140,7 @@ def __init__( event_timestamp_column: str, created_timestamp_column: Optional[str], field_mapping: Optional[Dict[str, str]], + materialization: Optional[Dict[str, str]] = None, ): super().__init__( event_timestamp_column, created_timestamp_column, field_mapping @@ -140,6 +148,7 @@ def __init__( self.project = project self.dataset = dataset self.table = table + self.materialization = materialization @property def spark_format(self) -> str: @@ -151,7 +160,15 @@ def spark_path(self) -> str: @property def spark_read_options(self) -> Dict[str, str]: - return {**super().spark_read_options, "viewsEnabled": "true"} + opts = {**super().spark_read_options, "viewsEnabled": "true"} + if self.materialization: + opts.update( + { + "materializationProject": self.materialization["project"], + "materializationDataset": self.materialization["dataset"], + } + ) + return opts def _source_from_dict(dct: Dict) -> Source: @@ -174,6 +191,7 @@ def _source_from_dict(dct: Dict) -> Source: field_mapping=dct["bq"].get("field_mapping", {}), event_timestamp_column=dct["bq"]["event_timestamp_column"], created_timestamp_column=dct["bq"].get("created_timestamp_column"), + materialization=dct["bq"].get("materialization"), ) @@ -273,8 +291,6 @@ def as_of_join( entity_event_timestamp_column: str, feature_table_df: DataFrame, feature_table: FeatureTable, - feature_event_timestamp_column: str, - feature_created_timestamp_column: str, ) -> DataFrame: """Perform an as of join between entity and feature table, given a maximum age tolerance. Join conditions: @@ -294,10 +310,6 @@ def as_of_join( feature_table_df (Dataframe): Spark dataframe representing the feature table. feature_table (FeatureTable): Feature table specification, which provide information on how the join should be performed, such as the entity primary keys and max age. - feature_event_timestamp_column (str): Column name in feature_table_df which represents - event timestamp. - feature_created_timestamp_column (str): Column name in feature_table_df which represents - when the feature is created. Returns: DataFrame: Join result, which contains all the original columns from entity_df, as well @@ -324,8 +336,7 @@ def as_of_join( None >>> feature_table_1.name 'table1' - >>> df = as_of_join(entity_df, "event_timestamp", feature_table_1_df, feature_table_1, - "event_timestamp", "created_timestamp") + >>> df = as_of_join(entity_df, "event_timestamp", feature_table_1_df, feature_table_1) >>> df.show() +------+-------------------+---------------+ |entity| event_timestamp|table1__feature| @@ -345,8 +356,7 @@ def as_of_join( 43200 >>> feature_table_2.name 'table2' - >>> df = as_of_join(entity_df, "event_timestamp", feature_table_2_df, feature_table_2, - "event_timestamp", "created_timestamp") + >>> df = as_of_join(entity_df, "event_timestamp", feature_table_2_df, feature_table_2) >>> df.show() +------+-------------------+---------------+ |entity| event_timestamp|table2__feature| @@ -358,10 +368,10 @@ def as_of_join( entity_with_id = entity_df.withColumn("_row_nr", monotonically_increasing_id()) feature_event_timestamp_column_with_prefix = ( - f"{feature_table.name}__{feature_event_timestamp_column}" + f"{feature_table.name}__{EVENT_TIMESTAMP_ALIAS}" ) feature_created_timestamp_column_with_prefix = ( - f"{feature_table.name}__{feature_created_timestamp_column}" + f"{feature_table.name}__{CREATED_TIMESTAMP_ALIAS}" ) projection = [ @@ -418,8 +428,6 @@ def join_entity_to_feature_tables( entity_event_timestamp_column: str, feature_table_dfs: List[DataFrame], feature_tables: List[FeatureTable], - feature_event_timestamp_columns: List[str], - feature_created_timestamp_columns: List[str], ) -> DataFrame: """Perform as of join between entity and multiple feature table. @@ -431,10 +439,6 @@ def join_entity_to_feature_tables( feature_table_dfs (List[Dataframe]): List of Spark dataframes representing the feature tables. feature_tables (List[FeatureTable]): List of feature table specification. The length and ordering of this argument must follow that of feature_table_dfs. - feature_event_timestamp_columns (List[str]): Column names which represent event timestamp for the - feature tables. The length and ordering of this argument must follow that of feature_table_dfs. - feature_created_timestamp_columns (str): Column names which represent when the feature is created. - The length and ordering of this argument must follow that of feature_table_dfs. Returns: DataFrame: Join result, which contains all the original columns from entity_df, as well @@ -482,8 +486,7 @@ def join_entity_to_feature_tables( tables, ) >>> joined_df = join_entity_to_feature_tables(entity_df, "event_timestamp", - [table1_df, table2_df], [table1, table2], - ["event_timestamp"] * 2, ["created_timestamp"] * 2) + [table1_df, table2_df], [table1, table2]) >>> joined_df.show() +------+-------------------+----------------+----------------+ @@ -494,24 +497,9 @@ def join_entity_to_feature_tables( """ joined_df = entity_df - for ( - feature_table_df, - feature_table, - feature_event_timestamp_column, - feature_created_timestamp_column, - ) in zip( - feature_table_dfs, - feature_tables, - feature_event_timestamp_columns, - feature_created_timestamp_columns, - ): + for (feature_table_df, feature_table,) in zip(feature_table_dfs, feature_tables): joined_df = as_of_join( - joined_df, - entity_event_timestamp_column, - feature_table_df, - feature_table, - feature_event_timestamp_column, - feature_created_timestamp_column, + joined_df, entity_event_timestamp_column, feature_table_df, feature_table, ) return joined_df @@ -583,6 +571,11 @@ def _read_and_verify_feature_table_df_from_source( mapped_source_df = _map_column(source_df, source.field_mapping) + if not source.created_timestamp_column: + raise SchemaError( + "Created timestamp column must not be none for feature table." + ) + column_selection = ( feature_table.feature_names + feature_table.entity_names @@ -614,9 +607,11 @@ def _read_and_verify_feature_table_df_from_source( ) return mapped_source_df.select( - feature_table.feature_names - + feature_table.entity_names - + [source.event_timestamp_column, source.created_timestamp_column] + [col(name) for name in feature_table.feature_names + feature_table.entity_names] + + [ + col(source.event_timestamp_column).alias(EVENT_TIMESTAMP_ALIAS), + col(source.created_timestamp_column).alias(CREATED_TIMESTAMP_ALIAS), + ] ) @@ -692,18 +687,6 @@ def retrieve_historical_features( for feature_table, source in zip(feature_tables, feature_tables_sources) ] - feature_event_timestamp_columns = [ - source.event_timestamp_column for source in feature_tables_sources - ] - feature_created_timestamp_columns: List[str] = [] - for source in feature_tables_sources: - if source.created_timestamp_column: - feature_created_timestamp_columns.append(source.created_timestamp_column) - else: - raise SchemaError( - "Created timestamp column must not be none for feature table." - ) - expected_entities = [] for feature_table in feature_tables: expected_entities.extend(feature_table.entities) @@ -733,8 +716,6 @@ def retrieve_historical_features( entity_source.event_timestamp_column, feature_table_dfs, feature_tables, - feature_event_timestamp_columns, - feature_created_timestamp_columns, ) @@ -748,7 +729,15 @@ def start_job( result = retrieve_historical_features( spark, entity_source_conf, feature_tables_sources_conf, feature_tables_conf ) + destination = FileDestination(**destination_conf) + if destination.format == "tfrecord": + entity_source = _source_from_dict(entity_source_conf) + result = result.withColumn( + entity_source.event_timestamp_column, + col(entity_source.event_timestamp_column).cast(LongType()), + ) + result.write.format(destination.format).mode("overwrite").save(destination.path) @@ -781,13 +770,17 @@ def _feature_table_from_dict(dct: Dict[str, Any]) -> FeatureTable: ) +def json_b64_decode(s: str) -> Any: + return json.loads(b64decode(s.encode("ascii"))) + + if __name__ == "__main__": spark = SparkSession.builder.getOrCreate() args = _get_args() - feature_tables_conf = json.loads(args.feature_tables) - feature_tables_sources_conf = json.loads(args.feature_tables_sources) - entity_source_conf = json.loads(args.entity_source) - destination_conf = json.loads(args.destination) + feature_tables_conf = json_b64_decode(args.feature_tables) + feature_tables_sources_conf = json_b64_decode(args.feature_tables_sources) + entity_source_conf = json_b64_decode(args.entity_source) + destination_conf = json_b64_decode(args.destination) start_job( spark, entity_source_conf, diff --git a/sdk/python/feast/pyspark/launcher.py b/sdk/python/feast/pyspark/launcher.py index e3e7c03e51d..6363ebcaffb 100644 --- a/sdk/python/feast/pyspark/launcher.py +++ b/sdk/python/feast/pyspark/launcher.py @@ -1,29 +1,12 @@ +import os +import tempfile from datetime import datetime -from typing import TYPE_CHECKING, List, Union +from typing import TYPE_CHECKING, List, Optional, Union +from urllib.parse import urlparse, urlunparse from feast.config import Config -from feast.constants import ( - CONFIG_DEADLETTER_PATH, - CONFIG_REDIS_HOST, - CONFIG_REDIS_PORT, - CONFIG_REDIS_SSL, - CONFIG_SPARK_DATAPROC_CLUSTER_NAME, - CONFIG_SPARK_DATAPROC_PROJECT, - CONFIG_SPARK_DATAPROC_REGION, - CONFIG_SPARK_EMR_CLUSTER_ID, - CONFIG_SPARK_EMR_CLUSTER_TEMPLATE_PATH, - CONFIG_SPARK_EMR_LOG_LOCATION, - CONFIG_SPARK_EMR_REGION, - CONFIG_SPARK_HOME, - CONFIG_SPARK_INGESTION_JOB_JAR, - CONFIG_SPARK_LAUNCHER, - CONFIG_SPARK_STAGING_LOCATION, - CONFIG_SPARK_STANDALONE_MASTER, - CONFIG_STATSD_ENABLED, - CONFIG_STATSD_HOST, - CONFIG_STATSD_PORT, - CONFIG_STENCIL_URL, -) +from feast.constants import ConfigOptions as opt +from feast.data_format import ParquetFormat from feast.data_source import BigQuerySource, DataSource, FileSource, KafkaSource from feast.feature_table import FeatureTable from feast.pyspark.abc import ( @@ -37,6 +20,7 @@ StreamIngestionJobParameters, ) from feast.staging.entities import create_bq_view_of_joined_features_and_entities +from feast.staging.storage_client import get_staging_client from feast.value_type import ValueType if TYPE_CHECKING: @@ -47,7 +31,7 @@ def _standalone_launcher(config: Config) -> JobLauncher: from feast.pyspark.launchers import standalone return standalone.StandaloneClusterLauncher( - config.get(CONFIG_SPARK_STANDALONE_MASTER), config.get(CONFIG_SPARK_HOME) + config.get(opt.SPARK_STANDALONE_MASTER), config.get(opt.SPARK_HOME), ) @@ -55,10 +39,13 @@ def _dataproc_launcher(config: Config) -> JobLauncher: from feast.pyspark.launchers import gcloud return gcloud.DataprocClusterLauncher( - config.get(CONFIG_SPARK_DATAPROC_CLUSTER_NAME), - config.get(CONFIG_SPARK_STAGING_LOCATION), - config.get(CONFIG_SPARK_DATAPROC_REGION), - config.get(CONFIG_SPARK_DATAPROC_PROJECT), + cluster_name=config.get(opt.DATAPROC_CLUSTER_NAME), + staging_location=config.get(opt.SPARK_STAGING_LOCATION), + region=config.get(opt.DATAPROC_REGION), + project_id=config.get(opt.DATAPROC_PROJECT), + executor_instances=config.get(opt.DATAPROC_EXECUTOR_INSTANCES), + executor_cores=config.get(opt.DATAPROC_EXECUTOR_CORES), + executor_memory=config.get(opt.DATAPROC_EXECUTOR_MEMORY), ) @@ -70,11 +57,29 @@ def _get_optional(option): return config.get(option) return aws.EmrClusterLauncher( - region=config.get(CONFIG_SPARK_EMR_REGION), - existing_cluster_id=_get_optional(CONFIG_SPARK_EMR_CLUSTER_ID), - new_cluster_template_path=_get_optional(CONFIG_SPARK_EMR_CLUSTER_TEMPLATE_PATH), - staging_location=config.get(CONFIG_SPARK_STAGING_LOCATION), - emr_log_location=config.get(CONFIG_SPARK_EMR_LOG_LOCATION), + region=config.get(opt.EMR_REGION), + existing_cluster_id=_get_optional(opt.EMR_CLUSTER_ID), + new_cluster_template_path=_get_optional(opt.EMR_CLUSTER_TEMPLATE_PATH), + staging_location=config.get(opt.SPARK_STAGING_LOCATION), + emr_log_location=config.get(opt.EMR_LOG_LOCATION), + ) + + +def _k8s_launcher(config: Config) -> JobLauncher: + from feast.pyspark.launchers import k8s + + staging_location = config.get(opt.SPARK_STAGING_LOCATION) + staging_uri = urlparse(staging_location) + + return k8s.KubernetesJobLauncher( + namespace=config.get(opt.SPARK_K8S_NAMESPACE), + resource_template_path=config.get(opt.SPARK_K8S_JOB_TEMPLATE_PATH, None), + staging_location=staging_location, + incluster=config.getboolean(opt.SPARK_K8S_USE_INCLUSTER_CONFIG), + staging_client=get_staging_client(staging_uri.scheme, config), + # azure-related arguments are None if not using Azure blob storage + azure_account_name=config.get(opt.AZURE_BLOB_ACCOUNT_NAME, None), + azure_account_key=config.get(opt.AZURE_BLOB_ACCOUNT_ACCESS_KEY, None), ) @@ -82,14 +87,15 @@ def _get_optional(option): "standalone": _standalone_launcher, "dataproc": _dataproc_launcher, "emr": _emr_launcher, + "k8s": _k8s_launcher, } def resolve_launcher(config: Config) -> JobLauncher: - return _launchers[config.get(CONFIG_SPARK_LAUNCHER)](config) + return _launchers[config.get(opt.SPARK_LAUNCHER)](config) -def _source_to_argument(source: DataSource): +def _source_to_argument(source: DataSource, config: Config): common_properties = { "field_mapping": dict(source.field_mapping), "event_timestamp_column": source.event_timestamp_column, @@ -112,6 +118,14 @@ def _source_to_argument(source: DataSource): properties["project"] = project properties["dataset"] = dataset properties["table"] = table + if config.exists(opt.SPARK_BQ_MATERIALIZATION_PROJECT) and config.exists( + opt.SPARK_BQ_MATERIALIZATION_DATASET + ): + properties["materialization"] = dict( + project=config.get(opt.SPARK_BQ_MATERIALIZATION_PROJECT), + dataset=config.get(opt.SPARK_BQ_MATERIALIZATION_DATASET), + ) + return {"bq": properties} if isinstance(source, KafkaSource): @@ -141,6 +155,7 @@ def _feature_table_to_argument( for n in feature_table.entities ], "max_age": feature_table.max_age.ToSeconds() if feature_table.max_age else None, + "labels": dict(feature_table.labels), } @@ -159,9 +174,9 @@ def start_historical_feature_retrieval_spark_session( spark_session = SparkSession.builder.getOrCreate() return retrieve_historical_features( spark=spark_session, - entity_source_conf=_source_to_argument(entity_source), + entity_source_conf=_source_to_argument(entity_source, client._config), feature_tables_sources_conf=[ - _source_to_argument(feature_table.batch_source) + _source_to_argument(feature_table.batch_source, client._config) for feature_table in feature_tables ], feature_tables_conf=[ @@ -182,20 +197,26 @@ def start_historical_feature_retrieval_job( launcher = resolve_launcher(client._config) feature_sources = [ _source_to_argument( - replace_bq_table_with_joined_view(feature_table, entity_source) + replace_bq_table_with_joined_view(feature_table, entity_source), + client._config, ) for feature_table in feature_tables ] + extra_packages = [] + if output_format == "tfrecord": + extra_packages.append("com.linkedin.sparktfrecord:spark-tfrecord_2.12:0.3.0") + return launcher.historical_feature_retrieval( RetrievalJobParameters( - entity_source=_source_to_argument(entity_source), + entity_source=_source_to_argument(entity_source, client._config), feature_tables_sources=feature_sources, feature_tables=[ _feature_table_to_argument(client, project, feature_table) for feature_table in feature_tables ], destination={"format": output_format, "path": output_path}, + extra_packages=extra_packages, ) ) @@ -241,28 +262,49 @@ def start_offline_to_online_ingestion( return launcher.offline_to_online_ingestion( BatchIngestionJobParameters( - jar=client._config.get(CONFIG_SPARK_INGESTION_JOB_JAR), - source=_source_to_argument(feature_table.batch_source), + jar=client._config.get(opt.SPARK_INGESTION_JAR), + source=_source_to_argument(feature_table.batch_source, client._config), feature_table=_feature_table_to_argument(client, project, feature_table), start=start, end=end, - redis_host=client._config.get(CONFIG_REDIS_HOST), - redis_port=client._config.getint(CONFIG_REDIS_PORT), - redis_ssl=client._config.getboolean(CONFIG_REDIS_SSL), + redis_host=client._config.get(opt.REDIS_HOST), + redis_port=client._config.getint(opt.REDIS_PORT), + redis_ssl=client._config.getboolean(opt.REDIS_SSL), statsd_host=( - client._config.getboolean(CONFIG_STATSD_ENABLED) - and client._config.get(CONFIG_STATSD_HOST) + client._config.getboolean(opt.STATSD_ENABLED) + and client._config.get(opt.STATSD_HOST) ), statsd_port=( - client._config.getboolean(CONFIG_STATSD_ENABLED) - and client._config.getint(CONFIG_STATSD_PORT) + client._config.getboolean(opt.STATSD_ENABLED) + and client._config.getint(opt.STATSD_PORT) ), - deadletter_path=client._config.get(CONFIG_DEADLETTER_PATH), - stencil_url=client._config.get(CONFIG_STENCIL_URL), + deadletter_path=client._config.get(opt.DEADLETTER_PATH), + stencil_url=client._config.get(opt.STENCIL_URL), ) ) +def get_stream_to_online_ingestion_params( + client: "Client", project: str, feature_table: FeatureTable, extra_jars: List[str] +) -> StreamIngestionJobParameters: + return StreamIngestionJobParameters( + jar=client._config.get(opt.SPARK_INGESTION_JAR), + extra_jars=extra_jars, + source=_source_to_argument(feature_table.stream_source, client._config), + feature_table=_feature_table_to_argument(client, project, feature_table), + redis_host=client._config.get(opt.REDIS_HOST), + redis_port=client._config.getint(opt.REDIS_PORT), + redis_ssl=client._config.getboolean(opt.REDIS_SSL), + statsd_host=client._config.getboolean(opt.STATSD_ENABLED) + and client._config.get(opt.STATSD_HOST), + statsd_port=client._config.getboolean(opt.STATSD_ENABLED) + and client._config.getint(opt.STATSD_PORT), + deadletter_path=client._config.get(opt.DEADLETTER_PATH), + stencil_url=client._config.get(opt.STENCIL_URL), + drop_invalid_rows=client._config.get(opt.INGESTION_DROP_INVALID_ROWS), + ) + + def start_stream_to_online_ingestion( client: "Client", project: str, feature_table: FeatureTable, extra_jars: List[str] ) -> StreamIngestionJob: @@ -270,27 +312,19 @@ def start_stream_to_online_ingestion( launcher = resolve_launcher(client._config) return launcher.start_stream_to_online_ingestion( - StreamIngestionJobParameters( - jar=client._config.get(CONFIG_SPARK_INGESTION_JOB_JAR), - extra_jars=extra_jars, - source=_source_to_argument(feature_table.stream_source), - feature_table=_feature_table_to_argument(client, project, feature_table), - redis_host=client._config.get(CONFIG_REDIS_HOST), - redis_port=client._config.getint(CONFIG_REDIS_PORT), - redis_ssl=client._config.getboolean(CONFIG_REDIS_SSL), - statsd_host=client._config.getboolean(CONFIG_STATSD_ENABLED) - and client._config.get(CONFIG_STATSD_HOST), - statsd_port=client._config.getboolean(CONFIG_STATSD_ENABLED) - and client._config.getint(CONFIG_STATSD_PORT), - deadletter_path=client._config.get(CONFIG_DEADLETTER_PATH), - stencil_url=client._config.get(CONFIG_STENCIL_URL), + get_stream_to_online_ingestion_params( + client, project, feature_table, extra_jars ) ) -def list_jobs(include_terminated: bool, client: "Client") -> List[SparkJob]: +def list_jobs( + include_terminated: bool, client: "Client", table_name: Optional[str] = None +) -> List[SparkJob]: launcher = resolve_launcher(client._config) - return launcher.list_jobs(include_terminated=include_terminated) + return launcher.list_jobs( + include_terminated=include_terminated, table_name=table_name + ) def get_job_by_id(job_id: str, client: "Client") -> SparkJob: @@ -298,6 +332,32 @@ def get_job_by_id(job_id: str, client: "Client") -> SparkJob: return launcher.get_job_by_id(job_id) -def stage_dataframe(df, event_timestamp_column: str, client: "Client") -> FileSource: - launcher = resolve_launcher(client._config) - return launcher.stage_dataframe(df, event_timestamp_column) +def stage_dataframe(df, event_timestamp_column: str, config: Config) -> FileSource: + """ + Helper function to upload a pandas dataframe in parquet format to a temporary location (under + SPARK_STAGING_LOCATION) and return it wrapped in a FileSource. + + Args: + event_timestamp_column(str): the name of the timestamp column in the dataframe. + config(Config): feast config. + """ + staging_location = config.get(opt.SPARK_STAGING_LOCATION) + staging_uri = urlparse(staging_location) + + with tempfile.NamedTemporaryFile() as f: + df.to_parquet(f) + + file_url = urlunparse( + get_staging_client(staging_uri.scheme, config).upload_fileobj( + f, + f.name, + remote_path_prefix=os.path.join(staging_location, "dataframes"), + remote_path_suffix=".parquet", + ) + ) + + return FileSource( + event_timestamp_column=event_timestamp_column, + file_format=ParquetFormat(), + file_url=file_url, + ) diff --git a/sdk/python/feast/pyspark/launchers/aws/emr.py b/sdk/python/feast/pyspark/launchers/aws/emr.py index 02cb59c12c6..d9ddb7cdae2 100644 --- a/sdk/python/feast/pyspark/launchers/aws/emr.py +++ b/sdk/python/feast/pyspark/launchers/aws/emr.py @@ -1,14 +1,12 @@ import os -import tempfile +from datetime import datetime from io import BytesIO from typing import Any, Dict, List, Optional +from urllib.parse import urlunparse import boto3 -import pandas from botocore.config import Config as BotoConfig -from feast.data_format import ParquetFormat -from feast.data_source import FileSource from feast.pyspark.abc import ( BatchIngestionJob, BatchIngestionJobParameters, @@ -21,6 +19,7 @@ StreamIngestionJob, StreamIngestionJobParameters, ) +from feast.staging.storage_client import get_staging_client from .emr_utils import ( FAILED_STEP_STATES, @@ -33,13 +32,13 @@ EmrJobRef, JobInfo, _cancel_job, + _get_job_creation_time, _get_job_state, _historical_retrieval_step, _job_ref_to_str, _list_jobs, _load_new_cluster_template, _random_string, - _s3_upload, _stream_ingestion_step, _sync_offline_to_online_step, _upload_jar, @@ -75,6 +74,9 @@ def get_status(self) -> SparkJobStatus: def cancel(self): _cancel_job(self._emr_client, self._job_ref) + def get_start_time(self) -> datetime: + return _get_job_creation_time(self._emr_client, self._job_ref) + class EmrRetrievalJob(EmrJobMixin, RetrievalJob): """ @@ -109,8 +111,12 @@ class EmrBatchIngestionJob(EmrJobMixin, BatchIngestionJob): Ingestion job result for a EMR cluster """ - def __init__(self, emr_client, job_ref: EmrJobRef): + def __init__(self, emr_client, job_ref: EmrJobRef, table_name: str): super().__init__(emr_client, job_ref) + self._table_name = table_name + + def get_feature_table(self) -> str: + return self._table_name class EmrStreamIngestionJob(EmrJobMixin, StreamIngestionJob): @@ -118,8 +124,16 @@ class EmrStreamIngestionJob(EmrJobMixin, StreamIngestionJob): Ingestion streaming job for a EMR cluster """ - def __init__(self, emr_client, job_ref: EmrJobRef): + def __init__(self, emr_client, job_ref: EmrJobRef, job_hash: str, table_name: str): super().__init__(emr_client, job_ref) + self._job_hash = job_hash + self._table_name = table_name + + def get_hash(self) -> str: + return self._job_hash + + def get_feature_table(self) -> str: + return self._table_name class EmrClusterLauncher(JobLauncher): @@ -217,17 +231,20 @@ def historical_feature_retrieval( with open(job_params.get_main_file_path()) as f: pyspark_script = f.read() - pyspark_script_path = _s3_upload( - BytesIO(pyspark_script.encode("utf8")), - local_path="historical_retrieval.py", - remote_path_prefix=self._staging_location, - remote_path_suffix=".py", + pyspark_script_path = urlunparse( + get_staging_client("s3").upload_fileobj( + BytesIO(pyspark_script.encode("utf8")), + local_path="historical_retrieval.py", + remote_path_prefix=self._staging_location, + remote_path_suffix=".py", + ) ) step = _historical_retrieval_step( pyspark_script_path, args=job_params.get_arguments(), output_file_uri=job_params.get_destination_path(), + packages=job_params.get_extra_packages(), ) job_ref = self._submit_emr_job(step) @@ -261,7 +278,9 @@ def offline_to_online_ingestion( job_ref = self._submit_emr_job(step) - return EmrBatchIngestionJob(self._emr_client(), job_ref) + return EmrBatchIngestionJob( + self._emr_client(), job_ref, ingestion_job_params.get_feature_table_name() + ) def start_stream_to_online_ingestion( self, ingestion_job_params: StreamIngestionJobParameters @@ -283,30 +302,23 @@ def start_stream_to_online_ingestion( else: extra_jar_paths.append(_upload_jar(self._staging_location, extra_jar)) + job_hash = ingestion_job_params.get_job_hash() + step = _stream_ingestion_step( jar_s3_path, extra_jar_paths, ingestion_job_params.get_feature_table_name(), args=ingestion_job_params.get_arguments(), + job_hash=job_hash, ) job_ref = self._submit_emr_job(step) - return EmrStreamIngestionJob(self._emr_client(), job_ref) - - def stage_dataframe(self, df: pandas.DataFrame, event_timestamp: str) -> FileSource: - with tempfile.NamedTemporaryFile() as f: - df.to_parquet(f) - file_url = _s3_upload( - f, - f.name, - remote_path_prefix=os.path.join(self._staging_location, "dataframes"), - remote_path_suffix=".parquet", - ) - return FileSource( - event_timestamp_column=event_timestamp, - file_format=ParquetFormat(), - file_url=file_url, + return EmrStreamIngestionJob( + self._emr_client(), + job_ref, + job_hash, + ingestion_job_params.get_feature_table_name(), ) def _job_from_job_info(self, job_info: JobInfo) -> SparkJob: @@ -318,23 +330,37 @@ def _job_from_job_info(self, job_info: JobInfo) -> SparkJob: output_file_uri=job_info.output_file_uri, ) elif job_info.job_type == OFFLINE_TO_ONLINE_JOB_TYPE: + table_name = job_info.table_name if job_info.table_name else "" + assert table_name is not None return EmrBatchIngestionJob( - emr_client=self._emr_client(), job_ref=job_info.job_ref, + emr_client=self._emr_client(), + job_ref=job_info.job_ref, + table_name=table_name, ) elif job_info.job_type == STREAM_TO_ONLINE_JOB_TYPE: + table_name = job_info.table_name if job_info.table_name else "" + assert table_name is not None + # job_hash must not be None for stream ingestion jobs + assert job_info.job_hash is not None return EmrStreamIngestionJob( - emr_client=self._emr_client(), job_ref=job_info.job_ref, + emr_client=self._emr_client(), + job_ref=job_info.job_ref, + job_hash=job_info.job_hash, + table_name=table_name, ) else: # We should never get here raise ValueError(f"Unknown job type {job_info.job_type}") - def list_jobs(self, include_terminated: bool) -> List[SparkJob]: + def list_jobs( + self, include_terminated: bool, table_name: Optional[str] = None + ) -> List[SparkJob]: """ Find EMR job by a string id. Args: include_terminated: whether to include terminated jobs. + table_name: FeatureTable name to filter by Returns: A list of SparkJob instances. @@ -343,7 +369,7 @@ def list_jobs(self, include_terminated: bool) -> List[SparkJob]: jobs = _list_jobs( emr_client=self._emr_client(), job_type=None, - table_name=None, + table_name=table_name, active_only=not include_terminated, ) diff --git a/sdk/python/feast/pyspark/launchers/aws/emr_utils.py b/sdk/python/feast/pyspark/launchers/aws/emr_utils.py index 634d82ce787..15127b38f11 100644 --- a/sdk/python/feast/pyspark/launchers/aws/emr_utils.py +++ b/sdk/python/feast/pyspark/launchers/aws/emr_utils.py @@ -1,17 +1,42 @@ -import hashlib import logging import os import random import string -import tempfile import time -from typing import IO, Any, Dict, List, NamedTuple, Optional, Tuple +from datetime import datetime +from typing import Any, Dict, List, NamedTuple, Optional +from urllib.parse import urlparse, urlunparse -import boto3 -import botocore -import pandas +import pytz import yaml +from feast.pyspark.abc import BQ_SPARK_PACKAGE + +__all__ = [ + "FAILED_STEP_STATES", + "HISTORICAL_RETRIEVAL_JOB_TYPE", + "IN_PROGRESS_STEP_STATES", + "OFFLINE_TO_ONLINE_JOB_TYPE", + "STREAM_TO_ONLINE_JOB_TYPE", + "SUCCEEDED_STEP_STATES", + "TERMINAL_STEP_STATES", + "EmrJobRef", + "JobInfo", + "_cancel_job", + "_get_job_creation_time", + "_get_job_state", + "_historical_retrieval_step", + "_job_ref_to_str", + "_list_jobs", + "_load_new_cluster_template", + "_random_string", + "_stream_ingestion_step", + "_sync_offline_to_online_step", + "_upload_jar", + "_wait_for_job_state", +] +from feast.staging.storage_client import get_staging_client + log = logging.getLogger("aws") SUPPORTED_EMR_VERSION = "emr-6.0.0" @@ -55,82 +80,17 @@ def _random_string(length) -> str: return "".join(random.choice(string.ascii_lowercase) for _ in range(length)) -def _s3_split_path(path: str) -> Tuple[str, str]: - """ Convert s3:// url to (bucket, key) """ - assert path.startswith("s3://") - _, _, bucket, key = path.split("/", 3) - return bucket, key - - -def _hash_fileobj(fileobj: IO[bytes]) -> str: - """ Compute sha256 hash of a file. File pointer will be reset to 0 on return. """ - fileobj.seek(0) - h = hashlib.sha256() - for block in iter(lambda: fileobj.read(2 ** 20), b""): - h.update(block) - fileobj.seek(0) - return h.hexdigest() - - -def _s3_upload( - fileobj: IO[bytes], - local_path: str, - *, - remote_path: Optional[str] = None, - remote_path_prefix: Optional[str] = None, - remote_path_suffix: Optional[str] = None, -) -> str: - """ - Upload a local file to S3. We store the file sha256 sum in S3 metadata and skip the upload - if the file hasn't changed. - - You can either specify remote_path or remote_path_prefix+remote_path_suffix. In the latter case, - the remote path will be computed as $remote_path_prefix/$sha256$remote_path_suffix - """ - - assert (remote_path is not None) or ( - remote_path_prefix is not None and remote_path_suffix is not None - ) - - sha256sum = _hash_fileobj(fileobj) - - if remote_path is None: - assert remote_path_prefix is not None - remote_path = os.path.join( - remote_path_prefix, f"{sha256sum}{remote_path_suffix}" - ) - - bucket, key = _s3_split_path(remote_path) - client = boto3.client("s3") - - try: - head_response = client.head_object(Bucket=bucket, Key=key) - if head_response["Metadata"]["sha256sum"] == sha256sum: - # File already exists - return remote_path - else: - log.info("Uploading {local_path} to {remote_path}") - client.upload_fileobj( - fileobj, bucket, key, ExtraArgs={"Metadata": {"sha256sum": sha256sum}}, - ) - return remote_path - except botocore.exceptions.ClientError as e: - if e.response["Error"]["Code"] == "404": - log.info("Uploading {local_path} to {remote_path}") - client.upload_fileobj( - fileobj, bucket, key, ExtraArgs={"Metadata": {"sha256sum": sha256sum}}, - ) - return remote_path - else: - raise - - -def _upload_jar(jar_s3_prefix: str, local_path: str) -> str: - with open(local_path, "rb") as f: - return _s3_upload( - f, - local_path, - remote_path=os.path.join(jar_s3_prefix, os.path.basename(local_path)), +def _upload_jar(jar_s3_prefix: str, jar_path: str) -> str: + if ( + jar_path.startswith("s3://") + or jar_path.startswith("s3a://") + or jar_path.startswith("https://") + ): + return jar_path + with open(jar_path, "rb") as f: + uri = urlparse(os.path.join(jar_s3_prefix, os.path.basename(jar_path))) + return urlunparse( + get_staging_client(uri.scheme).upload_fileobj(f, jar_path, remote_uri=uri) ) @@ -156,7 +116,7 @@ def _sync_offline_to_online_step( "--class", "feast.ingestion.IngestionJob", "--packages", - "com.google.cloud.spark:spark-bigquery-with-dependencies_2.12:0.17.2", + BQ_SPARK_PACKAGE, jar_path, ] + args, @@ -183,6 +143,7 @@ class JobInfo(NamedTuple): state: str table_name: Optional[str] output_file_uri: Optional[str] + job_hash: Optional[str] def _list_jobs( @@ -228,6 +189,8 @@ def _list_jobs( "feast.step_metadata.historical_retrieval.output_file_uri" ) + job_hash = props.get("feast.step_metadata.job_hash") + if table_name and step_table_name != table_name: continue @@ -241,20 +204,12 @@ def _list_jobs( state=step["Status"]["State"], table_name=step_table_name, output_file_uri=output_file_uri, + job_hash=job_hash, ) ) return res -def _get_stream_to_online_job(emr_client, table_name: str) -> List[JobInfo]: - return _list_jobs( - emr_client, - job_type=STREAM_TO_ONLINE_JOB_TYPE, - table_name=table_name, - active_only=True, - ) - - def _get_first_step_id(emr_client, cluster_id: str) -> str: response = emr_client.list_steps(ClusterId=cluster_id,) assert len(response["Steps"]) == 1 @@ -292,6 +247,21 @@ def _get_step_state(emr_client, cluster_id: str, step_id: str) -> str: return state +def _get_job_creation_time(emr_client, job: EmrJobRef) -> datetime: + if job.step_id is None: + step_id = _get_first_step_id(emr_client, job.cluster_id) + else: + step_id = job.step_id + + return _get_step_creation_time(emr_client, job.cluster_id, step_id) + + +def _get_step_creation_time(emr_client, cluster_id: str, step_id: str) -> datetime: + response = emr_client.describe_step(ClusterId=cluster_id, StepId=step_id) + step_creation_time = response["Step"]["Status"]["Timeline"]["CreationDateTime"] + return step_creation_time.astimezone(pytz.utc).replace(tzinfo=None) + + def _wait_for_step_state( emr_client, cluster_id: str, @@ -324,7 +294,7 @@ def _cancel_job(emr_client, job: EmrJobRef): emr_client.cancel_steps( ClusterId=job.cluster_id, StepIds=[step_id], - StepCancellationOption="TERMINATE_PROCESS", + StepCancellationOption="SEND_INTERRUPT", ) _wait_for_job_state( @@ -332,16 +302,11 @@ def _cancel_job(emr_client, job: EmrJobRef): ) -def _upload_dataframe(s3prefix: str, df: pandas.DataFrame) -> str: - with tempfile.NamedTemporaryFile() as f: - df.to_parquet(f) - return _s3_upload( - f, f.name, remote_path_prefix=s3prefix, remote_path_suffix=".parquet" - ) - - def _historical_retrieval_step( - pyspark_script_path: str, args: List[str], output_file_uri: str, + pyspark_script_path: str, + args: List[str], + output_file_uri: str, + packages: List[str] = None, ) -> Dict[str, Any]: return { @@ -357,14 +322,21 @@ def _historical_retrieval_step( "Value": output_file_uri, }, ], - "Args": ["spark-submit", pyspark_script_path] + args, + "Args": ["spark-submit"] + + (["--packages", ",".join(packages)] if packages else []) + + [pyspark_script_path] + + args, "Jar": "command-runner.jar", }, } def _stream_ingestion_step( - jar_path: str, extra_jar_paths: List[str], feature_table_name: str, args: List[str], + jar_path: str, + extra_jar_paths: List[str], + feature_table_name: str, + args: List[str], + job_hash: str, ) -> Dict[str, Any]: if extra_jar_paths: @@ -384,14 +356,12 @@ def _stream_ingestion_step( "Key": "feast.step_metadata.stream_to_online.table_name", "Value": feature_table_name, }, + {"Key": "feast.step_metadata.job_hash", "Value": job_hash}, ], "Args": ["spark-submit", "--class", "feast.ingestion.IngestionJob"] + jars_args - + [ - "--packages", - "com.google.cloud.spark:spark-bigquery-with-dependencies_2.12:0.17.2", - jar_path, - ] + + ["--conf", "spark.yarn.isPython=true"] + + ["--packages", BQ_SPARK_PACKAGE, jar_path] + args, "Jar": "command-runner.jar", }, diff --git a/sdk/python/feast/pyspark/launchers/gcloud/dataproc.py b/sdk/python/feast/pyspark/launchers/gcloud/dataproc.py index 1dfce4ce444..e4e9b167fc6 100644 --- a/sdk/python/feast/pyspark/launchers/gcloud/dataproc.py +++ b/sdk/python/feast/pyspark/launchers/gcloud/dataproc.py @@ -1,4 +1,3 @@ -import json import os import time import uuid @@ -25,9 +24,18 @@ from feast.staging.storage_client import get_staging_client +def _truncate_label(label: str) -> str: + return label[:63] + + class DataprocJobMixin: def __init__( - self, job: Job, refresh_fn: Callable[[], Job], cancel_fn: Callable[[], None] + self, + job: Job, + refresh_fn: Callable[[], Job], + cancel_fn: Callable[[], None], + project: str, + region: str, ): """ Implementation of common methods for different types of SparkJob running on Dataproc cluster. @@ -40,6 +48,8 @@ def __init__( self._job = job self._refresh_fn = refresh_fn self._cancel_fn = cancel_fn + self._project = project + self._region = region def get_id(self) -> str: """ @@ -131,6 +141,15 @@ def block_polling(self, interval_sec=30, timeout_sec=3600) -> SparkJobStatus: time.sleep(interval_sec) return status + def get_start_time(self): + return self._job.status.state_start_time + + def get_log_uri(self) -> Optional[str]: + return ( + f"https://console.cloud.google.com/dataproc/jobs/{self.get_id()}" + f"?region={self._region}&project={self._project}" + ) + class DataprocRetrievalJob(DataprocJobMixin, RetrievalJob): """ @@ -142,6 +161,8 @@ def __init__( job: Job, refresh_fn: Callable[[], Job], cancel_fn: Callable[[], None], + project: str, + region: str, output_file_uri: str, ): """ @@ -150,7 +171,7 @@ def __init__( Args: output_file_uri (str): Uri to the historical feature retrieval job output file. """ - super().__init__(job, refresh_fn, cancel_fn) + super().__init__(job, refresh_fn, cancel_fn, project, region) self._output_file_uri = output_file_uri def get_output_file_uri(self, timeout_sec=None, block=True): @@ -168,12 +189,33 @@ class DataprocBatchIngestionJob(DataprocJobMixin, BatchIngestionJob): Batch Ingestion job result for a Dataproc cluster """ + def get_feature_table(self) -> str: + return self._job.labels.get(DataprocClusterLauncher.FEATURE_TABLE_LABEL_KEY, "") + class DataprocStreamingIngestionJob(DataprocJobMixin, StreamIngestionJob): """ Streaming Ingestion job result for a Dataproc cluster """ + def __init__( + self, + job: Job, + refresh_fn: Callable[[], Job], + cancel_fn: Callable[[], None], + project: str, + region: str, + job_hash: str, + ) -> None: + super().__init__(job, refresh_fn, cancel_fn, project, region) + self._job_hash = job_hash + + def get_hash(self) -> str: + return self._job_hash + + def get_feature_table(self) -> str: + return self._job.labels.get(DataprocClusterLauncher.FEATURE_TABLE_LABEL_KEY, "") + class DataprocClusterLauncher(JobLauncher): """ @@ -184,9 +226,18 @@ class DataprocClusterLauncher(JobLauncher): EXTERNAL_JARS = ["gs://spark-lib/bigquery/spark-bigquery-latest_2.12.jar"] JOB_TYPE_LABEL_KEY = "feast_job_type" + JOB_HASH_LABEL_KEY = "feast_job_hash" + FEATURE_TABLE_LABEL_KEY = "feast_feature_tables" def __init__( - self, cluster_name: str, staging_location: str, region: str, project_id: str, + self, + cluster_name: str, + staging_location: str, + region: str, + project_id: str, + executor_instances: str, + executor_cores: str, + executor_memory: str, ): """ Initialize a dataproc job controller client, used internally for job submission and result @@ -199,8 +250,14 @@ def __init__( GCS directory for the storage of files generated by the launcher, such as the pyspark scripts. region (str): Dataproc cluster region. - project_id (str: + project_id (str): GCP project id for the dataproc cluster. + executor_instances (str): + Number of executor instances for dataproc job. + executor_cores (str): + Number of cores for dataproc job. + executor_memory (str): + Amount of memory for dataproc job. """ self.cluster_name = cluster_name @@ -217,19 +274,28 @@ def __init__( self.job_client = JobControllerClient( client_options={"api_endpoint": f"{region}-dataproc.googleapis.com:443"} ) + self.executor_instances = executor_instances + self.executor_cores = executor_cores + self.executor_memory = executor_memory def _stage_file(self, file_path: str, job_id: str) -> str: if not os.path.isfile(file_path): return file_path staging_client = get_staging_client("gs") - blob_path = os.path.join(self.remote_path, job_id, os.path.basename(file_path),) - staging_client.upload_file(file_path, self.staging_bucket, blob_path) + blob_path = os.path.join( + self.remote_path, job_id, os.path.basename(file_path), + ).lstrip("/") + blob_uri_str = f"gs://{self.staging_bucket}/{blob_path}" + with open(file_path, "rb") as f: + staging_client.upload_fileobj( + f, file_path, remote_uri=urlparse(blob_uri_str) + ) - return f"gs://{self.staging_bucket}/{blob_path}" + return blob_uri_str def dataproc_submit( - self, job_params: SparkJobParameters + self, job_params: SparkJobParameters, extra_properties: Dict[str, str] ) -> Tuple[Job, Callable[[], Job], Callable[[], None]]: local_job_id = str(uuid.uuid4()) main_file_uri = self._stage_file(job_params.get_main_file_path(), local_job_id) @@ -238,14 +304,50 @@ def dataproc_submit( "placement": {"cluster_name": self.cluster_name}, "labels": {self.JOB_TYPE_LABEL_KEY: job_params.get_job_type().name.lower()}, } + + maven_package_properties = { + "spark.jars.packages": ",".join(job_params.get_extra_packages()) + } + common_properties = { + "spark.executor.instances": self.executor_instances, + "spark.executor.cores": self.executor_cores, + "spark.executor.memory": self.executor_memory, + } + + if isinstance(job_params, StreamIngestionJobParameters): + job_config["labels"][self.FEATURE_TABLE_LABEL_KEY] = _truncate_label( + job_params.get_feature_table_name() + ) + # Add job hash to labels only for the stream ingestion job + job_config["labels"][self.JOB_HASH_LABEL_KEY] = job_params.get_job_hash() + + if isinstance(job_params, BatchIngestionJobParameters): + job_config["labels"][self.FEATURE_TABLE_LABEL_KEY] = _truncate_label( + job_params.get_feature_table_name() + ) + if job_params.get_class_name(): + scala_job_properties = { + "spark.yarn.user.classpath.first": "true", + "spark.executor.instances": self.executor_instances, + "spark.executor.cores": self.executor_cores, + "spark.executor.memory": self.executor_memory, + "spark.pyspark.driver.python": "python3.7", + "spark.pyspark.python": "python3.7", + } + job_config.update( { "spark_job": { "jar_file_uris": [main_file_uri] + self.EXTERNAL_JARS, "main_class": job_params.get_class_name(), "args": job_params.get_arguments(), - "properties": {"spark.yarn.user.classpath.first": "true"}, + "properties": { + **scala_job_properties, + **common_properties, + **maven_package_properties, + **extra_properties, + }, } } ) @@ -256,6 +358,11 @@ def dataproc_submit( "main_python_file_uri": main_file_uri, "jar_file_uris": self.EXTERNAL_JARS, "args": job_params.get_arguments(), + "properties": { + **common_properties, + **maven_package_properties, + **extra_properties, + }, } } ) @@ -286,25 +393,43 @@ def dataproc_cancel(self, job_id): def historical_feature_retrieval( self, job_params: RetrievalJobParameters ) -> RetrievalJob: - job, refresh_fn, cancel_fn = self.dataproc_submit(job_params) + job, refresh_fn, cancel_fn = self.dataproc_submit( + job_params, {"dev.feast.outputuri": job_params.get_destination_path()} + ) return DataprocRetrievalJob( - job, refresh_fn, cancel_fn, job_params.get_destination_path() + job=job, + refresh_fn=refresh_fn, + cancel_fn=cancel_fn, + project=self.project_id, + region=self.region, + output_file_uri=job_params.get_destination_path(), ) def offline_to_online_ingestion( self, ingestion_job_params: BatchIngestionJobParameters ) -> BatchIngestionJob: - job, refresh_fn, cancel_fn = self.dataproc_submit(ingestion_job_params) - return DataprocBatchIngestionJob(job, refresh_fn, cancel_fn) + job, refresh_fn, cancel_fn = self.dataproc_submit(ingestion_job_params, {}) + return DataprocBatchIngestionJob( + job=job, + refresh_fn=refresh_fn, + cancel_fn=cancel_fn, + project=self.project_id, + region=self.region, + ) def start_stream_to_online_ingestion( self, ingestion_job_params: StreamIngestionJobParameters ) -> StreamIngestionJob: - job, refresh_fn, cancel_fn = self.dataproc_submit(ingestion_job_params) - return DataprocStreamingIngestionJob(job, refresh_fn, cancel_fn) - - def stage_dataframe(self, df, event_timestamp_column: str): - raise NotImplementedError + job, refresh_fn, cancel_fn = self.dataproc_submit(ingestion_job_params, {}) + job_hash = ingestion_job_params.get_job_hash() + return DataprocStreamingIngestionJob( + job=job, + refresh_fn=refresh_fn, + cancel_fn=cancel_fn, + project=self.project_id, + region=self.region, + job_hash=job_hash, + ) def get_job_by_id(self, job_id: str) -> SparkJob: job = self.job_client.get_job( @@ -324,21 +449,49 @@ def _dataproc_job_to_spark_job(self, job: Job) -> SparkJob: cancel_fn = partial(self.dataproc_cancel, job_id) if job_type == SparkJobType.HISTORICAL_RETRIEVAL.name.lower(): - output_path = json.loads(job.pyspark_job.args[-1])["path"] - return DataprocRetrievalJob(job, refresh_fn, cancel_fn, output_path) + output_path = job.pyspark_job.properties.get("dev.feast.outputuri", "") + return DataprocRetrievalJob( + job=job, + refresh_fn=refresh_fn, + cancel_fn=cancel_fn, + project=self.project_id, + region=self.region, + output_file_uri=output_path, + ) if job_type == SparkJobType.BATCH_INGESTION.name.lower(): - return DataprocBatchIngestionJob(job, refresh_fn, cancel_fn) + return DataprocBatchIngestionJob( + job=job, + refresh_fn=refresh_fn, + cancel_fn=cancel_fn, + project=self.project_id, + region=self.region, + ) if job_type == SparkJobType.STREAM_INGESTION.name.lower(): - return DataprocStreamingIngestionJob(job, refresh_fn, cancel_fn) + job_hash = job.labels[self.JOB_HASH_LABEL_KEY] + return DataprocStreamingIngestionJob( + job=job, + refresh_fn=refresh_fn, + cancel_fn=cancel_fn, + project=self.project_id, + region=self.region, + job_hash=job_hash, + ) raise ValueError(f"Unrecognized job type: {job_type}") - def list_jobs(self, include_terminated: bool) -> List[SparkJob]: + def list_jobs( + self, include_terminated: bool, table_name: Optional[str] = None + ) -> List[SparkJob]: job_filter = f"labels.{self.JOB_TYPE_LABEL_KEY} = * AND clusterName = {self.cluster_name}" + if table_name: + job_filter = ( + job_filter + + f" AND labels.{self.FEATURE_TABLE_LABEL_KEY} = {_truncate_label(table_name)}" + ) if not include_terminated: - job_filter = job_filter + "AND status.state = ACTIVE" + job_filter = job_filter + " AND status.state = ACTIVE" return [ self._dataproc_job_to_spark_job(job) for job in self.job_client.list_jobs( diff --git a/sdk/python/feast/pyspark/launchers/k8s/__init__.py b/sdk/python/feast/pyspark/launchers/k8s/__init__.py new file mode 100644 index 00000000000..816d9880904 --- /dev/null +++ b/sdk/python/feast/pyspark/launchers/k8s/__init__.py @@ -0,0 +1,13 @@ +from .k8s import ( + KubernetesBatchIngestionJob, + KubernetesJobLauncher, + KubernetesRetrievalJob, + KubernetesStreamIngestionJob, +) + +__all__ = [ + "KubernetesRetrievalJob", + "KubernetesBatchIngestionJob", + "KubernetesStreamIngestionJob", + "KubernetesJobLauncher", +] diff --git a/sdk/python/feast/pyspark/launchers/k8s/k8s.py b/sdk/python/feast/pyspark/launchers/k8s/k8s.py new file mode 100644 index 00000000000..398ebd49373 --- /dev/null +++ b/sdk/python/feast/pyspark/launchers/k8s/k8s.py @@ -0,0 +1,421 @@ +import hashlib +import random +import string +import time +from datetime import datetime +from io import BytesIO +from pathlib import Path +from typing import Any, Dict, List, Optional, cast +from urllib.parse import urlparse, urlunparse + +import yaml +from kubernetes.client.api import CustomObjectsApi + +from feast.pyspark.abc import ( + BQ_SPARK_PACKAGE, + BatchIngestionJob, + BatchIngestionJobParameters, + JobLauncher, + RetrievalJob, + RetrievalJobParameters, + SparkJob, + SparkJobFailure, + SparkJobStatus, + StreamIngestionJob, + StreamIngestionJobParameters, +) +from feast.staging.storage_client import AbstractStagingClient + +from .k8s_utils import ( + DEFAULT_JOB_TEMPLATE, + HISTORICAL_RETRIEVAL_JOB_TYPE, + LABEL_FEATURE_TABLE, + LABEL_FEATURE_TABLE_HASH, + METADATA_JOBHASH, + METADATA_OUTPUT_URI, + OFFLINE_TO_ONLINE_JOB_TYPE, + STREAM_TO_ONLINE_JOB_TYPE, + JobInfo, + _cancel_job_by_id, + _get_api, + _get_job_by_id, + _list_jobs, + _prepare_job_resource, + _submit_job, +) + + +def _load_resource_template(job_template_path: Path) -> Dict[str, Any]: + with open(job_template_path, "rt") as f: + return yaml.safe_load(f) + + +def _generate_job_id() -> str: + return "feast-" + "".join( + random.choice(string.ascii_lowercase + string.digits) for _ in range(8) + ) + + +def _truncate_label(label: str) -> str: + return label[:63] + + +def _generate_table_hash(table_name: str) -> str: + return hashlib.md5(table_name.encode()).hexdigest() + + +class KubernetesJobMixin: + def __init__(self, api: CustomObjectsApi, namespace: str, job_id: str): + self._api = api + self._job_id = job_id + self._namespace = namespace + + def get_id(self) -> str: + return self._job_id + + def get_status(self) -> SparkJobStatus: + job = _get_job_by_id(self._api, self._namespace, self._job_id) + assert job is not None + return job.state + + def get_start_time(self) -> datetime: + job = _get_job_by_id(self._api, self._namespace, self._job_id) + assert job is not None + return job.start_time + + def cancel(self): + _cancel_job_by_id(self._api, self._namespace, self._job_id) + + def _wait_for_complete(self, timeout_seconds: Optional[float]) -> bool: + """ Returns true if the job completed successfully """ + start_time = time.time() + while (timeout_seconds is None) or (time.time() - start_time < timeout_seconds): + status = self.get_status() + if status == SparkJobStatus.COMPLETED: + return True + elif status == SparkJobStatus.FAILED: + return False + else: + time.sleep(1) + else: + raise TimeoutError("Timeout waiting for job to complete") + + +class KubernetesRetrievalJob(KubernetesJobMixin, RetrievalJob): + """ + Historical feature retrieval job result for a k8s cluster + """ + + def __init__( + self, api: CustomObjectsApi, namespace: str, job_id: str, output_file_uri: str + ): + """ + This is the job object representing the historical retrieval job, returned by KubernetesClusterLauncher. + + Args: + output_file_uri (str): Uri to the historical feature retrieval job output file. + """ + super().__init__(api, namespace, job_id) + self._output_file_uri = output_file_uri + + def get_output_file_uri(self, timeout_sec=None, block=True): + if not block: + return self._output_file_uri + + if self._wait_for_complete(timeout_sec): + return self._output_file_uri + else: + raise SparkJobFailure("Spark job failed") + + +class KubernetesBatchIngestionJob(KubernetesJobMixin, BatchIngestionJob): + """ + Ingestion job result for a k8s cluster + """ + + def __init__( + self, api: CustomObjectsApi, namespace: str, job_id: str, feature_table: str + ): + super().__init__(api, namespace, job_id) + self._feature_table = feature_table + + def get_feature_table(self) -> str: + return self._feature_table + + +class KubernetesStreamIngestionJob(KubernetesJobMixin, StreamIngestionJob): + """ + Ingestion streaming job for a k8s cluster + """ + + def __init__( + self, + api: CustomObjectsApi, + namespace: str, + job_id: str, + job_hash: str, + feature_table: str, + ): + super().__init__(api, namespace, job_id) + self._job_hash = job_hash + self._feature_table = feature_table + + def get_hash(self) -> str: + return self._job_hash + + def get_feature_table(self) -> str: + return self._feature_table + + +class KubernetesJobLauncher(JobLauncher): + """ + Submits spark jobs to a spark cluster. Currently supports only historical feature retrieval jobs. + """ + + def __init__( + self, + namespace: str, + incluster: bool, + staging_location: str, + resource_template_path: Optional[Path], + staging_client: AbstractStagingClient, + azure_account_name: str, + azure_account_key: str, + ): + self._namespace = namespace + self._api = _get_api(incluster=incluster) + self._staging_location = staging_location + self._staging_client = staging_client + self._azure_account_name = azure_account_name + self._azure_account_key = azure_account_key + if resource_template_path is not None: + self._resource_template = _load_resource_template(resource_template_path) + else: + self._resource_template = yaml.safe_load(DEFAULT_JOB_TEMPLATE) + + def _job_from_job_info(self, job_info: JobInfo) -> SparkJob: + if job_info.job_type == HISTORICAL_RETRIEVAL_JOB_TYPE: + assert METADATA_OUTPUT_URI in job_info.extra_metadata + return KubernetesRetrievalJob( + api=self._api, + namespace=job_info.namespace, + job_id=job_info.job_id, + output_file_uri=job_info.extra_metadata[METADATA_OUTPUT_URI], + ) + elif job_info.job_type == OFFLINE_TO_ONLINE_JOB_TYPE: + return KubernetesBatchIngestionJob( + api=self._api, + namespace=job_info.namespace, + job_id=job_info.job_id, + feature_table=job_info.labels.get(LABEL_FEATURE_TABLE, ""), + ) + elif job_info.job_type == STREAM_TO_ONLINE_JOB_TYPE: + # job_hash must not be None for stream ingestion jobs + assert METADATA_JOBHASH in job_info.extra_metadata + return KubernetesStreamIngestionJob( + api=self._api, + namespace=job_info.namespace, + job_id=job_info.job_id, + job_hash=job_info.extra_metadata[METADATA_JOBHASH], + feature_table=job_info.labels.get(LABEL_FEATURE_TABLE, ""), + ) + else: + # We should never get here + raise ValueError(f"Unknown job type {job_info.job_type}") + + def _get_azure_credentials(self): + uri = urlparse(self._staging_location) + if uri.scheme != "wasbs": + return {} + account_name = self._azure_account_name + account_key = self._azure_account_key + if account_name is None or account_key is None: + raise Exception( + "Using Azure blob storage requires Azure blob account name and access key to be set in config" + ) + return { + f"spark.hadoop.fs.azure.account.key.{account_name}.blob.core.windows.net": f"{account_key}" + } + + def historical_feature_retrieval( + self, job_params: RetrievalJobParameters + ) -> RetrievalJob: + """ + Submits a historical feature retrieval job to a Spark cluster. + + Raises: + SparkJobFailure: The spark job submission failed, encountered error + during execution, or timeout. + + Returns: + RetrievalJob: wrapper around remote job that returns file uri to the result file. + """ + + with open(job_params.get_main_file_path()) as f: + pyspark_script = f.read() + + pyspark_script_path = urlunparse( + self._staging_client.upload_fileobj( + BytesIO(pyspark_script.encode("utf8")), + local_path="historical_retrieval.py", + remote_path_prefix=self._staging_location, + remote_path_suffix=".py", + ) + ) + + job_id = _generate_job_id() + + resource = _prepare_job_resource( + job_template=self._resource_template, + job_id=job_id, + job_type=HISTORICAL_RETRIEVAL_JOB_TYPE, + main_application_file=pyspark_script_path, + main_class=None, + packages=[], + jars=[], + extra_metadata={METADATA_OUTPUT_URI: job_params.get_destination_path()}, + azure_credentials=self._get_azure_credentials(), + arguments=job_params.get_arguments(), + namespace=self._namespace, + ) + + job_info = _submit_job( + api=self._api, resource=resource, namespace=self._namespace, + ) + + return cast(RetrievalJob, self._job_from_job_info(job_info)) + + def _upload_jar(self, jar_path: str) -> str: + if ( + jar_path.startswith("s3://") + or jar_path.startswith("s3a://") + or jar_path.startswith("https://") + ): + return jar_path + elif jar_path.startswith("file://"): + local_jar_path = urlparse(jar_path).path + else: + local_jar_path = jar_path + with open(local_jar_path, "rb") as f: + return urlunparse( + self._staging_client.upload_fileobj( + f, + local_jar_path, + remote_path_prefix=self._staging_location, + remote_path_suffix=".jar", + ) + ) + + def offline_to_online_ingestion( + self, ingestion_job_params: BatchIngestionJobParameters + ) -> BatchIngestionJob: + """ + Submits a batch ingestion job to a Spark cluster. + + Raises: + SparkJobFailure: The spark job submission failed, encountered error + during execution, or timeout. + + Returns: + BatchIngestionJob: wrapper around remote job that can be used to check when job completed. + """ + + jar_s3_path = self._upload_jar(ingestion_job_params.get_main_file_path()) + + job_id = _generate_job_id() + + resource = _prepare_job_resource( + job_template=self._resource_template, + job_id=job_id, + job_type=OFFLINE_TO_ONLINE_JOB_TYPE, + main_application_file=jar_s3_path, + main_class=ingestion_job_params.get_class_name(), + packages=[BQ_SPARK_PACKAGE], + jars=[], + extra_metadata={}, + azure_credentials=self._get_azure_credentials(), + arguments=ingestion_job_params.get_arguments(), + namespace=self._namespace, + extra_labels={ + LABEL_FEATURE_TABLE: _truncate_label( + ingestion_job_params.get_feature_table_name() + ), + LABEL_FEATURE_TABLE_HASH: _generate_table_hash( + ingestion_job_params.get_feature_table_name() + ), + }, + ) + + job_info = _submit_job( + api=self._api, resource=resource, namespace=self._namespace, + ) + + return cast(BatchIngestionJob, self._job_from_job_info(job_info)) + + def start_stream_to_online_ingestion( + self, ingestion_job_params: StreamIngestionJobParameters + ) -> StreamIngestionJob: + """ + Starts a stream ingestion job to a Spark cluster. + + Raises: + SparkJobFailure: The spark job submission failed, encountered error + during execution, or timeout. + + Returns: + StreamIngestionJob: wrapper around remote job. + """ + + jar_s3_path = self._upload_jar(ingestion_job_params.get_main_file_path()) + + extra_jar_paths: List[str] = [] + for extra_jar in ingestion_job_params.get_extra_jar_paths(): + extra_jar_paths.append(self._upload_jar(extra_jar)) + + job_hash = ingestion_job_params.get_job_hash() + job_id = _generate_job_id() + + resource = _prepare_job_resource( + job_template=self._resource_template, + job_id=job_id, + job_type=STREAM_TO_ONLINE_JOB_TYPE, + main_application_file=jar_s3_path, + main_class=ingestion_job_params.get_class_name(), + packages=[BQ_SPARK_PACKAGE], + jars=extra_jar_paths, + extra_metadata={METADATA_JOBHASH: job_hash}, + azure_credentials=self._get_azure_credentials(), + arguments=ingestion_job_params.get_arguments(), + namespace=self._namespace, + extra_labels={ + LABEL_FEATURE_TABLE: _truncate_label( + ingestion_job_params.get_feature_table_name() + ), + LABEL_FEATURE_TABLE_HASH: _generate_table_hash( + ingestion_job_params.get_feature_table_name() + ), + }, + ) + + job_info = _submit_job( + api=self._api, resource=resource, namespace=self._namespace, + ) + + return cast(StreamIngestionJob, self._job_from_job_info(job_info)) + + def get_job_by_id(self, job_id: str) -> SparkJob: + job_info = _get_job_by_id(self._api, self._namespace, job_id) + if job_info is None: + raise KeyError(f"Job iwth id {job_id} not found") + else: + return self._job_from_job_info(job_info) + + def list_jobs( + self, include_terminated: bool, table_name: Optional[str] = None + ) -> List[SparkJob]: + return [ + self._job_from_job_info(job) + for job in _list_jobs(self._api, self._namespace, table_name) + if include_terminated + or job.state not in (SparkJobStatus.COMPLETED, SparkJobStatus.FAILED) + ] diff --git a/sdk/python/feast/pyspark/launchers/k8s/k8s_utils.py b/sdk/python/feast/pyspark/launchers/k8s/k8s_utils.py new file mode 100644 index 00000000000..19338bc313f --- /dev/null +++ b/sdk/python/feast/pyspark/launchers/k8s/k8s_utils.py @@ -0,0 +1,318 @@ +import hashlib +from copy import deepcopy +from datetime import datetime +from typing import Any, Dict, List, NamedTuple, Optional, Tuple + +from kubernetes import client, config +from kubernetes.client.api import CustomObjectsApi + +from feast.pyspark.abc import SparkJobStatus + +__all__ = [ + "_get_api", + "_cancel_job_by_id", + "_prepare_job_resource", + "_list_jobs", + "_get_job_by_id", + "STREAM_TO_ONLINE_JOB_TYPE", + "OFFLINE_TO_ONLINE_JOB_TYPE", + "HISTORICAL_RETRIEVAL_JOB_TYPE", + "METADATA_JOBHASH", + "METADATA_OUTPUT_URI", + "JobInfo", +] + +STREAM_TO_ONLINE_JOB_TYPE = "STREAM_TO_ONLINE_JOB" +OFFLINE_TO_ONLINE_JOB_TYPE = "OFFLINE_TO_ONLINE_JOB" +HISTORICAL_RETRIEVAL_JOB_TYPE = "HISTORICAL_RETRIEVAL_JOB" + +LABEL_JOBID = "feast.dev/jobid" +LABEL_JOBTYPE = "feast.dev/type" +LABEL_FEATURE_TABLE = "feast.dev/table" +LABEL_FEATURE_TABLE_HASH = "feast.dev/tablehash" + +# Can't store these bits of info in k8s labels due to 64-character limit, so we store them as +# sparkConf +METADATA_OUTPUT_URI = "dev.feast.outputuri" +METADATA_JOBHASH = "dev.feast.jobhash" + +METADATA_KEYS = set((METADATA_JOBHASH, METADATA_OUTPUT_URI)) + + +def _append_items(resource: Dict[str, Any], path: Tuple[str, ...], items: List[Any]): + """ A helper function to manipulate k8s resource configs. It updates an array in resource + definition given a jsonpath-like path. Will not update resource if items is empty. + Note that it updates resource dict in-place. + + Examples: + >>> _append_items({}, ("foo", "bar"), ["A", "B"]) + {'foo': {'bar': ['A', 'B']}} + + >>> _append_items({"foo": {"bar" : ["C"]}}, ("foo", "bar"), ["A", "B"]) + {'foo': {'bar': ['C', 'A', 'B']}} + + >>> _append_items({}, ("foo", "bar"), []) + {} + """ + + if not items: + return resource + + obj = resource + for i, p in enumerate(path): + if p not in obj: + if i == len(path) - 1: + obj[p] = [] + else: + obj[p] = {} + obj = obj[p] + assert isinstance(obj, list) + obj.extend(items) + return resource + + +def _add_keys(resource: Dict[str, Any], path: Tuple[str, ...], items: Dict[str, Any]): + """ A helper function to manipulate k8s resource configs. It will update a dict in resource + definition given a path (think jsonpath). Will ignore items set to None. Will not update + resource if all items are None. Note that it updates resource dict in-place. + + Examples: + >>> _add_keys({}, ("foo", "bar"), {"A": 1, "B": 2}) + {'foo': {'bar': {'A': 1, 'B': 2}}} + + >>> _add_keys({}, ("foo", "bar"), {"A": 1, "B": None}) + {'foo': {'bar': {'A': 1}}} + + >>> _add_keys({}, ("foo", "bar"), {"A": None, "B": None}) + {} + """ + + if not any(i is not None for i in items.values()): + return resource + + obj = resource + for p in path: + if p not in obj: + obj[p] = {} + obj = obj[p] + + for k, v in items.items(): + if v is not None: + obj[k] = v + return resource + + +def _job_id_to_resource_name(job_id: str) -> str: + return job_id + + +def _prepare_job_resource( + job_template: Dict[str, Any], + job_id: str, + job_type: str, + main_application_file: str, + main_class: Optional[str], + packages: List[str], + jars: List[str], + extra_metadata: Dict[str, str], + azure_credentials: Dict[str, str], + arguments: List[str], + namespace: str, + extra_labels: Dict[str, str] = None, +) -> Dict[str, Any]: + """ Prepare SparkApplication custom resource configs """ + job = deepcopy(job_template) + + labels = {LABEL_JOBID: job_id, LABEL_JOBTYPE: job_type} + if extra_labels: + labels = {**labels, **extra_labels} + + _add_keys(job, ("metadata", "labels"), labels) + _add_keys( + job, + ("metadata",), + dict(name=_job_id_to_resource_name(job_id), namespace=namespace), + ) + _add_keys(job, ("spec",), dict(mainClass=main_class)) + _add_keys(job, ("spec",), dict(mainApplicationFile=main_application_file)) + _add_keys(job, ("spec",), dict(arguments=arguments)) + + _add_keys(job, ("spec", "sparkConf"), extra_metadata) + _add_keys(job, ("spec", "sparkConf"), azure_credentials) + + _append_items(job, ("spec", "deps", "packages"), packages) + _append_items(job, ("spec", "deps", "jars"), jars) + + return job + + +def _get_api(incluster: bool) -> CustomObjectsApi: + # Configs can be set in Configuration class directly or using helper utility + if not incluster: + config.load_kube_config() + else: + config.load_incluster_config() + + return client.CustomObjectsApi() + + +def _crd_args(namespace: str) -> Dict[str, str]: + return dict( + group="sparkoperator.k8s.io", + version="v1beta2", + namespace=namespace, + plural="sparkapplications", + ) + + +class JobInfo(NamedTuple): + job_id: str + job_type: str + namespace: str + extra_metadata: Dict[str, str] + state: SparkJobStatus + labels: Dict[str, str] + start_time: datetime + + +STATE_MAP = { + "": SparkJobStatus.STARTING, + "SUBMITTED": SparkJobStatus.STARTING, + "RUNNING": SparkJobStatus.IN_PROGRESS, + "COMPLETED": SparkJobStatus.COMPLETED, + "FAILED": SparkJobStatus.FAILED, + "SUBMISSION_FAILED": SparkJobStatus.FAILED, + "PENDING_RERUN": SparkJobStatus.STARTING, + "INVALIDATING": SparkJobStatus.STARTING, + "SUCCEEDING": SparkJobStatus.IN_PROGRESS, + "FAILING": SparkJobStatus.FAILED, +} + + +def _k8s_state_to_feast(k8s_state: str) -> SparkJobStatus: + return STATE_MAP[k8s_state] + + +def _resource_to_job_info(resource: Dict[str, Any]) -> JobInfo: + labels = resource["metadata"]["labels"] + start_time = datetime.strptime( + resource["metadata"].get("creationTimestamp"), "%Y-%m-%dT%H:%M:%SZ" + ) + sparkConf = resource["spec"].get("sparkConf", {}) + + if "status" in resource: + state = _k8s_state_to_feast(resource["status"]["applicationState"]["state"]) + else: + state = _k8s_state_to_feast("") + + return JobInfo( + job_id=labels[LABEL_JOBID], + job_type=labels.get(LABEL_JOBTYPE, ""), + namespace=resource["metadata"].get("namespace", "default"), + extra_metadata={k: v for k, v in sparkConf.items() if k in METADATA_KEYS}, + state=state, + labels=labels, + start_time=start_time, + ) + + +def _submit_job(api: CustomObjectsApi, resource, namespace: str) -> JobInfo: + # create the resource + response = api.create_namespaced_custom_object( + **_crd_args(namespace), body=resource, + ) + return _resource_to_job_info(response) + + +def _list_jobs( + api: CustomObjectsApi, namespace: str, table_name: Optional[str] = None +) -> List[JobInfo]: + result = [] + + # Batch, Streaming Ingestion jobs + if table_name: + table_name_hash = hashlib.md5(table_name.encode()).hexdigest() + response = api.list_namespaced_custom_object( + **_crd_args(namespace), + label_selector=f"{LABEL_FEATURE_TABLE_HASH}={table_name_hash}", + ) + else: + # Retrieval jobs + response = api.list_namespaced_custom_object( + **_crd_args(namespace), label_selector=LABEL_JOBID, + ) + + for item in response["items"]: + result.append(_resource_to_job_info(item)) + return result + + +def _get_job_by_id( + api: CustomObjectsApi, namespace: str, job_id: str +) -> Optional[JobInfo]: + try: + response = api.get_namespaced_custom_object( + **_crd_args(namespace), name=_job_id_to_resource_name(job_id) + ) + + return _resource_to_job_info(response) + except client.ApiException as e: + if e.status == 404: + return None + else: + raise + + +def _cancel_job_by_id(api: CustomObjectsApi, namespace: str, job_id: str): + try: + api.delete_namespaced_custom_object( + **_crd_args(namespace), name=_job_id_to_resource_name(job_id), + ) + except client.ApiException as e: + if e.status == 404: + return None + else: + raise + + +DEFAULT_JOB_TEMPLATE = """ + +apiVersion: "sparkoperator.k8s.io/v1beta2" +kind: SparkApplication +metadata: + namespace: default +spec: + type: Scala + mode: cluster + image: "gcr.io/kf-feast/spark-py:v3.0.1" + imagePullPolicy: Always + sparkVersion: "3.0.1" + timeToLiveSeconds: 3600 + pythonVersion: "3" + restartPolicy: + type: Never + volumes: + - name: "test-volume" + hostPath: + path: "/tmp" + type: Directory + driver: + cores: 1 + coreLimit: "1200m" + memory: "512m" + labels: + version: 3.0.1 + serviceAccount: spark + volumeMounts: + - name: "test-volume" + mountPath: "/tmp" + executor: + cores: 1 + instances: 1 + memory: "512m" + labels: + version: 3.0.1 + volumeMounts: + - name: "test-volume" + mountPath: "/tmp" +""" diff --git a/sdk/python/feast/pyspark/launchers/standalone/__init__.py b/sdk/python/feast/pyspark/launchers/standalone/__init__.py index 1c44e5497fb..433d9ed1246 100644 --- a/sdk/python/feast/pyspark/launchers/standalone/__init__.py +++ b/sdk/python/feast/pyspark/launchers/standalone/__init__.py @@ -1,3 +1,11 @@ -from .local import StandaloneClusterLauncher, StandaloneClusterRetrievalJob +from .local import ( + StandaloneClusterLauncher, + StandaloneClusterRetrievalJob, + reset_job_cache, +) -__all__ = ["StandaloneClusterRetrievalJob", "StandaloneClusterLauncher"] +__all__ = [ + "StandaloneClusterRetrievalJob", + "StandaloneClusterLauncher", + "reset_job_cache", +] diff --git a/sdk/python/feast/pyspark/launchers/standalone/local.py b/sdk/python/feast/pyspark/launchers/standalone/local.py index 821a962ed3e..11f2f8fa4ed 100644 --- a/sdk/python/feast/pyspark/launchers/standalone/local.py +++ b/sdk/python/feast/pyspark/launchers/standalone/local.py @@ -1,14 +1,17 @@ import os import socket import subprocess +import threading import uuid from contextlib import closing -from typing import Dict, List +from datetime import datetime +from typing import Dict, List, Optional import requests from requests.exceptions import RequestException from feast.pyspark.abc import ( + BQ_SPARK_PACKAGE, BatchIngestionJob, BatchIngestionJobParameters, JobLauncher, @@ -22,9 +25,66 @@ StreamIngestionJobParameters, ) -# In-memory cache of Spark jobs -# This is necessary since we can't query Spark jobs in local mode -JOB_CACHE: Dict[str, SparkJob] = {} + +class JobCache: + """ + A *global* in-memory cache of Spark jobs. + + This is necessary since we can't easily keep track of running Spark jobs in local mode, since + there is no external state (unlike EMR and Dataproc which keep track of the running jobs for + us). + """ + + # Map of job_id -> spark job + job_by_id: Dict[str, SparkJob] + + # Map of job_id -> job_hash. The value can be None, indicating this job was + # manually created and Job Service isn't maintaining the state of this job + hash_by_id: Dict[str, Optional[str]] + + # This reentrant lock is necessary for multi-threading access + lock: threading.RLock + + def __init__(self): + self.job_by_id = {} + self.hash_by_id = {} + self.lock = threading.RLock() + + def add_job(self, job: SparkJob) -> None: + """Add a Spark job to the cache. + + Args: + job (SparkJob): The new Spark job to add. + """ + with self.lock: + self.job_by_id[job.get_id()] = job + if isinstance(job, StreamIngestionJob): + self.hash_by_id[job.get_id()] = job.get_hash() + + def list_jobs(self) -> List[SparkJob]: + """List all Spark jobs in the cache.""" + with self.lock: + return list(self.job_by_id.values()) + + def get_job_by_id(self, job_id: str) -> SparkJob: + """Get a Spark job with the given ID. Throws an exception if such job doesn't exist. + + Args: + job_id (str): External ID of the Spark job to get. + + Returns: + SparkJob: The Spark job with the given ID. + """ + with self.lock: + return self.job_by_id[job_id] + + +global_job_cache = JobCache() + + +def reset_job_cache(): + global global_job_cache + global_job_cache = JobCache() def _find_free_port(): @@ -42,6 +102,7 @@ def __init__( self._job_name = job_name self._process = process self._ui_port = ui_port + self._start_time = datetime.utcnow() def get_id(self) -> str: return self._job_id @@ -63,10 +124,10 @@ def check_if_started(self): if not app: return False - stages = requests.get( - f"http://localhost:{self._ui_port}/api/v1/applications/{app['id']}/stages" - ).json() - return bool(stages) + return True + + def get_start_time(self) -> datetime: + return self._start_time def get_status(self) -> SparkJobStatus: code = self._process.poll() @@ -90,7 +151,19 @@ class StandaloneClusterBatchIngestionJob(StandaloneClusterJobMixin, BatchIngesti Batch Ingestion job result for a standalone spark cluster """ - pass + def __init__( + self, + job_id: str, + job_name: str, + process: subprocess.Popen, + ui_port: int, + feature_table: str, + ) -> None: + super().__init__(job_id, job_name, process, ui_port) + self._feature_table = feature_table + + def get_feature_table(self) -> str: + return self._feature_table class StandaloneClusterStreamingIngestionJob( @@ -100,7 +173,24 @@ class StandaloneClusterStreamingIngestionJob( Streaming Ingestion job result for a standalone spark cluster """ - pass + def __init__( + self, + job_id: str, + job_name: str, + process: subprocess.Popen, + ui_port: int, + job_hash: str, + feature_table: str, + ) -> None: + super().__init__(job_id, job_name, process, ui_port) + self._job_hash = job_hash + self._feature_table = feature_table + + def get_hash(self) -> str: + return self._job_hash + + def get_feature_table(self) -> str: + return self._feature_table class StandaloneClusterRetrievalJob(StandaloneClusterJobMixin, RetrievalJob): @@ -152,8 +242,6 @@ class StandaloneClusterLauncher(JobLauncher): Submits jobs to a standalone Spark cluster in client mode. """ - BQ_CONNECTOR_VERSION = "2.12:0.17.3" - def __init__(self, master_url: str, spark_home: str = None): """ This launcher executes the spark-submit script in a subprocess. The subprocess @@ -203,7 +291,7 @@ def spark_submit( "--conf", "spark.sql.session.timeZone=UTC", # ignore local timezone "--packages", - f"com.google.cloud.spark:spark-bigquery-with-dependencies_{self.BQ_CONNECTOR_VERSION}", + ",".join([BQ_SPARK_PACKAGE] + job_params.get_extra_packages()), "--jars", "https://storage.googleapis.com/hadoop-lib/gcs/gcs-connector-hadoop2-latest.jar," "https://repo1.maven.org/maven2/org/apache/hadoop/hadoop-aws/2.7.3/hadoop-aws-2.7.3.jar," @@ -230,7 +318,7 @@ def historical_feature_retrieval( self.spark_submit(job_params), job_params.get_destination_path(), ) - JOB_CACHE[job_id] = job + global_job_cache.add_job(job) return job def offline_to_online_ingestion( @@ -243,8 +331,9 @@ def offline_to_online_ingestion( ingestion_job_params.get_name(), self.spark_submit(ingestion_job_params, ui_port), ui_port, + ingestion_job_params.get_feature_table_name(), ) - JOB_CACHE[job_id] = job + global_job_cache.add_job(job) return job def start_stream_to_online_ingestion( @@ -257,23 +346,24 @@ def start_stream_to_online_ingestion( ingestion_job_params.get_name(), self.spark_submit(ingestion_job_params, ui_port), ui_port, + ingestion_job_params.get_job_hash(), + ingestion_job_params.get_feature_table_name(), ) - JOB_CACHE[job_id] = job + global_job_cache.add_job(job) return job - def stage_dataframe(self, df, event_timestamp_column: str): - raise NotImplementedError - def get_job_by_id(self, job_id: str) -> SparkJob: - return JOB_CACHE[job_id] + return global_job_cache.get_job_by_id(job_id) - def list_jobs(self, include_terminated: bool) -> List[SparkJob]: + def list_jobs( + self, include_terminated: bool, table_name: Optional[str] + ) -> List[SparkJob]: if include_terminated is True: - return list(JOB_CACHE.values()) + return global_job_cache.list_jobs() else: return [ job - for job in JOB_CACHE.values() + for job in global_job_cache.list_jobs() if job.get_status() in (SparkJobStatus.STARTING, SparkJobStatus.IN_PROGRESS) ] diff --git a/sdk/python/feast/remote_job.py b/sdk/python/feast/remote_job.py index 0a766cf7961..8e1e12a70fe 100644 --- a/sdk/python/feast/remote_job.py +++ b/sdk/python/feast/remote_job.py @@ -1,5 +1,6 @@ import time -from typing import Any, Callable, Dict, List +from datetime import datetime +from typing import Any, Callable, Dict, List, Optional from feast.core.JobService_pb2 import CancelJobRequest, GetJobRequest from feast.core.JobService_pb2 import Job as JobProto @@ -23,6 +24,8 @@ def __init__( service: JobServiceStub, grpc_extra_param_provider: GrpcExtraParamProvider, job_id: str, + start_time: datetime, + log_uri: Optional[str], ): """ Args: @@ -32,6 +35,8 @@ def __init__( self._job_id = job_id self._service = service self._grpc_extra_param_provider = grpc_extra_param_provider + self._start_time = start_time + self._log_uri = log_uri def get_id(self) -> str: return self._job_id @@ -53,6 +58,9 @@ def get_status(self) -> SparkJobStatus: # we should never get here raise Exception(f"Invalid remote job state {response.job.status}") + def get_start_time(self) -> datetime: + return self._start_time + def cancel(self): self._service.CancelJob( CancelJobRequest(job_id=self._job_id), **self._grpc_extra_param_provider() @@ -72,6 +80,9 @@ def _wait_for_job_status( else: raise TimeoutError("Timed out waiting for job status") + def get_log_uri(self) -> Optional[str]: + return self._log_uri + class RemoteRetrievalJob(RemoteJobMixin, RetrievalJob): """ @@ -84,6 +95,8 @@ def __init__( grpc_extra_param_provider: GrpcExtraParamProvider, job_id: str, output_file_uri: str, + start_time: datetime, + log_uri: Optional[str], ): """ This is the job object representing the historical retrieval job. @@ -91,7 +104,9 @@ def __init__( Args: output_file_uri (str): Uri to the historical feature retrieval job output file. """ - super().__init__(service, grpc_extra_param_provider, job_id) + super().__init__( + service, grpc_extra_param_provider, job_id, start_time, log_uri + ) self._output_file_uri = output_file_uri def get_output_file_uri(self, timeout_sec=None): @@ -115,8 +130,17 @@ def __init__( service: JobServiceStub, grpc_extra_param_provider: GrpcExtraParamProvider, job_id: str, + feature_table: str, + start_time: datetime, + log_uri: Optional[str], ): - super().__init__(service, grpc_extra_param_provider, job_id) + super().__init__( + service, grpc_extra_param_provider, job_id, start_time, log_uri + ) + self._feature_table = feature_table + + def get_feature_table(self) -> str: + return self._feature_table class RemoteStreamIngestionJob(RemoteJobMixin, StreamIngestionJob): @@ -129,8 +153,24 @@ def __init__( service: JobServiceStub, grpc_extra_param_provider: GrpcExtraParamProvider, job_id: str, + feature_table: str, + start_time: datetime, + log_uri: Optional[str], ): - super().__init__(service, grpc_extra_param_provider, job_id) + super().__init__( + service, grpc_extra_param_provider, job_id, start_time, log_uri + ) + self._feature_table = feature_table + + def get_hash(self) -> str: + response = self._service.GetJob( + GetJobRequest(job_id=self._job_id), **self._grpc_extra_param_provider() + ) + + return response.job.hash + + def get_feature_table(self) -> str: + return self._feature_table def get_remote_job_from_proto( @@ -148,14 +188,34 @@ def get_remote_job_from_proto( Returns: (SparkJob): A remote job object for the given job """ + if job.type == JobType.RETRIEVAL_JOB: return RemoteRetrievalJob( - service, grpc_extra_param_provider, job.id, job.retrieval.output_location + service, + grpc_extra_param_provider, + job.id, + job.retrieval.output_location, + job.start_time.ToDatetime(), + job.log_uri, ) elif job.type == JobType.BATCH_INGESTION_JOB: - return RemoteBatchIngestionJob(service, grpc_extra_param_provider, job.id) + return RemoteBatchIngestionJob( + service, + grpc_extra_param_provider, + job.id, + job.batch_ingestion.table_name, + job.start_time.ToDatetime(), + job.log_uri, + ) elif job.type == JobType.STREAM_INGESTION_JOB: - return RemoteStreamIngestionJob(service, grpc_extra_param_provider, job.id) + return RemoteStreamIngestionJob( + service, + grpc_extra_param_provider, + job.id, + job.stream_ingestion.table_name, + job.start_time.ToDatetime(), + job.log_uri, + ) else: raise ValueError( f"Invalid Job Type {job.type}, has to be one of " diff --git a/sdk/python/feast/staging/entities.py b/sdk/python/feast/staging/entities.py index 8a4745fe24c..dbb9095a58d 100644 --- a/sdk/python/feast/staging/entities.py +++ b/sdk/python/feast/staging/entities.py @@ -7,6 +7,7 @@ import pandas as pd +from feast.config import Config from feast.data_format import ParquetFormat from feast.data_source import BigQuerySource, FileSource from feast.staging.storage_client import get_staging_client @@ -18,7 +19,7 @@ def stage_entities_to_fs( - entity_source: pd.DataFrame, staging_location: str + entity_source: pd.DataFrame, staging_location: str, config: Config ) -> FileSource: """ Dumps given (entities) dataframe as parquet file and stage it to remote file storage (subdirectory of staging_location) @@ -26,16 +27,20 @@ def stage_entities_to_fs( :return: FileSource with remote destination path """ entity_staging_uri = urlparse(os.path.join(staging_location, str(uuid.uuid4()))) - staging_client = get_staging_client(entity_staging_uri.scheme) + staging_client = get_staging_client(entity_staging_uri.scheme, config) with tempfile.NamedTemporaryFile() as df_export_path: - entity_source.to_parquet(df_export_path.name) - bucket = ( - None if entity_staging_uri.scheme == "file" else entity_staging_uri.netloc - ) - staging_client.upload_file( - df_export_path.name, bucket, entity_staging_uri.path.lstrip("/") + # prevent casting ns -> ms exception inside pyarrow + entity_source["event_timestamp"] = entity_source["event_timestamp"].dt.floor( + "ms" ) + entity_source.to_parquet(df_export_path.name) + + with open(df_export_path.name, "rb") as f: + staging_client.upload_fileobj( + f, df_export_path.name, remote_uri=entity_staging_uri + ) + # ToDo: support custom event_timestamp_column return FileSource( event_timestamp_column="event_timestamp", @@ -69,6 +74,9 @@ def stage_entities_to_bq( f"_entities_{datetime.now():%Y%m%d%H%M%s}", ) + # prevent casting ns -> ms exception inside pyarrow + entity_source["event_timestamp"] = entity_source["event_timestamp"].dt.floor("ms") + load_job: bigquery.LoadJob = bq_client.load_table_from_dataframe( entity_source, destination ) @@ -115,7 +123,7 @@ def create_bq_view_of_joined_features_and_entities( view.view_query = JOIN_TEMPLATE.format( entities=entities_ref, source=source_ref, - entity_key=",".join([f"source.{e} = entities.{e}" for e in entity_names]), + entity_key=" AND ".join([f"source.{e} = entities.{e}" for e in entity_names]), ) view.expires = datetime.now() + timedelta(days=1) bq_client.create_table(view) diff --git a/sdk/python/feast/staging/storage_client.py b/sdk/python/feast/staging/storage_client.py index 1cb250a598b..63d574460fd 100644 --- a/sdk/python/feast/staging/storage_client.py +++ b/sdk/python/feast/staging/storage_client.py @@ -18,17 +18,52 @@ import shutil from abc import ABC, ABCMeta, abstractmethod from tempfile import TemporaryFile -from typing import List +from typing import List, Optional, Tuple from typing.io import IO -from urllib.parse import ParseResult +from urllib.parse import ParseResult, urlparse from google.auth.exceptions import DefaultCredentialsError +from feast.config import Config +from feast.constants import ConfigOptions as opt + GS = "gs" S3 = "s3" +S3A = "s3a" +AZURE_SCHEME = "wasbs" LOCAL_FILE = "file" +def _hash_fileobj(fileobj: IO[bytes]) -> str: + """ Compute sha256 hash of a file. File pointer will be reset to 0 on return. """ + fileobj.seek(0) + h = hashlib.sha256() + for block in iter(lambda: fileobj.read(2 ** 20), b""): + h.update(block) + fileobj.seek(0) + return h.hexdigest() + + +def _gen_remote_uri( + fileobj: IO[bytes], + remote_uri: Optional[ParseResult], + remote_path_prefix: Optional[str], + remote_path_suffix: Optional[str], + sha256sum: Optional[str], +) -> ParseResult: + if remote_uri is None: + assert remote_path_prefix is not None and remote_path_suffix is not None + + if sha256sum is None: + sha256sum = _hash_fileobj(fileobj) + + return urlparse( + os.path.join(remote_path_prefix, f"{sha256sum}{remote_path_suffix}") + ) + else: + return remote_uri + + class AbstractStagingClient(ABC): """ Client used to stage files in order to upload or download datasets into a historical store. @@ -48,16 +83,43 @@ def download_file(self, uri: ParseResult) -> IO[bytes]: pass @abstractmethod - def list_files(self, bucket: str, path: str) -> List[str]: + def list_files(self, uri: ParseResult) -> List[str]: """ Lists all the files under a directory in an object store. """ pass @abstractmethod - def upload_file(self, local_path: str, bucket: str, remote_path: str): + def upload_fileobj( + self, + fileobj: IO[bytes], + local_path: str, + *, + remote_uri: Optional[ParseResult] = None, + remote_path_prefix: Optional[str] = None, + remote_path_suffix: Optional[str] = None, + ) -> ParseResult: """ - Uploads a file to an object store. + Uploads a file to an object store. You can either specify the destination object URI, + or destination suffix+prefix. In the latter case, this interface will work as a + content-addressable storage and the remote path will be computed using sha256 of the + uploaded content as `$remote_path_prefix/$sha256$remote_path_suffix` + + Args: + fileobj (IO[bytes]): file-like object containing the data to be uploaded. It needs to + supports seek() operation in addition to read/write. + local_path (str): a file name associated with fileobj. This param is only used for + diagnostic messages. If `fileobj` is a local file, pass its filename here. + remote_uri (ParseResult or None): destination object URI to upload to + remote_path_prefix (str or None): destination path prefix to upload to when using + content-addressable storage mode + remote_path_suffix (str or None): destination path suffix to upload to when using + content-addressable storage mode + + Returns: + ParseResult: the URI to the uploaded file. It would be the same as `remote_uri` if + `remote_uri` was passed in. Otherwise it will be the path computed from + `remote_path_prefix` and `remote_path_suffix`. """ pass @@ -96,19 +158,19 @@ def download_file(self, uri: ParseResult) -> IO[bytes]: file_obj.seek(0) return file_obj - def list_files(self, bucket: str, path: str) -> List[str]: + def list_files(self, uri: ParseResult) -> List[str]: """ Lists all the files under a directory in google cloud storage if path has wildcard(*) character. Args: - bucket (str): google cloud storage bucket name - path (str): object location in google cloud storage. + uri (urllib.parse.ParseResult): Parsed uri of this location Returns: List[str]: A list containing the full path to the file(s) in the remote staging location. """ + bucket, path = self._uri_to_bucket_key(uri) gs_bucket = self.gcs_client.get_bucket(bucket) if "*" in path: @@ -123,20 +185,29 @@ def list_files(self, bucket: str, path: str) -> List[str]: if re.match(regex, file) and file not in path ] else: - return [f"{GS}://{bucket}/{path.lstrip('/')}"] - - def upload_file(self, local_path: str, bucket: str, remote_path: str): - """ - Uploads file to google cloud storage. - - Args: - local_path (str): Path to the local file that needs to be uploaded/staged - bucket (str): gs Bucket name - remote_path (str): relative path to the folder to which the files need to be uploaded - """ + return [f"{GS}://{bucket}/{path}"] + + def _uri_to_bucket_key(self, remote_path: ParseResult) -> Tuple[str, str]: + assert remote_path.hostname is not None + return remote_path.hostname, remote_path.path.lstrip("/") + + def upload_fileobj( + self, + fileobj: IO[bytes], + local_path: str, + *, + remote_uri: Optional[ParseResult] = None, + remote_path_prefix: Optional[str] = None, + remote_path_suffix: Optional[str] = None, + ) -> ParseResult: + remote_uri = _gen_remote_uri( + fileobj, remote_uri, remote_path_prefix, remote_path_suffix, None + ) + bucket, key = self._uri_to_bucket_key(remote_uri) gs_bucket = self.gcs_client.get_bucket(bucket) - blob = gs_bucket.blob(remote_path.lstrip("/")) - blob.upload_from_filename(local_path) + blob = gs_bucket.blob(key) + blob.upload_from_file(fileobj) + return remote_uri class S3Client(AbstractStagingClient): @@ -144,7 +215,7 @@ class S3Client(AbstractStagingClient): Implementation of AbstractStagingClient for Aws S3 storage """ - def __init__(self): + def __init__(self, endpoint_url: str = None, url_scheme="s3"): try: import boto3 except ImportError: @@ -152,7 +223,8 @@ def __init__(self): "Install package boto3 for s3 staging support" "run ```pip install boto3```" ) - self.s3_client = boto3.client("s3") + self.s3_client = boto3.client("s3", endpoint_url=endpoint_url) + self.url_scheme = url_scheme def download_file(self, uri: ParseResult) -> IO[bytes]: """ @@ -163,25 +235,24 @@ def download_file(self, uri: ParseResult) -> IO[bytes]: Returns: TemporaryFile object """ - url = uri.path.lstrip("/") - bucket = uri.hostname + bucket, url = self._uri_to_bucket_key(uri) file_obj = TemporaryFile() self.s3_client.download_fileobj(bucket, url, file_obj) return file_obj - def list_files(self, bucket: str, path: str) -> List[str]: + def list_files(self, uri: ParseResult) -> List[str]: """ Lists all the files under a directory in s3 if path has wildcard(*) character. Args: - bucket (str): s3 bucket name. - path (str): Object location in s3. + uri (urllib.parse.ParseResult): Parsed uri of this location Returns: List[str]: A list containing the full path to the file(s) in the remote staging location. """ + bucket, path = self._uri_to_bucket_key(uri) if "*" in path: regex = re.compile(path.replace("*", ".*?").strip("/")) blob_list = self.s3_client.list_objects( @@ -189,59 +260,145 @@ def list_files(self, bucket: str, path: str) -> List[str]: ) # File path should not be in path (file path must be longer than path) return [ - f"{S3}://{bucket}/{file}" + f"{self.url_scheme}://{bucket}/{file}" for file in [x["Key"] for x in blob_list["Contents"]] if re.match(regex, file) and file not in path ] else: - return [f"{S3}://{bucket}/{path.lstrip('/')}"] - - def _hash_file(self, local_path: str): - h = hashlib.sha256() - with open(local_path, "rb") as f: - for block in iter(lambda: f.read(2 ** 20), b""): - h.update(block) - return h.hexdigest() - - def upload_file(self, local_path: str, bucket: str, remote_path: str): - """ - Uploads file to s3. - - Args: - local_path (str): Path to the local file that needs to be uploaded/staged - bucket (str): s3 Bucket name - remote_path (str): relative path to the folder to which the files need to be uploaded - """ - - sha256sum = self._hash_file(local_path) + return [f"{self.url_scheme}://{bucket}/{path}"] + + def _uri_to_bucket_key(self, remote_path: ParseResult) -> Tuple[str, str]: + assert remote_path.hostname is not None + return remote_path.hostname, remote_path.path.lstrip("/") + + def upload_fileobj( + self, + fileobj: IO[bytes], + local_path: str, + *, + remote_uri: Optional[ParseResult] = None, + remote_path_prefix: Optional[str] = None, + remote_path_suffix: Optional[str] = None, + ) -> ParseResult: + sha256sum = _hash_fileobj(fileobj) + remote_uri = _gen_remote_uri( + fileobj, remote_uri, remote_path_prefix, remote_path_suffix, sha256sum + ) import botocore + bucket, key = self._uri_to_bucket_key(remote_uri) + try: - head_response = self.s3_client.head_object(Bucket=bucket, Key=remote_path) + head_response = self.s3_client.head_object(Bucket=bucket, Key=key) if head_response["Metadata"]["sha256sum"] == sha256sum: # File already exists - return remote_path + return remote_uri else: - print(f"Uploading {local_path} to {remote_path}") - self.s3_client.upload_file( - local_path, + print(f"Uploading {local_path} to {remote_uri}") + self.s3_client.upload_fileobj( + fileobj, bucket, - remote_path, + key, ExtraArgs={"Metadata": {"sha256sum": sha256sum}}, ) - return remote_path + return remote_uri except botocore.exceptions.ClientError as e: if e.response["Error"]["Code"] != "404": raise - self.s3_client.upload_file( - local_path, - bucket, - remote_path, - ExtraArgs={"Metadata": {"sha256sum": sha256sum}}, + self.s3_client.upload_fileobj( + fileobj, bucket, key, ExtraArgs={"Metadata": {"sha256sum": sha256sum}}, + ) + return remote_uri + + +class AzureBlobClient(AbstractStagingClient): + """ + Implementation of AbstractStagingClient for Azure Blob storage + """ + + def __init__(self, account_name: str, account_access_key: str): + try: + from azure.storage.blob import BlobServiceClient + except ImportError: + raise ImportError( + "Install package azure-storage-blob for azure blob staging support" + "run ```pip install azure-storage-blob```" + ) + self.account_name = account_name + account_url = f"https://{account_name}.blob.core.windows.net" + self.blob_service_client = BlobServiceClient( + account_url=account_url, credential=account_access_key ) - return remote_path + + def download_file(self, uri: ParseResult) -> IO[bytes]: + """ + Downloads a file from Azure blob storage and returns a TemporaryFile object + + Args: + uri (urllib.parse.ParseResult): Parsed uri of the file ex: urlparse("wasbs://bucket@account_name.blob.core.windows.net/file.avro") + + Returns: + TemporaryFile object + """ + bucket, path = self._uri_to_bucket_key(uri) + container_client = self.blob_service_client.get_container_client(bucket) + return container_client.download_blob(path).readall() + + def list_files(self, uri: ParseResult) -> List[str]: + """ + Lists all the files under a directory in azure blob storage if path has wildcard(*) character. + + Args: + uri (urllib.parse.ParseResult): Parsed uri of this location + + Returns: + List[str]: A list containing the full path to the file(s) in the + remote staging location. + """ + + bucket, path = self._uri_to_bucket_key(uri) + if "*" in path: + regex = re.compile(path.replace("*", ".*?").strip("/")) + container_client = self.blob_service_client.get_container_client(bucket) + blob_list = container_client.list_blobs( + name_starts_with=path.strip("/").split("*")[0] + ) + # File path should not be in path (file path must be longer than path) + return [ + f"wasbs://{bucket}@{self.account_name}.blob.core.windows.net/{file}" + for file in [x.name for x in blob_list] + if re.match(regex, file) and file not in path + ] + else: + return [ + f"wasbs://{bucket}@{self.account_name}.blob.core.windows.net/{path}" + ] + + def _uri_to_bucket_key(self, uri: ParseResult) -> Tuple[str, str]: + assert uri.hostname == f"{self.account_name}.blob.core.windows.net" + assert uri.username + bucket = uri.username + key = uri.path.lstrip("/") + return bucket, key + + def upload_fileobj( + self, + fileobj: IO[bytes], + local_path: str, + *, + remote_uri: Optional[ParseResult] = None, + remote_path_prefix: Optional[str] = None, + remote_path_suffix: Optional[str] = None, + ) -> ParseResult: + remote_uri = _gen_remote_uri( + fileobj, remote_uri, remote_path_prefix, remote_path_suffix, None + ) + bucket, key = self._uri_to_bucket_key(remote_uri) + container_client = self.blob_service_client.get_container_client(bucket) + container_client.upload_blob(name=key, data=fileobj, overwrite=True) + return remote_uri class LocalFSClient(AbstractStagingClient): @@ -258,7 +415,7 @@ def download_file(self, uri: ParseResult) -> IO[bytes]: Reads a local file from the disk Args: - uri (urllib.parse.ParseResult): Parsed uri of the file ex: urlparse("file://folder/file.avro") + uri (urllib.parse.ParseResult): Parsed uri of the file ex: urlparse("file:///folder/file.avro") Returns: TemporaryFile object """ @@ -266,31 +423,91 @@ def download_file(self, uri: ParseResult) -> IO[bytes]: file_obj = open(url, "rb") return file_obj - def list_files(self, bucket: str, path: str) -> List[str]: + def list_files(self, uri: ParseResult) -> List[str]: raise NotImplementedError("list files not implemented for Local file") - def upload_file(self, local_path: str, bucket: str, remote_path: str): - dest_fpath = remote_path if remote_path.startswith("/") else "/" + remote_path - os.makedirs(os.path.dirname(dest_fpath), exist_ok=True) - shutil.copy(local_path, dest_fpath) + def _uri_to_path(self, uri: ParseResult) -> str: + return uri.path + + def upload_fileobj( + self, + fileobj: IO[bytes], + local_path: str, + *, + remote_uri: Optional[ParseResult] = None, + remote_path_prefix: Optional[str] = None, + remote_path_suffix: Optional[str] = None, + ) -> ParseResult: + + remote_uri = _gen_remote_uri( + fileobj, remote_uri, remote_path_prefix, remote_path_suffix, None + ) + remote_file_path = self._uri_to_path(remote_uri) + os.makedirs(os.path.dirname(remote_file_path), exist_ok=True) + with open(remote_file_path, "wb") as fdest: + shutil.copyfileobj(fileobj, fdest) + return remote_uri + + +def _s3_client(config: Config = None): + if config is None: + endpoint_url = None + else: + endpoint_url = config.get(opt.S3_ENDPOINT_URL, None) + return S3Client(endpoint_url=endpoint_url) + + +def _s3a_client(config: Config = None): + if config is None: + endpoint_url = None + else: + endpoint_url = config.get(opt.S3_ENDPOINT_URL, None) + return S3Client(endpoint_url=endpoint_url, url_scheme="s3a") + + +def _gcs_client(config: Config = None): + return GCSClient() + + +def _azure_blob_client(config: Config = None): + if config is None: + raise Exception("Azure blob client requires config") + account_name = config.get(opt.AZURE_BLOB_ACCOUNT_NAME, None) + account_access_key = config.get(opt.AZURE_BLOB_ACCOUNT_ACCESS_KEY, None) + if account_name is None or account_access_key is None: + raise Exception( + f"Azure blob client requires {opt.AZURE_BLOB_ACCOUNT_NAME} and {opt.AZURE_BLOB_ACCOUNT_ACCESS_KEY} set in config" + ) + return AzureBlobClient(account_name, account_access_key) + + +def _local_fs_client(config: Config = None): + return LocalFSClient() -storage_clients = {GS: GCSClient, S3: S3Client, LOCAL_FILE: LocalFSClient} +storage_clients = { + GS: _gcs_client, + S3: _s3_client, + S3A: _s3a_client, + AZURE_SCHEME: _azure_blob_client, + LOCAL_FILE: _local_fs_client, +} -def get_staging_client(scheme): +def get_staging_client(scheme, config: Config = None) -> AbstractStagingClient: """ Initialization of a specific client object(GCSClient, S3Client etc.) Args: scheme (str): uri scheme: s3, gs or file + config (Config): additional configuration Returns: An object of concrete implementation of AbstractStagingClient """ try: - return storage_clients[scheme]() + return storage_clients[scheme](config) except ValueError: raise Exception( - f"Could not identify file scheme {scheme}. Only gs://, file:// and s3:// are supported" + f"Could not identify file scheme {scheme}. Only gs://, file://, s3:// and wasbs:// (for Azure) are supported" ) diff --git a/sdk/python/feast/third_party/__init__.py b/sdk/python/feast/third_party/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/sdk/python/feast/third_party/grpc/__init__.py b/sdk/python/feast/third_party/grpc/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/sdk/python/feast/third_party/grpc/health/__init__.py b/sdk/python/feast/third_party/grpc/health/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/sdk/python/feast/third_party/grpc/health/v1/__init__.py b/sdk/python/feast/third_party/grpc/health/v1/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/sdk/python/feast/wait.py b/sdk/python/feast/wait.py index c32897606ec..daa0b0b7dc5 100644 --- a/sdk/python/feast/wait.py +++ b/sdk/python/feast/wait.py @@ -15,15 +15,14 @@ import time from typing import Any, Callable, Optional, Tuple -from feast.constants import CONFIG_MAX_WAIT_INTERVAL_KEY -from feast.constants import FEAST_DEFAULT_OPTIONS as defaults +from feast.constants import MAX_WAIT_INTERVAL def wait_retry_backoff( retry_fn: Callable[[], Tuple[Any, bool]], timeout_secs: int = 0, timeout_msg: Optional[str] = "Timeout while waiting for retry_fn() to return True", - max_interval_secs: int = int(defaults[CONFIG_MAX_WAIT_INTERVAL_KEY]), + max_interval_secs: int = int(MAX_WAIT_INTERVAL), ) -> Any: """ Repeatedly try calling given retry_fn until it returns a True boolean success flag. diff --git a/sdk/python/requirements-ci.txt b/sdk/python/requirements-ci.txt index 258bca8aab4..e3a7fd2d95f 100644 --- a/sdk/python/requirements-ci.txt +++ b/sdk/python/requirements-ci.txt @@ -4,12 +4,12 @@ black==19.10b0 isort>=5 grpcio-tools==1.31.0 mypy-protobuf -pyspark==2.4.2 +pyspark==3.0.1 pandas~=1.0.0 mock==2.0.0 pandavro==1.5.* moto -mypy +mypy==0.790 mypy-protobuf avro==1.10.0 gcsfs @@ -19,4 +19,7 @@ pytest==6.0.0 pytest-lazy-fixture==0.6.3 pytest-timeout==1.4.2 pytest-ordering==0.6.* -pytest-mock==1.10.4 \ No newline at end of file +pytest-mock==1.10.4 +PyYAML==5.3.1 +great-expectations==0.13.2 +adlfs==0.5.9 diff --git a/sdk/python/requirements-dev.txt b/sdk/python/requirements-dev.txt index ca845e7f5ae..0822db76f94 100644 --- a/sdk/python/requirements-dev.txt +++ b/sdk/python/requirements-dev.txt @@ -38,5 +38,7 @@ flake8 black==19.10b0 boto3 moto -pyspark==2.4.2 -pyspark-stubs==2.4.0.post9 +pyspark==3.0.1 +pyspark-stubs==3.0.0.post1 +kubernetes==12.0.* +azure-storage-blob==12.6.0 diff --git a/sdk/python/setup.py b/sdk/python/setup.py index b8481f23d5f..cebd5155819 100644 --- a/sdk/python/setup.py +++ b/sdk/python/setup.py @@ -27,7 +27,6 @@ REQUIRED = [ "Click==7.*", "google-api-core==1.22.4", - "google-auth<2.0dev,>=1.14.0", "google-cloud-bigquery==1.18.*", "google-cloud-storage==1.20.*", "google-cloud-core==1.0.*", @@ -37,14 +36,15 @@ "pandas~=1.0.0", "pandavro==1.5.*", "protobuf>=3.10", - "PyYAML==5.1.*", + "PyYAML==5.3.*", "fastavro>=0.22.11,<0.23", "tabulate==0.8.*", "toml==0.10.*", "tqdm==4.*", - "pyarrow<0.16.0,>=0.15.1", + "pyarrow==2.0.0", "numpy", "google", + "kubernetes==12.0.*", ] # README file from Feast repo root directory @@ -77,7 +77,10 @@ install_requires=REQUIRED, # https://stackoverflow.com/questions/28509965/setuptools-development-requirements # Install dev requirements with: pip install -e .[dev] - extras_require={"dev": ["mypy-protobuf==1.*", "grpcio-testing==1.*"]}, + extras_require={ + "dev": ["mypy-protobuf==1.*", "grpcio-testing==1.*"], + "validation": ["great_expectations==0.13.2", "pyspark==3.0.1"], + }, include_package_data=True, license="Apache", classifiers=[ diff --git a/sdk/python/tests/feast_core_server.py b/sdk/python/tests/feast_core_server.py index 85f09175bf3..0c7191f35b4 100644 --- a/sdk/python/tests/feast_core_server.py +++ b/sdk/python/tests/feast_core_server.py @@ -11,6 +11,8 @@ ApplyEntityResponse, ApplyFeatureTableRequest, ApplyFeatureTableResponse, + DeleteFeatureTableRequest, + DeleteFeatureTableResponse, GetEntityRequest, GetEntityResponse, GetFeastCoreVersionResponse, @@ -20,6 +22,7 @@ ListEntitiesResponse, ListFeatureTablesRequest, ListFeatureTablesResponse, + ListProjectsResponse, ) from feast.core.Entity_pb2 import Entity as EntityProto from feast.core.Entity_pb2 import EntityMeta @@ -66,6 +69,7 @@ class CoreServicer(Core.CoreServiceServicer): def __init__(self): self._feature_tables = dict() self._entities = dict() + self._projects = ["default"] def GetFeastCoreVersion(self, request, context): return GetFeastCoreVersionResponse(version="0.10.0") @@ -105,6 +109,10 @@ def ApplyFeatureTable(self, request: ApplyFeatureTableRequest, context): return ApplyFeatureTableResponse(table=applied_feature_table,) + def DeleteFeatureTable(self, request: DeleteFeatureTableRequest, context): + del self._feature_tables[request.name] + return DeleteFeatureTableResponse() + def GetEntity(self, request: GetEntityRequest, context): filtered_entities = [ entity @@ -119,6 +127,9 @@ def ListEntities(self, request: ListEntitiesRequest, context): return ListEntitiesResponse(entities=filtered_entities_response) + def ListProjects(self, request, context): + return ListProjectsResponse(projects=self._projects) + def ApplyEntity(self, request: ApplyEntityRequest, context): entity_spec = request.spec diff --git a/sdk/python/tests/grpc/test_auth.py b/sdk/python/tests/grpc/test_auth.py index 7f023aabcfd..29f781cbedb 100644 --- a/sdk/python/tests/grpc/test_auth.py +++ b/sdk/python/tests/grpc/test_auth.py @@ -14,6 +14,7 @@ # limitations under the License. import json +from configparser import NoOptionError from http import HTTPStatus from unittest.mock import call, patch @@ -141,7 +142,7 @@ def test_get_auth_metadata_plugin_oauth_should_raise_when_response_is_not_200( def test_get_auth_metadata_plugin_oauth_should_raise_when_config_is_incorrect( config_with_missing_variable, ): - with raises(RuntimeError): + with raises((RuntimeError, NoOptionError)): get_auth_metadata_plugin(config_with_missing_variable) diff --git a/sdk/python/tests/loaders/test_file.py b/sdk/python/tests/loaders/test_file.py index 9d02447ab35..dae1006aaed 100644 --- a/sdk/python/tests/loaders/test_file.py +++ b/sdk/python/tests/loaders/test_file.py @@ -32,7 +32,7 @@ FOLDER_NAME = "test_folder" FILE_NAME = "test.avro" -LOCAL_FILE = "file://tmp/tmp" +LOCAL_FILE = "file:///tmp/tmp" S3_LOCATION = f"s3://{BUCKET}/{FOLDER_NAME}" TEST_DATA_FRAME = pd.DataFrame( diff --git a/sdk/python/tests/test_as_of_join.py b/sdk/python/tests/test_as_of_join.py index 23ce94e176a..31cd150cbf9 100644 --- a/sdk/python/tests/test_as_of_join.py +++ b/sdk/python/tests/test_as_of_join.py @@ -229,12 +229,7 @@ def test_join_without_max_age( ) joined_df = as_of_join( - entity_df, - "event_timestamp", - feature_table_df, - feature_table, - "event_timestamp", - "created_timestamp", + entity_df, "event_timestamp", feature_table_df, feature_table, ) expected_joined_schema = StructType( @@ -298,12 +293,7 @@ def test_join_with_max_age( ) joined_df = as_of_join( - entity_df, - "event_timestamp", - feature_table_df, - feature_table, - "event_timestamp", - "created_timestamp", + entity_df, "event_timestamp", feature_table_df, feature_table, ) expected_joined_schema = StructType( @@ -377,12 +367,7 @@ def test_join_with_composite_entity( ) joined_df = as_of_join( - entity_df, - "event_timestamp", - feature_table_df, - feature_table, - "event_timestamp", - "created_timestamp", + entity_df, "event_timestamp", feature_table_df, feature_table, ) expected_joined_schema = StructType( @@ -444,12 +429,7 @@ def test_select_subset_of_columns_as_entity_primary_keys( ) joined_df = as_of_join( - entity_df, - "event_timestamp", - feature_table_df, - feature_table, - "event_timestamp", - "created_timestamp", + entity_df, "event_timestamp", feature_table_df, feature_table, ) expected_joined_schema = StructType( @@ -552,8 +532,6 @@ def test_multiple_join( "event_timestamp", [customer_table_df, driver_table_df], [customer_table, driver_table], - ["event_timestamp"] * 2, - ["created_timestamp"] * 2, ) expected_joined_schema = StructType( diff --git a/sdk/python/tests/test_client.py b/sdk/python/tests/test_client.py index 564a7b671cc..91f98267a89 100644 --- a/sdk/python/tests/test_client.py +++ b/sdk/python/tests/test_client.py @@ -35,6 +35,7 @@ from feast.core.CoreService_pb2 import ( GetFeastCoreVersionResponse, GetFeatureTableResponse, + ListFeaturesResponse, ) from feast.core.DataSource_pb2 import DataSource as DataSourceProto from feast.core.Feature_pb2 import FeatureSpecV2 as FeatureSpecProto @@ -376,7 +377,7 @@ def test_apply_entity_success(self, test_client): ) # Register Entity with Core - test_client.apply_entity(entity) + test_client.apply(entity) entities = test_client.list_entities() @@ -428,7 +429,7 @@ def test_apply_feature_table_success(self, test_client): ) # Register Feature Table with Core - test_client.apply_feature_table(ft1) + test_client.apply(ft1) feature_tables = test_client.list_feature_tables() @@ -447,6 +448,45 @@ def test_apply_feature_table_success(self, test_client): and feature_tables[0].entities[0] == "fs1-my-entity-1" ) + @pytest.mark.parametrize( + "test_client", [lazy_fixture("client"), lazy_fixture("secure_client")] + ) + def test_list_features(self, test_client, mocker): + mocker.patch.object( + test_client, + "_core_service_stub", + return_value=Core.CoreServiceStub(grpc.insecure_channel("")), + ) + + feature1_proto = FeatureSpecProto( + name="feature_1", value_type=ValueProto.ValueType.FLOAT + ) + feature2_proto = FeatureSpecProto( + name="feature_2", value_type=ValueProto.ValueType.STRING + ) + + mocker.patch.object( + test_client._core_service_stub, + "ListFeatures", + return_value=ListFeaturesResponse( + features={ + "driver_car:feature_1": feature1_proto, + "driver_car:feature_2": feature2_proto, + } + ), + ) + + features = test_client.list_features_by_ref(project="test") + assert len(features) == 2 + + native_feature_list = [] + for _, feature_proto in features.items(): + native_feature_list.append(feature_proto) + + assert sorted(native_feature_list) == sorted( + [Feature.from_proto(feature1_proto), Feature.from_proto(feature2_proto)] + ) + @pytest.mark.parametrize( "mocked_client", [lazy_fixture("mock_client")], ) diff --git a/sdk/python/tests/test_config.py b/sdk/python/tests/test_config.py index 9ed34a736a2..3c2f66dd01a 100644 --- a/sdk/python/tests/test_config.py +++ b/sdk/python/tests/test_config.py @@ -103,6 +103,17 @@ def test_default_options(self): config = Config(path=path) assert config.get("CORE_URL") == "localhost:6565" + def test_defaults_are_not_written(self): + """ + default values are not written to config file + """ + fd, path = mkstemp() + config = Config(path=path) + config.set("option", "value") + config.save() + with open(path) as f: + assert f.read() == "[general]\noption = value\n\n" + def test_type_casting(self): """ Test type casting of strings to other types @@ -117,6 +128,16 @@ def test_type_casting(self): assert config.getfloat("FLOAT_VAR") == 1.0 assert config.getboolean("BOOLEAN_VAR") is True + def test_type_casting_of_defaults(self): + """ + default values are casted as expected + """ + fd, path = mkstemp() + config = Config(path=path) + assert isinstance(config.getboolean("enable_auth"), bool) + assert isinstance(config.getint("DATAPROC_EXECUTOR_INSTANCES"), int) + assert isinstance(config.getfloat("DATAPROC_EXECUTOR_INSTANCES"), float) + def test_set_value(self): """ Test type casting of strings to other types diff --git a/sdk/python/tests/test_historical_feature_retrieval.py b/sdk/python/tests/test_historical_feature_retrieval.py index 0aa5079d46f..c7212418119 100644 --- a/sdk/python/tests/test_historical_feature_retrieval.py +++ b/sdk/python/tests/test_historical_feature_retrieval.py @@ -28,6 +28,7 @@ from feast import Client, Entity, Feature, FeatureTable, FileSource, ValueType from feast.core import CoreService_pb2_grpc as Core from feast.data_format import ParquetFormat +from feast.pyspark.abc import SparkJobStatus from tests.feast_core_server import CoreServicer @@ -107,14 +108,34 @@ def client_with_local_spark(tmpdir): ) +@pytest.fixture() +def client_with_tfrecord_output(tmpdir): + import pyspark + + spark_staging_location = f"file://{os.path.join(tmpdir, 'staging')}" + historical_feature_output_location = ( + f"file://{os.path.join(tmpdir, 'historical_feature_retrieval_tfrecord_output')}" + ) + + return Client( + core_url=f"localhost:{free_port}", + spark_launcher="standalone", + spark_standalone_master="local", + spark_home=os.path.dirname(pyspark.__file__), + spark_staging_location=spark_staging_location, + historical_feature_output_location=historical_feature_output_location, + historical_feature_output_format="tfrecord", + ) + + @pytest.fixture() def driver_entity(client): - return client.apply_entity(Entity("driver_id", "description", ValueType.INT32)) + return client.apply(Entity("driver_id", "description", ValueType.INT32)) @pytest.fixture() def customer_entity(client): - return client.apply_entity(Entity("customer_id", "description", ValueType.INT32)) + return client.apply(Entity("customer_id", "description", ValueType.INT32)) def create_temp_parquet_file( @@ -191,7 +212,7 @@ def transactions_feature_table(spark, client): feature_table = FeatureTable( "transactions", ["customer_id"], features, batch_source=file_source ) - yield client.apply_feature_table(feature_table) + yield client.apply(feature_table) shutil.rmtree(temp_dir) @@ -239,7 +260,7 @@ def bookings_feature_table(spark, client): feature_table = FeatureTable( "bookings", ["driver_id"], features, batch_source=file_source, max_age=max_age ) - yield client.apply_feature_table(feature_table) + yield client.apply(feature_table) shutil.rmtree(temp_dir) @@ -288,7 +309,7 @@ def bookings_feature_table_with_mapping(spark, client): feature_table = FeatureTable( "bookings", ["driver_id"], features, batch_source=file_source, max_age=max_age ) - yield client.apply_feature_table(feature_table) + yield client.apply(feature_table) shutil.rmtree(temp_dir) @@ -466,3 +487,39 @@ def test_historical_feature_retrieval_with_pandas_dataframe_input( by=["customer_id", "driver_id", "event_timestamp"] ).reset_index(drop=True), ) + + +@pytest.mark.usefixtures( + "driver_entity", + "customer_entity", + "bookings_feature_table", + "transactions_feature_table", +) +def test_historical_feature_retrieval_with_tfrecord_output( + client_with_tfrecord_output, +): + + customer_driver_pairs_pandas_df = pd.DataFrame( + np.array( + [ + [1001, 8001, datetime(year=2020, month=9, day=1, tzinfo=utc)], + [2001, 8001, datetime(year=2020, month=9, day=2, tzinfo=utc)], + [2001, 8002, datetime(year=2020, month=9, day=1, tzinfo=utc)], + [1001, 8001, datetime(year=2020, month=9, day=2, tzinfo=utc)], + [1001, 8001, datetime(year=2020, month=9, day=3, tzinfo=utc)], + [1001, 8001, datetime(year=2020, month=9, day=4, tzinfo=utc)], + ] + ), + columns=["customer_id", "driver_id", "event_timestamp"], + ) + customer_driver_pairs_pandas_df = customer_driver_pairs_pandas_df.astype( + {"customer_id": "int32", "driver_id": "int32"} + ) + + job_output = client_with_tfrecord_output.get_historical_features( + ["transactions:total_transactions", "bookings:total_completed_bookings"], + customer_driver_pairs_pandas_df, + ) + + job_output.get_output_file_uri() + assert job_output.get_status() == SparkJobStatus.COMPLETED diff --git a/sdk/python/tests/test_remote_job.py b/sdk/python/tests/test_remote_job.py index 98c18642c59..37bbb504145 100644 --- a/sdk/python/tests/test_remote_job.py +++ b/sdk/python/tests/test_remote_job.py @@ -1,6 +1,7 @@ from collections import defaultdict from concurrent import futures from contextlib import contextmanager +from datetime import datetime import grpc @@ -57,7 +58,9 @@ def GetJob(self, request, context): mock_servicer = MockServicer() with mock_server(mock_servicer) as service: - remote_job = RemoteRetrievalJob(service, lambda: {}, "test", "foo") + remote_job = RemoteRetrievalJob( + service, lambda: {}, "test", "foo", datetime.now(), None + ) assert remote_job.get_output_file_uri(timeout_sec=2) == "foo" assert mock_servicer._call_count["GetJob"] == 2 diff --git a/sdk/python/tests/test_streaming_control_loop.py b/sdk/python/tests/test_streaming_control_loop.py new file mode 100644 index 00000000000..b71fae69258 --- /dev/null +++ b/sdk/python/tests/test_streaming_control_loop.py @@ -0,0 +1,188 @@ +import os +import subprocess +from concurrent import futures +from contextlib import contextmanager +from typing import List +from unittest.mock import patch + +import grpc +import pyspark + +from feast.client import Client +from feast.core.CoreService_pb2_grpc import add_CoreServiceServicer_to_server +from feast.data_format import ParquetFormat, ProtoFormat +from feast.data_source import FileSource, KafkaSource +from feast.entity import Entity +from feast.feature import Feature +from feast.feature_table import FeatureTable +from feast.job_service import ensure_stream_ingestion_jobs +from feast.pyspark.launchers.standalone import ( + StandaloneClusterLauncher, + reset_job_cache, +) +from feast.value_type import ValueType +from tests.feast_core_server import CoreServicer as MockCoreServicer + + +@contextmanager +def mock_server(servicer, add_fn): + """Instantiate a server and return its address for use in tests""" + server = grpc.server(futures.ThreadPoolExecutor(max_workers=10)) + add_fn(servicer, server) + port = server.add_insecure_port("[::]:0") + server.start() + + try: + address = "localhost:%d" % port + with grpc.insecure_channel(address): + yield address + finally: + server.stop(None) + + +SERVING_URL = "serving.example.com" + + +class TestStreamingControlLoop: + table_name = "my-feature-table-1" + + features_1 = [ + Feature(name="fs1-my-feature-1", dtype=ValueType.INT64), + Feature(name="fs1-my-feature-2", dtype=ValueType.STRING), + Feature(name="fs1-my-feature-3", dtype=ValueType.STRING_LIST), + Feature(name="fs1-my-feature-4", dtype=ValueType.BYTES_LIST), + ] + + features_2 = features_1 + [ + Feature(name="fs1-my-feature-5", dtype=ValueType.BYTES_LIST), + ] + + def _create_ft(self, client: Client, features) -> None: + entity = Entity( + name="driver_car_id", + description="Car driver id", + value_type=ValueType.STRING, + labels={"team": "matchmaking"}, + ) + + # Register Entity with Core + client.apply(entity) + + # Create Feature Tables + batch_source = FileSource( + file_format=ParquetFormat(), + file_url="file://feast/*", + event_timestamp_column="ts_col", + created_timestamp_column="timestamp", + date_partition_column="date_partition_col", + ) + + stream_source = KafkaSource( + bootstrap_servers="localhost:9094", + message_format=ProtoFormat("class.path"), + topic="test_topic", + event_timestamp_column="ts_col", + created_timestamp_column="timestamp", + ) + + ft1 = FeatureTable( + name=self.table_name, + features=features, + entities=["driver_car_id"], + labels={"team": "matchmaking"}, + batch_source=batch_source, + stream_source=stream_source, + ) + + # Register Feature Table with Core + client.apply(ft1) + + def _delete_ft(self, client: Client): + client.delete_feature_table(self.table_name) + + def test_streaming_job_control_loop(self) -> None: + """ Test streaming job control loop logic. """ + + reset_job_cache() + + core_servicer = MockCoreServicer() + + processes: List[subprocess.Popen] = [] + + def _mock_spark_submit(self, *args, **kwargs) -> subprocess.Popen: + # We mock StandaloneClusterLauncher.spark_submit to run a dummy process and pretend + # that this is a spark structured streaming process. In addition, this implementation + # will keep track of launched processes in an array. + result = subprocess.Popen(args=["/bin/bash", "-c", "sleep 600"]) + processes.append(result) + return result + + with patch.object( + StandaloneClusterLauncher, "spark_submit", new=_mock_spark_submit + ), mock_server( + core_servicer, add_CoreServiceServicer_to_server + ) as core_service_url: + client = Client( + core_url=core_service_url, + serving_url=SERVING_URL, + spark_launcher="standalone", + spark_home=os.path.dirname(pyspark.__file__), + ) + + # Run one iteration of the control loop. It should do nothing since we have no + # feature tables. + ensure_stream_ingestion_jobs(client=client, all_projects=True) + + # No jobs should be running at this point. + assert len(client.list_jobs(include_terminated=True)) == 0 + + # Now, create a new feature table. + self._create_ft(client, self.features_1) + + # Run another iteration of the control loop. + ensure_stream_ingestion_jobs(client=client, all_projects=True) + + # We expect a streaming job to be created for the new Feature Table. + assert len(client.list_jobs(include_terminated=False)) == 1 + assert len(processes) == 1 + + first_job_id = client.list_jobs(include_terminated=False)[0].get_id() + + # Pretend that the streaming job has terminated for no reason. + processes[0].kill() + + # The control loop is expected to notice the killed job and start it again. + ensure_stream_ingestion_jobs(client=client, all_projects=True) + + # We expect to find one terminated job and one restarted job. + assert len(client.list_jobs(include_terminated=False)) == 1 + assert len(client.list_jobs(include_terminated=True)) == 2 + + id_after_restart = client.list_jobs(include_terminated=False)[0].get_id() + + # Indeed it is a new job with a new id. + assert id_after_restart != first_job_id + + # Update the feature table. + self._create_ft(client, self.features_2) + + # Run another iteration of the job control loop. We expect to restart the streaming + # job since the feature table has changed. + ensure_stream_ingestion_jobs(client=client, all_projects=True) + + # We expect to find two terminated job and one live job. + assert len(client.list_jobs(include_terminated=False)) == 1 + assert len(client.list_jobs(include_terminated=True)) == 3 + + id_after_change = client.list_jobs(include_terminated=False)[0].get_id() + assert id_after_restart != id_after_change + + # Delete the feature table. + self._delete_ft(client) + + # Run another iteration of the job control loop. We expect it to terminate the streaming + # job. + ensure_stream_ingestion_jobs(client=client, all_projects=True) + + assert len(client.list_jobs(include_terminated=False)) == 0 + assert len(client.list_jobs(include_terminated=True)) == 3 diff --git a/serving/pom.xml b/serving/pom.xml index b216aa29ecd..b8f675dd305 100644 --- a/serving/pom.xml +++ b/serving/pom.xml @@ -30,14 +30,6 @@ Feast Serving Feature serving API service - - - spring-plugins - Spring Plugins - https://repo.spring.io/plugins-release - - - @@ -92,12 +84,6 @@ ${project.version} - - dev.feast - feast-storage-connector-bigquery - ${project.version} - - dev.feast feast-common @@ -169,21 +155,26 @@ joda-time joda-time - + io.jaegertracing jaeger-client - 0.31.0 + 1.3.2 io.opentracing opentracing-api - 0.31.0 + 0.33.0 io.opentracing opentracing-noop - 0.31.0 + 0.33.0 + + + io.opentracing.contrib + opentracing-grpc + 0.2.3 @@ -307,13 +298,13 @@ org.testcontainers testcontainers - 1.14.3 + 1.15.1 test org.testcontainers junit-jupiter - 1.14.3 + 1.15.1 test @@ -334,6 +325,12 @@ ${project.version} test + + com.squareup.okhttp + okhttp + 2.7.4 + test + diff --git a/serving/src/main/java/feast/serving/config/FeastProperties.java b/serving/src/main/java/feast/serving/config/FeastProperties.java index 6c0b33fa0a8..bf048459cba 100644 --- a/serving/src/main/java/feast/serving/config/FeastProperties.java +++ b/serving/src/main/java/feast/serving/config/FeastProperties.java @@ -331,16 +331,6 @@ public StoreProto.Store toProto() StoreProto.Store.RedisConfig.newBuilder(); JsonFormat.parser().merge(jsonWriter.writeValueAsString(config), redisConfig); return storeProtoBuilder.setRedisConfig(redisConfig.build()).build(); - case BIGQUERY: - StoreProto.Store.BigQueryConfig.Builder bqConfig = - StoreProto.Store.BigQueryConfig.newBuilder(); - JsonFormat.parser().merge(jsonWriter.writeValueAsString(config), bqConfig); - return storeProtoBuilder.setBigqueryConfig(bqConfig.build()).build(); - case CASSANDRA: - StoreProto.Store.CassandraConfig.Builder cassandraConfig = - StoreProto.Store.CassandraConfig.newBuilder(); - JsonFormat.parser().merge(jsonWriter.writeValueAsString(config), cassandraConfig); - return storeProtoBuilder.setCassandraConfig(cassandraConfig.build()).build(); default: throw new InvalidProtocolBufferException("Invalid store set"); } diff --git a/serving/src/main/java/feast/serving/config/InstrumentationConfig.java b/serving/src/main/java/feast/serving/config/InstrumentationConfig.java index 30269c5d0ec..295b263f664 100644 --- a/serving/src/main/java/feast/serving/config/InstrumentationConfig.java +++ b/serving/src/main/java/feast/serving/config/InstrumentationConfig.java @@ -17,6 +17,7 @@ package feast.serving.config; import io.opentracing.Tracer; +import io.opentracing.contrib.grpc.TracingServerInterceptor; import io.opentracing.noop.NoopTracerFactory; import io.prometheus.client.exporter.MetricsServlet; import io.prometheus.client.hotspot.DefaultExports; @@ -54,4 +55,9 @@ public Tracer tracer() { return io.jaegertracing.Configuration.fromEnv(feastProperties.getTracing().getServiceName()) .getTracer(); } + + @Bean + public TracingServerInterceptor tracingInterceptor(Tracer tracer) { + return TracingServerInterceptor.newBuilder().withTracer(tracer).build(); + } } diff --git a/serving/src/main/java/feast/serving/config/JobServiceConfig.java b/serving/src/main/java/feast/serving/config/JobServiceConfig.java deleted file mode 100644 index b85e24062da..00000000000 --- a/serving/src/main/java/feast/serving/config/JobServiceConfig.java +++ /dev/null @@ -1,39 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2019 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.serving.config; - -import com.fasterxml.jackson.core.JsonProcessingException; -import com.google.protobuf.InvalidProtocolBufferException; -import feast.proto.core.StoreProto.Store.StoreType; -import feast.serving.service.JobService; -import feast.serving.service.NoopJobService; -import feast.serving.service.RedisBackedJobService; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; - -@Configuration -public class JobServiceConfig { - - @Bean - public JobService jobService(FeastProperties feastProperties) - throws InvalidProtocolBufferException, JsonProcessingException { - if (!feastProperties.getActiveStore().toProto().getType().equals(StoreType.BIGQUERY)) { - return new NoopJobService(); - } - return new RedisBackedJobService(feastProperties.getJobStore()); - } -} diff --git a/serving/src/main/java/feast/serving/config/ServingServiceConfig.java b/serving/src/main/java/feast/serving/config/ServingServiceConfig.java deleted file mode 100644 index 41a92e4bc64..00000000000 --- a/serving/src/main/java/feast/serving/config/ServingServiceConfig.java +++ /dev/null @@ -1,84 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2019 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.serving.config; - -import com.fasterxml.jackson.core.JsonProcessingException; -import com.google.protobuf.InvalidProtocolBufferException; -import feast.proto.core.StoreProto; -import feast.serving.service.HistoricalServingService; -import feast.serving.service.JobService; -import feast.serving.service.NoopJobService; -import feast.serving.service.OnlineServingService; -import feast.serving.service.ServingService; -import feast.serving.specs.CachedSpecService; -import feast.storage.api.retriever.HistoricalRetriever; -import feast.storage.api.retriever.OnlineRetriever; -import feast.storage.connectors.bigquery.retriever.BigQueryHistoricalRetriever; -import feast.storage.connectors.redis.retriever.RedisClusterOnlineRetriever; -import feast.storage.connectors.redis.retriever.RedisOnlineRetriever; -import io.opentracing.Tracer; -import java.util.Map; -import org.slf4j.Logger; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; - -@Configuration -public class ServingServiceConfig { - - private static final Logger log = org.slf4j.LoggerFactory.getLogger(ServingServiceConfig.class); - - @Bean - public ServingService servingService( - FeastProperties feastProperties, - CachedSpecService specService, - JobService jobService, - Tracer tracer) - throws InvalidProtocolBufferException, JsonProcessingException { - ServingService servingService = null; - FeastProperties.Store store = feastProperties.getActiveStore(); - StoreProto.Store.StoreType storeType = store.toProto().getType(); - Map config = store.getConfig(); - - switch (storeType) { - case REDIS_CLUSTER: - OnlineRetriever redisClusterRetriever = RedisClusterOnlineRetriever.create(config); - servingService = new OnlineServingService(redisClusterRetriever, specService, tracer); - break; - case REDIS: - OnlineRetriever redisRetriever = RedisOnlineRetriever.create(config); - servingService = new OnlineServingService(redisRetriever, specService, tracer); - break; - case BIGQUERY: - if (jobService.getClass() == NoopJobService.class) { - throw new IllegalArgumentException( - "Unable to instantiate JobService which is required by BigQueryHistoricalRetriever."); - } - HistoricalRetriever bqRetriever = BigQueryHistoricalRetriever.create(config); - servingService = new HistoricalServingService(bqRetriever, specService, jobService); - break; - case CASSANDRA: - case UNRECOGNIZED: - case INVALID: - throw new IllegalArgumentException( - String.format( - "Unsupported store type '%s' for store name '%s'", - store.getType(), store.getName())); - } - - return servingService; - } -} diff --git a/serving/src/main/java/feast/serving/config/ServingServiceConfigV2.java b/serving/src/main/java/feast/serving/config/ServingServiceConfigV2.java index 3508c64648b..9ec35c33e19 100644 --- a/serving/src/main/java/feast/serving/config/ServingServiceConfigV2.java +++ b/serving/src/main/java/feast/serving/config/ServingServiceConfigV2.java @@ -25,7 +25,6 @@ import feast.storage.api.retriever.OnlineRetrieverV2; import feast.storage.connectors.redis.retriever.*; import io.opentracing.Tracer; -import java.util.Map; import org.slf4j.Logger; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; @@ -41,20 +40,19 @@ public ServingServiceV2 servingServiceV2( ServingServiceV2 servingService = null; FeastProperties.Store store = feastProperties.getActiveStore(); StoreProto.Store.StoreType storeType = store.toProto().getType(); - Map config = store.getConfig(); switch (storeType) { case REDIS_CLUSTER: - RedisClientAdapter redisClusterClient = RedisClusterClient.create(config); + RedisClientAdapter redisClusterClient = + RedisClusterClient.create(store.toProto().getRedisClusterConfig()); OnlineRetrieverV2 redisClusterRetriever = new OnlineRetriever(redisClusterClient); servingService = new OnlineServingServiceV2(redisClusterRetriever, specService, tracer); break; case REDIS: - RedisClientAdapter redisClient = RedisClient.create(config); + RedisClientAdapter redisClient = RedisClient.create(store.toProto().getRedisConfig()); OnlineRetrieverV2 redisRetriever = new OnlineRetriever(redisClient); servingService = new OnlineServingServiceV2(redisRetriever, specService, tracer); break; - case CASSANDRA: case UNRECOGNIZED: case INVALID: throw new IllegalArgumentException( diff --git a/serving/src/main/java/feast/serving/controller/HealthServiceController.java b/serving/src/main/java/feast/serving/controller/HealthServiceController.java index 97a5fd9f93a..6615cf56eb6 100644 --- a/serving/src/main/java/feast/serving/controller/HealthServiceController.java +++ b/serving/src/main/java/feast/serving/controller/HealthServiceController.java @@ -19,7 +19,7 @@ import feast.proto.core.StoreProto.Store; import feast.proto.serving.ServingAPIProto.GetFeastServingInfoRequest; import feast.serving.interceptors.GrpcMonitoringInterceptor; -import feast.serving.service.ServingService; +import feast.serving.service.ServingServiceV2; import feast.serving.specs.CachedSpecService; import io.grpc.health.v1.HealthGrpc.HealthImplBase; import io.grpc.health.v1.HealthProto.HealthCheckRequest; @@ -34,10 +34,10 @@ @GrpcService(interceptors = {GrpcMonitoringInterceptor.class}) public class HealthServiceController extends HealthImplBase { private CachedSpecService specService; - private ServingService servingService; + private ServingServiceV2 servingService; @Autowired - public HealthServiceController(CachedSpecService specService, ServingService servingService) { + public HealthServiceController(CachedSpecService specService, ServingServiceV2 servingService) { this.specService = specService; this.servingService = servingService; } @@ -45,9 +45,10 @@ public HealthServiceController(CachedSpecService specService, ServingService ser @Override public void check( HealthCheckRequest request, StreamObserver responseObserver) { - // TODO: Implement proper logic to determine if ServingService is healthy e.g. - // if it's online service check that it the service can retrieve dummy/random feature set. - // Implement similary for batch service. + // TODO: Implement proper logic to determine if ServingServiceV2 is healthy e.g. + // if it's online service check that it the service can retrieve dummy/random + // feature table. + // Implement similary for batch service. try { Store store = specService.getStore(); diff --git a/serving/src/main/java/feast/serving/controller/ServingServiceGRpcController.java b/serving/src/main/java/feast/serving/controller/ServingServiceGRpcController.java index 3c8a10abedb..531be39f9d6 100644 --- a/serving/src/main/java/feast/serving/controller/ServingServiceGRpcController.java +++ b/serving/src/main/java/feast/serving/controller/ServingServiceGRpcController.java @@ -19,42 +19,36 @@ import feast.common.auth.service.AuthorizationService; import feast.common.logging.interceptors.GrpcMessageInterceptor; import feast.proto.serving.ServingAPIProto; -import feast.proto.serving.ServingAPIProto.FeatureReference; -import feast.proto.serving.ServingAPIProto.GetBatchFeaturesRequest; -import feast.proto.serving.ServingAPIProto.GetBatchFeaturesResponse; import feast.proto.serving.ServingAPIProto.GetFeastServingInfoRequest; import feast.proto.serving.ServingAPIProto.GetFeastServingInfoResponse; -import feast.proto.serving.ServingAPIProto.GetJobRequest; -import feast.proto.serving.ServingAPIProto.GetJobResponse; -import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesRequest; import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesResponse; import feast.proto.serving.ServingServiceGrpc.ServingServiceImplBase; import feast.serving.config.FeastProperties; import feast.serving.exception.SpecRetrievalException; import feast.serving.interceptors.GrpcMonitoringInterceptor; -import feast.serving.service.ServingService; import feast.serving.service.ServingServiceV2; import feast.serving.util.RequestHelper; import io.grpc.Status; import io.grpc.stub.StreamObserver; -import io.opentracing.Scope; import io.opentracing.Span; import io.opentracing.Tracer; -import java.util.List; -import java.util.Set; -import java.util.stream.Collectors; +import io.opentracing.contrib.grpc.TracingServerInterceptor; import net.devh.boot.grpc.server.service.GrpcService; import org.slf4j.Logger; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.security.access.AccessDeniedException; import org.springframework.security.core.context.SecurityContextHolder; -@GrpcService(interceptors = {GrpcMessageInterceptor.class, GrpcMonitoringInterceptor.class}) +@GrpcService( + interceptors = { + TracingServerInterceptor.class, + GrpcMessageInterceptor.class, + GrpcMonitoringInterceptor.class + }) public class ServingServiceGRpcController extends ServingServiceImplBase { private static final Logger log = org.slf4j.LoggerFactory.getLogger(ServingServiceGRpcController.class); - private final ServingService servingService; private final ServingServiceV2 servingServiceV2; private final String version; private final Tracer tracer; @@ -63,12 +57,10 @@ public class ServingServiceGRpcController extends ServingServiceImplBase { @Autowired public ServingServiceGRpcController( AuthorizationService authorizationService, - ServingService servingService, ServingServiceV2 servingServiceV2, FeastProperties feastProperties, Tracer tracer) { this.authorizationService = authorizationService; - this.servingService = servingService; this.servingServiceV2 = servingServiceV2; this.version = feastProperties.getVersion(); this.tracer = tracer; @@ -78,109 +70,17 @@ public ServingServiceGRpcController( public void getFeastServingInfo( GetFeastServingInfoRequest request, StreamObserver responseObserver) { - GetFeastServingInfoResponse feastServingInfo = servingService.getFeastServingInfo(request); + GetFeastServingInfoResponse feastServingInfo = servingServiceV2.getFeastServingInfo(request); feastServingInfo = feastServingInfo.toBuilder().setVersion(version).build(); responseObserver.onNext(feastServingInfo); responseObserver.onCompleted(); } - @Override - public void getOnlineFeatures( - GetOnlineFeaturesRequest request, - StreamObserver responseObserver) { - Span span = tracer.buildSpan("getOnlineFeatures").start(); - try (Scope scope = tracer.scopeManager().activate(span, false)) { - // authorize for the project in request object. - if (request.getProject() != null && !request.getProject().isEmpty()) { - // project set at root level overrides the project set at feature set level - this.authorizationService.authorizeRequest( - SecurityContextHolder.getContext(), request.getProject()); - } else { - // authorize for projects set in feature list, backward compatibility for - // <=v0.5.X - this.checkProjectAccess(request.getFeaturesList()); - } - RequestHelper.validateOnlineRequest(request); - GetOnlineFeaturesResponse onlineFeatures = servingService.getOnlineFeatures(request); - responseObserver.onNext(onlineFeatures); - responseObserver.onCompleted(); - } catch (SpecRetrievalException e) { - log.error("Failed to retrieve specs in SpecService", e); - responseObserver.onError( - Status.NOT_FOUND.withDescription(e.getMessage()).withCause(e).asException()); - } catch (AccessDeniedException e) { - log.info(String.format("User prevented from accessing one of the projects in request")); - responseObserver.onError( - Status.PERMISSION_DENIED - .withDescription(e.getMessage()) - .withCause(e) - .asRuntimeException()); - } catch (Exception e) { - log.warn("Failed to get Online Features", e); - responseObserver.onError(e); - } - span.finish(); - } - - @Override - public void getBatchFeatures( - GetBatchFeaturesRequest request, StreamObserver responseObserver) { - try { - RequestHelper.validateBatchRequest(request); - this.checkProjectAccess(request.getFeaturesList()); - GetBatchFeaturesResponse batchFeatures = servingService.getBatchFeatures(request); - responseObserver.onNext(batchFeatures); - responseObserver.onCompleted(); - } catch (SpecRetrievalException e) { - log.error("Failed to retrieve specs in SpecService", e); - responseObserver.onError( - Status.NOT_FOUND.withDescription(e.getMessage()).withCause(e).asException()); - } catch (AccessDeniedException e) { - log.info(String.format("User prevented from accessing one of the projects in request")); - responseObserver.onError( - Status.PERMISSION_DENIED - .withDescription(e.getMessage()) - .withCause(e) - .asRuntimeException()); - } catch (Exception e) { - log.warn("Failed to get Batch Features", e); - responseObserver.onError(e); - } - } - - @Override - public void getJob(GetJobRequest request, StreamObserver responseObserver) { - try { - GetJobResponse response = servingService.getJob(request); - responseObserver.onNext(response); - responseObserver.onCompleted(); - } catch (Exception e) { - log.warn("Failed to get Job", e); - responseObserver.onError(e); - } - } - - private void checkProjectAccess(List featureList) { - Set projectList = - featureList.stream().map(FeatureReference::getProject).collect(Collectors.toSet()); - if (projectList.isEmpty()) { - authorizationService.authorizeRequest(SecurityContextHolder.getContext(), "default"); - } else { - projectList.stream() - .forEach( - project -> { - this.authorizationService.authorizeRequest( - SecurityContextHolder.getContext(), project); - }); - } - } - @Override public void getOnlineFeaturesV2( ServingAPIProto.GetOnlineFeaturesRequestV2 request, StreamObserver responseObserver) { - Span span = tracer.buildSpan("getOnlineFeaturesV2").start(); - try (Scope scope = tracer.scopeManager().activate(span, false)) { + try { // authorize for the project in request object. if (request.getProject() != null && !request.getProject().isEmpty()) { // project set at root level overrides the project set at feature table level @@ -188,7 +88,11 @@ public void getOnlineFeaturesV2( SecurityContextHolder.getContext(), request.getProject()); } RequestHelper.validateOnlineRequest(request); + Span span = tracer.buildSpan("getOnlineFeaturesV2").start(); GetOnlineFeaturesResponse onlineFeatures = servingServiceV2.getOnlineFeatures(request); + if (span != null) { + span.finish(); + } responseObserver.onNext(onlineFeatures); responseObserver.onCompleted(); } catch (SpecRetrievalException e) { @@ -206,6 +110,5 @@ public void getOnlineFeaturesV2( log.warn("Failed to get Online Features", e); responseObserver.onError(e); } - span.finish(); } } diff --git a/serving/src/main/java/feast/serving/controller/ServingServiceRestController.java b/serving/src/main/java/feast/serving/controller/ServingServiceRestController.java index df71112ef78..8a198a82019 100644 --- a/serving/src/main/java/feast/serving/controller/ServingServiceRestController.java +++ b/serving/src/main/java/feast/serving/controller/ServingServiceRestController.java @@ -20,12 +20,11 @@ import feast.proto.serving.ServingAPIProto.GetFeastServingInfoRequest; import feast.proto.serving.ServingAPIProto.GetFeastServingInfoResponse; -import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesRequest; +import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesRequestV2; import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesResponse; import feast.serving.config.FeastProperties; -import feast.serving.service.ServingService; +import feast.serving.service.ServingServiceV2; import feast.serving.util.RequestHelper; -import io.opentracing.Tracer; import java.util.List; import java.util.Map; import org.springframework.beans.factory.annotation.Autowired; @@ -36,16 +35,14 @@ @RestController public class ServingServiceRestController { - private final ServingService servingService; + private final ServingServiceV2 servingService; private final String version; - private final Tracer tracer; @Autowired public ServingServiceRestController( - ServingService servingService, FeastProperties feastProperties, Tracer tracer) { + ServingServiceV2 servingService, FeastProperties feastProperties) { this.servingService = servingService; this.version = feastProperties.getVersion(); - this.tracer = tracer; } @RequestMapping(value = "/api/v1/info", produces = "application/json") @@ -60,7 +57,7 @@ public GetFeastServingInfoResponse getInfo() { produces = "application/json", consumes = "application/json") public List> getOnlineFeatures( - @RequestBody GetOnlineFeaturesRequest request) { + @RequestBody GetOnlineFeaturesRequestV2 request) { RequestHelper.validateOnlineRequest(request); GetOnlineFeaturesResponse onlineFeatures = servingService.getOnlineFeatures(request); return mapGetOnlineFeaturesResponse(onlineFeatures); diff --git a/serving/src/main/java/feast/serving/service/HistoricalServingService.java b/serving/src/main/java/feast/serving/service/HistoricalServingService.java deleted file mode 100644 index b63f05d0af7..00000000000 --- a/serving/src/main/java/feast/serving/service/HistoricalServingService.java +++ /dev/null @@ -1,124 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2019 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.serving.service; - -import feast.proto.serving.ServingAPIProto; -import feast.proto.serving.ServingAPIProto.*; -import feast.proto.serving.ServingAPIProto.Job.Builder; -import feast.serving.specs.CachedSpecService; -import feast.storage.api.retriever.FeatureSetRequest; -import feast.storage.api.retriever.HistoricalRetrievalResult; -import feast.storage.api.retriever.HistoricalRetriever; -import io.grpc.Status; -import java.util.List; -import java.util.Optional; -import java.util.UUID; -import org.slf4j.Logger; - -public class HistoricalServingService implements ServingService { - - private static final Logger log = - org.slf4j.LoggerFactory.getLogger(HistoricalServingService.class); - - private final HistoricalRetriever retriever; - private final CachedSpecService specService; - private final JobService jobService; - - public HistoricalServingService( - HistoricalRetriever retriever, CachedSpecService specService, JobService jobService) { - this.retriever = retriever; - this.specService = specService; - this.jobService = jobService; - } - - /** {@inheritDoc} */ - @Override - public GetFeastServingInfoResponse getFeastServingInfo( - GetFeastServingInfoRequest getFeastServingInfoRequest) { - return GetFeastServingInfoResponse.newBuilder() - .setType(FeastServingType.FEAST_SERVING_TYPE_BATCH) - .setJobStagingLocation(retriever.getStagingLocation()) - .build(); - } - - /** {@inheritDoc} */ - @Override - public GetOnlineFeaturesResponse getOnlineFeatures(GetOnlineFeaturesRequest getFeaturesRequest) { - throw Status.UNIMPLEMENTED.withDescription("Method not implemented").asRuntimeException(); - } - - /** {@inheritDoc} */ - @Override - public GetBatchFeaturesResponse getBatchFeatures(GetBatchFeaturesRequest getFeaturesRequest) { - List featureSetRequests = - specService.getFeatureSets(getFeaturesRequest.getFeaturesList()); - String retrievalId = UUID.randomUUID().toString(); - Job runningJob = - Job.newBuilder() - .setId(retrievalId) - .setType(JobType.JOB_TYPE_DOWNLOAD) - .setStatus(JobStatus.JOB_STATUS_RUNNING) - .build(); - jobService.upsert(runningJob); - Thread thread = - new Thread( - new Runnable() { - @Override - public void run() { - HistoricalRetrievalResult result = - retriever.getHistoricalFeatures( - retrievalId, - getFeaturesRequest.getDatasetSource(), - featureSetRequests, - getFeaturesRequest.getComputeStatistics()); - jobService.upsert(resultToJob(result)); - } - }); - thread.start(); - - return GetBatchFeaturesResponse.newBuilder().setJob(runningJob).build(); - } - - /** {@inheritDoc} */ - @Override - public GetJobResponse getJob(GetJobRequest getJobRequest) { - Optional job = jobService.get(getJobRequest.getJob().getId()); - if (!job.isPresent()) { - throw Status.NOT_FOUND - .withDescription(String.format("Job not found: %s", getJobRequest.getJob().getId())) - .asRuntimeException(); - } - return GetJobResponse.newBuilder().setJob(job.get()).build(); - } - - private Job resultToJob(HistoricalRetrievalResult result) { - Builder builder = - Job.newBuilder() - .setId(result.getId()) - .setType(JobType.JOB_TYPE_DOWNLOAD) - .setStatus(result.getStatus()); - if (result.hasError()) { - return builder.setError(result.getError()).build(); - } - Builder jobBuilder = - builder.addAllFileUris(result.getFileUris()).setDataFormat(result.getDataFormat()); - if (result.getStats() != null) { - jobBuilder.setDatasetFeatureStatisticsList(result.getStats()); - } - return builder.build(); - } -} diff --git a/serving/src/main/java/feast/serving/service/JobService.java b/serving/src/main/java/feast/serving/service/JobService.java deleted file mode 100644 index 3198fffd4cc..00000000000 --- a/serving/src/main/java/feast/serving/service/JobService.java +++ /dev/null @@ -1,40 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2019 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.serving.service; - -import feast.proto.serving.ServingAPIProto.Job; -import java.util.Optional; - -// JobService interface specifies the operations to manage Job instances internally in Feast - -public interface JobService { - - /** - * Get Job by job id. - * - * @param id job id - * @return feast.serving.ServingAPIProto.Job - */ - Optional get(String id); - - /** - * Update or create a job (if not exists) - * - * @param job feast.serving.ServingAPIProto.Job - */ - void upsert(Job job); -} diff --git a/serving/src/main/java/feast/serving/service/NoopJobService.java b/serving/src/main/java/feast/serving/service/NoopJobService.java deleted file mode 100644 index 5407cdcbe51..00000000000 --- a/serving/src/main/java/feast/serving/service/NoopJobService.java +++ /dev/null @@ -1,32 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2019 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.serving.service; - -import feast.proto.serving.ServingAPIProto.Job; -import java.util.Optional; - -// No-op implementation of the JobService, for online serving stores. -public class NoopJobService implements JobService { - - @Override - public Optional get(String id) { - return Optional.empty(); - } - - @Override - public void upsert(Job job) {} -} diff --git a/serving/src/main/java/feast/serving/service/OnlineServingService.java b/serving/src/main/java/feast/serving/service/OnlineServingService.java deleted file mode 100644 index a7d9d284aa2..00000000000 --- a/serving/src/main/java/feast/serving/service/OnlineServingService.java +++ /dev/null @@ -1,326 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2019 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.serving.service; - -import com.google.common.collect.ImmutableMap; -import com.google.common.collect.Streams; -import com.google.protobuf.Duration; -import feast.common.models.Feature; -import feast.common.models.FeatureSet; -import feast.proto.serving.ServingAPIProto.*; -import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesRequest.EntityRow; -import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesResponse.FieldStatus; -import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesResponse.FieldValues; -import feast.proto.types.FeatureRowProto.FeatureRow; -import feast.proto.types.FieldProto.Field; -import feast.proto.types.ValueProto.Value; -import feast.serving.specs.CachedSpecService; -import feast.serving.util.Metrics; -import feast.storage.api.retriever.FeatureSetRequest; -import feast.storage.api.retriever.OnlineRetriever; -import io.grpc.Status; -import io.opentracing.Scope; -import io.opentracing.Tracer; -import java.util.*; -import java.util.stream.Collectors; -import org.apache.commons.lang3.tuple.Pair; -import org.slf4j.Logger; - -public class OnlineServingService implements ServingService { - - private static final Logger log = org.slf4j.LoggerFactory.getLogger(OnlineServingService.class); - private final CachedSpecService specService; - private final Tracer tracer; - private final OnlineRetriever retriever; - - public OnlineServingService( - OnlineRetriever retriever, CachedSpecService specService, Tracer tracer) { - this.retriever = retriever; - this.specService = specService; - this.tracer = tracer; - } - - /** {@inheritDoc} */ - @Override - public GetFeastServingInfoResponse getFeastServingInfo( - GetFeastServingInfoRequest getFeastServingInfoRequest) { - return GetFeastServingInfoResponse.newBuilder() - .setType(FeastServingType.FEAST_SERVING_TYPE_ONLINE) - .build(); - } - - /** {@inheritDoc} */ - @Override - public GetOnlineFeaturesResponse getOnlineFeatures(GetOnlineFeaturesRequest request) { - try (Scope scope = tracer.buildSpan("getOnlineFeatures").startActive(true)) { - List entityRows = request.getEntityRowsList(); - // Collect the feature/entity value for each entity row in entityValueMap - Map> entityValuesMap = - entityRows.stream().collect(Collectors.toMap(row -> row, row -> new HashMap<>())); - // Collect the feature/entity status metadata for each entity row in entityValueMap - Map> entityStatusesMap = - entityRows.stream().collect(Collectors.toMap(row -> row, row -> new HashMap<>())); - // Collect featureRows retrieved for logging/tracing - List>> logFeatureRows = new LinkedList<>(); - - if (!request.getOmitEntitiesInResponse()) { - // Add entity row's fields as response fields - entityRows.forEach( - entityRow -> { - Map valueMap = entityRow.getFieldsMap(); - entityValuesMap.get(entityRow).putAll(valueMap); - entityStatusesMap.get(entityRow).putAll(getMetadataMap(valueMap, false, false)); - }); - } - - List featureSetRequests = - specService.getFeatureSets(request.getFeaturesList(), request.getProject()); - for (FeatureSetRequest featureSetRequest : featureSetRequests) { - // Pull feature rows for given entity rows from the feature/featureset specified in feature - // set request. - // from the configured online - List> featureRows = - retriever.getOnlineFeatures(entityRows, featureSetRequest); - // Check that feature row returned corresponds to a given entity row. - if (featureRows.size() != entityRows.size()) { - throw Status.INTERNAL - .withDescription( - "The no. of FeatureRow obtained from OnlineRetriever" - + "does not match no. of entityRow passed.") - .asRuntimeException(); - } - - Streams.zip(entityRows.stream(), featureRows.stream(), Pair::of) - .forEach( - entityFeaturePair -> { - EntityRow entityRow = entityFeaturePair.getLeft(); - Optional featureRow = entityFeaturePair.getRight(); - // Unpack feature field values and merge into entityValueMap - boolean isOutsideMaxAge = - checkOutsideMaxAge(featureSetRequest, entityRow, featureRow); - Map valueMap = - unpackValueMap(featureRow, featureSetRequest, isOutsideMaxAge); - entityValuesMap.get(entityRow).putAll(valueMap); - - // Generate metadata for feature values and merge into entityFieldsMap - boolean isNotFound = featureRow.isEmpty(); - Map statusMap = - getMetadataMap(valueMap, isNotFound, isOutsideMaxAge); - entityStatusesMap.get(entityRow).putAll(statusMap); - - // Populate metrics/log request - populateCountMetrics(statusMap, featureSetRequest); - }); - populateRequestCountMetrics(featureSetRequest); - logFeatureRows.add(featureRows); - } - if (scope != null) { - logFeatureRowsTrace(scope, logFeatureRows, featureSetRequests); - } - - // Build response field values from entityValuesMap and entityStatusesMap - // Reponse field values should be in the same order as the entityRows provided by the user. - List fieldValuesList = - entityRows.stream() - .map( - entityRow -> { - return FieldValues.newBuilder() - .putAllFields(entityValuesMap.get(entityRow)) - .putAllStatuses(entityStatusesMap.get(entityRow)) - .build(); - }) - .collect(Collectors.toList()); - return GetOnlineFeaturesResponse.newBuilder().addAllFieldValues(fieldValuesList).build(); - } - } - - /** - * Unpack feature values using data from the given feature row for features specified in the given - * feature set request. - * - * @param featureRow optional to unpack for feature values. - * @param featureSetRequest feature set request for which the feature row is retrieved for. - * @param isOutsideMaxAge whether which the feature row contains values that is outside max age. - * @return valueMap mapping string feature name to feature value for the given feature set - * request. - */ - private static Map unpackValueMap( - Optional featureRow, - FeatureSetRequest featureSetRequest, - boolean isOutsideMaxAge) { - Map valueMap = new HashMap<>(); - // In order to return values containing the same feature references provided by the user, - // we reuse the feature references in the request as the keys in field builder map - Map nameRefMap = featureSetRequest.getFeatureRefsByName(); - - if (featureRow.isPresent()) { - // unpack feature row's feature values and populate value map - Map featureValueMap = - featureRow.get().getFieldsList().stream() - .filter(featureRowField -> nameRefMap.containsKey(featureRowField.getName())) - .collect( - Collectors.toMap( - featureRowField -> { - FeatureReference featureRef = nameRefMap.get(featureRowField.getName()); - return Feature.getFeatureStringWithProjectRef(featureRef); - }, - featureRowField -> { - // drop feature values with an age outside feature set's max age. - return (isOutsideMaxAge) - ? Value.newBuilder().build() - : featureRowField.getValue(); - })); - valueMap.putAll(featureValueMap); - } - // create empty values for features specified in request but not present in feature row. - Set missingFeatures = - nameRefMap.values().stream() - .map(ref -> Feature.getFeatureStringWithProjectRef(ref)) - .collect(Collectors.toSet()); - missingFeatures.removeAll(valueMap.keySet()); - missingFeatures.forEach(refString -> valueMap.put(refString, Value.newBuilder().build())); - - return valueMap; - } - - /** - * Generate Field level Status metadata for the given valueMap. - * - * @param valueMap map of field name to value to generate metadata for. - * @param isNotFound whether the given valueMap represents values that were not found in the - * online retriever. - * @param isOutsideMaxAge whether the given valueMap contains values with age outside feature - * set's max age. - * @return a 1:1 map keyed by field name containing field status metadata instead of values in the - * given valueMap. - */ - private static Map getMetadataMap( - Map valueMap, boolean isNotFound, boolean isOutsideMaxAge) { - return valueMap.entrySet().stream() - .collect( - Collectors.toMap( - es -> es.getKey(), - es -> { - Value fieldValue = es.getValue(); - if (isNotFound) { - return FieldStatus.NOT_FOUND; - } else if (isOutsideMaxAge) { - return FieldStatus.OUTSIDE_MAX_AGE; - } else if (fieldValue.getValCase().equals(Value.ValCase.VAL_NOT_SET)) { - return FieldStatus.NULL_VALUE; - } - return FieldStatus.PRESENT; - })); - } - - /** - * Determine if the feature data in the given feature row is outside maxAge. Data is outside - * maxAge to be when the difference ingestion time set in feature row and the retrieval time set - * in entity row exceeds featureset max age. - * - * @param featureSetRequest contains the spec where feature's max age is extracted. - * @param entityRow contains the retrieval timing of when features are pulled. - * @param featureRow contains the ingestion timing and feature data. - */ - private static boolean checkOutsideMaxAge( - FeatureSetRequest featureSetRequest, EntityRow entityRow, Optional featureRow) { - Duration maxAge = featureSetRequest.getSpec().getMaxAge(); - if (featureRow.isEmpty()) { // no data to consider - return false; - } - if (maxAge.equals(Duration.getDefaultInstance())) { // max age is not set - return false; - } - - long givenTimestamp = entityRow.getEntityTimestamp().getSeconds(); - if (givenTimestamp == 0) { - givenTimestamp = System.currentTimeMillis() / 1000; - } - long timeDifference = givenTimestamp - featureRow.get().getEventTimestamp().getSeconds(); - return timeDifference > maxAge.getSeconds(); - } - - private void logFeatureRowsTrace( - Scope scope, - List>> logFeatureRows, - List featureSetRequests) { - List> loggableFeatureRows = - Streams.zip( - logFeatureRows.stream(), - featureSetRequests.stream(), - (featureRows, featureSetRequest) -> { - FeatureRow.Builder nullFeatureRowBuilder = - FeatureRow.newBuilder() - .setFeatureSet( - FeatureSet.getFeatureSetStringRef(featureSetRequest.getSpec())); - for (FeatureReference featureReference : - featureSetRequest.getFeatureReferences()) { - nullFeatureRowBuilder.addFields( - Field.newBuilder().setName(featureReference.getName())); - } - - // log null feature row when feature row is empty - return featureRows.stream() - .map( - featureRow -> { - return (featureRow.isEmpty()) - ? nullFeatureRowBuilder.build() - : featureRow.get(); - }) - .collect(Collectors.toList()); - }) - .collect(Collectors.toList()); - - scope.span().log(ImmutableMap.of("event", "featureRows", "value", loggableFeatureRows)); - } - - private void populateCountMetrics( - Map statusMap, FeatureSetRequest featureSetRequest) { - String project = featureSetRequest.getSpec().getProject(); - statusMap - .entrySet() - .forEach( - es -> { - String featureRefString = es.getKey(); - FieldStatus status = es.getValue(); - if (status == FieldStatus.NOT_FOUND) { - Metrics.notFoundKeyCount.labels(project, featureRefString).inc(); - } - if (status == FieldStatus.OUTSIDE_MAX_AGE) { - Metrics.staleKeyCount.labels(project, featureRefString).inc(); - } - }); - } - - private void populateRequestCountMetrics(FeatureSetRequest featureSetRequest) { - String project = featureSetRequest.getSpec().getProject(); - featureSetRequest - .getFeatureReferences() - .parallelStream() - .forEach(ref -> Metrics.requestCount.labels(project, ref.getName()).inc()); - } - - @Override - public GetBatchFeaturesResponse getBatchFeatures(GetBatchFeaturesRequest getFeaturesRequest) { - throw Status.UNIMPLEMENTED.withDescription("Method not implemented").asRuntimeException(); - } - - @Override - public GetJobResponse getJob(GetJobRequest getJobRequest) { - throw Status.UNIMPLEMENTED.withDescription("Method not implemented").asRuntimeException(); - } -} diff --git a/serving/src/main/java/feast/serving/service/OnlineServingServiceV2.java b/serving/src/main/java/feast/serving/service/OnlineServingServiceV2.java index dca3159ce81..70dd6f73878 100644 --- a/serving/src/main/java/feast/serving/service/OnlineServingServiceV2.java +++ b/serving/src/main/java/feast/serving/service/OnlineServingServiceV2.java @@ -16,32 +16,59 @@ */ package feast.serving.service; +import static feast.common.models.FeatureTable.getFeatureTableStringRef; + import com.google.protobuf.Duration; import feast.common.models.FeatureV2; -import feast.proto.core.FeatureProto; -import feast.proto.core.FeatureTableProto.FeatureTableSpec; +import feast.proto.serving.ServingAPIProto.FeastServingType; import feast.proto.serving.ServingAPIProto.FeatureReferenceV2; +import feast.proto.serving.ServingAPIProto.GetFeastServingInfoRequest; +import feast.proto.serving.ServingAPIProto.GetFeastServingInfoResponse; import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesRequestV2; import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesResponse; import feast.proto.types.ValueProto; +import feast.serving.exception.SpecRetrievalException; import feast.serving.specs.CachedSpecService; import feast.serving.util.Metrics; import feast.storage.api.retriever.Feature; import feast.storage.api.retriever.OnlineRetrieverV2; import io.grpc.Status; -import io.opentracing.Scope; +import io.opentracing.Span; import io.opentracing.Tracer; import java.util.*; +import java.util.function.Function; import java.util.stream.Collectors; +import java.util.stream.IntStream; import org.slf4j.Logger; public class OnlineServingServiceV2 implements ServingServiceV2 { - private static final Logger log = org.slf4j.LoggerFactory.getLogger(OnlineServingService.class); + private static final Logger log = org.slf4j.LoggerFactory.getLogger(OnlineServingServiceV2.class); private final CachedSpecService specService; private final Tracer tracer; private final OnlineRetrieverV2 retriever; + private static final HashMap + TYPE_TO_VAL_CASE = + new HashMap<>() { + { + put(ValueProto.ValueType.Enum.BYTES, ValueProto.Value.ValCase.BYTES_VAL); + put(ValueProto.ValueType.Enum.STRING, ValueProto.Value.ValCase.STRING_VAL); + put(ValueProto.ValueType.Enum.INT32, ValueProto.Value.ValCase.INT32_VAL); + put(ValueProto.ValueType.Enum.INT64, ValueProto.Value.ValCase.INT64_VAL); + put(ValueProto.ValueType.Enum.DOUBLE, ValueProto.Value.ValCase.DOUBLE_VAL); + put(ValueProto.ValueType.Enum.FLOAT, ValueProto.Value.ValCase.FLOAT_VAL); + put(ValueProto.ValueType.Enum.BOOL, ValueProto.Value.ValCase.BOOL_VAL); + put(ValueProto.ValueType.Enum.BYTES_LIST, ValueProto.Value.ValCase.BYTES_LIST_VAL); + put(ValueProto.ValueType.Enum.STRING_LIST, ValueProto.Value.ValCase.STRING_LIST_VAL); + put(ValueProto.ValueType.Enum.INT32_LIST, ValueProto.Value.ValCase.INT32_LIST_VAL); + put(ValueProto.ValueType.Enum.INT64_LIST, ValueProto.Value.ValCase.INT64_LIST_VAL); + put(ValueProto.ValueType.Enum.DOUBLE_LIST, ValueProto.Value.ValCase.DOUBLE_LIST_VAL); + put(ValueProto.ValueType.Enum.FLOAT_LIST, ValueProto.Value.ValCase.FLOAT_LIST_VAL); + put(ValueProto.ValueType.Enum.BOOL_LIST, ValueProto.Value.ValCase.BOOL_LIST_VAL); + } + }; + public OnlineServingServiceV2( OnlineRetrieverV2 retriever, CachedSpecService specService, Tracer tracer) { this.retriever = retriever; @@ -49,6 +76,15 @@ public OnlineServingServiceV2( this.tracer = tracer; } + /** {@inheritDoc} */ + @Override + public GetFeastServingInfoResponse getFeastServingInfo( + GetFeastServingInfoRequest getFeastServingInfoRequest) { + return GetFeastServingInfoResponse.newBuilder() + .setType(FeastServingType.FEAST_SERVING_TYPE_ONLINE) + .build(); + } + @Override public GetOnlineFeaturesResponse getOnlineFeatures(GetOnlineFeaturesRequestV2 request) { String projectName = request.getProject(); @@ -59,145 +95,149 @@ public GetOnlineFeaturesResponse getOnlineFeatures(GetOnlineFeaturesRequestV2 re projectName = "default"; } - try (Scope scope = tracer.buildSpan("getOnlineFeaturesV2").startActive(true)) { - List entityRows = request.getEntityRowsList(); - // Collect the feature/entity value for each entity row in entityValueMap - Map> entityValuesMap = - entityRows.stream().collect(Collectors.toMap(row -> row, row -> new HashMap<>())); - // Collect the feature/entity status metadata for each entity row in entityValueMap - Map> - entityStatusesMap = - entityRows.stream().collect(Collectors.toMap(row -> row, row -> new HashMap<>())); - - entityRows.forEach( - entityRow -> { - Map valueMap = entityRow.getFieldsMap(); - entityValuesMap.get(entityRow).putAll(valueMap); - entityStatusesMap.get(entityRow).putAll(getMetadataMap(valueMap, false, false)); - }); - - List>> entityRowsFeatures = - retriever.getOnlineFeatures(projectName, entityRows, featureReferences); - - if (entityRowsFeatures.size() != entityRows.size()) { - throw Status.INTERNAL - .withDescription( - "The no. of FeatureRow obtained from OnlineRetriever" - + "does not match no. of entityRow passed.") - .asRuntimeException(); - } + List entityRows = request.getEntityRowsList(); + List> values = + entityRows.stream().map(r -> new HashMap<>(r.getFieldsMap())).collect(Collectors.toList()); + List> statuses = + entityRows.stream() + .map(r -> getMetadataMap(r.getFieldsMap(), false, false)) + .collect(Collectors.toList()); - for (int i = 0; i < entityRows.size(); i++) { - GetOnlineFeaturesRequestV2.EntityRow entityRow = entityRows.get(i); - List> curEntityRowFeatures = entityRowsFeatures.get(i); - - Map> featureReferenceFeatureMap = - getFeatureRefFeatureMap(curEntityRowFeatures); - - Map allValueMaps = new HashMap<>(); - Map allStatusMaps = new HashMap<>(); - - for (FeatureReferenceV2 featureReference : featureReferences) { - if (featureReferenceFeatureMap.containsKey(featureReference)) { - Optional feature = featureReferenceFeatureMap.get(featureReference); - - FeatureTableSpec featureTableSpec = - specService.getFeatureTableSpec(projectName, feature.get().getFeatureReference()); - FeatureProto.FeatureSpecV2 featureSpec = - specService.getFeatureSpec(projectName, feature.get().getFeatureReference()); - ValueProto.ValueType.Enum valueTypeEnum = featureSpec.getValueType(); - ValueProto.Value.ValCase valueCase = feature.get().getFeatureValue().getValCase(); - boolean isMatchingFeatureSpec = checkSameFeatureSpec(valueTypeEnum, valueCase); - - boolean isOutsideMaxAge = checkOutsideMaxAge(featureTableSpec, entityRow, feature); - Map valueMap = - unpackValueMap(feature, isOutsideMaxAge, isMatchingFeatureSpec); - allValueMaps.putAll(valueMap); - - // Generate metadata for feature values and merge into entityFieldsMap - Map statusMap = - getMetadataMap(valueMap, !isMatchingFeatureSpec, isOutsideMaxAge); - allStatusMaps.putAll(statusMap); - - // Populate metrics/log request - populateCountMetrics(statusMap, projectName); - } else { - Map valueMap = - new HashMap<>() { - { - put( - FeatureV2.getFeatureStringRef(featureReference), - ValueProto.Value.newBuilder().build()); - } - }; - allValueMaps.putAll(valueMap); + Span storageRetrievalSpan = tracer.buildSpan("storageRetrieval").start(); + if (storageRetrievalSpan != null) { + storageRetrievalSpan.setTag("entities", entityRows.size()); + storageRetrievalSpan.setTag("features", featureReferences.size()); + } + List> entityRowsFeatures = + retriever.getOnlineFeatures(projectName, entityRows, featureReferences); + if (storageRetrievalSpan != null) { + storageRetrievalSpan.finish(); + } + + if (entityRowsFeatures.size() != entityRows.size()) { + throw Status.INTERNAL + .withDescription( + "The no. of FeatureRow obtained from OnlineRetriever" + + "does not match no. of entityRow passed.") + .asRuntimeException(); + } - Map statusMap = - getMetadataMap(valueMap, true, false); - allStatusMaps.putAll(statusMap); + String finalProjectName = projectName; + Map featureMaxAges = + featureReferences.stream() + .distinct() + .collect( + Collectors.toMap( + Function.identity(), + ref -> specService.getFeatureTableSpec(finalProjectName, ref).getMaxAge())); - // Populate metrics/log request - populateCountMetrics(statusMap, projectName); - } + Map featureValueTypes = + featureReferences.stream() + .distinct() + .collect( + Collectors.toMap( + Function.identity(), + ref -> { + try { + return specService.getFeatureSpec(finalProjectName, ref).getValueType(); + } catch (SpecRetrievalException e) { + return ValueProto.ValueType.Enum.INVALID; + } + })); + + Span postProcessingSpan = tracer.buildSpan("postProcessing").start(); + + for (int i = 0; i < entityRows.size(); i++) { + GetOnlineFeaturesRequestV2.EntityRow entityRow = entityRows.get(i); + List curEntityRowFeatures = entityRowsFeatures.get(i); + + Map featureReferenceFeatureMap = + getFeatureRefFeatureMap(curEntityRowFeatures); + + Map rowValues = values.get(i); + Map rowStatuses = statuses.get(i); + + for (FeatureReferenceV2 featureReference : featureReferences) { + if (featureReferenceFeatureMap.containsKey(featureReference)) { + Feature feature = featureReferenceFeatureMap.get(featureReference); + + ValueProto.Value.ValCase valueCase = feature.getFeatureValue().getValCase(); + + boolean isMatchingFeatureSpec = + checkSameFeatureSpec(featureValueTypes.get(feature.getFeatureReference()), valueCase); + boolean isOutsideMaxAge = + checkOutsideMaxAge( + feature, entityRow, featureMaxAges.get(feature.getFeatureReference())); + + Map valueMap = + unpackValueMap(feature, isOutsideMaxAge, isMatchingFeatureSpec); + rowValues.putAll(valueMap); + + // Generate metadata for feature values and merge into entityFieldsMap + Map statusMap = + getMetadataMap(valueMap, !isMatchingFeatureSpec, isOutsideMaxAge); + rowStatuses.putAll(statusMap); + + // Populate metrics/log request + populateCountMetrics(statusMap, projectName); + } else { + Map valueMap = + new HashMap<>() { + { + put( + FeatureV2.getFeatureStringRef(featureReference), + ValueProto.Value.newBuilder().build()); + } + }; + rowValues.putAll(valueMap); + + Map statusMap = + getMetadataMap(valueMap, true, false); + rowStatuses.putAll(statusMap); + + // Populate metrics/log request + populateCountMetrics(statusMap, projectName); } - entityValuesMap.get(entityRow).putAll(allValueMaps); - entityStatusesMap.get(entityRow).putAll(allStatusMaps); } + } - // Build response field values from entityValuesMap and entityStatusesMap - // Response field values should be in the same order as the entityRows provided by the user. - List fieldValuesList = - entityRows.stream() - .map( - entityRow -> { - return GetOnlineFeaturesResponse.FieldValues.newBuilder() - .putAllFields(entityValuesMap.get(entityRow)) - .putAllStatuses(entityStatusesMap.get(entityRow)) - .build(); - }) - .collect(Collectors.toList()); - return GetOnlineFeaturesResponse.newBuilder().addAllFieldValues(fieldValuesList).build(); + if (postProcessingSpan != null) { + postProcessingSpan.finish(); } + + populateHistogramMetrics(entityRows, featureReferences, projectName); + populateFeatureCountMetrics(featureReferences, projectName); + + // Build response field values from entityValuesMap and entityStatusesMap + // Response field values should be in the same order as the entityRows provided by the user. + List fieldValuesList = + IntStream.range(0, entityRows.size()) + .mapToObj( + entityRowIdx -> + GetOnlineFeaturesResponse.FieldValues.newBuilder() + .putAllFields(values.get(entityRowIdx)) + .putAllStatuses(statuses.get(entityRowIdx)) + .build()) + .collect(Collectors.toList()); + return GetOnlineFeaturesResponse.newBuilder().addAllFieldValues(fieldValuesList).build(); } private boolean checkSameFeatureSpec( ValueProto.ValueType.Enum valueTypeEnum, ValueProto.Value.ValCase valueCase) { - HashMap typingMap = - new HashMap<>() { - { - put(ValueProto.ValueType.Enum.BYTES, ValueProto.Value.ValCase.BYTES_VAL); - put(ValueProto.ValueType.Enum.STRING, ValueProto.Value.ValCase.STRING_VAL); - put(ValueProto.ValueType.Enum.INT32, ValueProto.Value.ValCase.INT32_VAL); - put(ValueProto.ValueType.Enum.INT64, ValueProto.Value.ValCase.INT64_VAL); - put(ValueProto.ValueType.Enum.DOUBLE, ValueProto.Value.ValCase.DOUBLE_VAL); - put(ValueProto.ValueType.Enum.FLOAT, ValueProto.Value.ValCase.FLOAT_VAL); - put(ValueProto.ValueType.Enum.BOOL, ValueProto.Value.ValCase.BOOL_VAL); - put(ValueProto.ValueType.Enum.BYTES_LIST, ValueProto.Value.ValCase.BYTES_LIST_VAL); - put(ValueProto.ValueType.Enum.STRING_LIST, ValueProto.Value.ValCase.STRING_LIST_VAL); - put(ValueProto.ValueType.Enum.INT32_LIST, ValueProto.Value.ValCase.INT32_LIST_VAL); - put(ValueProto.ValueType.Enum.INT64_LIST, ValueProto.Value.ValCase.INT64_LIST_VAL); - put(ValueProto.ValueType.Enum.DOUBLE_LIST, ValueProto.Value.ValCase.DOUBLE_LIST_VAL); - put(ValueProto.ValueType.Enum.FLOAT_LIST, ValueProto.Value.ValCase.FLOAT_LIST_VAL); - put(ValueProto.ValueType.Enum.BOOL_LIST, ValueProto.Value.ValCase.BOOL_LIST_VAL); - } - }; + if (valueTypeEnum.equals(ValueProto.ValueType.Enum.INVALID)) { + return false; + } + if (valueCase.equals(ValueProto.Value.ValCase.VAL_NOT_SET)) { return true; } - return typingMap.get(valueTypeEnum).equals(valueCase); + return TYPE_TO_VAL_CASE.get(valueTypeEnum).equals(valueCase); } - private static Map> getFeatureRefFeatureMap( - List> features) { - Map> featureReferenceFeatureMap = new HashMap<>(); - features.forEach( - feature -> { - FeatureReferenceV2 featureReference = feature.get().getFeatureReference(); - featureReferenceFeatureMap.put(featureReference, feature); - }); - - return featureReferenceFeatureMap; + private static Map getFeatureRefFeatureMap(List features) { + return features.stream() + .collect(Collectors.toMap(Feature::getFeatureReference, Function.identity())); } /** @@ -216,7 +256,7 @@ private static Map getMetadataMap return valueMap.entrySet().stream() .collect( Collectors.toMap( - es -> es.getKey(), + Map.Entry::getKey, es -> { ValueProto.Value fieldValue = es.getValue(); if (isNotFound) { @@ -231,20 +271,18 @@ private static Map getMetadataMap } private static Map unpackValueMap( - Optional feature, boolean isOutsideMaxAge, boolean isMatchingFeatureSpec) { + Feature feature, boolean isOutsideMaxAge, boolean isMatchingFeatureSpec) { Map valueMap = new HashMap<>(); - if (feature.isPresent()) { - if (!isOutsideMaxAge && isMatchingFeatureSpec) { - valueMap.put( - FeatureV2.getFeatureStringRef(feature.get().getFeatureReference()), - feature.get().getFeatureValue()); - } else { - valueMap.put( - FeatureV2.getFeatureStringRef(feature.get().getFeatureReference()), - ValueProto.Value.newBuilder().build()); - } + if (!isOutsideMaxAge && isMatchingFeatureSpec) { + valueMap.put( + FeatureV2.getFeatureStringRef(feature.getFeatureReference()), feature.getFeatureValue()); + } else { + valueMap.put( + FeatureV2.getFeatureStringRef(feature.getFeatureReference()), + ValueProto.Value.newBuilder().build()); } + return valueMap; } @@ -253,18 +291,13 @@ private static Map unpackValueMap( * maxAge to be when the difference ingestion time set in feature row and the retrieval time set * in entity row exceeds FeatureTable max age. * - * @param featureTableSpec contains the spec where feature's max age is extracted. - * @param entityRow contains the retrieval timing of when features are pulled. * @param feature contains the ingestion timing and feature data. + * @param entityRow contains the retrieval timing of when features are pulled. + * @param maxAge feature's max age. */ private static boolean checkOutsideMaxAge( - FeatureTableSpec featureTableSpec, - GetOnlineFeaturesRequestV2.EntityRow entityRow, - Optional feature) { - Duration maxAge = featureTableSpec.getMaxAge(); - if (feature.isEmpty()) { // no data to consider - return false; - } + Feature feature, GetOnlineFeaturesRequestV2.EntityRow entityRow, Duration maxAge) { + if (maxAge.equals(Duration.getDefaultInstance())) { // max age is not set return false; } @@ -273,10 +306,38 @@ private static boolean checkOutsideMaxAge( if (givenTimestamp == 0) { givenTimestamp = System.currentTimeMillis() / 1000; } - long timeDifference = givenTimestamp - feature.get().getEventTimestamp().getSeconds(); + long timeDifference = givenTimestamp - feature.getEventTimestamp().getSeconds(); return timeDifference > maxAge.getSeconds(); } + /** + * Populate histogram metrics that can be used for analysing online retrieval calls + * + * @param entityRows entity rows provided in request + * @param featureReferences feature references provided in request + * @param project project name provided in request + */ + private void populateHistogramMetrics( + List entityRows, + List featureReferences, + String project) { + Metrics.requestEntityCountDistribution + .labels(project) + .observe(Double.valueOf(entityRows.size())); + Metrics.requestFeatureCountDistribution + .labels(project) + .observe(Double.valueOf(featureReferences.size())); + + long countDistinctFeatureTables = + featureReferences.stream() + .map(featureReference -> getFeatureTableStringRef(project, featureReference)) + .distinct() + .count(); + Metrics.requestFeatureTableCountDistribution + .labels(project) + .observe(Double.valueOf(countDistinctFeatureTables)); + } + /** * Populate count metrics that can be used for analysing online retrieval calls * @@ -285,18 +346,23 @@ private static boolean checkOutsideMaxAge( */ private void populateCountMetrics( Map statusMap, String project) { - statusMap - .entrySet() - .forEach( - es -> { - String featureRefString = es.getKey(); - GetOnlineFeaturesResponse.FieldStatus status = es.getValue(); - if (status == GetOnlineFeaturesResponse.FieldStatus.NOT_FOUND) { - Metrics.notFoundKeyCount.labels(project, featureRefString).inc(); - } - if (status == GetOnlineFeaturesResponse.FieldStatus.OUTSIDE_MAX_AGE) { - Metrics.staleKeyCount.labels(project, featureRefString).inc(); - } - }); + statusMap.forEach( + (featureRefString, status) -> { + if (status == GetOnlineFeaturesResponse.FieldStatus.NOT_FOUND) { + Metrics.notFoundKeyCount.labels(project, featureRefString).inc(); + } + if (status == GetOnlineFeaturesResponse.FieldStatus.OUTSIDE_MAX_AGE) { + Metrics.staleKeyCount.labels(project, featureRefString).inc(); + } + }); + } + + private void populateFeatureCountMetrics( + List featureReferences, String project) { + featureReferences.forEach( + featureReference -> + Metrics.requestFeatureCount + .labels(project, FeatureV2.getFeatureStringRef(featureReference)) + .inc()); } } diff --git a/serving/src/main/java/feast/serving/service/RedisBackedJobService.java b/serving/src/main/java/feast/serving/service/RedisBackedJobService.java deleted file mode 100644 index 9081b03f518..00000000000 --- a/serving/src/main/java/feast/serving/service/RedisBackedJobService.java +++ /dev/null @@ -1,87 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2019 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.serving.service; - -import com.google.protobuf.util.JsonFormat; -import feast.proto.serving.ServingAPIProto.Job; -import feast.proto.serving.ServingAPIProto.Job.Builder; -import feast.serving.config.FeastProperties; -import io.lettuce.core.RedisClient; -import io.lettuce.core.RedisURI; -import io.lettuce.core.api.StatefulRedisConnection; -import io.lettuce.core.api.sync.RedisCommands; -import io.lettuce.core.codec.ByteArrayCodec; -import io.lettuce.core.resource.DefaultClientResources; -import java.util.Optional; -import org.joda.time.Duration; -import org.slf4j.Logger; - -// TODO: Do rate limiting, currently if clients call get() or upsert() -// and an exceedingly high rate e.g. they wrap job reload in a while loop with almost no wait -// Redis connection may break and need to restart Feast serving. Need to handle this. - -public class RedisBackedJobService implements JobService { - - private static final Logger log = org.slf4j.LoggerFactory.getLogger(RedisBackedJobService.class); - private final RedisCommands syncCommand; - // Remove job state info after "defaultExpirySeconds" to prevent filling up Redis memory - // and since users normally don't require info about relatively old jobs. - private final int defaultExpirySeconds = (int) Duration.standardDays(1).getStandardSeconds(); - - public RedisBackedJobService(FeastProperties.JobStoreProperties jobStoreProperties) { - RedisURI uri = - RedisURI.create(jobStoreProperties.getRedisHost(), jobStoreProperties.getRedisPort()); - - this.syncCommand = - RedisClient.create(DefaultClientResources.create(), uri) - .connect(new ByteArrayCodec()) - .sync(); - } - - public RedisBackedJobService(StatefulRedisConnection connection) { - this.syncCommand = connection.sync(); - } - - @Override - public Optional get(String id) { - Job job = null; - try { - String json = new String(syncCommand.get(id.getBytes())); - if (json.isEmpty()) { - return Optional.empty(); - } - Builder builder = Job.newBuilder(); - JsonFormat.parser().merge(json, builder); - job = builder.build(); - } catch (Exception e) { - log.error(String.format("Failed to parse JSON for Feast job: %s", e.getMessage())); - } - return Optional.ofNullable(job); - } - - @Override - public void upsert(Job job) { - try { - syncCommand.set( - job.getId().getBytes(), - JsonFormat.printer().omittingInsignificantWhitespace().print(job).getBytes()); - syncCommand.expire(job.getId().getBytes(), defaultExpirySeconds); - } catch (Exception e) { - log.error(String.format("Failed to upsert job: %s", e.getMessage())); - } - } -} diff --git a/serving/src/main/java/feast/serving/service/ServingService.java b/serving/src/main/java/feast/serving/service/ServingService.java deleted file mode 100644 index 1fe9840d594..00000000000 --- a/serving/src/main/java/feast/serving/service/ServingService.java +++ /dev/null @@ -1,100 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2019 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.serving.service; - -import feast.proto.serving.ServingAPIProto.GetBatchFeaturesRequest; -import feast.proto.serving.ServingAPIProto.GetBatchFeaturesResponse; -import feast.proto.serving.ServingAPIProto.GetFeastServingInfoRequest; -import feast.proto.serving.ServingAPIProto.GetFeastServingInfoResponse; -import feast.proto.serving.ServingAPIProto.GetJobRequest; -import feast.proto.serving.ServingAPIProto.GetJobResponse; -import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesRequest; -import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesResponse; - -public interface ServingService { - /** - * Get information about the Feast serving deployment. - * - *

For Bigquery deployments, this includes the default job staging location to load - * intermediate files to. Otherwise, this method only returns the current Feast Serving backing - * store type. - * - * @param getFeastServingInfoRequest {@link GetFeastServingInfoRequest} - * @return {@link GetFeastServingInfoResponse} - */ - GetFeastServingInfoResponse getFeastServingInfo( - GetFeastServingInfoRequest getFeastServingInfoRequest); - - /** - * Get features from an online serving store, given a list of {@link - * feast.proto.serving.ServingAPIProto.FeatureReference}s to retrieve, and list of {@link - * feast.proto.serving.ServingAPIProto.GetOnlineFeaturesRequest.EntityRow}s to join the retrieved - * values to. - * - *

Features can be queried across feature sets, but each {@link - * feast.proto.serving.ServingAPIProto.GetOnlineFeaturesRequest.EntityRow} must contain all - * entities for all feature sets included in the request. - * - *

This request is fulfilled synchronously. - * - * @param getFeaturesRequest {@link GetOnlineFeaturesRequest} containing list of {@link - * feast.proto.serving.ServingAPIProto.FeatureReference}s to retrieve and list of {@link - * feast.proto.serving.ServingAPIProto.GetOnlineFeaturesRequest.EntityRow}s to join the - * retrieved values to. - * @return {@link GetOnlineFeaturesResponse} with list of {@link - * feast.proto.serving.ServingAPIProto.GetOnlineFeaturesResponse.FieldValues} for each {@link - * feast.proto.serving.ServingAPIProto.GetOnlineFeaturesRequest.EntityRow} supplied. - */ - GetOnlineFeaturesResponse getOnlineFeatures(GetOnlineFeaturesRequest getFeaturesRequest); - - /** - * Get features from a batch serving store, given a list of {@link - * feast.proto.serving.ServingAPIProto.FeatureReference}s to retrieve, and {@link - * feast.proto.serving.ServingAPIProto.DatasetSource} pointing to remote location of dataset to - * join retrieved features to. All columns in the provided dataset will be preserved in the output - * dataset. - * - *

Due to the potential size of batch retrieval requests, this request is fulfilled - * asynchronously, and returns a retrieval job id, which when supplied to {@link - * #getJob(GetJobRequest)} will return the status of the retrieval job. - * - * @param getFeaturesRequest {@link GetBatchFeaturesRequest} containing a list of {@link - * feast.proto.serving.ServingAPIProto.FeatureReference}s to retrieve, and {@link - * feast.proto.serving.ServingAPIProto.DatasetSource} pointing to remote location of dataset - * to join retrieved features to. - * @return {@link GetBatchFeaturesResponse} containing reference to a retrieval {@link - * feast.proto.serving.ServingAPIProto.Job}. - */ - GetBatchFeaturesResponse getBatchFeatures(GetBatchFeaturesRequest getFeaturesRequest); - - /** - * Get the status of a retrieval job from a batch serving store. - * - *

The client should check the status of the returned job periodically by calling ReloadJob to - * determine if the job has completed successfully or with an error. If the job completes - * successfully i.e. status = JOB_STATUS_DONE with no error, then the client can check the - * file_uris for the location to download feature values data. The client is assumed to have - * access to these file URIs. - * - *

If an error occurred during retrieval, the {@link GetJobResponse} will also contain the - * error that resulted in termination. - * - * @param getJobRequest {@link GetJobRequest} containing reference to a retrieval job - * @return {@link GetJobResponse} - */ - GetJobResponse getJob(GetJobRequest getJobRequest); -} diff --git a/serving/src/main/java/feast/serving/service/ServingServiceV2.java b/serving/src/main/java/feast/serving/service/ServingServiceV2.java index 6164b93eb59..05acb31b78e 100644 --- a/serving/src/main/java/feast/serving/service/ServingServiceV2.java +++ b/serving/src/main/java/feast/serving/service/ServingServiceV2.java @@ -16,19 +16,33 @@ */ package feast.serving.service; +import feast.proto.serving.ServingAPIProto; import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesRequestV2; import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesResponse; public interface ServingServiceV2 { + /** + * Get information about the Feast serving deployment. + * + *

For Bigquery deployments, this includes the default job staging location to load + * intermediate files to. Otherwise, this method only returns the current Feast Serving backing + * store type. + * + * @param getFeastServingInfoRequest {@link ServingAPIProto.GetFeastServingInfoRequest} + * @return {@link ServingAPIProto.GetFeastServingInfoResponse} + */ + ServingAPIProto.GetFeastServingInfoResponse getFeastServingInfo( + ServingAPIProto.GetFeastServingInfoRequest getFeastServingInfoRequest); + /** * Get features from an online serving store, given a list of {@link * feast.proto.serving.ServingAPIProto.FeatureReferenceV2}s to retrieve, and list of {@link * feast.proto.serving.ServingAPIProto.GetOnlineFeaturesRequestV2.EntityRow}s to join the * retrieved values to. * - *

Features can be queried across feature sets, but each {@link + *

Features can be queried across feature tables, but each {@link * feast.proto.serving.ServingAPIProto.GetOnlineFeaturesRequestV2.EntityRow} must contain all - * entities for all feature sets included in the request. + * entities for all feature tables included in the request. * *

This request is fulfilled synchronously. * diff --git a/serving/src/main/java/feast/serving/specs/CachedSpecService.java b/serving/src/main/java/feast/serving/specs/CachedSpecService.java index bf7f106eed9..f54e08fc607 100644 --- a/serving/src/main/java/feast/serving/specs/CachedSpecService.java +++ b/serving/src/main/java/feast/serving/specs/CachedSpecService.java @@ -16,44 +16,27 @@ */ package feast.serving.specs; -import static feast.common.models.Feature.getFeatureStringWithProjectRef; -import static feast.common.models.FeatureSet.getFeatureSetStringRef; -import static feast.common.models.FeatureTable.getFeatureTableStringRef; -import static java.util.stream.Collectors.groupingBy; - import com.google.common.cache.CacheBuilder; import com.google.common.cache.CacheLoader; import com.google.common.cache.LoadingCache; -import feast.proto.core.CoreServiceProto.ListFeatureSetsRequest; -import feast.proto.core.CoreServiceProto.ListFeatureSetsResponse; +import feast.proto.core.CoreServiceProto; import feast.proto.core.CoreServiceProto.ListFeatureTablesRequest; import feast.proto.core.CoreServiceProto.ListFeatureTablesResponse; import feast.proto.core.CoreServiceProto.ListProjectsRequest; import feast.proto.core.FeatureProto; -import feast.proto.core.FeatureSetProto.FeatureSet; -import feast.proto.core.FeatureSetProto.FeatureSetSpec; -import feast.proto.core.FeatureSetProto.FeatureSpec; import feast.proto.core.FeatureTableProto.FeatureTable; import feast.proto.core.FeatureTableProto.FeatureTableSpec; import feast.proto.core.StoreProto; import feast.proto.core.StoreProto.Store; -import feast.proto.core.StoreProto.Store.Subscription; import feast.proto.serving.ServingAPIProto; -import feast.proto.serving.ServingAPIProto.FeatureReference; import feast.serving.exception.SpecRetrievalException; -import feast.storage.api.retriever.FeatureSetRequest; import io.grpc.StatusRuntimeException; import io.prometheus.client.Gauge; -import java.util.ArrayList; import java.util.HashMap; -import java.util.HashSet; import java.util.List; import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutionException; -import java.util.stream.Collectors; import org.apache.commons.lang3.tuple.ImmutablePair; -import org.apache.commons.lang3.tuple.Pair; import org.slf4j.Logger; /** In-memory cache of specs hosted in Feast Core. */ @@ -62,23 +45,10 @@ public class CachedSpecService { private static final int MAX_SPEC_COUNT = 1000; private static final Logger log = org.slf4j.LoggerFactory.getLogger(CachedSpecService.class); private static final String DEFAULT_PROJECT_NAME = "default"; - // flag to signal that multiple featuresets match a specific - // string feature reference in the feature to featureset mapping. - private static final String FEATURE_SET_CONFLICT_FLAG = "##CONFLICT##"; private final CoreSpecService coreService; - - private Map featureToFeatureSetMapping; - - private final LoadingCache featureSetCache; private Store store; - private static Gauge featureSetsCount = - Gauge.build() - .name("feature_set_count") - .subsystem("feast_serving") - .help("number of feature sets served by this instance") - .register(); private static Gauge cacheLastUpdated = Gauge.build() .name("cache_last_updated") @@ -86,43 +56,33 @@ public class CachedSpecService { .help("epoch time of the last time the cache was updated") .register(); - private final LoadingCache featureTableCache; private static Gauge featureTablesCount = Gauge.build() .name("feature_table_count") .subsystem("feast_serving") - .help("number of feature sets served by this instance") + .help("number of feature tables served by this instance") .register(); + private final LoadingCache, FeatureTableSpec> featureTableCache; + private final LoadingCache< - String, Map> + ImmutablePair, FeatureProto.FeatureSpecV2> featureCache; public CachedSpecService(CoreSpecService coreService, StoreProto.Store store) { this.coreService = coreService; this.store = coreService.registerStore(store); - Map featureSets = getFeatureSetMap(); - featureToFeatureSetMapping = - new ConcurrentHashMap<>(getFeatureToFeatureSetMapping(featureSets)); - CacheLoader featureSetCacheLoader = CacheLoader.from(featureSets::get); - featureSetCache = - CacheBuilder.newBuilder().maximumSize(MAX_SPEC_COUNT).build(featureSetCacheLoader); - featureSetCache.putAll(featureSets); - - Map featureTables = getFeatureTableMap().getLeft(); - CacheLoader featureTableCacheLoader = - CacheLoader.from(featureTables::get); + CacheLoader, FeatureTableSpec> featureTableCacheLoader = + CacheLoader.from(k -> retrieveSingleFeatureTable(k.getLeft(), k.getRight())); featureTableCache = CacheBuilder.newBuilder().maximumSize(MAX_SPEC_COUNT).build(featureTableCacheLoader); - featureTableCache.putAll(featureTables); - Map> features = - getFeatureTableMap().getRight(); - CacheLoader> - featureCacheLoader = CacheLoader.from(features::get); + CacheLoader< + ImmutablePair, FeatureProto.FeatureSpecV2> + featureCacheLoader = + CacheLoader.from(k -> retrieveSingleFeature(k.getLeft(), k.getRight())); featureCache = CacheBuilder.newBuilder().build(featureCacheLoader); - featureCache.putAll(features); } /** @@ -134,137 +94,25 @@ public Store getStore() { return this.store; } - public FeatureSetSpec getFeatureSetSpec(String featureSetRef) throws ExecutionException { - return featureSetCache.get(featureSetRef); - } - - /** - * Get FeatureSetSpecs for the given features references. See {@link #getFeatureSets(List, - * String)} for full documentation. - */ - public List getFeatureSets(List featureReferences) { - return getFeatureSets(featureReferences, ""); - } - - /** - * Get FeatureSetSpecs for the given features references. If the project is unspecified in the - * given references or project override, autofills the default project. Throws a {@link - * SpecRetrievalException} if multiple feature sets match given string reference. - * - * @param projectOverride If specified would take the spec request in the context of the given - * project only. Otherwise if "", will default to determining the project from the individual - * references. Has higher precedence compared to project specifed in Feature Reference if both - * are specified. - * @return FeatureSetRequest containing the specs, and their respective feature references - */ - public List getFeatureSets( - List featureReferences, String projectOverride) { - List featureSetRequests = new ArrayList<>(); - featureReferences.stream() - .map( - featureReference -> { - // apply project override when finding feature set for feature - FeatureReference queryFeatureRef = featureReference; - if (!projectOverride.isEmpty()) { - queryFeatureRef = featureReference.toBuilder().setProject(projectOverride).build(); - } - - String featureSetRefStr = mapFeatureToFeatureSetReference(queryFeatureRef); - return Pair.of(featureSetRefStr, featureReference); - }) - .collect(groupingBy(Pair::getLeft)) - .forEach( - (featureSetRefStr, fsRefStrAndFeatureRefs) -> { - List featureRefs = - fsRefStrAndFeatureRefs.stream().map(Pair::getRight).collect(Collectors.toList()); - featureSetRequests.add(buildFeatureSetRequest(featureSetRefStr, featureRefs)); - }); - return featureSetRequests; - } - - /** - * Build a Feature Set request from the Feature Set specified by given Feature Set reference and - * given Feature References. - * - * @param featureSetRefStr string feature set reference specifying the feature set that contains - * requested features - * @param featureReferences list of feature references specifying the containing feature set - * references. - */ - private FeatureSetRequest buildFeatureSetRequest( - String featureSetRefStr, List featureReferences) { - // get feature set for name - FeatureSetSpec featureSetSpec; - try { - featureSetSpec = featureSetCache.get(featureSetRefStr); - } catch (ExecutionException e) { - throw new SpecRetrievalException( - String.format("Unable to find featureSet with name: %s", featureSetRefStr), e); - } - - // check that requested features reference point to different features in the - // featureset. - HashSet featureNames = new HashSet<>(); - featureReferences.forEach( - ref -> { - if (featureNames.contains(ref.getName())) { - throw new SpecRetrievalException( - "Multiple Feature References referencing the same feature in a featureset is not allowed."); - } - featureNames.add(ref.getName()); - }); - - return FeatureSetRequest.newBuilder() - .setSpec(featureSetSpec) - .addAllFeatureReferences(featureReferences) - .build(); - } - - /** Maps given Feature Reference to the containing Feature Set's string reference */ - private String mapFeatureToFeatureSetReference(FeatureReference featureReference) { - // map feature reference to coresponding feature set string reference - String featureSetRefStr = - featureToFeatureSetMapping.get(getFeatureStringWithProjectRef(featureReference)); - if (featureSetRefStr == null) { - throw new SpecRetrievalException( - String.format( - "Unable to find Feature Set for the given Feature Reference: %s", - getFeatureStringWithProjectRef(featureReference))); - } else if (featureSetRefStr == FEATURE_SET_CONFLICT_FLAG) { - throw new SpecRetrievalException( - String.format( - "Given Feature Reference is amibigous as it matches multiple Feature Sets: %s." - + "Please specify a more specific Feature Reference (ie specify the project or feature set)", - getFeatureStringWithProjectRef(featureReference))); - } - return featureSetRefStr; - } - /** * Reload the store configuration from the given config path, then retrieve the necessary specs * from core to preload the cache. */ public void populateCache() { - Map featureSetMap = getFeatureSetMap(); - - featureSetCache.invalidateAll(); - featureSetCache.putAll(featureSetMap); - - featureToFeatureSetMapping = getFeatureToFeatureSetMapping(featureSetMap); - - featureSetsCount.set(featureSetCache.size()); - - Map featureTableMap = getFeatureTableMap().getLeft(); + ImmutablePair< + HashMap, FeatureTableSpec>, + HashMap< + ImmutablePair, + FeatureProto.FeatureSpecV2>> + specs = getFeatureTableMap(); featureTableCache.invalidateAll(); - featureTableCache.putAll(featureTableMap); + featureTableCache.putAll(specs.getLeft()); featureTablesCount.set(featureTableCache.size()); - Map> featureMap = - getFeatureTableMap().getRight(); featureCache.invalidateAll(); - featureCache.putAll(featureMap); + featureCache.putAll(specs.getRight()); cacheLastUpdated.set(System.currentTimeMillis()); } @@ -277,124 +125,20 @@ public void scheduledPopulateCache() { } } - private Map getFeatureSetMap() { - HashMap featureSets = new HashMap<>(); - - for (Subscription subscription : this.store.getSubscriptionsList()) { - try { - if (!subscription.getExclude()) { - ListFeatureSetsResponse featureSetsResponse = - coreService.listFeatureSets( - ListFeatureSetsRequest.newBuilder() - .setFilter( - ListFeatureSetsRequest.Filter.newBuilder() - .setProject(subscription.getProject()) - .setFeatureSetName(subscription.getName())) - .build()); - - for (FeatureSet featureSet : featureSetsResponse.getFeatureSetsList()) { - FeatureSetSpec spec = featureSet.getSpec(); - featureSets.put(getFeatureSetStringRef(spec), spec); - } - } - } catch (StatusRuntimeException e) { - throw new RuntimeException( - String.format("Unable to retrieve specs matching subscription %s", subscription), e); - } - } - return featureSets; - } - - /** - * Generate a feature to feature set mapping from the given feature sets map. Accounts for - * variations (missing project, feature_set) in string feature references generated by creating - * multiple entries in the returned mapping for each variation. - * - * @param featureSets map of feature set name to feature set specs - * @return mapping of string feature references to name of feature sets - */ - private Map getFeatureToFeatureSetMapping( - Map featureSets) { - Map mapping = new HashMap<>(); - - featureSets.values().stream() - .forEach( - featureSetSpec -> { - for (FeatureSpec featureSpec : featureSetSpec.getFeaturesList()) { - // Register the different permutations of string feature references - // that refers to this feature in the feature to featureset mapping. - - // Features in FeatureSets in default project can be referenced without project. - boolean isInDefaultProject = - featureSetSpec.getProject().equals(DEFAULT_PROJECT_NAME); - - for (boolean hasProject : new boolean[] {true, false}) { - if (!isInDefaultProject && !hasProject) continue; - // Features can be referenced without a featureset if there are no conflicts. - for (boolean hasFeatureSet : new boolean[] {true, false}) { - // Get mapping between string feature reference and featureset - Pair singleMapping = - this.generateFeatureToFeatureSetMapping( - featureSpec, featureSetSpec, hasProject, hasFeatureSet); - String featureRef = singleMapping.getKey(); - String featureSetRef = singleMapping.getValue(); - // Check if another feature set has already mapped to this - // string feature reference. if so mark the conflict. - if (mapping.containsKey(featureRef)) { - mapping.put(featureRef, FEATURE_SET_CONFLICT_FLAG); - } else { - mapping.put(featureRef, featureSetRef); - } - } - } - } - }); - - return mapping; - } - - /** - * Generate a single mapping between the given feature and the featureset. Maps a feature - * reference refering to the given feature to the corresponding featureset's name. - * - * @param featureSpec specifying the feature to create mapping for. - * @param featureSetSpec specifying the feature set to create mapping for. - * @param hasProject whether generated mapping's string feature ref has a project. - * @param hasFeatureSet whether generated mapping's string feature ref has a featureSet. - * @return a pair mapping a string feature reference to a featureset name. - */ - private Pair generateFeatureToFeatureSetMapping( - FeatureSpec featureSpec, - FeatureSetSpec featureSetSpec, - boolean hasProject, - boolean hasFeatureSet) { - FeatureReference.Builder featureRef = - FeatureReference.newBuilder() - .setProject(featureSetSpec.getProject()) - .setFeatureSet(featureSetSpec.getName()) - .setName(featureSpec.getName()); - if (!hasProject) { - featureRef = featureRef.clearProject(); - } - if (!hasFeatureSet) { - featureRef = featureRef.clearFeatureSet(); - } - return Pair.of( - getFeatureStringWithProjectRef(featureRef.build()), getFeatureSetStringRef(featureSetSpec)); - } - /** * Provides a map for easy retrieval of FeatureTable spec using FeatureTable reference * * @return Map in the format of */ private ImmutablePair< - Map, - Map>> + HashMap, FeatureTableSpec>, + HashMap< + ImmutablePair, + FeatureProto.FeatureSpecV2>> getFeatureTableMap() { - HashMap featureTables = new HashMap<>(); - HashMap> features = - new HashMap<>(); + HashMap, FeatureTableSpec> featureTables = new HashMap<>(); + HashMap, FeatureProto.FeatureSpecV2> + features = new HashMap<>(); List projects = coreService.listProjects(ListProjectsRequest.newBuilder().build()).getProjectsList(); @@ -410,8 +154,7 @@ private Pair generateFeatureToFeatureSetMapping( new HashMap<>(); for (FeatureTable featureTable : featureTablesResponse.getTablesList()) { FeatureTableSpec spec = featureTable.getSpec(); - // Key of Map is in the form of - featureTables.put(getFeatureTableStringRef(project, spec), spec); + featureTables.put(ImmutablePair.of(project, spec.getName()), spec); String featureTableName = spec.getName(); List featureSpecs = spec.getFeaturesList(); @@ -421,10 +164,10 @@ private Pair generateFeatureToFeatureSetMapping( .setFeatureTable(featureTableName) .setName(featureSpec.getName()) .build(); - featureRefSpecMap.put(featureReference, featureSpec); + features.put(ImmutablePair.of(project, featureReference), featureSpec); } } - features.put(project, featureRefSpecMap); + } catch (StatusRuntimeException e) { throw new RuntimeException( String.format("Unable to retrieve specs matching project %s", project), e); @@ -433,28 +176,53 @@ private Pair generateFeatureToFeatureSetMapping( return ImmutablePair.of(featureTables, features); } + private FeatureTableSpec retrieveSingleFeatureTable(String projectName, String tableName) { + FeatureTable table = + coreService + .getFeatureTable( + CoreServiceProto.GetFeatureTableRequest.newBuilder() + .setProject(projectName) + .setName(tableName) + .build()) + .getTable(); + return table.getSpec(); + } + + private FeatureProto.FeatureSpecV2 retrieveSingleFeature( + String projectName, ServingAPIProto.FeatureReferenceV2 featureReference) { + FeatureTableSpec featureTableSpec = + getFeatureTableSpec(projectName, featureReference); // don't stress core too much + if (featureTableSpec == null) { + return null; + } + return featureTableSpec.getFeaturesList().stream() + .filter(f -> f.getName().equals(featureReference.getName())) + .findFirst() + .orElse(null); + } + public FeatureTableSpec getFeatureTableSpec( - String project, ServingAPIProto.FeatureReferenceV2 featureReference) { - String featureTableRefStr = getFeatureTableStringRef(project, featureReference); + String projectName, ServingAPIProto.FeatureReferenceV2 featureReference) { FeatureTableSpec featureTableSpec; try { - featureTableSpec = featureTableCache.get(featureTableRefStr); - } catch (ExecutionException e) { + featureTableSpec = + featureTableCache.get(ImmutablePair.of(projectName, featureReference.getFeatureTable())); + } catch (ExecutionException | CacheLoader.InvalidCacheLoadException e) { throw new SpecRetrievalException( - String.format("Unable to find FeatureTable with name: %s", featureTableRefStr), e); + String.format( + "Unable to find FeatureTable %s/%s", projectName, featureReference.getFeatureTable()), + e); } return featureTableSpec; } public FeatureProto.FeatureSpecV2 getFeatureSpec( - String project, ServingAPIProto.FeatureReferenceV2 featureReference) { + String projectName, ServingAPIProto.FeatureReferenceV2 featureReference) { FeatureProto.FeatureSpecV2 featureSpec; try { - Map featureRefSpecMap = - featureCache.get(project); - featureSpec = featureRefSpecMap.get(featureReference); - } catch (ExecutionException e) { + featureSpec = featureCache.get(ImmutablePair.of(projectName, featureReference)); + } catch (ExecutionException | CacheLoader.InvalidCacheLoadException e) { throw new SpecRetrievalException( String.format("Unable to find Feature with name: %s", featureReference.getName()), e); } diff --git a/serving/src/main/java/feast/serving/specs/CoreSpecService.java b/serving/src/main/java/feast/serving/specs/CoreSpecService.java index 8bdf4fcdc06..5429d229313 100644 --- a/serving/src/main/java/feast/serving/specs/CoreSpecService.java +++ b/serving/src/main/java/feast/serving/specs/CoreSpecService.java @@ -17,10 +17,7 @@ package feast.serving.specs; import feast.proto.core.CoreServiceGrpc; -import feast.proto.core.CoreServiceProto.GetFeatureSetRequest; -import feast.proto.core.CoreServiceProto.GetFeatureSetResponse; -import feast.proto.core.CoreServiceProto.ListFeatureSetsRequest; -import feast.proto.core.CoreServiceProto.ListFeatureSetsResponse; +import feast.proto.core.CoreServiceProto; import feast.proto.core.CoreServiceProto.ListFeatureTablesRequest; import feast.proto.core.CoreServiceProto.ListFeatureTablesResponse; import feast.proto.core.CoreServiceProto.ListProjectsRequest; @@ -52,14 +49,6 @@ public CoreSpecService( } } - public GetFeatureSetResponse getFeatureSet(GetFeatureSetRequest getFeatureSetRequest) { - return blockingStub.getFeatureSet(getFeatureSetRequest); - } - - public ListFeatureSetsResponse listFeatureSets(ListFeatureSetsRequest ListFeatureSetsRequest) { - return blockingStub.listFeatureSets(ListFeatureSetsRequest); - } - public UpdateStoreResponse updateStore(UpdateStoreRequest updateStoreRequest) { return blockingStub.updateStore(updateStoreRequest); } @@ -92,4 +81,9 @@ public ListFeatureTablesResponse listFeatureTables( ListFeatureTablesRequest listFeatureTablesRequest) { return blockingStub.listFeatureTables(listFeatureTablesRequest); } + + public CoreServiceProto.GetFeatureTableResponse getFeatureTable( + CoreServiceProto.GetFeatureTableRequest getFeatureTableRequest) { + return blockingStub.getFeatureTable(getFeatureTableRequest); + } } diff --git a/serving/src/main/java/feast/serving/util/Metrics.java b/serving/src/main/java/feast/serving/util/Metrics.java index 13cbe0e1ccb..90b94930e71 100644 --- a/serving/src/main/java/feast/serving/util/Metrics.java +++ b/serving/src/main/java/feast/serving/util/Metrics.java @@ -29,7 +29,34 @@ public class Metrics { .labelNames("method") .register(); - public static final Counter requestCount = + public static final Histogram requestEntityCountDistribution = + Histogram.build() + .buckets(1, 2, 5, 10, 20, 50, 100, 200) + .name("request_entity_count_distribution") + .subsystem("feast_serving") + .help("Number of entity rows per request") + .labelNames("project") + .register(); + + public static final Histogram requestFeatureCountDistribution = + Histogram.build() + .buckets(1, 2, 5, 10, 15, 20, 30, 50) + .name("request_feature_count_distribution") + .subsystem("feast_serving") + .help("Number of feature rows per request") + .labelNames("project") + .register(); + + public static final Histogram requestFeatureTableCountDistribution = + Histogram.build() + .buckets(1, 2, 5, 10, 20) + .name("request_feature_table_count_distribution") + .subsystem("feast_serving") + .help("Number of feature tables per request") + .labelNames("project") + .register(); + + public static final Counter requestFeatureCount = Counter.build() .name("request_feature_count") .subsystem("feast_serving") diff --git a/serving/src/main/java/feast/serving/util/RequestHelper.java b/serving/src/main/java/feast/serving/util/RequestHelper.java index 6a3a79b03e0..4d478f430f2 100644 --- a/serving/src/main/java/feast/serving/util/RequestHelper.java +++ b/serving/src/main/java/feast/serving/util/RequestHelper.java @@ -16,26 +16,11 @@ */ package feast.serving.util; -import feast.proto.serving.ServingAPIProto.FeatureReference; import feast.proto.serving.ServingAPIProto.FeatureReferenceV2; -import feast.proto.serving.ServingAPIProto.GetBatchFeaturesRequest; -import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesRequest; import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesRequestV2; -import io.grpc.Status; -import java.util.Set; -import java.util.stream.Collectors; public class RequestHelper { - public static void validateOnlineRequest(GetOnlineFeaturesRequest request) { - // EntityDataSetRow shall not be empty - if (request.getEntityRowsCount() <= 0) { - throw Status.INVALID_ARGUMENT - .withDescription("Entity value must be provided") - .asRuntimeException(); - } - } - public static void validateOnlineRequest(GetOnlineFeaturesRequestV2 request) { // All EntityRows should not be empty if (request.getEntityRowsCount() <= 0) { @@ -47,30 +32,6 @@ public static void validateOnlineRequest(GetOnlineFeaturesRequestV2 request) { } } - public static void validateBatchRequest(GetBatchFeaturesRequest getFeaturesRequest) { - if (!getFeaturesRequest.hasDatasetSource()) { - throw Status.INVALID_ARGUMENT - .withDescription("Dataset source must be provided") - .asRuntimeException(); - } - - if (!getFeaturesRequest.getDatasetSource().hasFileSource()) { - throw Status.INVALID_ARGUMENT - .withDescription("Dataset source must be provided: only file source supported") - .asRuntimeException(); - } - - Set uniqueFeatureNames = - getFeaturesRequest.getFeaturesList().stream() - .map(FeatureReference::getName) - .collect(Collectors.toSet()); - if (uniqueFeatureNames.size() != getFeaturesRequest.getFeaturesList().size()) { - throw Status.INVALID_ARGUMENT - .withDescription("Feature names must be unique within the request") - .asRuntimeException(); - } - } - public static void validateOnlineRequestFeatureReference(FeatureReferenceV2 featureReference) { if (featureReference.getFeatureTable().isEmpty()) { throw new IllegalArgumentException("FeatureTable name must be provided in FeatureReference"); diff --git a/serving/src/main/resources/application.yml b/serving/src/main/resources/application.yml index 16837b46086..288ec7eb972 100644 --- a/serving/src/main/resources/application.yml +++ b/serving/src/main/resources/application.yml @@ -54,33 +54,12 @@ feast: type: REDIS_CLUSTER config: # Store specific configuration. # Connection string specifies the host:port of Redis instances in the redis cluster. - connection_string: "localhost:7000,localhost:7001,localhost:7002,localhost:7003,localhost:7004,localhost:7005" + connection_string: "localhost:7000,localhost:7001,localhost:7002,localhost:7003,localhost:7004,localhost:7005" + read_from: MASTER subscriptions: - name: "*" project: "*" version: "*" - - name: historical - type: BIGQUERY - config: # Store specific configuration. - # GCP Project - project_id: my_project - # BigQuery Dataset Id - dataset_id: my_dataset - # staging-location specifies the URI to store intermediate files for batch serving. - # Feast Serving client is expected to have read access to this staging location - # to download the batch features. - # For example: gs://mybucket/myprefix - # Please omit the trailing slash in the URI. - staging_location: gs://mybucket/myprefix - # Retry options for BigQuery retrieval jobs - initial_retry_delay_seconds: 1 - # BigQuery timeout for retrieval jobs - total_timeout_seconds: 21600 - # BigQuery sink write frequency - write_triggering_frequency_seconds: 600 - subscriptions: - - name: "*" - project: "*" tracing: # If true, Feast will provide tracing data (using OpenTracing API) for various RPC method calls @@ -92,14 +71,6 @@ feast: # The service name identifier for the tracing data service-name: feast_serving - # The job store is used to maintain job management state for Feast Serving. This is required when using certain - # historical stores like BigQuery. Only Redis is supported as a job store. - job_store: - # Redis host to connect to - redis_host: localhost - # Redis port to connect to - redis_port: 6379 - logging: # Audit logging provides a machine readable structured JSON log that can give better # insight into what is happening in Feast. diff --git a/serving/src/test/java/feast/serving/controller/ServingServiceGRpcControllerTest.java b/serving/src/test/java/feast/serving/controller/ServingServiceGRpcControllerTest.java index 671b57b2f65..c0901c48bcd 100644 --- a/serving/src/test/java/feast/serving/controller/ServingServiceGRpcControllerTest.java +++ b/serving/src/test/java/feast/serving/controller/ServingServiceGRpcControllerTest.java @@ -30,15 +30,13 @@ import feast.common.auth.config.SecurityProperties.AuthenticationProperties; import feast.common.auth.config.SecurityProperties.AuthorizationProperties; import feast.common.auth.service.AuthorizationService; -import feast.proto.serving.ServingAPIProto.FeatureReference; -import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesRequest; -import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesRequest.EntityRow; +import feast.proto.serving.ServingAPIProto.FeatureReferenceV2; +import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesRequestV2; +import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesRequestV2.EntityRow; import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesResponse; import feast.proto.types.ValueProto.Value; import feast.serving.config.FeastProperties; -import feast.serving.service.ServingService; import feast.serving.service.ServingServiceV2; -import io.grpc.StatusRuntimeException; import io.grpc.stub.StreamObserver; import io.jaegertracing.Configuration; import io.opentracing.Tracer; @@ -52,13 +50,11 @@ public class ServingServiceGRpcControllerTest { - @Mock private ServingService mockServingService; - @Mock private ServingServiceV2 mockServingServiceV2; @Mock private StreamObserver mockStreamObserver; - private GetOnlineFeaturesRequest validRequest; + private GetOnlineFeaturesRequestV2 validRequest; private ServingServiceGRpcController service; @@ -71,12 +67,20 @@ public void setUp() { initMocks(this); validRequest = - GetOnlineFeaturesRequest.newBuilder() - .addFeatures(FeatureReference.newBuilder().setName("feature1").build()) - .addFeatures(FeatureReference.newBuilder().setName("feature2").build()) + GetOnlineFeaturesRequestV2.newBuilder() + .addFeatures( + FeatureReferenceV2.newBuilder() + .setFeatureTable("featuretable_1") + .setName("feature1") + .build()) + .addFeatures( + FeatureReferenceV2.newBuilder() + .setFeatureTable("featuretable_1") + .setName("feature2") + .build()) .addEntityRows( EntityRow.newBuilder() - .setEntityTimestamp(Timestamp.newBuilder().setSeconds(100)) + .setTimestamp(Timestamp.newBuilder().setSeconds(100)) .putFields("entity1", Value.newBuilder().setInt64Val(1).build()) .putFields("entity2", Value.newBuilder().setInt64Val(1).build())) .build(); @@ -97,23 +101,23 @@ private ServingServiceGRpcController getServingServiceGRpcController(boolean ena AuthorizationService authorizationservice = new AuthorizationService(feastProperties.getSecurity(), authProvider); return new ServingServiceGRpcController( - authorizationservice, mockServingService, mockServingServiceV2, feastProperties, tracer); + authorizationservice, mockServingServiceV2, feastProperties, tracer); } @Test public void shouldPassValidRequestAsIs() { service = getServingServiceGRpcController(false); - service.getOnlineFeatures(validRequest, mockStreamObserver); - Mockito.verify(mockServingService).getOnlineFeatures(validRequest); + service.getOnlineFeaturesV2(validRequest, mockStreamObserver); + Mockito.verify(mockServingServiceV2).getOnlineFeatures(validRequest); } @Test public void shouldCallOnErrorIfEntityDatasetIsNotSet() { service = getServingServiceGRpcController(false); - GetOnlineFeaturesRequest missingEntityName = - GetOnlineFeaturesRequest.newBuilder(validRequest).clearEntityRows().build(); - service.getOnlineFeatures(missingEntityName, mockStreamObserver); - Mockito.verify(mockStreamObserver).onError(Mockito.any(StatusRuntimeException.class)); + GetOnlineFeaturesRequestV2 missingEntityName = + GetOnlineFeaturesRequestV2.newBuilder(validRequest).clearEntityRows().build(); + service.getOnlineFeaturesV2(missingEntityName, mockStreamObserver); + Mockito.verify(mockStreamObserver).onError(Mockito.any(IllegalArgumentException.class)); } @Test @@ -125,20 +129,7 @@ public void shouldPassValidRequestAsIsIfRequestIsAuthorized() { doReturn(AuthorizationResult.success()) .when(authProvider) .checkAccessToProject(anyString(), any(Authentication.class)); - service.getOnlineFeatures(validRequest, mockStreamObserver); - Mockito.verify(mockServingService).getOnlineFeatures(validRequest); - } - - @Test - public void shouldThrowErrorOnValidRequestIfRequestIsUnauthorized() { - service = getServingServiceGRpcController(true); - SecurityContext context = mock(SecurityContext.class); - SecurityContextHolder.setContext(context); - when(context.getAuthentication()).thenReturn(authentication); - doReturn(AuthorizationResult.failed(null)) - .when(authProvider) - .checkAccessToProject(anyString(), any(Authentication.class)); - service.getOnlineFeatures(validRequest, mockStreamObserver); - Mockito.verify(mockStreamObserver).onError(Mockito.any(StatusRuntimeException.class)); + service.getOnlineFeaturesV2(validRequest, mockStreamObserver); + Mockito.verify(mockServingServiceV2).getOnlineFeatures(validRequest); } } diff --git a/serving/src/test/java/feast/serving/it/AuthTestUtils.java b/serving/src/test/java/feast/serving/it/AuthTestUtils.java index e5e0def8018..a4c3db7defe 100644 --- a/serving/src/test/java/feast/serving/it/AuthTestUtils.java +++ b/serving/src/test/java/feast/serving/it/AuthTestUtils.java @@ -16,42 +16,27 @@ */ package feast.serving.it; -import static org.awaitility.Awaitility.waitAtMost; -import static org.hamcrest.CoreMatchers.equalTo; -import static org.hamcrest.beans.HasPropertyWithValue.hasProperty; -import static org.junit.jupiter.api.Assertions.assertEquals; - import com.google.gson.JsonArray; import com.google.gson.JsonObject; import com.google.protobuf.Timestamp; import feast.common.auth.credentials.OAuthCredentials; import feast.proto.core.CoreServiceGrpc; -import feast.proto.core.FeatureSetProto; -import feast.proto.core.FeatureSetProto.FeatureSetStatus; -import feast.proto.core.SourceProto; -import feast.proto.serving.ServingAPIProto.FeatureReference; -import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesRequest; -import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesRequest.EntityRow; +import feast.proto.serving.ServingAPIProto; +import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesRequestV2; import feast.proto.serving.ServingServiceGrpc; -import feast.proto.types.ValueProto; import feast.proto.types.ValueProto.Value; import io.grpc.CallCredentials; import io.grpc.Channel; import io.grpc.ManagedChannelBuilder; import java.io.IOException; -import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; -import java.util.List; import java.util.Map; -import java.util.concurrent.TimeUnit; -import java.util.stream.Collectors; import okhttp3.MediaType; import okhttp3.OkHttpClient; import okhttp3.Request; import okhttp3.RequestBody; import okhttp3.Response; -import org.apache.commons.lang3.tuple.Pair; import org.junit.runners.model.InitializationError; import sh.ory.keto.ApiClient; import sh.ory.keto.ApiException; @@ -64,99 +49,26 @@ public class AuthTestUtils { private static final String DEFAULT_FLAVOR = "glob"; - static SourceProto.Source defaultSource = - createSource("kafka:9092,localhost:9094", "feast-features"); - - public static SourceProto.Source getDefaultSource() { - return defaultSource; - } - - public static SourceProto.Source createSource(String server, String topic) { - return SourceProto.Source.newBuilder() - .setType(SourceProto.SourceType.KAFKA) - .setKafkaSourceConfig( - SourceProto.KafkaSourceConfig.newBuilder() - .setBootstrapServers(server) - .setTopic(topic) - .build()) - .build(); - } - - public static FeatureSetProto.FeatureSet createFeatureSet( - SourceProto.Source source, + public static GetOnlineFeaturesRequestV2 createOnlineFeatureRequest( String projectName, - String name, - List> entities, - List> features) { - return FeatureSetProto.FeatureSet.newBuilder() - .setSpec( - FeatureSetProto.FeatureSetSpec.newBuilder() - .setSource(source) - .setName(name) - .setProject(projectName) - .addAllEntities( - entities.stream() - .map( - pair -> - FeatureSetProto.EntitySpec.newBuilder() - .setName(pair.getLeft()) - .setValueType(pair.getRight()) - .build()) - .collect(Collectors.toList())) - .addAllFeatures( - features.stream() - .map( - pair -> - FeatureSetProto.FeatureSpec.newBuilder() - .setName(pair.getLeft()) - .setValueType(pair.getRight()) - .build()) - .collect(Collectors.toList())) - .build()) - .build(); - } - - public static GetOnlineFeaturesRequest createOnlineFeatureRequest( - String projectName, String featureName, String entityId, int entityValue) { - return GetOnlineFeaturesRequest.newBuilder() + String featureTableName, + String featureName, + String entityId, + int entityValue) { + return GetOnlineFeaturesRequestV2.newBuilder() .setProject(projectName) - .addFeatures(FeatureReference.newBuilder().setName(featureName).build()) + .addFeatures( + ServingAPIProto.FeatureReferenceV2.newBuilder() + .setFeatureTable(featureTableName) + .setName(featureName) + .build()) .addEntityRows( - EntityRow.newBuilder() - .setEntityTimestamp(Timestamp.newBuilder().setSeconds(100)) + GetOnlineFeaturesRequestV2.EntityRow.newBuilder() + .setTimestamp(Timestamp.newBuilder().setSeconds(100)) .putFields(entityId, Value.newBuilder().setInt64Val(entityValue).build())) .build(); } - public static void applyFeatureSet( - CoreSimpleAPIClient secureApiClient, - String projectName, - String entityId, - String featureName) { - List> entities = new ArrayList<>(); - entities.add(Pair.of(entityId, ValueProto.ValueType.Enum.INT64)); - List> features = new ArrayList<>(); - features.add(Pair.of(featureName, ValueProto.ValueType.Enum.INT64)); - String featureSetName = "test_1"; - FeatureSetProto.FeatureSet expectedFeatureSet = - AuthTestUtils.createFeatureSet( - AuthTestUtils.getDefaultSource(), projectName, featureSetName, entities, features); - secureApiClient.simpleApplyFeatureSet(expectedFeatureSet); - waitAtMost(2, TimeUnit.MINUTES) - .until( - () -> { - return secureApiClient.simpleGetFeatureSet(projectName, featureSetName).getMeta(); - }, - hasProperty("status", equalTo(FeatureSetStatus.STATUS_READY))); - FeatureSetProto.FeatureSet actualFeatureSet = - secureApiClient.simpleGetFeatureSet(projectName, featureSetName); - assertEquals( - expectedFeatureSet.getSpec().getProject(), actualFeatureSet.getSpec().getProject()); - assertEquals(expectedFeatureSet.getSpec().getName(), actualFeatureSet.getSpec().getName()); - assertEquals(expectedFeatureSet.getSpec().getSource(), actualFeatureSet.getSpec().getSource()); - assertEquals(FeatureSetStatus.STATUS_READY, actualFeatureSet.getMeta().getStatus()); - } - public static CoreSimpleAPIClient getSecureApiClientForCore( int feastCorePort, Map options) { CallCredentials callCredentials = null; diff --git a/serving/src/test/java/feast/serving/it/BaseAuthIT.java b/serving/src/test/java/feast/serving/it/BaseAuthIT.java index 98836f9156e..79d47737e71 100644 --- a/serving/src/test/java/feast/serving/it/BaseAuthIT.java +++ b/serving/src/test/java/feast/serving/it/BaseAuthIT.java @@ -27,6 +27,7 @@ @SpringBootTest public class BaseAuthIT { + static final String FEATURE_TABLE_NAME = "featuretable_1"; static final String FEATURE_NAME = "feature_1"; static final String ENTITY_ID = "entity_id"; static final String PROJECT_NAME = "project_1"; @@ -42,8 +43,6 @@ public class BaseAuthIT { static final String CORE = "core_1"; - static final String JOB_CONTROLLER = "jobcontroller_1"; - static final String HYDRA = "hydra_1"; static final int HYDRA_PORT = 4445; @@ -54,8 +53,6 @@ public class BaseAuthIT { static final int FEAST_CORE_PORT = 6565; - static final int FEAST_JOB_CONTROLLER_PORT = 6570; - @DynamicPropertySource static void properties(DynamicPropertyRegistry registry) { registry.add("feast.stores[0].name", () -> "online"); diff --git a/serving/src/test/java/feast/serving/it/CoreSimpleAPIClient.java b/serving/src/test/java/feast/serving/it/CoreSimpleAPIClient.java index 3886bfbb8e8..f7bc12f5fc6 100644 --- a/serving/src/test/java/feast/serving/it/CoreSimpleAPIClient.java +++ b/serving/src/test/java/feast/serving/it/CoreSimpleAPIClient.java @@ -19,7 +19,6 @@ import feast.proto.core.CoreServiceGrpc; import feast.proto.core.CoreServiceProto; import feast.proto.core.EntityProto; -import feast.proto.core.FeatureSetProto; import feast.proto.core.FeatureTableProto; public class CoreSimpleAPIClient { @@ -29,22 +28,12 @@ public CoreSimpleAPIClient(CoreServiceGrpc.CoreServiceBlockingStub stub) { this.stub = stub; } - public void simpleApplyFeatureSet(FeatureSetProto.FeatureSet featureSet) { - stub.applyFeatureSet( - CoreServiceProto.ApplyFeatureSetRequest.newBuilder().setFeatureSet(featureSet).build()); - } - - public FeatureSetProto.FeatureSet simpleGetFeatureSet(String projectName, String name) { - return stub.getFeatureSet( - CoreServiceProto.GetFeatureSetRequest.newBuilder() - .setName(name) - .setProject(projectName) - .build()) - .getFeatureSet(); - } - - public void simpleApplyEntity(EntityProto.EntitySpecV2 entitySpec) { - stub.applyEntity(CoreServiceProto.ApplyEntityRequest.newBuilder().setSpec(entitySpec).build()); + public void simpleApplyEntity(String projectName, EntityProto.EntitySpecV2 entitySpec) { + stub.applyEntity( + CoreServiceProto.ApplyEntityRequest.newBuilder() + .setProject(projectName) + .setSpec(entitySpec) + .build()); } public EntityProto.Entity getEntity(String projectName, String name) { @@ -56,9 +45,13 @@ public EntityProto.Entity getEntity(String projectName, String name) { .getEntity(); } - public void simpleApplyFeatureTable(FeatureTableProto.FeatureTableSpec featureTable) { + public void simpleApplyFeatureTable( + String projectName, FeatureTableProto.FeatureTableSpec featureTable) { stub.applyFeatureTable( - CoreServiceProto.ApplyFeatureTableRequest.newBuilder().setTableSpec(featureTable).build()); + CoreServiceProto.ApplyFeatureTableRequest.newBuilder() + .setProject(projectName) + .setTableSpec(featureTable) + .build()); } public FeatureTableProto.FeatureTable simpleGetFeatureTable(String projectName, String name) { diff --git a/serving/src/test/java/feast/serving/it/ServingServiceOauthAuthenticationIT.java b/serving/src/test/java/feast/serving/it/ServingServiceOauthAuthenticationIT.java index 476b017c6e1..8f2440d2475 100644 --- a/serving/src/test/java/feast/serving/it/ServingServiceOauthAuthenticationIT.java +++ b/serving/src/test/java/feast/serving/it/ServingServiceOauthAuthenticationIT.java @@ -20,21 +20,28 @@ import static org.junit.jupiter.api.Assertions.assertTrue; import static org.testcontainers.containers.wait.strategy.Wait.forHttp; +import com.google.common.collect.ImmutableMap; import com.squareup.okhttp.OkHttpClient; import com.squareup.okhttp.Request; import com.squareup.okhttp.Response; -import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesRequest; +import feast.common.it.DataGenerator; +import feast.proto.core.EntityProto; +import feast.proto.core.FeatureTableProto; +import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesRequestV2; import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesResponse; import feast.proto.serving.ServingServiceGrpc.ServingServiceBlockingStub; +import feast.proto.types.ValueProto; import feast.proto.types.ValueProto.Value; import io.grpc.ManagedChannel; import java.io.File; import java.io.IOException; import java.time.Duration; +import java.util.Arrays; import java.util.HashMap; import java.util.Map; import org.junit.ClassRule; import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.runners.model.InitializationError; import org.springframework.boot.test.context.SpringBootTest; @@ -58,6 +65,8 @@ @Testcontainers public class ServingServiceOauthAuthenticationIT extends BaseAuthIT { + CoreSimpleAPIClient coreClient; + FeatureTableProto.FeatureTableSpec expectedFeatureTableSpec; static final Map options = new HashMap<>(); static final int FEAST_SERVING_PORT = 6566; @@ -72,11 +81,6 @@ public class ServingServiceOauthAuthenticationIT extends BaseAuthIT { .withExposedService( CORE, FEAST_CORE_PORT, - Wait.forLogMessage(".*gRPC Server started.*\\n", 1) - .withStartupTimeout(Duration.ofMinutes(SERVICE_START_MAX_WAIT_TIME_IN_MINUTES))) - .withExposedService( - JOB_CONTROLLER, - FEAST_JOB_CONTROLLER_PORT, Wait.forLogMessage(".*gRPC Server started.*\\n", 1) .withStartupTimeout(Duration.ofMinutes(SERVICE_START_MAX_WAIT_TIME_IN_MINUTES))); @@ -96,6 +100,35 @@ static void globalSetup() throws IOException, InitializationError, InterruptedEx options.put("grant_type", GRANT_TYPE); } + @BeforeEach + public void initState() { + coreClient = AuthTestUtils.getSecureApiClientForCore(FEAST_CORE_PORT, options); + EntityProto.EntitySpecV2 entitySpec = + DataGenerator.createEntitySpecV2( + ENTITY_ID, + "Entity 1 description", + ValueProto.ValueType.Enum.STRING, + ImmutableMap.of("label_key", "label_value")); + coreClient.simpleApplyEntity(PROJECT_NAME, entitySpec); + + expectedFeatureTableSpec = + DataGenerator.createFeatureTableSpec( + FEATURE_TABLE_NAME, + Arrays.asList(ENTITY_ID), + new HashMap<>() { + { + put(FEATURE_NAME, ValueProto.ValueType.Enum.STRING); + } + }, + 7200, + ImmutableMap.of("feat_key2", "feat_value2")) + .toBuilder() + .setBatchSource( + DataGenerator.createFileDataSourceSpec("file:///path/to/file", "ts_col", "")) + .build(); + coreClient.simpleApplyFeatureTable(PROJECT_NAME, expectedFeatureTableSpec); + } + /** Test that Feast Serving metrics endpoint can be accessed with authentication enabled */ @Test public void shouldAllowUnauthenticatedAccessToMetricsEndpoint() throws IOException { @@ -111,37 +144,47 @@ public void shouldAllowUnauthenticatedAccessToMetricsEndpoint() throws IOExcepti @Test public void shouldAllowUnauthenticatedGetOnlineFeatures() { - // apply feature set - CoreSimpleAPIClient coreClient = - AuthTestUtils.getSecureApiClientForCore(FEAST_CORE_PORT, options); - AuthTestUtils.applyFeatureSet(coreClient, PROJECT_NAME, ENTITY_ID, FEATURE_NAME); + FeatureTableProto.FeatureTable actualFeatureTable = + coreClient.simpleGetFeatureTable(PROJECT_NAME, FEATURE_TABLE_NAME); + assertEquals(expectedFeatureTableSpec.getName(), actualFeatureTable.getSpec().getName()); + assertEquals( + expectedFeatureTableSpec.getBatchSource(), actualFeatureTable.getSpec().getBatchSource()); + ServingServiceBlockingStub servingStub = AuthTestUtils.getServingServiceStub(false, FEAST_SERVING_PORT, null); - GetOnlineFeaturesRequest onlineFeatureRequest = - AuthTestUtils.createOnlineFeatureRequest(PROJECT_NAME, FEATURE_NAME, ENTITY_ID, 1); - GetOnlineFeaturesResponse featureResponse = servingStub.getOnlineFeatures(onlineFeatureRequest); + GetOnlineFeaturesRequestV2 onlineFeatureRequestV2 = + AuthTestUtils.createOnlineFeatureRequest( + PROJECT_NAME, FEATURE_TABLE_NAME, FEATURE_NAME, ENTITY_ID, 1); + GetOnlineFeaturesResponse featureResponse = + servingStub.getOnlineFeaturesV2(onlineFeatureRequestV2); + assertEquals(1, featureResponse.getFieldValuesCount()); Map fieldsMap = featureResponse.getFieldValues(0).getFieldsMap(); assertTrue(fieldsMap.containsKey(ENTITY_ID)); - assertTrue(fieldsMap.containsKey(FEATURE_NAME)); + assertTrue(fieldsMap.containsKey(FEATURE_TABLE_NAME + ":" + FEATURE_NAME)); ((ManagedChannel) servingStub.getChannel()).shutdown(); } @Test void canGetOnlineFeaturesIfAuthenticated() { - // apply feature set - CoreSimpleAPIClient coreClient = - AuthTestUtils.getSecureApiClientForCore(FEAST_CORE_PORT, options); - AuthTestUtils.applyFeatureSet(coreClient, PROJECT_NAME, ENTITY_ID, FEATURE_NAME); + FeatureTableProto.FeatureTable actualFeatureTable = + coreClient.simpleGetFeatureTable(PROJECT_NAME, FEATURE_TABLE_NAME); + assertEquals(expectedFeatureTableSpec.getName(), actualFeatureTable.getSpec().getName()); + assertEquals( + expectedFeatureTableSpec.getBatchSource(), actualFeatureTable.getSpec().getBatchSource()); + ServingServiceBlockingStub servingStub = AuthTestUtils.getServingServiceStub(true, FEAST_SERVING_PORT, options); - GetOnlineFeaturesRequest onlineFeatureRequest = - AuthTestUtils.createOnlineFeatureRequest(PROJECT_NAME, FEATURE_NAME, ENTITY_ID, 1); - GetOnlineFeaturesResponse featureResponse = servingStub.getOnlineFeatures(onlineFeatureRequest); + GetOnlineFeaturesRequestV2 onlineFeatureRequest = + AuthTestUtils.createOnlineFeatureRequest( + PROJECT_NAME, FEATURE_TABLE_NAME, FEATURE_NAME, ENTITY_ID, 1); + + GetOnlineFeaturesResponse featureResponse = + servingStub.getOnlineFeaturesV2(onlineFeatureRequest); assertEquals(1, featureResponse.getFieldValuesCount()); Map fieldsMap = featureResponse.getFieldValues(0).getFieldsMap(); assertTrue(fieldsMap.containsKey(ENTITY_ID)); - assertTrue(fieldsMap.containsKey(FEATURE_NAME)); + assertTrue(fieldsMap.containsKey(FEATURE_TABLE_NAME + ":" + FEATURE_NAME)); ((ManagedChannel) servingStub.getChannel()).shutdown(); } } diff --git a/serving/src/test/java/feast/serving/it/ServingServiceOauthAuthorizationIT.java b/serving/src/test/java/feast/serving/it/ServingServiceOauthAuthorizationIT.java index 50111fcd155..64fe44b2dce 100644 --- a/serving/src/test/java/feast/serving/it/ServingServiceOauthAuthorizationIT.java +++ b/serving/src/test/java/feast/serving/it/ServingServiceOauthAuthorizationIT.java @@ -21,20 +21,22 @@ import static org.junit.jupiter.api.Assertions.assertTrue; import static org.testcontainers.containers.wait.strategy.Wait.forHttp; -import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesRequest; +import feast.common.it.DataGenerator; +import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesRequestV2; import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesResponse; import feast.proto.serving.ServingServiceGrpc.ServingServiceBlockingStub; +import feast.proto.types.ValueProto; import feast.proto.types.ValueProto.Value; import io.grpc.ManagedChannel; import io.grpc.StatusRuntimeException; import java.io.File; import java.io.IOException; import java.time.Duration; +import java.util.Collections; import java.util.HashMap; import java.util.Map; import org.junit.ClassRule; import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.runners.model.InitializationError; import org.springframework.boot.test.context.SpringBootTest; @@ -45,6 +47,8 @@ import org.testcontainers.containers.wait.strategy.Wait; import org.testcontainers.junit.jupiter.Container; import org.testcontainers.junit.jupiter.Testcontainers; +import org.testcontainers.shaded.com.google.common.collect.ImmutableList; +import org.testcontainers.shaded.com.google.common.collect.ImmutableMap; import sh.ory.keto.ApiException; @ActiveProfiles("it") @@ -80,11 +84,6 @@ public class ServingServiceOauthAuthorizationIT extends BaseAuthIT { FEAST_CORE_PORT, Wait.forLogMessage(".*gRPC Server started.*\\n", 1) .withStartupTimeout(Duration.ofMinutes(SERVICE_START_MAX_WAIT_TIME_IN_MINUTES))) - .withExposedService( - JOB_CONTROLLER, - FEAST_JOB_CONTROLLER_PORT, - Wait.forLogMessage(".*gRPC Server started.*\\n", 1) - .withStartupTimeout(Duration.ofMinutes(SERVICE_START_MAX_WAIT_TIME_IN_MINUTES))) .withExposedService("adaptor_1", KETO_ADAPTOR_PORT) .withExposedService("keto_1", KETO_PORT, forHttp("/health/ready").forStatusCode(200)); @@ -132,12 +131,18 @@ static void globalSetup() throws IOException, InitializationError, InterruptedEx adminCredentials.put("grant_type", GRANT_TYPE); coreClient = AuthTestUtils.getSecureApiClientForCore(FEAST_CORE_PORT, adminCredentials); - } - - @BeforeEach - public void setUp() { - // seed core - AuthTestUtils.applyFeatureSet(coreClient, PROJECT_NAME, ENTITY_ID, FEATURE_NAME); + coreClient.simpleApplyEntity( + PROJECT_NAME, + DataGenerator.createEntitySpecV2( + ENTITY_ID, "", ValueProto.ValueType.Enum.STRING, Collections.emptyMap())); + coreClient.simpleApplyFeatureTable( + PROJECT_NAME, + DataGenerator.createFeatureTableSpec( + FEATURE_TABLE_NAME, + ImmutableList.of(ENTITY_ID), + ImmutableMap.of(FEATURE_NAME, ValueProto.ValueType.Enum.STRING), + 0, + Collections.emptyMap())); } @Test @@ -145,13 +150,14 @@ public void shouldNotAllowUnauthenticatedGetOnlineFeatures() { ServingServiceBlockingStub servingStub = AuthTestUtils.getServingServiceStub(false, FEAST_SERVING_PORT, null); - GetOnlineFeaturesRequest onlineFeatureRequest = - AuthTestUtils.createOnlineFeatureRequest(PROJECT_NAME, FEATURE_NAME, ENTITY_ID, 1); + GetOnlineFeaturesRequestV2 onlineFeatureRequest = + AuthTestUtils.createOnlineFeatureRequest( + PROJECT_NAME, FEATURE_TABLE_NAME, FEATURE_NAME, ENTITY_ID, 1); Exception exception = assertThrows( StatusRuntimeException.class, () -> { - servingStub.getOnlineFeatures(onlineFeatureRequest); + servingStub.getOnlineFeaturesV2(onlineFeatureRequest); }); String expectedMessage = "UNAUTHENTICATED: Authentication failed"; @@ -162,16 +168,17 @@ public void shouldNotAllowUnauthenticatedGetOnlineFeatures() { @Test void canGetOnlineFeaturesIfAdmin() { - // apply feature set ServingServiceBlockingStub servingStub = AuthTestUtils.getServingServiceStub(true, FEAST_SERVING_PORT, adminCredentials); - GetOnlineFeaturesRequest onlineFeatureRequest = - AuthTestUtils.createOnlineFeatureRequest(PROJECT_NAME, FEATURE_NAME, ENTITY_ID, 1); - GetOnlineFeaturesResponse featureResponse = servingStub.getOnlineFeatures(onlineFeatureRequest); + GetOnlineFeaturesRequestV2 onlineFeatureRequest = + AuthTestUtils.createOnlineFeatureRequest( + PROJECT_NAME, FEATURE_TABLE_NAME, FEATURE_NAME, ENTITY_ID, 1); + GetOnlineFeaturesResponse featureResponse = + servingStub.getOnlineFeaturesV2(onlineFeatureRequest); assertEquals(1, featureResponse.getFieldValuesCount()); Map fieldsMap = featureResponse.getFieldValues(0).getFieldsMap(); assertTrue(fieldsMap.containsKey(ENTITY_ID)); - assertTrue(fieldsMap.containsKey(FEATURE_NAME)); + assertTrue(fieldsMap.containsKey(FEATURE_TABLE_NAME + ":" + FEATURE_NAME)); ((ManagedChannel) servingStub.getChannel()).shutdown(); } @@ -182,13 +189,15 @@ void canGetOnlineFeaturesIfProjectMember() { memberCredsOptions.put(CLIENT_ID, PROJECT_MEMBER_CLIENT_ID); ServingServiceBlockingStub servingStub = AuthTestUtils.getServingServiceStub(true, FEAST_SERVING_PORT, memberCredsOptions); - GetOnlineFeaturesRequest onlineFeatureRequest = - AuthTestUtils.createOnlineFeatureRequest(PROJECT_NAME, FEATURE_NAME, ENTITY_ID, 1); - GetOnlineFeaturesResponse featureResponse = servingStub.getOnlineFeatures(onlineFeatureRequest); + GetOnlineFeaturesRequestV2 onlineFeatureRequest = + AuthTestUtils.createOnlineFeatureRequest( + PROJECT_NAME, FEATURE_TABLE_NAME, FEATURE_NAME, ENTITY_ID, 1); + GetOnlineFeaturesResponse featureResponse = + servingStub.getOnlineFeaturesV2(onlineFeatureRequest); assertEquals(1, featureResponse.getFieldValuesCount()); Map fieldsMap = featureResponse.getFieldValues(0).getFieldsMap(); assertTrue(fieldsMap.containsKey(ENTITY_ID)); - assertTrue(fieldsMap.containsKey(FEATURE_NAME)); + assertTrue(fieldsMap.containsKey(FEATURE_TABLE_NAME + ":" + FEATURE_NAME)); ((ManagedChannel) servingStub.getChannel()).shutdown(); } @@ -199,12 +208,13 @@ void cantGetOnlineFeaturesIfNotProjectMember() { notMemberCredsOptions.put(CLIENT_ID, NOT_PROJECT_MEMBER_CLIENT_ID); ServingServiceBlockingStub servingStub = AuthTestUtils.getServingServiceStub(true, FEAST_SERVING_PORT, notMemberCredsOptions); - GetOnlineFeaturesRequest onlineFeatureRequest = - AuthTestUtils.createOnlineFeatureRequest(PROJECT_NAME, FEATURE_NAME, ENTITY_ID, 1); + GetOnlineFeaturesRequestV2 onlineFeatureRequest = + AuthTestUtils.createOnlineFeatureRequest( + PROJECT_NAME, FEATURE_TABLE_NAME, FEATURE_NAME, ENTITY_ID, 1); StatusRuntimeException exception = assertThrows( StatusRuntimeException.class, - () -> servingStub.getOnlineFeatures(onlineFeatureRequest)); + () -> servingStub.getOnlineFeaturesV2(onlineFeatureRequest)); String expectedMessage = String.format( diff --git a/serving/src/test/java/feast/serving/it/TestUtils.java b/serving/src/test/java/feast/serving/it/TestUtils.java index 0c72df1ea86..6772dade9c1 100644 --- a/serving/src/test/java/feast/serving/it/TestUtils.java +++ b/serving/src/test/java/feast/serving/it/TestUtils.java @@ -90,7 +90,7 @@ public static void applyFeatureTable( .setBatchSource( DataGenerator.createFileDataSourceSpec("file:///path/to/file", "ts_col", "dt_col")) .build(); - secureApiClient.simpleApplyFeatureTable(expectedFeatureTableSpec); + secureApiClient.simpleApplyFeatureTable(projectName, expectedFeatureTableSpec); FeatureTable actualFeatureTable = secureApiClient.simpleGetFeatureTable(projectName, featureTableName); assertEquals(expectedFeatureTableSpec.getName(), actualFeatureTable.getSpec().getName()); @@ -98,7 +98,7 @@ public static void applyFeatureTable( public static void applyEntity( CoreSimpleAPIClient coreApiClient, String projectName, EntitySpecV2 entitySpec) { - coreApiClient.simpleApplyEntity(entitySpec); + coreApiClient.simpleApplyEntity(projectName, entitySpec); String entityName = entitySpec.getName(); Entity actualEntity = coreApiClient.getEntity(projectName, entityName); assertEquals(entitySpec.getName(), actualEntity.getSpec().getName()); diff --git a/serving/src/test/java/feast/serving/service/CachedSpecServiceTest.java b/serving/src/test/java/feast/serving/service/CachedSpecServiceTest.java index b41a75918f4..57932d49297 100644 --- a/serving/src/test/java/feast/serving/service/CachedSpecServiceTest.java +++ b/serving/src/test/java/feast/serving/service/CachedSpecServiceTest.java @@ -17,7 +17,6 @@ package feast.serving.service; import static org.hamcrest.CoreMatchers.equalTo; -import static org.hamcrest.Matchers.containsInAnyOrder; import static org.junit.Assert.assertThat; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; @@ -27,29 +26,17 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import feast.common.it.DataGenerator; -import feast.proto.core.CoreServiceProto.ListFeatureSetsRequest; -import feast.proto.core.CoreServiceProto.ListFeatureSetsResponse; import feast.proto.core.CoreServiceProto.ListFeatureTablesRequest; import feast.proto.core.CoreServiceProto.ListFeatureTablesResponse; import feast.proto.core.CoreServiceProto.ListProjectsRequest; import feast.proto.core.CoreServiceProto.ListProjectsResponse; -import feast.proto.core.FeatureSetProto; -import feast.proto.core.FeatureSetProto.FeatureSetSpec; -import feast.proto.core.FeatureSetProto.FeatureSpec; import feast.proto.core.FeatureTableProto; import feast.proto.core.FeatureTableProto.FeatureTableSpec; import feast.proto.core.StoreProto.Store; -import feast.proto.core.StoreProto.Store.Subscription; -import feast.proto.serving.ServingAPIProto.FeatureReference; import feast.proto.serving.ServingAPIProto.FeatureReferenceV2; import feast.proto.types.ValueProto; -import feast.serving.exception.SpecRetrievalException; import feast.serving.specs.CachedSpecService; import feast.serving.specs.CoreSpecService; -import feast.storage.api.retriever.FeatureSetRequest; -import java.util.HashMap; -import java.util.List; -import java.util.Map; import org.junit.Before; import org.junit.Rule; import org.junit.Test; @@ -64,7 +51,6 @@ public class CachedSpecServiceTest { @Mock CoreSpecService coreService; - private Map featureSetSpecs; private CachedSpecService cachedSpecService; private ImmutableList featureTableEntities; @@ -78,25 +64,6 @@ public void setUp() { initMocks(this); this.store = Store.newBuilder().build(); - this.featureSetSpecs = new HashMap<>(); - - this.setupFeatureSetAndStoreSubscription( - "project", - "fs1", - List.of( - FeatureSpec.newBuilder().setName("feature").build(), - FeatureSpec.newBuilder().setName("feature2").build())); - - this.setupFeatureSetAndStoreSubscription( - "default", - "fs2", - List.of( - FeatureSpec.newBuilder().setName("feature3").build(), - FeatureSpec.newBuilder().setName("feature4").build(), - FeatureSpec.newBuilder().setName("feature5").build())); - - this.setupFeatureSetAndStoreSubscription( - "default", "fs3", List.of(FeatureSpec.newBuilder().setName("feature4").build())); this.setupProject("default"); this.featureTableEntities = ImmutableList.of("entity1"); @@ -137,9 +104,6 @@ private void setupProject(String project) { } private void setupFeatureTableAndProject(String project) { - ImmutableMap featureTable1Features = - this.featureTable1Features; - FeatureTableProto.FeatureTable featureTable1 = FeatureTableProto.FeatureTable.newBuilder().setSpec(this.featureTable1Spec).build(); FeatureTableProto.FeatureTable featureTable2 = @@ -156,37 +120,6 @@ private void setupFeatureTableAndProject(String project) { .build()); } - private void setupFeatureSetAndStoreSubscription( - String project, String name, List featureSpecs) { - FeatureSetSpec fsSpec = - FeatureSetSpec.newBuilder() - .setProject(project) - .setName(name) - .addAllFeatures(featureSpecs) - .build(); - this.featureSetSpecs.put(String.format("%s", name), fsSpec); - - this.store = - this.store - .toBuilder() - .addSubscriptions(Subscription.newBuilder().setProject(project).setName(name).build()) - .build(); - - // collect the different versions the featureset with the given name - FeatureSetProto.FeatureSet featureSet = - FeatureSetProto.FeatureSet.newBuilder().setSpec(fsSpec).build(); - - when(coreService.listFeatureSets( - ListFeatureSetsRequest.newBuilder() - .setFilter( - ListFeatureSetsRequest.Filter.newBuilder() - .setProject(project) - .setFeatureSetName(name) - .build()) - .build())) - .thenReturn(ListFeatureSetsResponse.newBuilder().addFeatureSets(featureSet).build()); - } - @Test public void shouldRegisterStoreWithCore() { verify(coreService, times(1)).registerStore(cachedSpecService.getStore()); @@ -229,132 +162,4 @@ public void shouldPopulateAndReturnDifferentFeatureTables() { cachedSpecService.getFeatureTableSpec("default", featureReference3), equalTo(this.featureTable2Spec)); } - - @Test - public void shouldPopulateAndReturnFeatureSets() { - // test that CachedSpecService can retrieve fully qualified feature references. - cachedSpecService.populateCache(); - FeatureReference fs1fr1 = - FeatureReference.newBuilder() - .setProject("project") - .setName("feature") - .setFeatureSet("fs1") - .build(); - FeatureReference fs1fr2 = - FeatureReference.newBuilder() - .setProject("project") - .setName("feature2") - .setFeatureSet("fs1") - .build(); - - assertThat( - cachedSpecService.getFeatureSets(List.of(fs1fr1, fs1fr2)), - equalTo( - List.of( - FeatureSetRequest.newBuilder() - .addFeatureReference(fs1fr1) - .addFeatureReference(fs1fr2) - .setSpec(featureSetSpecs.get("fs1")) - .build()))); - } - - @Test - public void shouldPopulateAndReturnFeatureSetWithDefaultProjectIfProjectNotSupplied() { - // test that CachedSpecService will use default project when project unspecified - FeatureReference fs2fr3 = - FeatureReference.newBuilder().setName("feature3").setFeatureSet("fs2").build(); - // check that this is true for references in where feature set is unspecified - FeatureReference fs2fr5 = FeatureReference.newBuilder().setName("feature5").build(); - - assertThat( - cachedSpecService.getFeatureSets(List.of(fs2fr3, fs2fr5)), - equalTo( - List.of( - FeatureSetRequest.newBuilder() - .addFeatureReference(fs2fr3) - .addFeatureReference(fs2fr5) - .setSpec(featureSetSpecs.get("fs2")) - .build()))); - } - - @Test - public void shouldPopulateAndReturnClosestFeatureSetIfFeatureSetNotSupplied() { - // test that CachedSpecService will try to match a featureset without a featureset name in - // reference - FeatureReference fs1fr1 = - FeatureReference.newBuilder().setProject("project").setName("feature").build(); - - // check that this is true for reference in which project is unspecified - FeatureReference fs2fr3 = FeatureReference.newBuilder().setName("feature3").build(); - - assertThat( - cachedSpecService.getFeatureSets(List.of(fs1fr1, fs2fr3)), - containsInAnyOrder( - List.of( - FeatureSetRequest.newBuilder() - .addFeatureReference(fs1fr1) - .setSpec(featureSetSpecs.get("fs1")) - .build(), - FeatureSetRequest.newBuilder() - .addFeatureReference(fs2fr3) - .setSpec(featureSetSpecs.get("fs2")) - .build()) - .toArray())); - } - - @Test - public void shouldPopulateAndReturnFeatureSetsGivenFeaturesFromDifferentFeatureSets() { - cachedSpecService.populateCache(); - FeatureReference fs1fr1 = - FeatureReference.newBuilder().setProject("project").setName("feature").build(); - - FeatureReference fs2fr3 = - FeatureReference.newBuilder().setProject("default").setName("feature3").build(); - - assertThat( - cachedSpecService.getFeatureSets(List.of(fs1fr1, fs2fr3)), - containsInAnyOrder( - List.of( - FeatureSetRequest.newBuilder() - .addFeatureReference(fs1fr1) - .setSpec(featureSetSpecs.get("fs1")) - .build(), - FeatureSetRequest.newBuilder() - .addFeatureReference(fs2fr3) - .setSpec(featureSetSpecs.get("fs2")) - .build()) - .toArray())); - } - - @Test - public void shouldPopulateAndReturnFeatureSetGivenFeaturesFromSameFeatureSet() { - cachedSpecService.populateCache(); - FeatureReference fr1 = - FeatureReference.newBuilder().setProject("project").setName("feature").build(); - FeatureReference fr2 = - FeatureReference.newBuilder().setProject("project").setName("feature2").build(); - - assertThat( - cachedSpecService.getFeatureSets(List.of(fr1, fr2)), - equalTo( - List.of( - FeatureSetRequest.newBuilder() - .addFeatureReference(fr1) - .addFeatureReference(fr2) - .setSpec(featureSetSpecs.get("fs1")) - .build()))); - } - - @Test - public void shouldThrowExceptionWhenMultipleFeatureSetMapToFeatureReference() - throws SpecRetrievalException { - // both fs2 and fs3 have the feature with the same name. - // using a generic feature reference only specifying the feature's name - // should cause a multiple feature sets to match and throw an error - FeatureReference fs2fr4 = FeatureReference.newBuilder().setName("feature4").build(); - FeatureReference fs3fr4 = FeatureReference.newBuilder().setName("feature4").build(); - - expectedException.expect(SpecRetrievalException.class); - cachedSpecService.getFeatureSets(List.of(fs2fr4, fs3fr4)); - } } diff --git a/serving/src/test/java/feast/serving/service/OnlineServingServiceTest.java b/serving/src/test/java/feast/serving/service/OnlineServingServiceTest.java index f2a8038a973..539ed398e16 100644 --- a/serving/src/test/java/feast/serving/service/OnlineServingServiceTest.java +++ b/serving/src/test/java/feast/serving/service/OnlineServingServiceTest.java @@ -16,34 +16,30 @@ */ package feast.serving.service; +import static feast.common.it.DataGenerator.*; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.equalTo; +import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.when; import static org.mockito.MockitoAnnotations.initMocks; -import com.google.common.collect.Lists; import com.google.protobuf.Duration; import com.google.protobuf.Timestamp; -import feast.proto.core.FeatureSetProto.EntitySpec; -import feast.proto.core.FeatureSetProto.FeatureSetSpec; -import feast.proto.serving.ServingAPIProto.FeatureReference; -import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesRequest; -import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesRequest.EntityRow; +import feast.proto.core.FeatureProto; +import feast.proto.core.FeatureTableProto.FeatureTableSpec; +import feast.proto.serving.ServingAPIProto; +import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesRequestV2; import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesResponse; import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesResponse.FieldStatus; import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesResponse.FieldValues; -import feast.proto.types.FeatureRowProto.FeatureRow; -import feast.proto.types.FieldProto; -import feast.proto.types.ValueProto.Value; +import feast.proto.types.ValueProto; import feast.serving.specs.CachedSpecService; -import feast.storage.api.retriever.FeatureSetRequest; -import feast.storage.connectors.redis.retriever.RedisOnlineRetriever; +import feast.storage.api.retriever.Feature; +import feast.storage.connectors.redis.retriever.OnlineRetriever; import io.opentracing.Tracer; import io.opentracing.Tracer.SpanBuilder; import java.util.ArrayList; -import java.util.Collections; import java.util.List; -import java.util.Optional; import org.junit.Before; import org.junit.Test; import org.mockito.ArgumentMatchers; @@ -53,364 +49,350 @@ public class OnlineServingServiceTest { @Mock CachedSpecService specService; - @Mock Tracer tracer; + @Mock OnlineRetriever retrieverV2; - @Mock RedisOnlineRetriever retriever; - - private OnlineServingService onlineServingService; + private OnlineServingServiceV2 onlineServingServiceV2; - List testFeatureRows; + List mockedFeatureRows; + List featureSpecs; @Before public void setUp() { initMocks(this); - onlineServingService = new OnlineServingService(retriever, specService, tracer); - - // create fake feature rows for testing. - testFeatureRows = new ArrayList<>(); - testFeatureRows.add( - FeatureRow.newBuilder() - .setEventTimestamp(Timestamp.newBuilder().setSeconds(100)) - .addAllFields( - Lists.newArrayList( - FieldProto.Field.newBuilder().setName("entity1").setValue(intValue(1)).build(), - FieldProto.Field.newBuilder() - .setName("entity2") - .setValue(strValue("a")) - .build(), - FieldProto.Field.newBuilder().setName("feature1").setValue(intValue(1)).build(), - FieldProto.Field.newBuilder() - .setName("feature2") - .setValue(intValue(1)) - .build())) - .setFeatureSet("featureSet") + onlineServingServiceV2 = new OnlineServingServiceV2(retrieverV2, specService, tracer); + + mockedFeatureRows = new ArrayList<>(); + mockedFeatureRows.add( + Feature.builder() + .setFeatureReference( + ServingAPIProto.FeatureReferenceV2.newBuilder() + .setFeatureTable("featuretable_1") + .setName("feature_1") + .build()) + .setFeatureValue(createStrValue("1")) + .setEventTimestamp(Timestamp.newBuilder().setSeconds(100).build()) .build()); - - testFeatureRows.add( - FeatureRow.newBuilder() - .setEventTimestamp(Timestamp.newBuilder().setSeconds(100)) - .addAllFields( - Lists.newArrayList( - FieldProto.Field.newBuilder().setName("entity1").setValue(intValue(2)).build(), - FieldProto.Field.newBuilder() - .setName("entity2") - .setValue(strValue("b")) - .build(), - FieldProto.Field.newBuilder().setName("feature1").setValue(intValue(2)).build(), - FieldProto.Field.newBuilder() - .setName("feature2") - .setValue(intValue(2)) - .build())) - .setFeatureSet("featureSet") + mockedFeatureRows.add( + Feature.builder() + .setFeatureReference( + ServingAPIProto.FeatureReferenceV2.newBuilder() + .setFeatureTable("featuretable_1") + .setName("feature_2") + .build()) + .setFeatureValue(createStrValue("2")) + .setEventTimestamp(Timestamp.newBuilder().setSeconds(100).build()) + .build()); + mockedFeatureRows.add( + Feature.builder() + .setFeatureReference( + ServingAPIProto.FeatureReferenceV2.newBuilder() + .setFeatureTable("featuretable_1") + .setName("feature_1") + .build()) + .setFeatureValue(createStrValue("3")) + .setEventTimestamp(Timestamp.newBuilder().setSeconds(100).build()) + .build()); + mockedFeatureRows.add( + Feature.builder() + .setFeatureReference( + ServingAPIProto.FeatureReferenceV2.newBuilder() + .setFeatureTable("featuretable_1") + .setName("feature_2") + .build()) + .setFeatureValue(createStrValue("4")) + .setEventTimestamp(Timestamp.newBuilder().setSeconds(100).build()) + .build()); + mockedFeatureRows.add( + Feature.builder() + .setFeatureReference( + ServingAPIProto.FeatureReferenceV2.newBuilder() + .setFeatureTable("featuretable_1") + .setName("feature_3") + .build()) + .setFeatureValue(createStrValue("5")) + .setEventTimestamp(Timestamp.newBuilder().setSeconds(100).build()) + .build()); + mockedFeatureRows.add( + Feature.builder() + .setFeatureReference( + ServingAPIProto.FeatureReferenceV2.newBuilder() + .setFeatureTable("featuretable_1") + .setName("feature_1") + .build()) + .setFeatureValue(createStrValue("6")) + .setEventTimestamp(Timestamp.newBuilder().setSeconds(50).build()) .build()); - testFeatureRows.add( - testFeatureRows - .get(1) - .toBuilder() - .setEventTimestamp(Timestamp.newBuilder().setSeconds(50)) + featureSpecs = new ArrayList<>(); + featureSpecs.add( + FeatureProto.FeatureSpecV2.newBuilder() + .setName("feature_1") + .setValueType(ValueProto.ValueType.Enum.STRING) + .build()); + featureSpecs.add( + FeatureProto.FeatureSpecV2.newBuilder() + .setName("feature_2") + .setValueType(ValueProto.ValueType.Enum.STRING) .build()); } @Test public void shouldReturnResponseWithValuesAndMetadataIfKeysPresent() { - GetOnlineFeaturesRequest request = - getOnlineFeaturesRequest( - List.of( - FeatureReference.newBuilder().setName("feature1").build(), - FeatureReference.newBuilder().setName("feature2").setProject("project").build())); - - FeatureSetRequest featureSetRequest = - FeatureSetRequest.newBuilder() - .addAllFeatureReferences(request.getFeaturesList()) - .setSpec(getFeatureSetSpec()) + String projectName = "default"; + ServingAPIProto.FeatureReferenceV2 featureReference1 = + ServingAPIProto.FeatureReferenceV2.newBuilder() + .setFeatureTable("featuretable_1") + .setName("feature_1") .build(); - - List> featureRows = - List.of(Optional.of(testFeatureRows.get(0)), Optional.of(testFeatureRows.get(1))); - - when(specService.getFeatureSets(request.getFeaturesList(), "")) - .thenReturn(Collections.singletonList(featureSetRequest)); - when(retriever.getOnlineFeatures(request.getEntityRowsList(), featureSetRequest)) - .thenReturn(featureRows); - - when(tracer.buildSpan(ArgumentMatchers.any())).thenReturn(Mockito.mock(SpanBuilder.class)); - - GetOnlineFeaturesResponse expected = - GetOnlineFeaturesResponse.newBuilder() - .addFieldValues( - FieldValues.newBuilder() - .putFields("entity1", intValue(1)) - .putStatuses("entity1", FieldStatus.PRESENT) - .putFields("entity2", strValue("a")) - .putStatuses("entity2", FieldStatus.PRESENT) - .putFields("feature1", intValue(1)) - .putStatuses("feature1", FieldStatus.PRESENT) - .putFields("project/feature2", intValue(1)) - .putStatuses("project/feature2", FieldStatus.PRESENT) - .build()) - .addFieldValues( - FieldValues.newBuilder() - .putFields("entity1", intValue(2)) - .putStatuses("entity1", FieldStatus.PRESENT) - .putFields("entity2", strValue("b")) - .putStatuses("entity2", FieldStatus.PRESENT) - .putFields("feature1", intValue(2)) - .putStatuses("feature1", FieldStatus.PRESENT) - .putFields("project/feature2", intValue(2)) - .putStatuses("project/feature2", FieldStatus.PRESENT) - .build()) + ServingAPIProto.FeatureReferenceV2 featureReference2 = + ServingAPIProto.FeatureReferenceV2.newBuilder() + .setFeatureTable("featuretable_1") + .setName("feature_2") .build(); - GetOnlineFeaturesResponse actual = onlineServingService.getOnlineFeatures(request); - assertThat(actual, equalTo(expected)); - } + List featureReferences = + List.of(featureReference1, featureReference2); + GetOnlineFeaturesRequestV2 request = getOnlineFeaturesRequestV2(projectName, featureReferences); + + List entityKeyList1 = new ArrayList<>(); + List entityKeyList2 = new ArrayList<>(); + entityKeyList1.add(mockedFeatureRows.get(0)); + entityKeyList1.add(mockedFeatureRows.get(1)); + entityKeyList2.add(mockedFeatureRows.get(2)); + entityKeyList2.add(mockedFeatureRows.get(3)); + + List> featureRows = List.of(entityKeyList1, entityKeyList2); + + when(retrieverV2.getOnlineFeatures(any(), any(), any())).thenReturn(featureRows); + when(specService.getFeatureTableSpec(any(), any())).thenReturn(getFeatureTableSpec()); + when(specService.getFeatureSpec(projectName, mockedFeatureRows.get(0).getFeatureReference())) + .thenReturn(featureSpecs.get(0)); + when(specService.getFeatureSpec(projectName, mockedFeatureRows.get(1).getFeatureReference())) + .thenReturn(featureSpecs.get(1)); + when(specService.getFeatureSpec(projectName, mockedFeatureRows.get(2).getFeatureReference())) + .thenReturn(featureSpecs.get(0)); + when(specService.getFeatureSpec(projectName, mockedFeatureRows.get(3).getFeatureReference())) + .thenReturn(featureSpecs.get(1)); - @Test - public void shouldReturnResponseWithUnsetValuesAndMetadataIfKeysNotPresent() { - // some keys not present, should have empty values - GetOnlineFeaturesRequest request = - getOnlineFeaturesRequest( - List.of( - FeatureReference.newBuilder().setName("feature1").build(), - FeatureReference.newBuilder().setName("feature2").setProject("project").build())); - - FeatureSetRequest featureSetRequest = - FeatureSetRequest.newBuilder() - .addAllFeatureReferences(request.getFeaturesList()) - .setSpec(getFeatureSetSpec()) - .build(); - - List> featureRows = - List.of(Optional.of(testFeatureRows.get(0)), Optional.empty()); - - when(specService.getFeatureSets(request.getFeaturesList(), "")) - .thenReturn(Collections.singletonList(featureSetRequest)); - when(retriever.getOnlineFeatures(request.getEntityRowsList(), featureSetRequest)) - .thenReturn(featureRows); when(tracer.buildSpan(ArgumentMatchers.any())).thenReturn(Mockito.mock(SpanBuilder.class)); GetOnlineFeaturesResponse expected = GetOnlineFeaturesResponse.newBuilder() .addFieldValues( FieldValues.newBuilder() - .putFields("entity1", intValue(1)) + .putFields("entity1", createInt64Value(1)) .putStatuses("entity1", FieldStatus.PRESENT) - .putFields("entity2", strValue("a")) + .putFields("entity2", createStrValue("a")) .putStatuses("entity2", FieldStatus.PRESENT) - .putFields("feature1", intValue(1)) - .putStatuses("feature1", FieldStatus.PRESENT) - .putFields("project/feature2", intValue(1)) - .putStatuses("project/feature2", FieldStatus.PRESENT) + .putFields("featuretable_1:feature_1", createStrValue("1")) + .putStatuses("featuretable_1:feature_1", FieldStatus.PRESENT) + .putFields("featuretable_1:feature_2", createStrValue("2")) + .putStatuses("featuretable_1:feature_2", FieldStatus.PRESENT) .build()) .addFieldValues( FieldValues.newBuilder() - .putFields("entity1", intValue(2)) + .putFields("entity1", createInt64Value(2)) .putStatuses("entity1", FieldStatus.PRESENT) - .putFields("entity2", strValue("b")) + .putFields("entity2", createStrValue("b")) .putStatuses("entity2", FieldStatus.PRESENT) - .putFields("feature1", Value.newBuilder().build()) - .putStatuses("feature1", FieldStatus.NOT_FOUND) - .putFields("project/feature2", Value.newBuilder().build()) - .putStatuses("project/feature2", FieldStatus.NOT_FOUND) + .putFields("featuretable_1:feature_1", createStrValue("3")) + .putStatuses("featuretable_1:feature_1", FieldStatus.PRESENT) + .putFields("featuretable_1:feature_2", createStrValue("4")) + .putStatuses("featuretable_1:feature_2", FieldStatus.PRESENT) .build()) .build(); - GetOnlineFeaturesResponse actual = onlineServingService.getOnlineFeatures(request); + GetOnlineFeaturesResponse actual = onlineServingServiceV2.getOnlineFeatures(request); assertThat(actual, equalTo(expected)); } @Test - public void shouldReturnResponseWithUnsetValuesAndMetadataIfMaxAgeIsExceeded() { - // keys present, but considered stale when compared to maxAge - GetOnlineFeaturesRequest request = - getOnlineFeaturesRequest( - List.of( - FeatureReference.newBuilder().setName("feature1").build(), - FeatureReference.newBuilder().setName("feature2").setProject("project").build())); - - List> featureRows = - List.of(Optional.of(testFeatureRows.get(0)), Optional.of(testFeatureRows.get(2))); - - FeatureSetSpec spec = - getFeatureSetSpec().toBuilder().setMaxAge(Duration.newBuilder().setSeconds(1)).build(); - FeatureSetRequest featureSetRequest = - FeatureSetRequest.newBuilder() - .addAllFeatureReferences(request.getFeaturesList()) - .setSpec(spec) + public void shouldReturnResponseWithUnsetValuesAndMetadataIfKeysNotPresent() { + String projectName = "default"; + ServingAPIProto.FeatureReferenceV2 featureReference1 = + ServingAPIProto.FeatureReferenceV2.newBuilder() + .setFeatureTable("featuretable_1") + .setName("feature_1") .build(); - - when(specService.getFeatureSets(request.getFeaturesList(), "")) - .thenReturn(Collections.singletonList(featureSetRequest)); - when(retriever.getOnlineFeatures(request.getEntityRowsList(), featureSetRequest)) - .thenReturn(featureRows); - when(tracer.buildSpan(ArgumentMatchers.any())).thenReturn(Mockito.mock(SpanBuilder.class)); - - GetOnlineFeaturesResponse expected = - GetOnlineFeaturesResponse.newBuilder() - .addFieldValues( - FieldValues.newBuilder() - .putFields("entity1", intValue(1)) - .putStatuses("entity1", FieldStatus.PRESENT) - .putFields("entity2", strValue("a")) - .putStatuses("entity2", FieldStatus.PRESENT) - .putFields("feature1", intValue(1)) - .putStatuses("feature1", FieldStatus.PRESENT) - .putFields("project/feature2", intValue(1)) - .putStatuses("project/feature2", FieldStatus.PRESENT) - .build()) - .addFieldValues( - FieldValues.newBuilder() - .putFields("entity1", intValue(2)) - .putStatuses("entity1", FieldStatus.PRESENT) - .putFields("entity2", strValue("b")) - .putStatuses("entity2", FieldStatus.PRESENT) - .putFields("feature1", Value.newBuilder().build()) - .putStatuses("feature1", FieldStatus.OUTSIDE_MAX_AGE) - .putFields("project/feature2", Value.newBuilder().build()) - .putStatuses("project/feature2", FieldStatus.OUTSIDE_MAX_AGE) - .build()) + ServingAPIProto.FeatureReferenceV2 featureReference2 = + ServingAPIProto.FeatureReferenceV2.newBuilder() + .setFeatureTable("featuretable_1") + .setName("feature_2") .build(); - GetOnlineFeaturesResponse actual = onlineServingService.getOnlineFeatures(request); - assertThat(actual, equalTo(expected)); - } + List featureReferences = + List.of(featureReference1, featureReference2); + GetOnlineFeaturesRequestV2 request = getOnlineFeaturesRequestV2(projectName, featureReferences); - @Test - public void shouldFilterOutUndesiredRows() { - // requested rows less than the rows available in the featureset - GetOnlineFeaturesRequest request = - getOnlineFeaturesRequest( - List.of(FeatureReference.newBuilder().setName("feature1").build())); + List entityKeyList1 = new ArrayList<>(); + List entityKeyList2 = new ArrayList<>(); + entityKeyList1.add(mockedFeatureRows.get(0)); + entityKeyList1.add(mockedFeatureRows.get(1)); + entityKeyList2.add(mockedFeatureRows.get(4)); - List> featureRows = - List.of(Optional.of(testFeatureRows.get(0)), Optional.of(testFeatureRows.get(1))); + List> featureRows = List.of(entityKeyList1, entityKeyList2); - FeatureSetRequest featureSetRequest = - FeatureSetRequest.newBuilder() - .addAllFeatureReferences(request.getFeaturesList()) - .setSpec(getFeatureSetSpec()) - .build(); + when(retrieverV2.getOnlineFeatures(any(), any(), any())).thenReturn(featureRows); + when(specService.getFeatureTableSpec(any(), any())).thenReturn(getFeatureTableSpec()); + when(specService.getFeatureSpec(projectName, mockedFeatureRows.get(0).getFeatureReference())) + .thenReturn(featureSpecs.get(0)); + when(specService.getFeatureSpec(projectName, mockedFeatureRows.get(1).getFeatureReference())) + .thenReturn(featureSpecs.get(1)); - when(specService.getFeatureSets(request.getFeaturesList(), "")) - .thenReturn(Collections.singletonList(featureSetRequest)); - when(retriever.getOnlineFeatures(request.getEntityRowsList(), featureSetRequest)) - .thenReturn(featureRows); when(tracer.buildSpan(ArgumentMatchers.any())).thenReturn(Mockito.mock(SpanBuilder.class)); GetOnlineFeaturesResponse expected = GetOnlineFeaturesResponse.newBuilder() .addFieldValues( FieldValues.newBuilder() - .putFields("entity1", intValue(1)) + .putFields("entity1", createInt64Value(1)) .putStatuses("entity1", FieldStatus.PRESENT) - .putFields("entity2", strValue("a")) + .putFields("entity2", createStrValue("a")) .putStatuses("entity2", FieldStatus.PRESENT) - .putFields("feature1", intValue(1)) - .putStatuses("feature1", FieldStatus.PRESENT) + .putFields("featuretable_1:feature_1", createStrValue("1")) + .putStatuses("featuretable_1:feature_1", FieldStatus.PRESENT) + .putFields("featuretable_1:feature_2", createStrValue("2")) + .putStatuses("featuretable_1:feature_2", FieldStatus.PRESENT) .build()) .addFieldValues( FieldValues.newBuilder() - .putFields("entity1", intValue(2)) + .putFields("entity1", createInt64Value(2)) .putStatuses("entity1", FieldStatus.PRESENT) - .putFields("entity2", strValue("b")) + .putFields("entity2", createStrValue("b")) .putStatuses("entity2", FieldStatus.PRESENT) - .putFields("feature1", intValue(2)) - .putStatuses("feature1", FieldStatus.PRESENT) + .putFields("featuretable_1:feature_1", createEmptyValue()) + .putStatuses("featuretable_1:feature_1", FieldStatus.NOT_FOUND) + .putFields("featuretable_1:feature_2", createEmptyValue()) + .putStatuses("featuretable_1:feature_2", FieldStatus.NOT_FOUND) .build()) .build(); - GetOnlineFeaturesResponse actual = onlineServingService.getOnlineFeatures(request); + GetOnlineFeaturesResponse actual = onlineServingServiceV2.getOnlineFeatures(request); assertThat(actual, equalTo(expected)); } @Test - public void shouldApplyProjectOverrideInRequest() { - GetOnlineFeaturesRequest request = - getOnlineFeaturesRequest( - List.of( - FeatureReference.newBuilder().setName("feature1").build(), - FeatureReference.newBuilder() - .setName("feature2") - .setProject("project") - .build())) - .toBuilder() - .setProject("project") + public void shouldReturnResponseWithUnsetValuesAndMetadataIfMaxAgeIsExceeded() { + String projectName = "default"; + ServingAPIProto.FeatureReferenceV2 featureReference1 = + ServingAPIProto.FeatureReferenceV2.newBuilder() + .setFeatureTable("featuretable_1") + .setName("feature_1") .build(); - - List> featureRows = - List.of(Optional.of(testFeatureRows.get(0)), Optional.of(testFeatureRows.get(1))); - - FeatureSetRequest featureSetRequest = - FeatureSetRequest.newBuilder() - .addAllFeatureReferences(request.getFeaturesList()) - .setSpec(getFeatureSetSpec()) + ServingAPIProto.FeatureReferenceV2 featureReference2 = + ServingAPIProto.FeatureReferenceV2.newBuilder() + .setFeatureTable("featuretable_1") + .setName("feature_2") .build(); + List featureReferences = + List.of(featureReference1, featureReference2); + GetOnlineFeaturesRequestV2 request = getOnlineFeaturesRequestV2(projectName, featureReferences); + + List entityKeyList1 = new ArrayList<>(); + List entityKeyList2 = new ArrayList<>(); + entityKeyList1.add(mockedFeatureRows.get(5)); + entityKeyList1.add(mockedFeatureRows.get(1)); + entityKeyList2.add(mockedFeatureRows.get(5)); + entityKeyList2.add(mockedFeatureRows.get(1)); + + List> featureRows = List.of(entityKeyList1, entityKeyList2); + + when(retrieverV2.getOnlineFeatures(any(), any(), any())).thenReturn(featureRows); + when(specService.getFeatureTableSpec(any(), any())) + .thenReturn( + FeatureTableSpec.newBuilder() + .setName("featuretable_1") + .addEntities("entity1") + .addEntities("entity2") + .addFeatures( + FeatureProto.FeatureSpecV2.newBuilder() + .setName("feature_1") + .setValueType(ValueProto.ValueType.Enum.STRING) + .build()) + .addFeatures( + FeatureProto.FeatureSpecV2.newBuilder() + .setName("feature_2") + .setValueType(ValueProto.ValueType.Enum.STRING) + .build()) + .setMaxAge(Duration.newBuilder().setSeconds(1)) + .build()); + when(specService.getFeatureSpec(projectName, mockedFeatureRows.get(1).getFeatureReference())) + .thenReturn(featureSpecs.get(1)); + when(specService.getFeatureSpec(projectName, mockedFeatureRows.get(5).getFeatureReference())) + .thenReturn(featureSpecs.get(0)); - when(specService.getFeatureSets(request.getFeaturesList(), "project")) - .thenReturn(Collections.singletonList(featureSetRequest)); - when(retriever.getOnlineFeatures(request.getEntityRowsList(), featureSetRequest)) - .thenReturn(featureRows); when(tracer.buildSpan(ArgumentMatchers.any())).thenReturn(Mockito.mock(SpanBuilder.class)); GetOnlineFeaturesResponse expected = GetOnlineFeaturesResponse.newBuilder() .addFieldValues( FieldValues.newBuilder() - .putFields("entity1", intValue(1)) + .putFields("entity1", createInt64Value(1)) .putStatuses("entity1", FieldStatus.PRESENT) - .putFields("entity2", strValue("a")) + .putFields("entity2", createStrValue("a")) .putStatuses("entity2", FieldStatus.PRESENT) - .putFields("feature1", intValue(1)) - .putStatuses("feature1", FieldStatus.PRESENT) - .putFields("project/feature2", intValue(1)) - .putStatuses("project/feature2", FieldStatus.PRESENT) + .putFields("featuretable_1:feature_1", createEmptyValue()) + .putStatuses("featuretable_1:feature_1", FieldStatus.OUTSIDE_MAX_AGE) + .putFields("featuretable_1:feature_2", createStrValue("2")) + .putStatuses("featuretable_1:feature_2", FieldStatus.PRESENT) .build()) .addFieldValues( FieldValues.newBuilder() - .putFields("entity1", intValue(2)) + .putFields("entity1", createInt64Value(2)) .putStatuses("entity1", FieldStatus.PRESENT) - .putFields("entity2", strValue("b")) + .putFields("entity2", createStrValue("b")) .putStatuses("entity2", FieldStatus.PRESENT) - .putFields("feature1", intValue(2)) - .putStatuses("feature1", FieldStatus.PRESENT) - .putFields("project/feature2", intValue(2)) - .putStatuses("project/feature2", FieldStatus.PRESENT) + .putFields("featuretable_1:feature_1", createEmptyValue()) + .putStatuses("featuretable_1:feature_1", FieldStatus.OUTSIDE_MAX_AGE) + .putFields("featuretable_1:feature_2", createStrValue("2")) + .putStatuses("featuretable_1:feature_2", FieldStatus.PRESENT) .build()) .build(); - GetOnlineFeaturesResponse actual = onlineServingService.getOnlineFeatures(request); + GetOnlineFeaturesResponse actual = onlineServingServiceV2.getOnlineFeatures(request); assertThat(actual, equalTo(expected)); } - private Value intValue(int val) { - return Value.newBuilder().setInt32Val(val).build(); - } - - private Value strValue(String val) { - return Value.newBuilder().setStringVal(val).build(); - } - - private FeatureSetSpec getFeatureSetSpec() { - return FeatureSetSpec.newBuilder() - .setName("featureSet") - .addEntities(EntitySpec.newBuilder().setName("entity1")) - .addEntities(EntitySpec.newBuilder().setName("entity2")) - .setMaxAge(Duration.newBuilder().setSeconds(30)) + private FeatureTableSpec getFeatureTableSpec() { + return FeatureTableSpec.newBuilder() + .setName("featuretable_1") + .addEntities("entity1") + .addEntities("entity2") + .addFeatures( + FeatureProto.FeatureSpecV2.newBuilder() + .setName("feature_1") + .setValueType(ValueProto.ValueType.Enum.STRING) + .build()) + .addFeatures( + FeatureProto.FeatureSpecV2.newBuilder() + .setName("feature_2") + .setValueType(ValueProto.ValueType.Enum.STRING) + .build()) + .setMaxAge(Duration.newBuilder().setSeconds(120)) .build(); } - private GetOnlineFeaturesRequest getOnlineFeaturesRequest( - List featureReferences) { - return GetOnlineFeaturesRequest.newBuilder() - .setOmitEntitiesInResponse(false) + private GetOnlineFeaturesRequestV2 getOnlineFeaturesRequestV2( + String projectName, List featureReferences) { + return GetOnlineFeaturesRequestV2.newBuilder() + .setProject(projectName) .addAllFeatures(featureReferences) .addEntityRows( - EntityRow.newBuilder() - .setEntityTimestamp(Timestamp.newBuilder().setSeconds(100)) - .putFields("entity1", intValue(1)) - .putFields("entity2", strValue("a"))) + GetOnlineFeaturesRequestV2.EntityRow.newBuilder() + .setTimestamp(Timestamp.newBuilder().setSeconds(100)) + .putFields("entity1", createInt64Value(1)) + .putFields("entity2", createStrValue("a"))) .addEntityRows( - EntityRow.newBuilder() - .setEntityTimestamp(Timestamp.newBuilder().setSeconds(100)) - .putFields("entity1", intValue(2)) - .putFields("entity2", strValue("b"))) + GetOnlineFeaturesRequestV2.EntityRow.newBuilder() + .setTimestamp(Timestamp.newBuilder().setSeconds(100)) + .putFields("entity1", createInt64Value(2)) + .putFields("entity2", createStrValue("b"))) + .addFeatures( + ServingAPIProto.FeatureReferenceV2.newBuilder() + .setFeatureTable("featuretable_1") + .setName("feature_1") + .build()) + .addFeatures( + ServingAPIProto.FeatureReferenceV2.newBuilder() + .setFeatureTable("featuretable_1") + .setName("feature_2") + .build()) .build(); } } diff --git a/serving/src/test/java/feast/serving/service/RedisBackedJobServiceTest.java b/serving/src/test/java/feast/serving/service/RedisBackedJobServiceTest.java deleted file mode 100644 index 23626c2cb85..00000000000 --- a/serving/src/test/java/feast/serving/service/RedisBackedJobServiceTest.java +++ /dev/null @@ -1,60 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.serving.service; - -import io.lettuce.core.RedisClient; -import io.lettuce.core.RedisURI; -import io.lettuce.core.codec.ByteArrayCodec; -import java.io.IOException; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; -import redis.embedded.RedisServer; - -public class RedisBackedJobServiceTest { - - private static Integer REDIS_PORT = 51235; - private RedisServer redis; - - @Before - public void setUp() throws IOException { - redis = new RedisServer(REDIS_PORT); - redis.start(); - } - - @After - public void teardown() { - redis.stop(); - } - - @Test - public void shouldRecoverIfRedisConnectionIsLost() { - RedisClient client = RedisClient.create(RedisURI.create("localhost", REDIS_PORT)); - RedisBackedJobService jobService = - new RedisBackedJobService(client.connect(new ByteArrayCodec())); - jobService.get("does not exist"); - redis.stop(); - try { - jobService.get("does not exist"); - } catch (Exception e) { - // pass, this should fail, and return a broken connection to the pool - } - redis.start(); - jobService.get("does not exist"); - client.shutdown(); - } -} diff --git a/serving/src/test/resources/docker-compose/docker-compose-it.yml b/serving/src/test/resources/docker-compose/docker-compose-it.yml index edef066fd52..fb7fb1f6adc 100644 --- a/serving/src/test/resources/docker-compose/docker-compose-it.yml +++ b/serving/src/test/resources/docker-compose/docker-compose-it.yml @@ -18,20 +18,6 @@ services: - -jar - /opt/feast/feast-core.jar - --spring.config.location=classpath:/application.yml,file:/etc/feast/application.yml - - jobcontroller: - image: gcr.io/kf-feast/feast-jobcontroller:latest - volumes: - - ./job-controller/application-it.yml:/etc/feast/application.yml - depends_on: - - kafka - ports: - - 6570:6570 - command: - - java - - -jar - - /opt/feast/feast-job-controller.jar - - --spring.config.location=classpath:/application.yml,file:/etc/feast/application.yml kafka: image: confluentinc/cp-kafka:5.2.1 diff --git a/serving/src/test/resources/docker-compose/job-controller/application-it.yml b/serving/src/test/resources/docker-compose/job-controller/application-it.yml deleted file mode 100644 index 118ce378726..00000000000 --- a/serving/src/test/resources/docker-compose/job-controller/application-it.yml +++ /dev/null @@ -1,16 +0,0 @@ -feast: - core-host: core - jobs: - enabled: true - polling_interval_milliseconds: 30000 - job_update_timeout_seconds: 240 - active_runner: direct - runners: - - name: direct - type: DirectRunner - options: {} - stream: - type: kafka - options: - topic: feast-features - bootstrapServers: "kafka:9092,localhost:9094" \ No newline at end of file diff --git a/spark/ingestion/pom.xml b/spark/ingestion/pom.xml index d78984837c9..0e0bf8385c5 100644 --- a/spark/ingestion/pom.xml +++ b/spark/ingestion/pom.xml @@ -33,7 +33,7 @@ 2.12 ${scala.version}.12 - 2.4.7 + 3.0.1 4.4.0 3.3.0 3.12.2 @@ -98,6 +98,13 @@ provided + + org.apache.spark + spark-catalyst_${scala.version} + ${spark.version} + provided + + org.codehaus.janino janino @@ -131,7 +138,7 @@ com.google.cloud.spark spark-bigquery_${scala.version} - 0.17.2 + 0.18.0 provided @@ -147,18 +154,18 @@ 2.5.0 - - org.apache.arrow - arrow-vector - 0.16.0 - - io.netty netty-all 4.1.52.Final + + commons-io + commons-io + 2.5 + + org.json4s json4s-ext_${scala.version} @@ -182,14 +189,14 @@ com.dimafeng testcontainers-scala-scalatest_${scala.version} - 0.38.3 + 0.38.8 test com.dimafeng testcontainers-scala-kafka_${scala.version} - 0.38.3 + 0.38.8 test @@ -251,6 +258,11 @@ test + + ${project.build.directory}/test-reports + . + WDF TestSuite.txt + @@ -359,6 +371,24 @@ true + + exec-maven-plugin + org.codehaus.mojo + + + Python UDF setup + generate-test-sources + + exec + + + ${skipITs} + ./setup.sh + ${basedir}/src/test/resources/python/ + + + + diff --git a/spark/ingestion/src/main/scala/feast/ingestion/BasePipeline.scala b/spark/ingestion/src/main/scala/feast/ingestion/BasePipeline.scala index cc1f451ae74..13dd51f280c 100644 --- a/spark/ingestion/src/main/scala/feast/ingestion/BasePipeline.scala +++ b/spark/ingestion/src/main/scala/feast/ingestion/BasePipeline.scala @@ -16,6 +16,7 @@ */ package feast.ingestion +import org.apache.log4j.{Level, Logger} import org.apache.spark.SparkConf import org.apache.spark.sql.{Column, SparkSession} import org.apache.spark.sql.functions.expr @@ -26,6 +27,8 @@ trait BasePipeline { // workaround for issue with arrow & netty // see https://github.com/apache/arrow/tree/master/java#java-properties System.setProperty("io.netty.tryReflectionSetAccessible", "true") + // suppress SubscriptionState logs + Logger.getLogger("org.apache.kafka").setLevel(Level.WARN) val conf = new SparkConf() @@ -41,12 +44,8 @@ trait BasePipeline { case Some(c: StatsDConfig) => conf .set( - "spark.metrics.conf.*.source.redis.class", - "org.apache.spark.metrics.source.RedisSinkMetricSource" - ) - .set( - "spark.metrics.conf.*.source.redis.labels", - s"feature_table=${jobConfig.featureTable.name}" + "spark.metrics.labels", + s"feature_table=${jobConfig.featureTable.name},project=${jobConfig.featureTable.project}" ) .set( "spark.metrics.conf.*.sink.statsd.class", @@ -56,7 +55,9 @@ trait BasePipeline { .set("spark.metrics.conf.*.sink.statsd.port", c.port.toString) .set("spark.metrics.conf.*.sink.statsd.period", "30") .set("spark.metrics.conf.*.sink.statsd.unit", "seconds") - .set("spark.metrics.namespace", jobConfig.mode.toString) + .set("spark.metrics.namespace", s"feast_${jobConfig.mode.toString.toLowerCase}") + // until proto parser udf will be fixed, we have to use this + .set("spark.sql.legacy.allowUntypedScalaUDF", "true") case None => () } diff --git a/spark/ingestion/src/main/scala/feast/ingestion/BatchPipeline.scala b/spark/ingestion/src/main/scala/feast/ingestion/BatchPipeline.scala index a54c83140f1..54b5a6d073e 100644 --- a/spark/ingestion/src/main/scala/feast/ingestion/BatchPipeline.scala +++ b/spark/ingestion/src/main/scala/feast/ingestion/BatchPipeline.scala @@ -16,12 +16,14 @@ */ package feast.ingestion +import feast.ingestion.metrics.IngestionPipelineMetrics import feast.ingestion.sources.bq.BigQueryReader import feast.ingestion.sources.file.FileReader import feast.ingestion.validation.{RowValidator, TypeCheck} import org.apache.commons.lang.StringUtils import org.apache.spark.SparkEnv -import org.apache.spark.sql.{SaveMode, SparkSession} +import org.apache.spark.sql.catalyst.encoders.RowEncoder +import org.apache.spark.sql.{Encoder, Row, SaveMode, SparkSession} /** * Batch Ingestion Flow: @@ -36,7 +38,8 @@ object BatchPipeline extends BasePipeline { val featureTable = config.featureTable val projection = inputProjection(config.source, featureTable.features, featureTable.entities) - val validator = new RowValidator(featureTable, config.source.eventTimestampColumn) + val rowValidator = new RowValidator(featureTable, config.source.eventTimestampColumn) + val metrics = new IngestionPipelineMetrics val input = config.source match { case source: BQSource => @@ -57,6 +60,8 @@ object BatchPipeline extends BasePipeline { val projected = input.select(projection: _*).cache() + implicit def rowEncoder: Encoder[Row] = RowEncoder(projected.schema) + TypeCheck.allTypesMatch(projected.schema, featureTable) match { case Some(error) => throw new RuntimeException(s"Dataframe columns don't match expected feature types: $error") @@ -64,7 +69,8 @@ object BatchPipeline extends BasePipeline { } val validRows = projected - .filter(validator.checkAll) + .map(metrics.incrementRead) + .filter(rowValidator.allChecks) validRows.write .format("feast.ingestion.stores.redis") @@ -72,12 +78,14 @@ object BatchPipeline extends BasePipeline { .option("namespace", featureTable.name) .option("project_name", featureTable.project) .option("timestamp_column", config.source.eventTimestampColumn) + .option("max_age", config.featureTable.maxAge.getOrElse(0L)) .save() config.deadLetterPath match { case Some(path) => projected - .filter(!validator.checkAll) + .filter(!rowValidator.allChecks) + .map(metrics.incrementDeadLetters) .write .format("parquet") .mode(SaveMode.Append) diff --git a/spark/ingestion/src/main/scala/feast/ingestion/IngestionJob.scala b/spark/ingestion/src/main/scala/feast/ingestion/IngestionJob.scala index f4d0dcc1159..290dc07fbfd 100644 --- a/spark/ingestion/src/main/scala/feast/ingestion/IngestionJob.scala +++ b/spark/ingestion/src/main/scala/feast/ingestion/IngestionJob.scala @@ -16,8 +16,8 @@ */ package feast.ingestion +import feast.ingestion.utils.JsonUtils import org.joda.time.DateTime - import org.json4s._ import org.json4s.jackson.JsonMethods.{parse => parseJSON} import org.json4s.ext.JavaEnumNameSerializer @@ -29,9 +29,9 @@ object IngestionJob { new JavaEnumNameSerializer[feast.proto.types.ValueProto.ValueType.Enum]() + ShortTypeHints(List(classOf[ProtoFormat], classOf[AvroFormat])) - val parser = new scopt.OptionParser[IngestionJobConfig]("IngestionJon") { + val parser = new scopt.OptionParser[IngestionJobConfig]("IngestionJob") { // ToDo: read version from Manifest - head("feast.ingestion.IngestionJob", "0.8.0") + head("feast.ingestion.IngestionJob", "0.9.0") opt[Modes]("mode") .action((x, c) => c.copy(mode = x)) @@ -39,18 +39,34 @@ object IngestionJob { .text("Mode to operate ingestion job (offline or online)") opt[String](name = "source") - .action((x, c) => - parseJSON(x).camelizeKeys.extract[Sources] match { + .action((x, c) => { + val json = parseJSON(x) + JsonUtils + .mapFieldWithParent(json) { + case (parent: String, (key: String, v: JValue)) if !parent.equals("field_mapping") => + JsonUtils.camelize(key) -> v + case (_, x) => x + } + .extract[Sources] match { case Sources(file: Some[FileSource], _, _) => c.copy(source = file.get) case Sources(_, bq: Some[BQSource], _) => c.copy(source = bq.get) case Sources(_, _, kafka: Some[KafkaSource]) => c.copy(source = kafka.get) } - ) + }) .required() .text("JSON-encoded source object (e.g. {\"kafka\":{\"bootstrapServers\":...}}") opt[String](name = "feature-table") - .action((x, c) => c.copy(featureTable = parseJSON(x).camelizeKeys.extract[FeatureTable])) + .action((x, c) => { + val ft = parseJSON(x).camelizeKeys.extract[FeatureTable] + + c.copy( + featureTable = ft, + streamingTriggeringSecs = ft.labels.getOrElse("_streaming_trigger_secs", "0").toInt, + validationConfig = + ft.labels.get("_validation").map(parseJSON(_).camelizeKeys.extract[ValidationConfig]) + ) + }) .required() .text("JSON-encoded FeatureTableSpec object") @@ -73,6 +89,9 @@ object IngestionJob { opt[String](name = "stencil-url") .action((x, c) => c.copy(stencilURL = Some(x))) + + opt[Unit](name = "drop-invalid") + .action((_, c) => c.copy(doNotIngestInvalidRows = true)) } def main(args: Array[String]): Unit = { diff --git a/spark/ingestion/src/main/scala/feast/ingestion/IngestionJobConfig.scala b/spark/ingestion/src/main/scala/feast/ingestion/IngestionJobConfig.scala index c922a1c096e..9b88e6a7e84 100644 --- a/spark/ingestion/src/main/scala/feast/ingestion/IngestionJobConfig.scala +++ b/spark/ingestion/src/main/scala/feast/ingestion/IngestionJobConfig.scala @@ -59,6 +59,8 @@ case class FileSource( override val datePartitionColumn: Option[String] = None ) extends BatchSource +case class BQMaterializationConfig(project: String, dataset: String) + case class BQSource( project: String, dataset: String, @@ -66,7 +68,8 @@ case class BQSource( override val fieldMapping: Map[String, String], override val eventTimestampColumn: String, override val createdTimestampColumn: Option[String] = None, - override val datePartitionColumn: Option[String] = None + override val datePartitionColumn: Option[String] = None, + materialization: Option[BQMaterializationConfig] = None ) extends BatchSource case class KafkaSource( @@ -91,7 +94,15 @@ case class FeatureTable( name: String, project: String, entities: Seq[Field], - features: Seq[Field] + features: Seq[Field], + maxAge: Option[Long] = None, + labels: Map[String, String] = Map.empty +) + +case class ValidationConfig( + name: String, + pickledCodePath: String, + includeArchivePath: String ) case class IngestionJobConfig( @@ -103,5 +114,8 @@ case class IngestionJobConfig( store: StoreConfig = RedisConfig("localhost", 6379, false), metrics: Option[MetricConfig] = None, deadLetterPath: Option[String] = None, - stencilURL: Option[String] = None + stencilURL: Option[String] = None, + streamingTriggeringSecs: Int = 0, + validationConfig: Option[ValidationConfig] = None, + doNotIngestInvalidRows: Boolean = false ) diff --git a/spark/ingestion/src/main/scala/feast/ingestion/StreamingPipeline.scala b/spark/ingestion/src/main/scala/feast/ingestion/StreamingPipeline.scala index 1945d4aa0f2..4b37950c2e8 100644 --- a/spark/ingestion/src/main/scala/feast/ingestion/StreamingPipeline.scala +++ b/spark/ingestion/src/main/scala/feast/ingestion/StreamingPipeline.scala @@ -16,15 +16,26 @@ */ package feast.ingestion +import java.io.File +import java.util.concurrent.TimeUnit + +import feast.ingestion.metrics.IngestionPipelineMetrics import feast.ingestion.registry.proto.ProtoRegistryFactory -import org.apache.spark.sql.{DataFrame, Row, SaveMode, SparkSession} -import org.apache.spark.sql.functions.udf +import org.apache.spark.sql.{DataFrame, Encoder, Row, SaveMode, SparkSession} +import org.apache.spark.sql.functions.{expr, struct, udf} import feast.ingestion.utils.ProtoReflection +import feast.ingestion.utils.testing.MemoryStreamingSource import feast.ingestion.validation.{RowValidator, TypeCheck} +import org.apache.commons.io.FileUtils import org.apache.commons.lang.StringUtils -import org.apache.spark.SparkEnv +import org.apache.spark.{SparkEnv, SparkFiles} +import org.apache.spark.api.python.DynamicPythonFunction import org.apache.spark.sql.streaming.StreamingQuery import org.apache.spark.sql.avro._ +import org.apache.spark.sql.execution.python.UserDefinedPythonFunction +import org.apache.spark.sql.execution.streaming.ProcessingTimeTrigger +import org.apache.spark.sql.types.BooleanType +import org.apache.spark.sql.catalyst.encoders.RowEncoder /** * Streaming pipeline (currently in micro-batches mode only, since we need to have multiple sinks: redis & deadletters). @@ -45,7 +56,9 @@ object StreamingPipeline extends BasePipeline with Serializable { val featureTable = config.featureTable val projection = inputProjection(config.source, featureTable.features, featureTable.entities) - val validator = new RowValidator(featureTable, config.source.eventTimestampColumn) + val rowValidator = new RowValidator(featureTable, config.source.eventTimestampColumn) + val metrics = new IngestionPipelineMetrics + val validationUDF = createValidationUDF(sparkSession, config) val input = config.source match { case source: KafkaSource => @@ -54,6 +67,8 @@ object StreamingPipeline extends BasePipeline with Serializable { .option("kafka.bootstrap.servers", source.bootstrapServers) .option("subscribe", source.topic) .load() + case source: MemoryStreamingSource => + source.read } val parsed = config.source.asInstanceOf[StreamingSource].format match { @@ -62,6 +77,9 @@ object StreamingPipeline extends BasePipeline with Serializable { input.withColumn("features", parser($"value")) case AvroFormat(schemaJson) => input.select(from_avro($"value", schemaJson).alias("features")) + case _ => + val columns = input.columns.map(input(_)) + input.select(struct(columns: _*).alias("features")) } val projected = parsed @@ -76,38 +94,51 @@ object StreamingPipeline extends BasePipeline with Serializable { val query = projected.writeStream .foreachBatch { (batchDF: DataFrame, batchID: Long) => - batchDF.persist() - - val validRows = batchDF - .filter(validator.checkAll) + val rowsAfterValidation = if (validationUDF.nonEmpty) { + val columns = batchDF.columns.map(batchDF(_)) + batchDF.withColumn( + "_isValid", + rowValidator.allChecks && validationUDF.get(struct(columns: _*)) + ) + } else { + batchDF.withColumn("_isValid", rowValidator.allChecks) + } + rowsAfterValidation.persist() + implicit def rowEncoder: Encoder[Row] = RowEncoder(rowsAfterValidation.schema) - validRows.write + rowsAfterValidation + .map(metrics.incrementRead) + .filter(if (config.doNotIngestInvalidRows) expr("_isValid") else rowValidator.allChecks) + .write .format("feast.ingestion.stores.redis") .option("entity_columns", featureTable.entities.map(_.name).mkString(",")) .option("namespace", featureTable.name) .option("project_name", featureTable.project) .option("timestamp_column", config.source.eventTimestampColumn) + .option("max_age", config.featureTable.maxAge.getOrElse(0L)) .save() config.deadLetterPath match { case Some(path) => - batchDF - .filter(!validator.checkAll) + rowsAfterValidation + .filter("!_isValid") + .map(metrics.incrementDeadLetters) .write .format("parquet") .mode(SaveMode.Append) .save(StringUtils.stripEnd(path, "/") + "/" + SparkEnv.get.conf.getAppId) case _ => - batchDF - .filter(!validator.checkAll) + rowsAfterValidation + .filter("!_isValid") .foreach(r => { println(s"Row failed validation $r") }) } - batchDF.unpersist() + rowsAfterValidation.unpersist() () // return Unit to avoid compile error with overloaded foreachBatch } + .trigger(ProcessingTimeTrigger.create(config.streamingTriggeringSecs, TimeUnit.SECONDS)) .start() Some(query) @@ -118,6 +149,45 @@ object StreamingPipeline extends BasePipeline with Serializable { val parser: Array[Byte] => Row = ProtoReflection.createMessageParser(protoRegistry, className) + // ToDo: create correctly typed parser + // spark deprecated returnType argument, instead it will infer it from udf function signature udf(parser, ProtoReflection.inferSchema(protoRegistry.getProtoDescriptor(className))) } + + private def createValidationUDF( + sparkSession: SparkSession, + config: IngestionJobConfig + ): Option[UserDefinedPythonFunction] = + config.validationConfig.map { validationConfig => + if (validationConfig.includeArchivePath.nonEmpty) { + val archivePath = + DynamicPythonFunction.libsPathWithPlatform(validationConfig.includeArchivePath) + sparkSession.sparkContext.addFile(archivePath) + } + + // this is the trick to download remote file on the driver + // after file added to sparkContext it will be immediately fetched to local dir (accessible via SparkFiles) + sparkSession.sparkContext.addFile(validationConfig.pickledCodePath) + val fileName = validationConfig.pickledCodePath.split("/").last + val pickledCode = FileUtils.readFileToByteArray(new File(SparkFiles.get(fileName))) + + val env = config.metrics match { + case Some(c: StatsDConfig) => + Map( + "STATSD_HOST" -> c.host, + "STATSD_PORT" -> c.port.toString, + "FEAST_INGESTION_FEATURE_TABLE" -> config.featureTable.name, + "FEAST_INGESTION_PROJECT_NAME" -> config.featureTable.project + ) + case _ => Map.empty[String, String] + } + + UserDefinedPythonFunction( + validationConfig.name, + DynamicPythonFunction.create(pickledCode, env), + BooleanType, + pythonEvalType = 200, // SQL_SCALAR_PANDAS_UDF (original constant is in private object) + udfDeterministic = true + ) + } } diff --git a/spark/ingestion/src/main/scala/feast/ingestion/metrics/IngestionPipelineMetrics.scala b/spark/ingestion/src/main/scala/feast/ingestion/metrics/IngestionPipelineMetrics.scala new file mode 100644 index 00000000000..99f112890ee --- /dev/null +++ b/spark/ingestion/src/main/scala/feast/ingestion/metrics/IngestionPipelineMetrics.scala @@ -0,0 +1,64 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * Copyright 2018-2020 The Feast Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package feast.ingestion.metrics + +import org.apache.spark.SparkEnv +import org.apache.spark.metrics.source.IngestionPipelineMetricSource +import org.apache.spark.sql.Row + +class IngestionPipelineMetrics extends Serializable { + + def incrementDeadLetters(row: Row): Row = { + if (metricSource.nonEmpty) + metricSource.get.METRIC_DEADLETTER_ROWS_INSERTED.inc() + + row + } + + def incrementRead(row: Row): Row = { + if (metricSource.nonEmpty) + metricSource.get.METRIC_ROWS_READ_FROM_SOURCE.inc() + + row + } + + def incrementRead(inc: Long): Unit = { + if (metricSource.nonEmpty) + metricSource.get.METRIC_ROWS_READ_FROM_SOURCE.inc(inc) + } + + def incrementDeadLetters(inc: Long): Unit = { + if (metricSource.nonEmpty) + metricSource.get.METRIC_DEADLETTER_ROWS_INSERTED.inc(inc) + } + + private lazy val metricSource: Option[IngestionPipelineMetricSource] = { + val metricsSystem = SparkEnv.get.metricsSystem + IngestionPipelineMetricsLock.synchronized { + if (metricsSystem.getSourcesByName(IngestionPipelineMetricSource.sourceName).isEmpty) { + metricsSystem.registerSource(new IngestionPipelineMetricSource) + } + } + + metricsSystem.getSourcesByName(IngestionPipelineMetricSource.sourceName) match { + case Seq(head) => Some(head.asInstanceOf[IngestionPipelineMetricSource]) + case _ => None + } + } +} + +private object IngestionPipelineMetricsLock diff --git a/spark/ingestion/src/main/scala/feast/ingestion/metrics/StatsdReporterWithTags.scala b/spark/ingestion/src/main/scala/feast/ingestion/metrics/StatsdReporterWithTags.scala index 66b48dd444e..880fc2b9829 100644 --- a/spark/ingestion/src/main/scala/feast/ingestion/metrics/StatsdReporterWithTags.scala +++ b/spark/ingestion/src/main/scala/feast/ingestion/metrics/StatsdReporterWithTags.scala @@ -125,8 +125,13 @@ class StatsdReporterWithTags( private def reportGauge(name: String, gauge: Gauge[_])(implicit socket: DatagramSocket): Unit = formatAny(gauge.getValue).foreach(v => send(fullName(name), v, GAUGE)) - private def reportCounter(name: String, counter: Counter)(implicit socket: DatagramSocket): Unit = - send(fullName(name), format(counter.getCount), COUNTER) + private def reportCounter(name: String, counter: Counter)(implicit + socket: DatagramSocket + ): Unit = { + val snapshot = counter.getCount + send(fullName(name), format(snapshot), COUNTER) + counter.dec(snapshot) // reset counter + } private def reportHistogram(name: String, histogram: Histogram)(implicit socket: DatagramSocket @@ -169,19 +174,28 @@ class StatsdReporterWithTags( reportMetered(name, timer) } + private val nameWithTag = """(\S+)#(\S+)""".r + private def send(name: String, value: String, metricType: String)(implicit socket: DatagramSocket ): Unit = { - val bytes = sanitize(s"$name:$value|$metricType").getBytes(UTF_8) + val bytes = name match { + case nameWithTag(name, tags) => + val tagsWithSemicolon = tags.replace('=', ':') + sanitize(s"$name:$value|$metricType|#$tagsWithSemicolon").getBytes(UTF_8) + case _ => + sanitize(s"$name:$value|$metricType").getBytes(UTF_8) + } val packet = new DatagramPacket(bytes, bytes.length, address) socket.send(packet) } - private val nameWithTag = """(\S+)#(\S+)""".r - private def fullName(name: String, suffixes: String*): String = name match { case nameWithTag(name, tags) => - MetricRegistry.name(prefix, name +: suffixes: _*) ++ "#" ++ tags + // filter out parts that consists only from numbers + // that could be executor-id for example + val stableName = name.split('.').filterNot(_ forall Character.isDigit).mkString(".") + MetricRegistry.name(prefix, stableName +: suffixes: _*) ++ "#" ++ tags case _ => MetricRegistry.name(prefix, name +: suffixes: _*) } diff --git a/spark/ingestion/src/main/scala/feast/ingestion/sources/bq/BigQueryReader.scala b/spark/ingestion/src/main/scala/feast/ingestion/sources/bq/BigQueryReader.scala index 1b0a57eea98..1b46139bcd5 100644 --- a/spark/ingestion/src/main/scala/feast/ingestion/sources/bq/BigQueryReader.scala +++ b/spark/ingestion/src/main/scala/feast/ingestion/sources/bq/BigQueryReader.scala @@ -30,9 +30,20 @@ object BigQueryReader { start: DateTime, end: DateTime ): DataFrame = { - sqlContext.read + val reader = sqlContext.read .format("bigquery") .option("viewsEnabled", "true") + + source.materialization match { + case Some(materializationConfig) => + reader + .option("materializationProject", materializationConfig.project) + .option("materializationDataset", materializationConfig.dataset) + + case _ => () + } + + reader .load(s"${source.project}.${source.dataset}.${source.table}") .filter(col(source.eventTimestampColumn) >= new Timestamp(start.getMillis)) .filter(col(source.eventTimestampColumn) < new Timestamp(end.getMillis)) diff --git a/spark/ingestion/src/main/scala/feast/ingestion/sources/file/FileReader.scala b/spark/ingestion/src/main/scala/feast/ingestion/sources/file/FileReader.scala index 5bad029b5b7..55d5c901eab 100644 --- a/spark/ingestion/src/main/scala/feast/ingestion/sources/file/FileReader.scala +++ b/spark/ingestion/src/main/scala/feast/ingestion/sources/file/FileReader.scala @@ -16,7 +16,7 @@ */ package feast.ingestion.sources.file -import java.sql.Timestamp +import java.sql.{Timestamp, Date} import feast.ingestion.FileSource import org.apache.spark.sql.functions.col @@ -30,9 +30,17 @@ object FileReader { start: DateTime, end: DateTime ): DataFrame = { - sqlContext.read + val reader = sqlContext.read .parquet(source.path) .filter(col(source.eventTimestampColumn) >= new Timestamp(start.getMillis)) .filter(col(source.eventTimestampColumn) < new Timestamp(end.getMillis)) + + source.datePartitionColumn match { + case Some(partitionColumn) if partitionColumn.nonEmpty => + reader + .filter(col(partitionColumn) >= new Date(start.getMillis)) + .filter(col(partitionColumn) <= new Date(end.getMillis)) + case _ => reader + } } } diff --git a/spark/ingestion/src/main/scala/feast/ingestion/stores/redis/HashTypePersistence.scala b/spark/ingestion/src/main/scala/feast/ingestion/stores/redis/HashTypePersistence.scala index b34f0667c0e..00ab8736302 100644 --- a/spark/ingestion/src/main/scala/feast/ingestion/stores/redis/HashTypePersistence.scala +++ b/spark/ingestion/src/main/scala/feast/ingestion/stores/redis/HashTypePersistence.scala @@ -16,16 +16,17 @@ */ package feast.ingestion.stores.redis -import org.apache.spark.sql.Row -import org.apache.spark.sql.types._ -import redis.clients.jedis.{Pipeline, Response} import java.nio.charset.StandardCharsets +import java.util import com.google.common.hash.Hashing - -import scala.jdk.CollectionConverters._ import com.google.protobuf.Timestamp import feast.ingestion.utils.TypeConversion +import org.apache.spark.sql.Row +import org.apache.spark.sql.types._ +import redis.clients.jedis.{Pipeline, Response} + +import scala.jdk.CollectionConverters._ /** * Use Redis hash type as storage layout. Every feature is stored as separate entry in Hash. @@ -35,10 +36,10 @@ import feast.ingestion.utils.TypeConversion * Values are serialized with protobuf (`ValueProto`). */ class HashTypePersistence(config: SparkRedisConfig) extends Persistence with Serializable { - def encodeRow( - keyColumns: Array[String], - timestampField: String, - value: Row + + private def encodeRow( + value: Row, + maxExpiryTimestamp: java.sql.Timestamp ): Map[Array[Byte], Array[Byte]] = { val fields = value.schema.fields.map(_.name) val types = value.schema.fields.map(f => (f.name, f.dataType)).toMap @@ -51,49 +52,87 @@ class HashTypePersistence(config: SparkRedisConfig) extends Persistence with Ser } .filter { case (k, _) => // don't store entities & timestamp - !keyColumns.contains(k) && k != config.timestampColumn + !config.entityColumns.contains(k) && k != config.timestampColumn } .map { case (k, v) => encodeKey(k) -> encodeValue(v, types(k)) } - val timestamp = Seq( + val timestampHash = Seq( ( - timestampField.getBytes, + timestampHashKey(config.namespace).getBytes, encodeValue(value.getAs[Timestamp](config.timestampColumn), TimestampType) ) ) - values ++ timestamp + val expiryUnixTimestamp = { + if (config.maxAge > 0) + value.getAs[java.sql.Timestamp](config.timestampColumn).getTime + config.maxAge * 1000 + else maxExpiryTimestamp.getTime + } + val expiryTimestamp = new java.sql.Timestamp(expiryUnixTimestamp) + val expiryTimestampHash = Seq( + ( + expiryTimestampHashKey(config.namespace).getBytes, + encodeValue(expiryTimestamp, TimestampType) + ) + ) + + values ++ timestampHash ++ expiryTimestampHash } - def encodeValue(value: Any, `type`: DataType): Array[Byte] = { + private def encodeValue(value: Any, `type`: DataType): Array[Byte] = { TypeConversion.sqlTypeToProtoValue(value, `type`).toByteArray } - def encodeKey(key: String): Array[Byte] = { + private def encodeKey(key: String): Array[Byte] = { val fullFeatureReference = s"${config.namespace}:$key" Hashing.murmur3_32.hashString(fullFeatureReference, StandardCharsets.UTF_8).asBytes() } - def save( + private def timestampHashKey(namespace: String): String = { + s"${config.timestampPrefix}:${namespace}" + } + + private def expiryTimestampHashKey(namespace: String): String = { + s"${config.expiryPrefix}:${namespace}" + } + + private def decodeTimestamp(encodedTimestamp: Array[Byte]): java.sql.Timestamp = { + new java.sql.Timestamp(Timestamp.parseFrom(encodedTimestamp).getSeconds * 1000) + } + + override def save( pipeline: Pipeline, key: Array[Byte], - value: Map[Array[Byte], Array[Byte]], - ttl: Int + row: Row, + expiryTimestamp: java.sql.Timestamp, + maxExpiryTimestamp: java.sql.Timestamp ): Unit = { - pipeline.hset(key, value.asJava) - if (ttl > 0) { - pipeline.expire(key, ttl) + val value = encodeRow(row, maxExpiryTimestamp).asJava + pipeline.hset(key, value) + if (expiryTimestamp.equals(maxExpiryTimestamp)) { + pipeline.persist(key) + } else { + pipeline.expireAt(key, expiryTimestamp.getTime / 1000) } } - def getTimestamp( + override def get( pipeline: Pipeline, - key: Array[Byte], - timestampField: String - ): Response[Array[Byte]] = { - pipeline.hget(key, timestampField.getBytes) + key: Array[Byte] + ): Response[util.Map[Array[Byte], Array[Byte]]] = { + pipeline.hgetAll(key) } + override def storedTimestamp( + value: util.Map[Array[Byte], Array[Byte]] + ): Option[java.sql.Timestamp] = { + value.asScala.toMap + .map { case (key, value) => + (key.map(_.toChar).mkString, value) + } + .get(timestampHashKey(config.namespace)) + .map(value => decodeTimestamp(value)) + } } diff --git a/spark/ingestion/src/main/scala/feast/ingestion/stores/redis/Persistence.scala b/spark/ingestion/src/main/scala/feast/ingestion/stores/redis/Persistence.scala index 47161358c2e..4c4b1690c0b 100644 --- a/spark/ingestion/src/main/scala/feast/ingestion/stores/redis/Persistence.scala +++ b/spark/ingestion/src/main/scala/feast/ingestion/stores/redis/Persistence.scala @@ -16,26 +16,58 @@ */ package feast.ingestion.stores.redis +import java.sql.Timestamp +import java.util + import org.apache.spark.sql.Row import redis.clients.jedis.{Pipeline, Response} +/** + * Determine how a Spark row should be serialized and stored on Redis. + */ trait Persistence { - def encodeRow( - keyColumns: Array[String], - timestampField: String, - value: Row - ): Map[Array[Byte], Array[Byte]] + /** + * Persist a Spark row to Redis + * + * @param pipeline Redis pipeline + * @param key Redis key in serialized bytes format + * @param row Row representing the value to be persist + * @param expiryTimestamp Expiry timestamp for the row + * @param maxExpiryTimestamp No ttl should be set if the expiry timestamp + * is equal to the maxExpiryTimestamp + */ def save( pipeline: Pipeline, key: Array[Byte], - value: Map[Array[Byte], Array[Byte]], - ttl: Int + row: Row, + expiryTimestamp: Timestamp, + maxExpiryTimestamp: Timestamp ): Unit - def getTimestamp( + /** + * Returns a Redis response, which can be used by `storedTimestamp` and `newExpiryTimestamp` to + * derive the currently stored event timestamp, and the updated expiry timestamp. This method will + * be called prior to persisting the row to Redis, so that `RedisSinkRelation` can decide whether + * the currently stored value should be updated. + * + * @param pipeline Redis pipeline + * @param key Redis key in serialized bytes format + * @return Redis response representing the row value + */ + def get( pipeline: Pipeline, - key: Array[Byte], - timestampField: String - ): Response[Array[Byte]] + key: Array[Byte] + ): Response[util.Map[Array[Byte], Array[Byte]]] + + /** + * Returns the currently stored event timestamp for the key and the feature table associated with the ingestion job. + * + * @param value Response returned from `get` + * @return Stored event timestamp associated with the key. Returns `None` if + * the key is not present in Redis, or if timestamp information is + * unavailable on the stored value. + */ + def storedTimestamp(value: util.Map[Array[Byte], Array[Byte]]): Option[Timestamp] + } diff --git a/spark/ingestion/src/main/scala/feast/ingestion/stores/redis/RedisSinkRelation.scala b/spark/ingestion/src/main/scala/feast/ingestion/stores/redis/RedisSinkRelation.scala index d880a6461c8..2f6c758b5eb 100644 --- a/spark/ingestion/src/main/scala/feast/ingestion/stores/redis/RedisSinkRelation.scala +++ b/spark/ingestion/src/main/scala/feast/ingestion/stores/redis/RedisSinkRelation.scala @@ -16,21 +16,24 @@ */ package feast.ingestion.stores.redis +import java.util + import com.google.protobuf.Timestamp -import com.redislabs.provider.redis.{ReadWriteConfig, RedisConfig, RedisEndpoint, RedisNode} -import redis.clients.jedis.util.JedisClusterCRC16 +import com.google.protobuf.util.Timestamps import com.redislabs.provider.redis.util.PipelineUtils.{foreachWithPipeline, mapWithPipeline} +import com.redislabs.provider.redis.{ReadWriteConfig, RedisConfig, RedisEndpoint, RedisNode} import feast.ingestion.utils.TypeConversion +import feast.proto.storage.RedisProto.RedisKeyV2 +import feast.proto.types.ValueProto import org.apache.spark.SparkEnv import org.apache.spark.metrics.source.RedisSinkMetricSource +import org.apache.spark.sql.functions.col import org.apache.spark.sql.sources.{BaseRelation, InsertableRelation} import org.apache.spark.sql.types.StructType -import org.apache.spark.sql.functions.col import org.apache.spark.sql.{DataFrame, Row, SQLContext} +import redis.clients.jedis.util.JedisClusterCRC16 -import collection.JavaConverters._ -import feast.proto.storage.RedisProto.RedisKeyV2 -import feast.proto.types.ValueProto +import scala.collection.JavaConverters._ /** * High-level writer to Redis. Relies on `Persistence` implementation for actual storage layout. @@ -45,6 +48,9 @@ class RedisSinkRelation(override val sqlContext: SQLContext, config: SparkRedisC extends BaseRelation with InsertableRelation with Serializable { + + import RedisSinkRelation._ + private implicit val redisConfig: RedisConfig = { new RedisConfig( new RedisEndpoint(sqlContext.sparkContext.getConf) @@ -57,13 +63,17 @@ class RedisSinkRelation(override val sqlContext: SQLContext, config: SparkRedisC override def schema: StructType = ??? + val MAX_EXPIRED_TIMESTAMP = new java.sql.Timestamp(Timestamps.MAX_VALUE.getSeconds * 1000) + val persistence: Persistence = new HashTypePersistence(config) override def insert(data: DataFrame, overwrite: Boolean): Unit = { // repartition for deduplication val dataToStore = - if (config.repartitionByEntity) - data.repartition(config.entityColumns.map(col): _*) + if (config.repartitionByEntity && data.rdd.getNumPartitions > 1) + data + .repartition(data.rdd.getNumPartitions, config.entityColumns.map(col): _*) + .localCheckpoint() else data dataToStore.foreachPartition { partition: Iterator[Row] => @@ -75,27 +85,27 @@ class RedisSinkRelation(override val sqlContext: SQLContext, config: SparkRedisC groupKeysByNode(redisConfig.hosts, rowsWithKey.keysIterator).foreach { case (node, keys) => val conn = node.connect() - // retrieve latest stored timestamp per key - val timestamps = mapWithPipeline(conn, keys) { (pipeline, key) => - persistence.getTimestamp(pipeline, key.toByteArray, timestampField) - } - - val timestampByKey = timestamps - .map(_.asInstanceOf[Array[Byte]]) - .map( - Option(_) - .map(Timestamp.parseFrom) - .map(t => new java.sql.Timestamp(t.getSeconds * 1000)) - ) - .zip(keys) - .map(_.swap) + // retrieve latest stored values + val storedValues = mapWithPipeline(conn, keys) { (pipeline, key) => + persistence.get(pipeline, key.toByteArray) + }.map(_.asInstanceOf[util.Map[Array[Byte], Array[Byte]]]) + + val timestamps = storedValues.map(persistence.storedTimestamp) + val timestampByKey = keys.zip(timestamps).toMap + + val expiryTimestampByKey = keys + .zip(storedValues) + .map { case (key, storedValue) => + (key, newExpiryTimestamp(rowsWithKey(key), storedValue)) + } .toMap foreachWithPipeline(conn, keys) { (pipeline, key) => val row = rowsWithKey(key) timestampByKey(key) match { - case Some(t) if !t.before(row.getAs[java.sql.Timestamp](config.timestampColumn)) => () + case Some(t) if (t.after(row.getAs[java.sql.Timestamp](config.timestampColumn))) => + () case _ => if (metricSource.nonEmpty) { val lag = System.currentTimeMillis() - row @@ -105,9 +115,13 @@ class RedisSinkRelation(override val sqlContext: SQLContext, config: SparkRedisC metricSource.get.METRIC_TOTAL_ROWS_INSERTED.inc() metricSource.get.METRIC_ROWS_LAG.update(lag) } - - val encodedRow = persistence.encodeRow(config.entityColumns, timestampField, row) - persistence.save(pipeline, key.toByteArray, encodedRow, ttl = 0) + persistence.save( + pipeline, + key.toByteArray, + row, + expiryTimestampByKey(key), + MAX_EXPIRED_TIMESTAMP + ) } } conn.close() @@ -142,15 +156,21 @@ class RedisSinkRelation(override val sqlContext: SQLContext, config: SparkRedisC .build } - private def timestampField: String = { - s"${config.timestampPrefix}:${config.namespace}" - } + private lazy val metricSource: Option[RedisSinkMetricSource] = { + MetricInitializationLock.synchronized { + // RedisSinkMetricSource needs to be registered on executor and SparkEnv must already exist. + // Which is problematic, since metrics system is initialized before SparkEnv set. + // That's why I moved source registering here + if (SparkEnv.get.metricsSystem.getSourcesByName(RedisSinkMetricSource.sourceName).isEmpty) { + SparkEnv.get.metricsSystem.registerSource(new RedisSinkMetricSource) + } + } - private lazy val metricSource: Option[RedisSinkMetricSource] = SparkEnv.get.metricsSystem.getSourcesByName(RedisSinkMetricSource.sourceName) match { case Seq(head) => Some(head.asInstanceOf[RedisSinkMetricSource]) case _ => None } + } private def groupKeysByNode( nodes: Array[RedisNode], @@ -169,4 +189,35 @@ class RedisSinkRelation(override val sqlContext: SQLContext, config: SparkRedisC nodes.filter { node => node.startSlot <= slot && node.endSlot >= slot }.filter(_.idx == 0)(0) } + + private def newExpiryTimestamp( + row: Row, + value: util.Map[Array[Byte], Array[Byte]] + ): java.sql.Timestamp = { + val maxExpiryOtherFeatureTables: Long = value.asScala.toMap + .map { case (key, value) => + (key.map(_.toChar).mkString, value) + } + .filterKeys(_.startsWith(config.expiryPrefix)) + .filterKeys(_.split(":").last != config.namespace) + .values + .map(value => Timestamp.parseFrom(value).getSeconds * 1000) + .reduceOption(_ max _) + .getOrElse(0) + + val rowExpiry: Long = + if (config.maxAge > 0) + (row + .getAs[java.sql.Timestamp](config.timestampColumn) + .getTime + config.maxAge * 1000) + else MAX_EXPIRED_TIMESTAMP.getTime + + val maxExpiry = maxExpiryOtherFeatureTables max rowExpiry + new java.sql.Timestamp(maxExpiry) + + } +} + +object RedisSinkRelation { + object MetricInitializationLock } diff --git a/spark/ingestion/src/main/scala/feast/ingestion/stores/redis/SparkRedisConfig.scala b/spark/ingestion/src/main/scala/feast/ingestion/stores/redis/SparkRedisConfig.scala index 389607ce99e..8892b486437 100644 --- a/spark/ingestion/src/main/scala/feast/ingestion/stores/redis/SparkRedisConfig.scala +++ b/spark/ingestion/src/main/scala/feast/ingestion/stores/redis/SparkRedisConfig.scala @@ -23,7 +23,9 @@ case class SparkRedisConfig( timestampColumn: String, iteratorGroupingSize: Int = 1000, timestampPrefix: String = "_ts", - repartitionByEntity: Boolean = true + repartitionByEntity: Boolean = true, + maxAge: Long = 0, + expiryPrefix: String = "_ex" ) object SparkRedisConfig { @@ -32,6 +34,7 @@ object SparkRedisConfig { val TS_COLUMN = "timestamp_column" val ENTITY_REPARTITION = "entity_repartition" val PROJECT_NAME = "project_name" + val MAX_AGE = "max_age" def parse(parameters: Map[String, String]): SparkRedisConfig = SparkRedisConfig( @@ -39,6 +42,7 @@ object SparkRedisConfig { projectName = parameters.getOrElse(PROJECT_NAME, "default"), entityColumns = parameters.getOrElse(ENTITY_COLUMNS, "").split(","), timestampColumn = parameters.getOrElse(TS_COLUMN, "event_timestamp"), - repartitionByEntity = parameters.getOrElse(ENTITY_REPARTITION, "true") == "true" + repartitionByEntity = parameters.getOrElse(ENTITY_REPARTITION, "true") == "true", + maxAge = parameters.get(MAX_AGE).map(_.toLong).getOrElse(0) ) } diff --git a/spark/ingestion/src/main/scala/feast/ingestion/utils/JsonUtils.scala b/spark/ingestion/src/main/scala/feast/ingestion/utils/JsonUtils.scala new file mode 100644 index 00000000000..6eeedad508f --- /dev/null +++ b/spark/ingestion/src/main/scala/feast/ingestion/utils/JsonUtils.scala @@ -0,0 +1,47 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * Copyright 2018-2021 The Feast Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package feast.ingestion.utils + +import java.util.Locale.ENGLISH + +import org.json4s.{JArray, JField, JObject, JValue} + +object JsonUtils { + def mapFieldWithParent(jv: JValue)(f: (String, JField) => JField): JValue = { + def rec(v: JValue, parent: String = ""): JValue = v match { + case JObject(l) => JObject(l.map { case (key, va) => f(parent, key -> rec(va, key)) }) + case JArray(l) => JArray(l.map(rec(_, parent))) + case x => x + } + rec(jv) + } + + def camelize(word: String): String = { + if (word.nonEmpty) { + val w = pascalize(word) + w.substring(0, 1).toLowerCase(ENGLISH) + w.substring(1) + } else { + word + } + } + + def pascalize(word: String): String = { + val lst = word.split("_").toList + (lst.headOption.map(s => s.substring(0, 1).toUpperCase(ENGLISH) + s.substring(1)).get :: + lst.tail.map(s => s.substring(0, 1).toUpperCase + s.substring(1))).mkString("") + } +} diff --git a/ingestion/src/main/java/feast/ingestion/options/OptionCompressor.java b/spark/ingestion/src/main/scala/feast/ingestion/utils/testing/MemoryStreamingSource.scala similarity index 50% rename from ingestion/src/main/java/feast/ingestion/options/OptionCompressor.java rename to spark/ingestion/src/main/scala/feast/ingestion/utils/testing/MemoryStreamingSource.scala index b2345fc3eb1..02e8a7a678b 100644 --- a/ingestion/src/main/java/feast/ingestion/options/OptionCompressor.java +++ b/spark/ingestion/src/main/scala/feast/ingestion/utils/testing/MemoryStreamingSource.scala @@ -14,18 +14,21 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package feast.ingestion.options; +package feast.ingestion.utils.testing -import java.io.IOException; +import feast.ingestion.{DataFormat, StreamingSource} +import org.apache.spark.sql.DataFrame +import org.apache.spark.sql.execution.streaming.MemoryStream -public interface OptionCompressor { +// For test purposes +case class MemoryStreamingSource( + stream: MemoryStream[_], + override val fieldMapping: Map[String, String] = Map.empty, + override val eventTimestampColumn: String = "timestamp", + override val createdTimestampColumn: Option[String] = None, + override val datePartitionColumn: Option[String] = None +) extends StreamingSource { + def read: DataFrame = stream.toDF() - /** - * Compress pipeline option into bytes format. This is necessary as some Beam runner has - * limitation in terms of pipeline option size. - * - * @param option Pipeline option value - * @return Compressed values of the option, as byte array - */ - byte[] compress(T option) throws IOException; + override def format: DataFormat = null } diff --git a/spark/ingestion/src/main/scala/feast/ingestion/validation/RowValidator.scala b/spark/ingestion/src/main/scala/feast/ingestion/validation/RowValidator.scala index 218e7ab6d05..7bb829e3ab2 100644 --- a/spark/ingestion/src/main/scala/feast/ingestion/validation/RowValidator.scala +++ b/spark/ingestion/src/main/scala/feast/ingestion/validation/RowValidator.scala @@ -30,6 +30,6 @@ class RowValidator(featureTable: FeatureTable, timestampColumn: String) extends def timestampPresent: Column = col(timestampColumn).isNotNull - def checkAll: Column = + def allChecks: Column = allEntitiesPresent && atLeastOneFeatureNotNull && timestampPresent } diff --git a/spark/ingestion/src/main/scala/org/apache/spark/api/python/DynamicPythonFunction.scala b/spark/ingestion/src/main/scala/org/apache/spark/api/python/DynamicPythonFunction.scala new file mode 100644 index 00000000000..59565ac8935 --- /dev/null +++ b/spark/ingestion/src/main/scala/org/apache/spark/api/python/DynamicPythonFunction.scala @@ -0,0 +1,94 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * Copyright 2018-2020 The Feast Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.api.python + +import java.io.File +import java.util.{ArrayList => JArrayList, HashMap => JHashMap} + +import org.apache.commons.io.IOUtils +import org.apache.spark.SparkEnv +import org.apache.spark.broadcast.Broadcast +import org.apache.spark.internal.config.{PYSPARK_DRIVER_PYTHON, PYSPARK_PYTHON} + +import collection.JavaConverters._ + +object DynamicPythonFunction { + private val conf = SparkEnv.get.conf + + val pythonExec = conf + .get(PYSPARK_DRIVER_PYTHON) + .orElse(conf.get(PYSPARK_PYTHON)) + .orElse(sys.env.get("PYSPARK_DRIVER_PYTHON")) + .orElse(sys.env.get("PYSPARK_PYTHON")) + .getOrElse("python3") + + private def runCommand(cmd: List[String]): String = { + val pb = new ProcessBuilder(cmd.asJava) + val p = pb.start() + p.waitFor() + IOUtils.toByteArray(p.getInputStream).map(_.toChar).mkString.trim + } + + def pythonVersion: String = { + runCommand( + List(pythonExec, "-c", "import sys; print(\"{0.major}.{0.minor}\".format(sys.version_info))") + ) + } + + def pythonPlatform: String = { + runCommand( + List(pythonExec, "-c", "import platform; print(platform.system().lower())") + ) + } + + def sparkHome: String = { + runCommand( + List(pythonExec, "-c", "import os; import pyspark; print(os.path.dirname(pyspark.__file__))") + ) + } + + def create( + pickledCode: Array[Byte], + env: Map[String, String] = Map.empty, + includePath: String = "libs/" + ): PythonFunction = { + val envVars = new JHashMap[String, String](env.asJava) + val broadcasts = new JArrayList[Broadcast[PythonBroadcast]]() + + if (!sys.env.contains("SPARK_HOME")) { + // in tests there's no SPARK_HOME + val libraries = List( + Seq(sparkHome, "python", "lib", "pyspark.zip").mkString(File.separator), + Seq(sparkHome, "python", "lib", "py4j-0.10.9-src.zip").mkString(File.separator) + ) + envVars.put("PYTHONPATH", libraries.mkString(File.pathSeparator)) + } + + PythonFunction( + pickledCode, + envVars, + List(includePath).asJava, + pythonExec, + pythonVersion, + broadcasts, + null + ) + } + + def libsPathWithPlatform(libsPath: String): String = + libsPath.replace("%(platform)s", s"py$pythonVersion-$pythonPlatform") +} diff --git a/spark/ingestion/src/main/scala/org/apache/spark/metrics/source/BaseMetricSource.scala b/spark/ingestion/src/main/scala/org/apache/spark/metrics/source/BaseMetricSource.scala new file mode 100644 index 00000000000..fd5c232a1c1 --- /dev/null +++ b/spark/ingestion/src/main/scala/org/apache/spark/metrics/source/BaseMetricSource.scala @@ -0,0 +1,50 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * Copyright 2018-2020 The Feast Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.metrics.source + +import com.codahale.metrics.MetricRegistry +import org.apache.spark.SparkEnv + +class BaseMetricSource extends Source { + override val sourceName: String = "" + + override val metricRegistry: MetricRegistry = new MetricRegistry + + private val sparkConfig = SparkEnv.get.conf + + private val metricLabels = sparkConfig.get("spark.metrics.labels", "") + + private val appId = sparkConfig.get("spark.app.id", "") + + private val executorId = sparkConfig.get("spark.executor.id", "") + + protected def metricWithLabels(name: String) = { + if (metricLabels.isEmpty) { + name + } else { + s"$name#$metricLabels,job_id=$appId-$executorId" + } + } + + protected def counterWithLabels(name: String) = { + if (metricLabels.isEmpty) { + name + } else { + s"$name#$metricLabels" + } + } +} diff --git a/ingestion/src/main/java/feast/ingestion/options/InputStreamConverter.java b/spark/ingestion/src/main/scala/org/apache/spark/metrics/source/IngestionPipelineMetricSource.scala similarity index 57% rename from ingestion/src/main/java/feast/ingestion/options/InputStreamConverter.java rename to spark/ingestion/src/main/scala/org/apache/spark/metrics/source/IngestionPipelineMetricSource.scala index e2fef732368..6710619f4c7 100644 --- a/ingestion/src/main/java/feast/ingestion/options/InputStreamConverter.java +++ b/spark/ingestion/src/main/scala/org/apache/spark/metrics/source/IngestionPipelineMetricSource.scala @@ -14,18 +14,18 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package feast.ingestion.options; +package org.apache.spark.metrics.source -import java.io.IOException; -import java.io.InputStream; +class IngestionPipelineMetricSource extends BaseMetricSource { + override val sourceName: String = IngestionPipelineMetricSource.sourceName -public interface InputStreamConverter { + val METRIC_DEADLETTER_ROWS_INSERTED = + metricRegistry.counter(counterWithLabels("deadletter_count")) - /** - * Used in conjunction with {@link OptionDecompressor} to decompress the pipeline option - * - * @param inputStream Input byte stream in compressed format - * @return Decompressed pipeline option value - */ - T readStream(InputStream inputStream) throws IOException; + val METRIC_ROWS_READ_FROM_SOURCE = + metricRegistry.counter(counterWithLabels("read_from_source_count")) +} + +object IngestionPipelineMetricSource { + val sourceName = "ingestion_pipeline" } diff --git a/spark/ingestion/src/main/scala/org/apache/spark/metrics/source/RedisSinkMetricSource.scala b/spark/ingestion/src/main/scala/org/apache/spark/metrics/source/RedisSinkMetricSource.scala index 77c9218a7ec..e5949d47bb4 100644 --- a/spark/ingestion/src/main/scala/org/apache/spark/metrics/source/RedisSinkMetricSource.scala +++ b/spark/ingestion/src/main/scala/org/apache/spark/metrics/source/RedisSinkMetricSource.scala @@ -16,30 +16,14 @@ */ package org.apache.spark.metrics.source -import com.codahale.metrics.MetricRegistry -import org.apache.spark.{SparkConf, SparkEnv} - -class RedisSinkMetricSource extends Source { +class RedisSinkMetricSource extends BaseMetricSource { override val sourceName: String = RedisSinkMetricSource.sourceName - override val metricRegistry: MetricRegistry = new MetricRegistry - - private val sparkConfig = Option(SparkEnv.get).map(_.conf).getOrElse(new SparkConf(true)) - - private val metricLabels = sparkConfig.get("spark.metrics.conf.*.source.redis.labels") - - private def nameWithLabels(name: String) = - if (metricLabels.isEmpty) { - name - } else { - s"$name#$metricLabels" - } - val METRIC_TOTAL_ROWS_INSERTED = - metricRegistry.counter(nameWithLabels("feast_ingestion_feature_row_ingested_count")) + metricRegistry.counter(counterWithLabels("feature_row_ingested_count")) val METRIC_ROWS_LAG = - metricRegistry.histogram(nameWithLabels("feast_ingestion_feature_row_lag_ms")) + metricRegistry.histogram(metricWithLabels("feature_row_lag_ms")) } object RedisSinkMetricSource { diff --git a/spark/ingestion/src/test/resources/python/setup.sh b/spark/ingestion/src/test/resources/python/setup.sh new file mode 100755 index 00000000000..4363d1fea49 --- /dev/null +++ b/spark/ingestion/src/test/resources/python/setup.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash + +CURRENT_PATH=$PWD +DESTINATION=${DESTINATION:-$CURRENT_PATH} + +# 1. Create libraries (dependencies) package +if [[ -f "$DESTINATION/libs.tar.gz" ]]; then + echo "$DESTINATION/libs.tar.gz exists." +else + tmp_dir=$(mktemp -d) + pip3 install -t ${tmp_dir}/libs great-expectations pyarrow==2.0.0 + cd $tmp_dir && tar -czf libs.tar.gz libs/ && mv libs.tar.gz $DESTINATION/libs.tar.gz +fi + +# 2. Pickle python udf +cd $CURRENT_PATH +pip3 install great-expectations setuptools pyspark==3.0.1 +python3 udf.py $DESTINATION/udf.pickle \ No newline at end of file diff --git a/spark/ingestion/src/test/resources/python/udf.py b/spark/ingestion/src/test/resources/python/udf.py new file mode 100644 index 00000000000..0d512e8ba3c --- /dev/null +++ b/spark/ingestion/src/test/resources/python/udf.py @@ -0,0 +1,49 @@ +import sys + +from pyspark import cloudpickle +from pyspark.sql.types import BooleanType + +import pandas as pd +import numpy as np + +from great_expectations.dataset import PandasDataset + + +def create_suite(): + df = pd.DataFrame() + df['num'] = np.random.randint(0, 10, 100) + df['num2'] = np.random.randint(0, 20, 100) + ds = PandasDataset.from_dataset(df) + + ds.expect_column_values_to_be_between('num', 0, 10) + ds.expect_column_values_to_be_between('num2', 0, 20) + + return ds.get_expectation_suite() + + +def create_validator(suite): + def validate(df) -> pd.DataFrame: + ds = PandasDataset.from_dataset(df) + # print(ds, ds.shape) + result = ds.validate(suite, result_format='COMPLETE') + valid_rows = pd.Series([True] * ds.shape[0]) + # print(result) + for check in result.results: + if check.success: + continue + + valid_rows.iloc[check.result['unexpected_index_list']] = False + return valid_rows + + return validate + + +def main(dest_path): + with open(dest_path, 'wb') as f: + fun = create_validator(create_suite()) + command = (fun, BooleanType()) + cloudpickle.dump(command, f) + + +if __name__ == '__main__': + main(sys.argv[1]) diff --git a/spark/ingestion/src/test/scala/feast/ingestion/BatchPipelineIT.scala b/spark/ingestion/src/test/scala/feast/ingestion/BatchPipelineIT.scala index 6ccfe9ee345..e0f57f6f0d6 100644 --- a/spark/ingestion/src/test/scala/feast/ingestion/BatchPipelineIT.scala +++ b/spark/ingestion/src/test/scala/feast/ingestion/BatchPipelineIT.scala @@ -17,19 +17,24 @@ package feast.ingestion import java.nio.file.Paths +import java.sql.Timestamp import collection.JavaConverters._ import com.dimafeng.testcontainers.{ForAllTestContainer, GenericContainer} +import com.google.protobuf.util.Timestamps import feast.proto.types.ValueProto.ValueType -import org.apache.spark.SparkConf +import org.apache.spark.{SparkConf, SparkEnv} import org.joda.time.{DateTime, Seconds} import org.scalacheck._ import org.scalatest._ import redis.clients.jedis.Jedis import feast.ingestion.helpers.RedisStorageHelper._ import feast.ingestion.helpers.DataHelper._ +import feast.ingestion.metrics.StatsDStub import feast.proto.storage.RedisProto.RedisKeyV2 import feast.proto.types.ValueProto +import org.apache.spark.sql.Encoder +import org.apache.spark.sql.catalyst.encoders.ExpressionEncoder case class TestRow( customer: String, @@ -41,22 +46,28 @@ case class TestRow( class BatchPipelineIT extends SparkSpec with ForAllTestContainer { override val container = GenericContainer("redis:6.0.8", exposedPorts = Seq(6379)) + val statsDStub = new StatsDStub override def withSparkConfOverrides(conf: SparkConf): SparkConf = conf .set("spark.redis.host", container.host) .set("spark.redis.port", container.mappedPort(6379).toString) + .set("spark.metrics.conf.*.sink.statsd.port", statsDStub.port.toString) trait Scope { val jedis = new Jedis("localhost", container.mappedPort(6379)) jedis.flushAll() + statsDStub.receivedMetrics // clean the buffer + + implicit def testRowEncoder: Encoder[TestRow] = ExpressionEncoder() + def rowGenerator(start: DateTime, end: DateTime, customerGen: Option[Gen[String]] = None) = for { customer <- customerGen.getOrElse(Gen.asciiPrintableStr) feature1 <- Gen.choose(0, 100) feature2 <- Gen.choose[Float](0, 1) eventTimestamp <- Gen - .choose(0, Seconds.secondsBetween(start, end).getSeconds) + .choose(0, Seconds.secondsBetween(start, end).getSeconds - 1) .map(start.withMillisOfSecond(0).plusSeconds) } yield TestRow( customer, @@ -89,7 +100,8 @@ class BatchPipelineIT extends SparkSpec with ForAllTestContainer { ) ), startTime = DateTime.parse("2020-08-01"), - endTime = DateTime.parse("2020-09-01") + endTime = DateTime.parse("2020-09-01"), + metrics = Some(StatsDConfig(host = "localhost", port = statsDStub.port)) ) } @@ -98,7 +110,7 @@ class BatchPipelineIT extends SparkSpec with ForAllTestContainer { val rows = generateDistinctRows(gen, 10000, groupByEntity) val tempPath = storeAsParquet(sparkSession, rows) val configWithOfflineSource = config.copy( - source = FileSource(tempPath, Map.empty, "eventTimestamp") + source = FileSource(tempPath, Map.empty, "eventTimestamp", datePartitionColumn = Some("date")) ) BatchPipeline.createPipeline(sparkSession, configWithOfflineSource) @@ -106,14 +118,261 @@ class BatchPipelineIT extends SparkSpec with ForAllTestContainer { val featureKeyEncoder: String => String = encodeFeatureKey(config.featureTable) rows.foreach(r => { - val storedValues = jedis.hgetAll(encodeEntityKey(r, config.featureTable)).asScala.toMap + val encodedEntityKey = encodeEntityKey(r, config.featureTable) + val storedValues = jedis.hgetAll(encodedEntityKey).asScala.toMap storedValues should beStoredRow( Map( featureKeyEncoder("feature1") -> r.feature1, featureKeyEncoder("feature2") -> r.feature2, - "_ts:test-fs" -> r.eventTimestamp + "_ts:test-fs" -> r.eventTimestamp, + "_ex:test-fs" -> new Timestamp(Timestamps.MAX_VALUE.getSeconds * 1000) ) ) + val keyTTL = jedis.ttl(encodedEntityKey).toInt + keyTTL shouldEqual -1 + + }) + + SparkEnv.get.metricsSystem.report() + statsDStub.receivedMetrics should contain.allElementsOf( + Map( + "driver.ingestion_pipeline.read_from_source_count" -> rows.length, + "driver.redis_sink.feature_row_ingested_count" -> rows.length + ) + ) + } + + "Parquet source file" should "be ingested in redis with expiry time equal to the largest of (event_timestamp + max_age) for" + + "all feature tables associated with the entity" in new Scope { + val startDate = new DateTime().minusDays(1).withTimeAtStartOfDay() + val endDate = new DateTime().withTimeAtStartOfDay() + val gen = rowGenerator(startDate, endDate) + val rows = generateDistinctRows(gen, 1000, groupByEntity) + val tempPath = storeAsParquet(sparkSession, rows) + val maxAge = 86400L * 30 + val configWithMaxAge = config.copy( + source = FileSource(tempPath, Map.empty, "eventTimestamp"), + featureTable = config.featureTable.copy(maxAge = Some(maxAge)), + startTime = startDate, + endTime = endDate + ) + + val ingestionTimeUnix = System.currentTimeMillis() + BatchPipeline.createPipeline(sparkSession, configWithMaxAge) + + val featureKeyEncoder: String => String = encodeFeatureKey(config.featureTable) + + rows.foreach(r => { + val encodedEntityKey = encodeEntityKey(r, config.featureTable) + val storedValues = jedis.hgetAll(encodedEntityKey).asScala.toMap + val expectedExpiryTimestamp = + new java.sql.Timestamp(r.eventTimestamp.getTime + 1000 * maxAge) + storedValues should beStoredRow( + Map( + featureKeyEncoder("feature1") -> r.feature1, + featureKeyEncoder("feature2") -> r.feature2, + "_ts:test-fs" -> r.eventTimestamp, + "_ex:test-fs" -> expectedExpiryTimestamp + ) + ) + val keyTTL = jedis.ttl(encodedEntityKey).toLong + keyTTL should (be <= (expectedExpiryTimestamp.getTime - ingestionTimeUnix) / 1000 and be > 0L) + + }) + + val increasedMaxAge = 86400L * 60 + val configWithSecondFeatureTable = config.copy( + source = FileSource(tempPath, Map.empty, "eventTimestamp"), + featureTable = config.featureTable.copy( + name = "test-fs-2", + maxAge = Some(increasedMaxAge) + ), + startTime = startDate, + endTime = endDate + ) + + val secondIngestionTimeUnix = System.currentTimeMillis() + BatchPipeline.createPipeline(sparkSession, configWithSecondFeatureTable) + + val featureKeyEncoderSecondTable: String => String = + encodeFeatureKey(configWithSecondFeatureTable.featureTable) + + rows.foreach(r => { + val encodedEntityKey = encodeEntityKey(r, config.featureTable) + val storedValues = jedis.hgetAll(encodedEntityKey).asScala.toMap + val expectedExpiryTimestamp1 = + new java.sql.Timestamp(r.eventTimestamp.getTime + 1000 * maxAge) + val expectedExpiryTimestamp2 = + new java.sql.Timestamp(r.eventTimestamp.getTime + 1000 * increasedMaxAge) + storedValues should beStoredRow( + Map( + featureKeyEncoder("feature1") -> r.feature1, + featureKeyEncoder("feature2") -> r.feature2, + featureKeyEncoderSecondTable("feature1") -> r.feature1, + featureKeyEncoderSecondTable("feature2") -> r.feature2, + "_ts:test-fs" -> r.eventTimestamp, + "_ts:test-fs-2" -> r.eventTimestamp, + "_ex:test-fs" -> expectedExpiryTimestamp1, + "_ex:test-fs-2" -> expectedExpiryTimestamp2 + ) + ) + val keyTTL = jedis.ttl(encodedEntityKey).toLong + keyTTL should (be <= (expectedExpiryTimestamp2.getTime - secondIngestionTimeUnix) / 1000 and be > (expectedExpiryTimestamp1.getTime - secondIngestionTimeUnix) / 1000) + + }) + } + + "Redis key TTL" should "not be updated, when a second feature table associated with the same entity is registered and ingested, if (event_timestamp + max_age) of the second " + + "Feature Table is not later than the expiry timestamp of the first feature table" in new Scope { + val startDate = new DateTime().minusDays(1).withTimeAtStartOfDay() + val endDate = new DateTime().withTimeAtStartOfDay() + val gen = rowGenerator(startDate, endDate) + val rows = generateDistinctRows(gen, 1000, groupByEntity) + val tempPath = storeAsParquet(sparkSession, rows) + val maxAge = 86400 * 3 + val configWithMaxAge = config.copy( + source = FileSource(tempPath, Map.empty, "eventTimestamp"), + featureTable = config.featureTable.copy(maxAge = Some(maxAge)), + startTime = startDate, + endTime = endDate + ) + + val ingestionTimeUnix = System.currentTimeMillis() + BatchPipeline.createPipeline(sparkSession, configWithMaxAge) + + val reducedMaxAge = 86400 * 2 + val configWithSecondFeatureTable = config.copy( + source = FileSource(tempPath, Map.empty, "eventTimestamp"), + featureTable = config.featureTable.copy( + name = "test-fs-2", + maxAge = Some(reducedMaxAge) + ), + startTime = startDate, + endTime = endDate + ) + + BatchPipeline.createPipeline(sparkSession, configWithSecondFeatureTable) + + val featureKeyEncoder: String => String = encodeFeatureKey(config.featureTable) + val featureKeyEncoderSecondTable: String => String = + encodeFeatureKey(configWithSecondFeatureTable.featureTable) + + rows.foreach(r => { + val encodedEntityKey = encodeEntityKey(r, config.featureTable) + val storedValues = jedis.hgetAll(encodedEntityKey).asScala.toMap + val expectedExpiryTimestamp1 = + new java.sql.Timestamp(r.eventTimestamp.getTime + 1000 * maxAge) + val expectedExpiryTimestamp2 = + new java.sql.Timestamp(r.eventTimestamp.getTime + 1000 * reducedMaxAge) + storedValues should beStoredRow( + Map( + featureKeyEncoder("feature1") -> r.feature1, + featureKeyEncoder("feature2") -> r.feature2, + featureKeyEncoderSecondTable("feature1") -> r.feature1, + featureKeyEncoderSecondTable("feature2") -> r.feature2, + "_ts:test-fs" -> r.eventTimestamp, + "_ts:test-fs-2" -> r.eventTimestamp, + "_ex:test-fs" -> expectedExpiryTimestamp1, + "_ex:test-fs-2" -> expectedExpiryTimestamp2 + ) + ) + val keyTTL = jedis.ttl(encodedEntityKey).toLong + keyTTL should (be <= (expectedExpiryTimestamp1.getTime - ingestionTimeUnix) / 1000 and + be > (expectedExpiryTimestamp2.getTime - ingestionTimeUnix) / 1000) + + }) + } + + "Redis key TTL" should "be updated, when the same feature table is re-ingested, with a smaller max age" in new Scope { + val startDate = new DateTime().minusDays(1).withTimeAtStartOfDay() + val endDate = new DateTime().withTimeAtStartOfDay() + val gen = rowGenerator(startDate, endDate) + val rows = generateDistinctRows(gen, 1000, groupByEntity) + val tempPath = storeAsParquet(sparkSession, rows) + val maxAge = 86400 * 3 + val configWithMaxAge = config.copy( + source = FileSource(tempPath, Map.empty, "eventTimestamp"), + featureTable = config.featureTable.copy(maxAge = Some(maxAge)), + startTime = startDate, + endTime = endDate + ) + + val ingestionTimeUnix = System.currentTimeMillis() + BatchPipeline.createPipeline(sparkSession, configWithMaxAge) + + val reducedMaxAge = 86400 * 2 + val configWithUpdatedFeatureTable = config.copy( + source = FileSource(tempPath, Map.empty, "eventTimestamp"), + featureTable = config.featureTable.copy( + maxAge = Some(reducedMaxAge) + ), + startTime = startDate, + endTime = endDate + ) + + BatchPipeline.createPipeline(sparkSession, configWithUpdatedFeatureTable) + + val featureKeyEncoder: String => String = encodeFeatureKey(config.featureTable) + + rows.foreach(r => { + val encodedEntityKey = encodeEntityKey(r, config.featureTable) + val storedValues = jedis.hgetAll(encodedEntityKey).asScala.toMap + val expiryTimestampAfterUpdate = + new java.sql.Timestamp(r.eventTimestamp.getTime + 1000 * reducedMaxAge) + storedValues should beStoredRow( + Map( + featureKeyEncoder("feature1") -> r.feature1, + featureKeyEncoder("feature2") -> r.feature2, + "_ts:test-fs" -> r.eventTimestamp, + "_ex:test-fs" -> expiryTimestampAfterUpdate + ) + ) + val keyTTL = jedis.ttl(encodedEntityKey).toLong + keyTTL should (be <= (expiryTimestampAfterUpdate.getTime - ingestionTimeUnix) / 1000 and be > 0L) + + }) + } + + "Redis key TTL" should "be removed, when the same feature table is re-ingested without max age" in new Scope { + val startDate = new DateTime().minusDays(1).withTimeAtStartOfDay() + val endDate = new DateTime().withTimeAtStartOfDay() + val gen = rowGenerator(startDate, endDate) + val rows = generateDistinctRows(gen, 1000, groupByEntity) + val tempPath = storeAsParquet(sparkSession, rows) + val maxAge = 86400 * 3 + val configWithMaxAge = config.copy( + source = FileSource(tempPath, Map.empty, "eventTimestamp"), + featureTable = config.featureTable.copy(maxAge = Some(maxAge)), + startTime = startDate, + endTime = endDate + ) + + BatchPipeline.createPipeline(sparkSession, configWithMaxAge) + + val configWithoutMaxAge = config.copy( + source = FileSource(tempPath, Map.empty, "eventTimestamp"), + startTime = startDate, + endTime = endDate + ) + + BatchPipeline.createPipeline(sparkSession, configWithoutMaxAge) + + val featureKeyEncoder: String => String = encodeFeatureKey(config.featureTable) + + rows.foreach(r => { + val encodedEntityKey = encodeEntityKey(r, config.featureTable) + val storedValues = jedis.hgetAll(encodedEntityKey).asScala.toMap + storedValues should beStoredRow( + Map( + featureKeyEncoder("feature1") -> r.feature1, + featureKeyEncoder("feature2") -> r.feature2, + "_ts:test-fs" -> r.eventTimestamp, + "_ex:test-fs" -> new Timestamp(Timestamps.MAX_VALUE.getSeconds * 1000) + ) + ) + val keyTTL = jedis.ttl(encodedEntityKey).toInt + keyTTL shouldEqual -1 + }) } @@ -218,6 +477,13 @@ class BatchPipelineIT extends SparkSpec with ForAllTestContainer { .toString ) .count() should be(rows.length) + + SparkEnv.get.metricsSystem.report() + statsDStub.receivedMetrics should contain.allElementsOf( + Map( + "driver.ingestion_pipeline.deadletter_count" -> rows.length + ) + ) } "Columns from source" should "be mapped according to configuration" in new Scope { diff --git a/spark/ingestion/src/test/scala/feast/ingestion/PandasUDF.scala b/spark/ingestion/src/test/scala/feast/ingestion/PandasUDF.scala new file mode 100644 index 00000000000..6a3d15d7c7d --- /dev/null +++ b/spark/ingestion/src/test/scala/feast/ingestion/PandasUDF.scala @@ -0,0 +1,163 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * Copyright 2018-2020 The Feast Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package feast.ingestion + +import java.nio.file.Paths +import java.sql.Timestamp +import java.util.Date + +import com.dimafeng.testcontainers.{ForAllTestContainer, GenericContainer} +import feast.ingestion.helpers.DataHelper.generateTempPath +import feast.ingestion.utils.testing.MemoryStreamingSource +import feast.proto.storage.RedisProto.RedisKeyV2 +import feast.proto.types.ValueProto +import feast.proto.types.ValueProto.ValueType +import org.apache.spark.SparkConf +import org.apache.spark.api.python.DynamicPythonFunction +import org.apache.spark.sql.{Encoder, Row, SQLContext} +import org.apache.spark.sql.catalyst.encoders.ExpressionEncoder +import org.apache.spark.sql.execution.python.UserDefinedPythonFunction +import org.apache.spark.sql.execution.streaming.MemoryStream +import org.apache.spark.sql.types.BooleanType +import org.apache.spark.sql.functions.struct +import redis.clients.jedis.Jedis + +import scala.jdk.CollectionConverters._ +import scala.util.Random + +class PandasUDF extends SparkSpec with ForAllTestContainer { + case class TestRow(key: String, num: Int, num2: Int, timestamp: java.sql.Timestamp) + + override val container = GenericContainer("redis:6.0.8", exposedPorts = Seq(6379)) + + override def withSparkConfOverrides(conf: SparkConf): SparkConf = conf + .set("spark.redis.host", container.host) + .set("spark.redis.port", container.mappedPort(6379).toString) + + trait Scope { + implicit def testRowEncoder: Encoder[TestRow] = ExpressionEncoder() + implicit def sqlContext: SQLContext = sparkSession.sqlContext + + val jedis = new Jedis("localhost", container.mappedPort(6379)) + jedis.flushAll() + + val SQL_SCALAR_PANDAS_UDF = 200 + val rand = new Random() + + def encodeEntityKey(key: String): Array[Byte] = + RedisKeyV2 + .newBuilder() + .setProject("default") + .addAllEntityNames(Seq("key").asJava) + .addEntityValues(ValueProto.Value.newBuilder().setStringVal(key)) + .build + .toByteArray + + // Function checks that num between 0 and 10 and num2 between 0 and 20 + // See the code test/resources/python/udf.py + val pickledCode = getClass.getResourceAsStream("/python/udf.pickle").readAllBytes() + val pythonFun = DynamicPythonFunction.create(pickledCode) + + val udf: UserDefinedPythonFunction = UserDefinedPythonFunction( + "validate", + pythonFun, + BooleanType, + SQL_SCALAR_PANDAS_UDF, + udfDeterministic = true + ) + + val inputData = MemoryStream[TestRow] + val config = IngestionJobConfig( + featureTable = FeatureTable( + name = "test-fs", + project = "default", + entities = Seq(Field("key", ValueType.Enum.STRING)), + features = Seq( + Field("num", ValueType.Enum.INT32), + Field("num2", ValueType.Enum.INT32) + ) + ), + source = MemoryStreamingSource(inputData), + validationConfig = Some( + ValidationConfig( + name = "testFun", + pickledCodePath = getClass.getResource("/python/udf.pickle").getPath, + includeArchivePath = getClass.getResource("/python/libs.tar.gz").getPath + ) + ), + doNotIngestInvalidRows = true, + deadLetterPath = Some(generateTempPath("deadletters")) + ) + } + + "Custom Python code" should "be applied in streaming pipeline" in new Scope { + sparkSession.sparkContext.addFile(getClass.getResource("/python/libs.tar.gz").getPath) + + val df = inputData.toDF() + + val cols = df.columns.map(df(_)) + val streamingQuery = df + .withColumn("valid", udf(struct(cols: _*))) + .writeStream + .format("memory") + .queryName("sink") + .start() + + val data = + (1 to 100).map(_ => TestRow(rand.nextString(5), rand.nextInt(100), rand.nextInt(100), null)) + inputData.addData(data) + streamingQuery.processAllAvailable() + + val expected = data + .map(testRow => Row(testRow.num, testRow.num2, testRow.num <= 10 && testRow.num2 <= 20)) + .toArray + + val output = sparkSession.sql("select num, num2, valid from sink").collect() + output should be(expected) + } + + "Custom Python code" should "be used for data validation in StreamingPipeline" in new Scope { + val query = StreamingPipeline.createPipeline(sparkSession, config).get + + val ts = new Timestamp(new Date().getTime) + val data = + (1 to 1000).map(_ => TestRow(rand.nextString(5), rand.nextInt(100), rand.nextInt(100), ts)) + inputData.addData(data) + query.processAllAvailable() + + // Map (key -> isValid) + val expected = data + .map(testRow => testRow.key -> (testRow.num <= 10 && testRow.num2 <= 20)) + .toMap + + // Invalid Rows stored to DeadLetters + sparkSession.read + .parquet( + Paths + .get(config.deadLetterPath.get, sparkSession.conf.get("spark.app.id")) + .toString + ) + .count() should be(expected.count(!_._2)) + + // Valid rows saved to Storage + expected.filter(_._2).keys.foreach { key => + { + jedis.hgetAll(encodeEntityKey(key)).asScala.toMap should not be (Map.empty) + } + } + } +} diff --git a/spark/ingestion/src/test/scala/feast/ingestion/SparkSpec.scala b/spark/ingestion/src/test/scala/feast/ingestion/SparkSpec.scala index 7d2b153c383..025a3b8be1d 100644 --- a/spark/ingestion/src/test/scala/feast/ingestion/SparkSpec.scala +++ b/spark/ingestion/src/test/scala/feast/ingestion/SparkSpec.scala @@ -21,6 +21,8 @@ import org.apache.spark.sql.SparkSession import org.scalatest.BeforeAndAfter class SparkSpec extends UnitSpec with BeforeAndAfter { + System.setProperty("io.netty.tryReflectionSetAccessible", "true") + var sparkSession: SparkSession = null def withSparkConfOverrides(conf: SparkConf): SparkConf = conf @@ -29,16 +31,17 @@ class SparkSpec extends UnitSpec with BeforeAndAfter { .setMaster("local[4]") .setAppName("Testing") .set("spark.default.parallelism", "8") - .set( - "spark.metrics.conf.*.source.redis.class", - "org.apache.spark.metrics.source.RedisSinkMetricSource" - ) .set( "spark.metrics.conf.*.sink.statsd.class", "org.apache.spark.metrics.sink.StatsdSinkWithTags" ) .set("spark.metrics.conf.*.sink.statsd.host", "localhost") - .set("spark.metrics.conf.*.sink.statsd.port", "8125") + .set("spark.metrics.conf.*.sink.statsd.period", "999") // disable scheduled reporting + .set("spark.metrics.conf.*.sink.statsd.unit", "minutes") + .set("spark.metrics.labels", "job_id=test") + .set("spark.metrics.namespace", "") + .set("spark.sql.legacy.allowUntypedScalaUDF", "true") + .set("spark.sql.execution.arrow.maxRecordsPerBatch", "50000") sparkSession = SparkSession .builder() diff --git a/spark/ingestion/src/test/scala/feast/ingestion/StreamingPipelineIT.scala b/spark/ingestion/src/test/scala/feast/ingestion/StreamingPipelineIT.scala index 39c44ad55a8..64ad30aa4d2 100644 --- a/spark/ingestion/src/test/scala/feast/ingestion/StreamingPipelineIT.scala +++ b/spark/ingestion/src/test/scala/feast/ingestion/StreamingPipelineIT.scala @@ -30,6 +30,7 @@ import org.apache.spark.SparkConf import org.joda.time.DateTime import org.apache.kafka.clients.producer._ import com.example.protos.{AllTypesMessage, InnerMessage, TestMessage, VehicleType} +import com.google.protobuf.util.Timestamps import com.google.protobuf.{AbstractMessage, ByteString, Timestamp} import org.scalacheck.Gen import redis.clients.jedis.Jedis @@ -39,12 +40,19 @@ import feast.ingestion.helpers.RedisStorageHelper._ import feast.ingestion.helpers.DataHelper._ import feast.proto.storage.RedisProto.RedisKeyV2 import feast.proto.types.ValueProto -import org.apache.spark.sql.Row +import org.apache.spark.sql.Encoder import org.apache.spark.sql.avro.to_avro +import org.apache.spark.sql.catalyst.encoders.ExpressionEncoder import org.apache.spark.sql.functions.{col, struct} -import org.apache.spark.sql.types.StructType class StreamingPipelineIT extends SparkSpec with ForAllTestContainer { + case class TestRow( + customer: String, + feature1: Int, + feature2: Float, + eventTimestamp: java.sql.Timestamp + ) + val redisContainer = GenericContainer("redis:6.0.8", exposedPorts = Seq(6379)) val kafkaContainer = KafkaContainer() @@ -69,6 +77,8 @@ class StreamingPipelineIT extends SparkSpec with ForAllTestContainer { } trait Scope extends KafkaPublisher { + implicit def testRowEncoder: Encoder[TestRow] = ExpressionEncoder() + val jedis = new Jedis("localhost", redisContainer.mappedPort(6379)) jedis.flushAll() @@ -136,16 +146,86 @@ class StreamingPipelineIT extends SparkSpec with ForAllTestContainer { query.processAllAvailable() rows.foreach { r => - val storedValues = jedis.hgetAll(encodeEntityKey(r, config.featureTable)).asScala.toMap + val encodedEntityKey = encodeEntityKey(r, config.featureTable) + val storedValues = jedis.hgetAll(encodedEntityKey).asScala.toMap storedValues should beStoredRow( Map( featureKeyEncoder("unique_drivers") -> r.getUniqueDrivers, - "_ts:driver-fs" -> new java.sql.Timestamp(r.getEventTimestamp.getSeconds * 1000) + "_ts:driver-fs" -> new java.sql.Timestamp(r.getEventTimestamp.getSeconds * 1000), + "_ex:driver-fs" -> new java.sql.Timestamp(Timestamps.MAX_VALUE.getSeconds * 1000) ) ) + val keyTTL = jedis.ttl(encodedEntityKey).toInt + keyTTL shouldEqual -1 } } + "Streaming pipeline" should "store messages from kafka to redis with expiry time equal to the largest of (event_timestamp + max_age) for all feature " + + "tables associated with the entity" in new Scope { + val maxAge = 86400 + val configWithMaxAge = config.copy( + source = kafkaSource, + featureTable = config.featureTable.copy(maxAge = Some(maxAge)) + ) + val query = StreamingPipeline.createPipeline(sparkSession, configWithMaxAge).get + query.processAllAvailable() // to init kafka consumer + + val rows = generateDistinctRows(rowGenerator, 100, groupByEntity) + + val ingestionTimeUnix = System.currentTimeMillis() + rows.foreach(sendToKafka(kafkaSource.topic, _)) + + query.processAllAvailable() + + rows.foreach { r => + val encodedEntityKey = encodeEntityKey(r, config.featureTable) + val storedValues = jedis.hgetAll(encodedEntityKey).asScala.toMap + storedValues should beStoredRow( + Map( + featureKeyEncoder("unique_drivers") -> r.getUniqueDrivers, + "_ts:driver-fs" -> new java.sql.Timestamp(r.getEventTimestamp.getSeconds * 1000), + "_ex:driver-fs" -> new java.sql.Timestamp( + (r.getEventTimestamp.getSeconds + maxAge) * 1000 + ) + ) + ) + val keyTTL = jedis.ttl(encodedEntityKey).toLong + keyTTL should (be <= (r.getEventTimestamp.getSeconds + maxAge - ingestionTimeUnix / 1000) and be > 0L) + } + + val kafkaSourceSecondFeatureTable = kafkaSource.copy(topic = "topic-2") + val configWithSecondFeatureTable = config.copy( + source = kafkaSourceSecondFeatureTable, + featureTable = config.featureTable.copy(name = "driver-fs-2") + ) + val querySecondFeatureTable = + StreamingPipeline.createPipeline(sparkSession, configWithSecondFeatureTable).get + querySecondFeatureTable.processAllAvailable() // to init kafka consumer + rows.foreach(sendToKafka(kafkaSourceSecondFeatureTable.topic, _)) + querySecondFeatureTable.processAllAvailable() + + val featureKeyEncoderSecondFeatureTable: String => String = + encodeFeatureKey(configWithSecondFeatureTable.featureTable) + rows.foreach { r => + val encodedEntityKey = encodeEntityKey(r, config.featureTable) + val storedValues = jedis.hgetAll(encodedEntityKey).asScala.toMap + storedValues should beStoredRow( + Map( + featureKeyEncoder("unique_drivers") -> r.getUniqueDrivers, + featureKeyEncoderSecondFeatureTable("unique_drivers") -> r.getUniqueDrivers, + "_ts:driver-fs" -> new java.sql.Timestamp(r.getEventTimestamp.getSeconds * 1000), + "_ex:driver-fs" -> new java.sql.Timestamp( + (r.getEventTimestamp.getSeconds + maxAge) * 1000 + ), + "_ex:driver-fs-2" -> new java.sql.Timestamp(Timestamps.MAX_VALUE.getSeconds * 1000) + ) + ) + val keyTTL = jedis.ttl(encodedEntityKey).toInt + keyTTL shouldEqual -1 + } + + } + "Streaming pipeline" should "store invalid proto messages to deadletter path" in new Scope { val configWithDeadletter = config.copy( source = kafkaSource, diff --git a/spark/ingestion/src/test/scala/feast/ingestion/helpers/RedisStorageHelper.scala b/spark/ingestion/src/test/scala/feast/ingestion/helpers/RedisStorageHelper.scala index 921d65d4778..d15126d9d37 100644 --- a/spark/ingestion/src/test/scala/feast/ingestion/helpers/RedisStorageHelper.scala +++ b/spark/ingestion/src/test/scala/feast/ingestion/helpers/RedisStorageHelper.scala @@ -38,14 +38,15 @@ object RedisStorageHelper { m compose { (_: Map[Array[Byte], Array[Byte]]) - .map { case (k, v) => - if (k.length == 4) + .map { + case (k, v) if k.length == 4 => ( ByteBuffer.wrap(k).order(ByteOrder.LITTLE_ENDIAN).getInt.toHexString, ValueProto.Value.parseFrom(v).asScala ) - else + case (k, v) if k.startsWith("_ts".getBytes) || k.startsWith("_ex".getBytes) => (new String(k), Timestamp.parseFrom(v).asScala) + case (k, v) => (new String(k), ValueProto.Value.parseFrom(v).asScala) } } } diff --git a/spark/ingestion/src/test/scala/feast/ingestion/metrics/StatsDStub.scala b/spark/ingestion/src/test/scala/feast/ingestion/metrics/StatsDStub.scala new file mode 100644 index 00000000000..62a6d5a8e30 --- /dev/null +++ b/spark/ingestion/src/test/scala/feast/ingestion/metrics/StatsDStub.scala @@ -0,0 +1,61 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * Copyright 2018-2020 The Feast Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package feast.ingestion.metrics + +import java.net.{DatagramPacket, DatagramSocket, SocketTimeoutException} + +import scala.collection.mutable.ArrayBuffer + +class StatsDStub { + val socket = new DatagramSocket() + socket.setSoTimeout(100) + + def port: Int = socket.getLocalPort + + def receive: Array[String] = { + val messages: ArrayBuffer[String] = ArrayBuffer() + var finished = false + + do { + val buf = new Array[Byte](65535) + val p = new DatagramPacket(buf, buf.length) + try { + socket.receive(p) + } catch { + case _: SocketTimeoutException => + finished = true + } + messages += new String(p.getData, 0, p.getLength) + } while (!finished) + + messages.toArray + } + + private val metricLine = """(.+):(.+)\|(.+)#(.+)""".r + + def receivedMetrics: Map[String, Float] = { + receive + .flatMap { + case metricLine(name, value, type_, tags) => + Seq(name -> value.toFloat) + case s: String => + Seq() + } + .groupBy(_._1) + .mapValues(_.map(_._2).sum) + } +} diff --git a/spark/ingestion/src/test/scala/feast/ingestion/metrics/StatsReporterSpec.scala b/spark/ingestion/src/test/scala/feast/ingestion/metrics/StatsReporterSpec.scala index 1ae61724ed1..b531b87a40a 100644 --- a/spark/ingestion/src/test/scala/feast/ingestion/metrics/StatsReporterSpec.scala +++ b/spark/ingestion/src/test/scala/feast/ingestion/metrics/StatsReporterSpec.scala @@ -16,46 +16,17 @@ */ package feast.ingestion.metrics -import java.net.{DatagramPacket, DatagramSocket, SocketTimeoutException} import java.util import java.util.Collections import com.codahale.metrics.{Gauge, Histogram, MetricRegistry, UniformReservoir} import feast.ingestion.UnitSpec -import scala.collection.mutable.ArrayBuffer import scala.jdk.CollectionConverters._ class StatsReporterSpec extends UnitSpec { - class SimpleServer { - val socket = new DatagramSocket() - socket.setSoTimeout(100) - - def port: Int = socket.getLocalPort - - def receive: Array[String] = { - val messages: ArrayBuffer[String] = ArrayBuffer() - var finished = false - - do { - val buf = new Array[Byte](65535) - val p = new DatagramPacket(buf, buf.length) - try { - socket.receive(p) - } catch { - case _: SocketTimeoutException => { - finished = true - } - } - messages += new String(p.getData, 0, p.getLength) - } while (!finished) - - messages.toArray - } - } - trait Scope { - val server = new SimpleServer + val server = new StatsDStub val reporter = new StatsdReporterWithTags( new MetricRegistry, "127.0.0.1", @@ -89,19 +60,19 @@ class StatsReporterSpec extends UnitSpec { server.receive should contain("test:0|g") } - "Statsd reporter" should "keep tags part in the name's end" in new Scope { + "Statsd reporter" should "keep tags part in the message's end" in new Scope { reporter.report( gauges = Collections.emptySortedMap(), counters = Collections.emptySortedMap(), histograms = new util.TreeMap( Map( - "test#fs=name" -> histogram((1 to 100)) + "prefix.1111.test#fs=name,job=aaa" -> histogram((1 to 100)) ).asJava ), meters = Collections.emptySortedMap(), timers = Collections.emptySortedMap() ) - server.receive should contain("test.p95#fs=name:95.95|ms") + server.receive should contain("prefix.test.p95:95.95|ms|#fs:name,job:aaa") } } diff --git a/storage/api/pom.xml b/storage/api/pom.xml index 231471e9cbf..cc2f84ecb17 100644 --- a/storage/api/pom.xml +++ b/storage/api/pom.xml @@ -42,12 +42,6 @@ ${project.version} - - org.apache.beam - beam-sdks-java-core - ${org.apache.beam.version} - - com.google.auto.value auto-value-annotations diff --git a/storage/api/src/main/java/feast/storage/api/retriever/FeatureSetRequest.java b/storage/api/src/main/java/feast/storage/api/retriever/FeatureSetRequest.java deleted file mode 100644 index 3482529a97d..00000000000 --- a/storage/api/src/main/java/feast/storage/api/retriever/FeatureSetRequest.java +++ /dev/null @@ -1,60 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2019 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.storage.api.retriever; - -import com.google.auto.value.AutoValue; -import com.google.common.collect.ImmutableSet; -import feast.proto.core.FeatureSetProto.FeatureSetSpec; -import feast.proto.serving.ServingAPIProto.FeatureReference; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; - -@AutoValue -public abstract class FeatureSetRequest { - public abstract FeatureSetSpec getSpec(); - - public abstract ImmutableSet getFeatureReferences(); - - public static Builder newBuilder() { - return new AutoValue_FeatureSetRequest.Builder(); - } - - @AutoValue.Builder - public abstract static class Builder { - public abstract Builder setSpec(FeatureSetSpec spec); - - abstract ImmutableSet.Builder featureReferencesBuilder(); - - public Builder addAllFeatureReferences(List featureReferenceList) { - featureReferencesBuilder().addAll(featureReferenceList); - return this; - } - - public Builder addFeatureReference(FeatureReference featureReference) { - featureReferencesBuilder().add(featureReference); - return this; - } - - public abstract FeatureSetRequest build(); - } - - public Map getFeatureRefsByName() { - return getFeatureReferences().stream() - .collect(Collectors.toMap(FeatureReference::getName, featureReference -> featureReference)); - } -} diff --git a/storage/api/src/main/java/feast/storage/api/retriever/HistoricalRetrievalResult.java b/storage/api/src/main/java/feast/storage/api/retriever/HistoricalRetrievalResult.java deleted file mode 100644 index 325a05e5140..00000000000 --- a/storage/api/src/main/java/feast/storage/api/retriever/HistoricalRetrievalResult.java +++ /dev/null @@ -1,125 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.storage.api.retriever; - -import com.google.auto.value.AutoValue; -import feast.proto.serving.ServingAPIProto.DataFormat; -import feast.proto.serving.ServingAPIProto.JobStatus; -import java.io.Serializable; -import java.util.List; -import javax.annotation.Nullable; -import org.tensorflow.metadata.v0.DatasetFeatureStatisticsList; - -/** Result of a historical feature retrieval request. */ -@AutoValue -public abstract class HistoricalRetrievalResult implements Serializable { - - public abstract String getId(); - - public abstract JobStatus getStatus(); - - @Nullable - public abstract String getError(); - - @Nullable - public abstract List getFileUris(); - - @Nullable - public abstract DataFormat getDataFormat(); - - @Nullable - public abstract DatasetFeatureStatisticsList getStats(); - - /** - * Instantiates a {@link HistoricalRetrievalResult} indicating that the retrieval was a failure, - * together with its associated error. - * - * @param id retrieval id identifying the retrieval request. - * @param error error that occurred - * @return {@link HistoricalRetrievalResult} - */ - public static HistoricalRetrievalResult error(String id, Exception error) { - return newBuilder() - .setId(id) - .setStatus(JobStatus.JOB_STATUS_DONE) - .setError(error.getMessage()) - .build(); - } - - /** - * Instantiates a {@link HistoricalRetrievalResult} indicating that the retrieval was a success, - * together with the location of the output. - * - * @param id retrieval id identifying the retrieval request - * @param fileUris list of output file URIs - * @param dataFormat data format of the output files - * @return - */ - public static HistoricalRetrievalResult success( - String id, List fileUris, DataFormat dataFormat) { - return newBuilder() - .setId(id) - .setStatus(JobStatus.JOB_STATUS_DONE) - .setFileUris(fileUris) - .setDataFormat(dataFormat) - .build(); - } - - /** - * Adds statistics to the result - * - * @param stats {@link DatasetFeatureStatisticsList} for the retrieved dataset - * @return {@link HistoricalRetrievalResult} - */ - public HistoricalRetrievalResult withStats(DatasetFeatureStatisticsList stats) { - return toBuilder().setStats(stats).build(); - } - - static Builder newBuilder() { - return new AutoValue_HistoricalRetrievalResult.Builder(); - } - - Builder toBuilder() { - return newBuilder() - .setId(getId()) - .setStatus(getStatus()) - .setFileUris(getFileUris()) - .setError(getError()) - .setDataFormat(getDataFormat()); - } - - @AutoValue.Builder - abstract static class Builder { - abstract Builder setId(String id); - - abstract Builder setStatus(JobStatus jobStatus); - - abstract Builder setError(String error); - - abstract Builder setFileUris(List fileUris); - - abstract Builder setDataFormat(DataFormat dataFormat); - - abstract Builder setStats(DatasetFeatureStatisticsList stats); - - abstract HistoricalRetrievalResult build(); - } - - public boolean hasError() { - return getError() != null; - } -} diff --git a/storage/api/src/main/java/feast/storage/api/retriever/HistoricalRetriever.java b/storage/api/src/main/java/feast/storage/api/retriever/HistoricalRetriever.java deleted file mode 100644 index 678b747da16..00000000000 --- a/storage/api/src/main/java/feast/storage/api/retriever/HistoricalRetriever.java +++ /dev/null @@ -1,53 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.storage.api.retriever; - -import feast.proto.serving.ServingAPIProto.DatasetSource; -import java.util.List; - -/** - * A historical retriever is a feature retriever that retrieves feature data corresponding to - * provided entities over a given period of time. - */ -public interface HistoricalRetriever { - - /** - * Get temporary staging location if applicable. If not applicable to this store, returns an empty - * string. - * - * @return staging location uri - */ - String getStagingLocation(); - - /** - * Get all features corresponding to the provided batch features request. - * - * @param retrievalId String that uniquely identifies this retrieval request. - * @param datasetSource {@link DatasetSource} containing source to load the dataset containing - * entity columns. - * @param featureSetRequests List of {@link FeatureSetRequest} to feature references in the - * request tied to that feature set. - * @param computeStatistics whether to compute statistics over the resultant dataset. - * @return {@link HistoricalRetrievalResult} if successful, contains the location of the results, - * else contains the error to be returned to the user. - */ - HistoricalRetrievalResult getHistoricalFeatures( - String retrievalId, - DatasetSource datasetSource, - List featureSetRequests, - boolean computeStatistics); -} diff --git a/storage/api/src/main/java/feast/storage/api/retriever/OnlineRetriever.java b/storage/api/src/main/java/feast/storage/api/retriever/OnlineRetriever.java deleted file mode 100644 index f56cf6ca022..00000000000 --- a/storage/api/src/main/java/feast/storage/api/retriever/OnlineRetriever.java +++ /dev/null @@ -1,43 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.storage.api.retriever; - -import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesRequest.EntityRow; -import feast.proto.types.FeatureRowProto.FeatureRow; -import java.util.List; -import java.util.Optional; - -/** An online retriever is a feature retriever that retrieves the latest feature data. */ -public interface OnlineRetriever { - - /** - * Get online features for the given entity rows using data retrieved from the feature/featureset - * specified in feature set request. - * - *

Each {@link FeatureRow} optional in the returned list then corresponds to an {@link - * EntityRow} provided by the user. If feature for a given entity row is not found, will return an - * empty optional instead. The no. of {@link FeatureRow} returned should match the no. of given - * {@link EntityRow}s - * - * @param entityRows list of entity rows to request features for. - * @param featureSetRequest specifies the features/feature set to retrieve data from - * @return list of {@link FeatureRow}s corresponding to data retrieved for each entity row from - * feature/featureset specified in featureset request. - */ - List> getOnlineFeatures( - List entityRows, FeatureSetRequest featureSetRequest); -} diff --git a/storage/api/src/main/java/feast/storage/api/retriever/OnlineRetrieverV2.java b/storage/api/src/main/java/feast/storage/api/retriever/OnlineRetrieverV2.java index 224cf5fe44c..9be66a7b1fb 100644 --- a/storage/api/src/main/java/feast/storage/api/retriever/OnlineRetrieverV2.java +++ b/storage/api/src/main/java/feast/storage/api/retriever/OnlineRetrieverV2.java @@ -18,7 +18,6 @@ import feast.proto.serving.ServingAPIProto; import java.util.List; -import java.util.Optional; public interface OnlineRetrieverV2 { /** @@ -37,7 +36,7 @@ public interface OnlineRetrieverV2 { * @return list of {@link Feature}s corresponding to data retrieved for each entity row from * FeatureTable specified in FeatureTable request. */ - List>> getOnlineFeatures( + List> getOnlineFeatures( String project, List entityRows, List featureReferences); diff --git a/storage/api/src/main/java/feast/storage/api/statistics/FeatureStatistics.java b/storage/api/src/main/java/feast/storage/api/statistics/FeatureStatistics.java deleted file mode 100644 index 355d856bd40..00000000000 --- a/storage/api/src/main/java/feast/storage/api/statistics/FeatureStatistics.java +++ /dev/null @@ -1,48 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.storage.api.statistics; - -import com.google.auto.value.AutoValue; -import com.google.common.collect.ImmutableList; -import org.tensorflow.metadata.v0.FeatureNameStatistics; - -/** Feature statistics over a bounded set of data. */ -@AutoValue -public abstract class FeatureStatistics { - - public abstract long getNumExamples(); - - public abstract ImmutableList getFeatureNameStatistics(); - - public static Builder newBuilder() { - return new AutoValue_FeatureStatistics.Builder(); - } - - @AutoValue.Builder - public abstract static class Builder { - public abstract Builder setNumExamples(long numExamples); - - protected abstract ImmutableList.Builder featureNameStatisticsBuilder(); - - public Builder addFeatureNameStatistics(FeatureNameStatistics featureNameStatistics) { - featureNameStatisticsBuilder().add(featureNameStatistics); - return this; - } - - public abstract FeatureStatistics build(); - } -} diff --git a/storage/api/src/main/java/feast/storage/api/statistics/StatisticsRetriever.java b/storage/api/src/main/java/feast/storage/api/statistics/StatisticsRetriever.java deleted file mode 100644 index b20dfbeaf1d..00000000000 --- a/storage/api/src/main/java/feast/storage/api/statistics/StatisticsRetriever.java +++ /dev/null @@ -1,46 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.storage.api.statistics; - -import com.google.protobuf.Timestamp; -import feast.proto.core.FeatureSetProto.FeatureSetSpec; -import java.util.List; - -public interface StatisticsRetriever { - - /** - * Get feature statistics for a single feature set, for a single dataset id. - * - * @param featureSetSpec feature set spec of the requested feature set - * @param features subset of features to retrieve. - * @param ingestionId ingestion id to filter the data by - * @return {@link FeatureStatistics} containing statistics for the requested features. - */ - FeatureStatistics getFeatureStatistics( - FeatureSetSpec featureSetSpec, List features, String ingestionId); - - /** - * Get feature statistics for a single feature set, for a single day. - * - * @param featureSetSpec feature set spec of the requested feature set - * @param features subset of features to retrieve. - * @param date date to filter the data by - * @return {@link FeatureStatistics} containing statistics for the requested features. - */ - FeatureStatistics getFeatureStatistics( - FeatureSetSpec featureSetSpec, List features, Timestamp date); -} diff --git a/storage/api/src/main/java/feast/storage/api/writer/DeadletterSink.java b/storage/api/src/main/java/feast/storage/api/writer/DeadletterSink.java deleted file mode 100644 index a07254bddb0..00000000000 --- a/storage/api/src/main/java/feast/storage/api/writer/DeadletterSink.java +++ /dev/null @@ -1,38 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.storage.api.writer; - -import org.apache.beam.sdk.transforms.PTransform; -import org.apache.beam.sdk.values.PCollection; -import org.apache.beam.sdk.values.PDone; - -/** Interface for for implementing user defined deadletter sinks to write failed elements to. */ -public interface DeadletterSink { - - /** - * Set up the deadletter sink for writes. This method will be called once during pipeline - * initialisation. - */ - void prepareWrite(); - - /** - * Get a {@link PTransform} that writes a collection of FailedElements to the deadletter sink. - * - * @return {@link PTransform} - */ - PTransform, PDone> write(); -} diff --git a/storage/api/src/main/java/feast/storage/api/writer/FailedElement.java b/storage/api/src/main/java/feast/storage/api/writer/FailedElement.java deleted file mode 100644 index c6db877216f..00000000000 --- a/storage/api/src/main/java/feast/storage/api/writer/FailedElement.java +++ /dev/null @@ -1,78 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2019 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.storage.api.writer; - -import com.google.auto.value.AutoValue; -import javax.annotation.Nullable; -import org.apache.beam.sdk.schemas.AutoValueSchema; -import org.apache.beam.sdk.schemas.annotations.DefaultSchema; -import org.joda.time.Instant; - -@AutoValue -// Use DefaultSchema annotation so this AutoValue class can be serialized by Beam -// https://issues.apache.org/jira/browse/BEAM-1891 -// https://github.com/apache/beam/pull/7334 -@DefaultSchema(AutoValueSchema.class) -public abstract class FailedElement { - public abstract Instant getTimestamp(); - - @Nullable - public abstract String getJobName(); - - @Nullable - public abstract String getProjectName(); - - @Nullable - public abstract String getFeatureSetName(); - - @Nullable - public abstract String getTransformName(); - - @Nullable - public abstract String getPayload(); - - @Nullable - public abstract String getErrorMessage(); - - @Nullable - public abstract String getStackTrace(); - - public static Builder newBuilder() { - return new AutoValue_FailedElement.Builder().setTimestamp(Instant.now()); - } - - @AutoValue.Builder - public abstract static class Builder { - public abstract Builder setTimestamp(Instant timestamp); - - public abstract Builder setProjectName(String projectName); - - public abstract Builder setFeatureSetName(String featureSetName); - - public abstract Builder setJobName(String jobName); - - public abstract Builder setTransformName(String transformName); - - public abstract Builder setPayload(String payload); - - public abstract Builder setErrorMessage(String errorMessage); - - public abstract Builder setStackTrace(String stackTrace); - - public abstract FailedElement build(); - } -} diff --git a/storage/api/src/main/java/feast/storage/api/writer/FeatureSink.java b/storage/api/src/main/java/feast/storage/api/writer/FeatureSink.java deleted file mode 100644 index 8734398a68b..00000000000 --- a/storage/api/src/main/java/feast/storage/api/writer/FeatureSink.java +++ /dev/null @@ -1,51 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.storage.api.writer; - -import feast.common.models.FeatureSetReference; -import feast.proto.core.FeatureSetProto; -import feast.proto.types.FeatureRowProto.FeatureRow; -import java.io.Serializable; -import org.apache.beam.sdk.transforms.PTransform; -import org.apache.beam.sdk.values.KV; -import org.apache.beam.sdk.values.PCollection; - -/** Interface for implementing user defined feature sink functionality. */ -public interface FeatureSink extends Serializable { - - /** - * Set up storage backend for write. This method will be called once during pipeline - * initialisation. - * - *

Should create transformation that would update sink's state based on given FeatureSetSpec - * stream. Returning stream should notify subscribers about successful installation of new - * FeatureSetSpec referenced by {@link FeatureSetReference}. - * - * @param featureSetSpecs specs stream - * @return stream of state updated events - */ - PCollection prepareWrite( - PCollection> featureSetSpecs); - - /** - * Get a {@link PTransform} that writes feature rows to the store, and returns a {@link - * WriteResult} that splits successful and failed inserts to be separately logged. - * - * @return {@link PTransform} - */ - PTransform, WriteResult> writer(); -} diff --git a/storage/api/src/main/java/feast/storage/api/writer/WriteResult.java b/storage/api/src/main/java/feast/storage/api/writer/WriteResult.java deleted file mode 100644 index abe06e41ee8..00000000000 --- a/storage/api/src/main/java/feast/storage/api/writer/WriteResult.java +++ /dev/null @@ -1,97 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.storage.api.writer; - -import com.google.common.collect.ImmutableMap; -import feast.proto.types.FeatureRowProto.FeatureRow; -import java.io.Serializable; -import java.util.Map; -import org.apache.beam.sdk.Pipeline; -import org.apache.beam.sdk.transforms.PTransform; -import org.apache.beam.sdk.values.*; - -/** The result of a write transform. */ -public final class WriteResult implements Serializable, POutput { - - private final Pipeline pipeline; - private final PCollection successfulInserts; - private final PCollection failedInserts; - - private static TupleTag successfulInsertsTag = new TupleTag<>("successfulInserts"); - private static TupleTag failedInsertsTupleTag = new TupleTag<>("failedInserts"); - - /** - * Creates a {@link WriteResult} in the given {@link Pipeline}. - * - * @param pipeline {@link Pipeline} - * @param successfulInserts {@link PCollection} of {@link FeatureRow}s successfully inserted into - * the store - * @param failedInserts {@link PCollection} of {@link FailedElement}s - * @return {@link WriteResult} - */ - public static WriteResult in( - Pipeline pipeline, - PCollection successfulInserts, - PCollection failedInserts) { - return new WriteResult(pipeline, successfulInserts, failedInserts); - } - - private WriteResult( - Pipeline pipeline, - PCollection successfulInserts, - PCollection failedInserts) { - - this.pipeline = pipeline; - this.successfulInserts = successfulInserts; - this.failedInserts = failedInserts; - } - - /** - * Gets set of feature rows that were unsuccessfully written to the store. The failed feature rows - * are wrapped in FailedElement objects so implementations of WriteResult can be flexible in how - * errors are stored. - * - * @return FailedElements of unsuccessfully written feature rows - */ - public PCollection getFailedInserts() { - return failedInserts; - } - - /** - * Gets set of successfully written feature rows. - * - * @return PCollection of feature rows successfully written to the store - */ - public PCollection getSuccessfulInserts() { - return successfulInserts; - } - - @Override - public Pipeline getPipeline() { - return pipeline; - } - - @Override - public Map, PValue> expand() { - return ImmutableMap.of( - successfulInsertsTag, successfulInserts, failedInsertsTupleTag, failedInserts); - } - - @Override - public void finishSpecifyingOutput( - String transformName, PInput input, PTransform transform) {} -} diff --git a/storage/api/src/main/java/feast/storage/common/retry/BackOffExecutor.java b/storage/api/src/main/java/feast/storage/common/retry/BackOffExecutor.java deleted file mode 100644 index 296582f8b35..00000000000 --- a/storage/api/src/main/java/feast/storage/common/retry/BackOffExecutor.java +++ /dev/null @@ -1,58 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.storage.common.retry; - -import java.io.Serializable; -import org.apache.beam.sdk.util.BackOff; -import org.apache.beam.sdk.util.BackOffUtils; -import org.apache.beam.sdk.util.FluentBackoff; -import org.apache.beam.sdk.util.Sleeper; -import org.joda.time.Duration; - -public class BackOffExecutor implements Serializable { - - private final Integer maxRetries; - private final Duration initialBackOff; - - public BackOffExecutor(Integer maxRetries, Duration initialBackOff) { - this.maxRetries = maxRetries; - this.initialBackOff = initialBackOff; - } - - public void execute(Retriable retriable) throws Exception { - FluentBackoff backoff = - FluentBackoff.DEFAULT.withMaxRetries(maxRetries).withInitialBackoff(initialBackOff); - execute(retriable, backoff); - } - - private void execute(Retriable retriable, FluentBackoff backoff) throws Exception { - Sleeper sleeper = Sleeper.DEFAULT; - BackOff backOff = backoff.backoff(); - while (true) { - try { - retriable.execute(); - break; - } catch (Exception e) { - if (retriable.isExceptionRetriable(e) && BackOffUtils.next(sleeper, backOff)) { - retriable.cleanUpAfterFailure(); - } else { - throw e; - } - } - } - } -} diff --git a/storage/api/src/main/java/feast/storage/common/retry/Retriable.java b/storage/api/src/main/java/feast/storage/common/retry/Retriable.java deleted file mode 100644 index 2c92c851758..00000000000 --- a/storage/api/src/main/java/feast/storage/common/retry/Retriable.java +++ /dev/null @@ -1,25 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.storage.common.retry; - -public interface Retriable { - void execute() throws Exception; - - Boolean isExceptionRetriable(Exception e); - - void cleanUpAfterFailure(); -} diff --git a/storage/api/src/main/java/feast/storage/common/testing/TestUtil.java b/storage/api/src/main/java/feast/storage/common/testing/TestUtil.java deleted file mode 100644 index 773abd57d61..00000000000 --- a/storage/api/src/main/java/feast/storage/common/testing/TestUtil.java +++ /dev/null @@ -1,200 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2019 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.storage.common.testing; - -import com.google.common.hash.Hashing; -import com.google.protobuf.ByteString; -import com.google.protobuf.Timestamp; -import feast.proto.core.FeatureSetProto.FeatureSet; -import feast.proto.core.FeatureSetProto.FeatureSetSpec; -import feast.proto.types.FeatureRowProto.FeatureRow; -import feast.proto.types.FeatureRowProto.FeatureRow.Builder; -import feast.proto.types.FieldProto.Field; -import feast.proto.types.ValueProto.*; -import java.nio.charset.StandardCharsets; -import java.time.Instant; -import java.util.concurrent.ThreadLocalRandom; -import org.apache.commons.lang3.RandomStringUtils; - -@SuppressWarnings("WeakerAccess") -public class TestUtil { - - /** - * Create a Feature Row with random value according to the FeatureSetSpec - * - * @param featureSet {@link FeatureSet} - * @return {@link FeatureRow} - */ - public static FeatureRow createRandomFeatureRow(FeatureSet featureSet) { - ThreadLocalRandom random = ThreadLocalRandom.current(); - int randomStringSizeMaxSize = 12; - return createRandomFeatureRow(featureSet, random.nextInt(0, randomStringSizeMaxSize) + 4); - } - - /** - * Create a Feature Row with random value according to the FeatureSet. - * - *

The Feature Row created contains fields according to the entities and features defined in - * FeatureSet, matching the value type of the field, with randomized value for testing. - * - * @param featureSet {@link FeatureSet} - * @param randomStringSize number of characters for the generated random string - * @return {@link FeatureRow} - */ - public static FeatureRow createRandomFeatureRow(FeatureSet featureSet, int randomStringSize) { - - Instant time = Instant.now(); - Timestamp timestamp = - Timestamp.newBuilder().setSeconds(time.getEpochSecond()).setNanos(time.getNano()).build(); - - Builder builder = - FeatureRow.newBuilder() - .setFeatureSet(getFeatureSetReference(featureSet)) - .setEventTimestamp(timestamp); - - featureSet - .getSpec() - .getEntitiesList() - .forEach( - field -> { - builder.addFields( - Field.newBuilder() - .setName(field.getName()) - .setValue(createRandomValue(field.getValueType(), randomStringSize)) - .build()); - }); - - featureSet - .getSpec() - .getFeaturesList() - .forEach( - field -> { - builder.addFields( - Field.newBuilder() - .setName(field.getName()) - .setValue(createRandomValue(field.getValueType(), randomStringSize)) - .build()); - }); - - return builder.build(); - } - - private static String getFeatureSetReference(FeatureSet featureSet) { - FeatureSetSpec spec = featureSet.getSpec(); - return String.format("%s/%s:%d", spec.getProject(), spec.getName()); - } - - /** - * Create a random Feast {@link Value} of {@link ValueType.Enum}. - * - * @param type {@link ValueType.Enum} - * @param randomStringSize number of characters for the generated random string - * @return {@link Value} - */ - public static Value createRandomValue(ValueType.Enum type, int randomStringSize) { - Value.Builder builder = Value.newBuilder(); - ThreadLocalRandom random = ThreadLocalRandom.current(); - - switch (type) { - case INVALID: - case UNRECOGNIZED: - throw new IllegalArgumentException("Invalid ValueType: " + type); - case BYTES: - builder.setBytesVal( - ByteString.copyFrom(RandomStringUtils.randomAlphanumeric(randomStringSize).getBytes())); - break; - case STRING: - builder.setStringVal(RandomStringUtils.randomAlphanumeric(randomStringSize)); - break; - case INT32: - builder.setInt32Val(random.nextInt()); - break; - case INT64: - builder.setInt64Val(random.nextLong()); - break; - case DOUBLE: - builder.setDoubleVal(random.nextDouble()); - break; - case FLOAT: - builder.setFloatVal(random.nextFloat()); - break; - case BOOL: - builder.setBoolVal(random.nextBoolean()); - break; - case BYTES_LIST: - builder.setBytesListVal( - BytesList.newBuilder() - .addVal( - ByteString.copyFrom( - RandomStringUtils.randomAlphanumeric(randomStringSize).getBytes())) - .build()); - break; - case STRING_LIST: - builder.setStringListVal( - StringList.newBuilder() - .addVal(RandomStringUtils.randomAlphanumeric(randomStringSize)) - .build()); - break; - case INT32_LIST: - builder.setInt32ListVal(Int32List.newBuilder().addVal(random.nextInt()).build()); - break; - case INT64_LIST: - builder.setInt64ListVal(Int64List.newBuilder().addVal(random.nextLong()).build()); - break; - case DOUBLE_LIST: - builder.setDoubleListVal(DoubleList.newBuilder().addVal(random.nextDouble()).build()); - break; - case FLOAT_LIST: - builder.setFloatListVal(FloatList.newBuilder().addVal(random.nextFloat()).build()); - break; - case BOOL_LIST: - builder.setBoolListVal(BoolList.newBuilder().addVal(random.nextBoolean()).build()); - break; - } - return builder.build(); - } - - /** - * Create a field object with given name and type. - * - * @param name of the field. - * @param value of the field. Should be compatible with the valuetype given. - * @param valueType type of the field. - * @return Field object - */ - public static Field field(String name, Object value, ValueType.Enum valueType) { - Field.Builder fieldBuilder = Field.newBuilder().setName(name); - switch (valueType) { - case INT32: - return fieldBuilder.setValue(Value.newBuilder().setInt32Val((int) value)).build(); - case INT64: - return fieldBuilder.setValue(Value.newBuilder().setInt64Val((int) value)).build(); - case FLOAT: - return fieldBuilder.setValue(Value.newBuilder().setFloatVal((float) value)).build(); - case DOUBLE: - return fieldBuilder.setValue(Value.newBuilder().setDoubleVal((double) value)).build(); - case STRING: - return fieldBuilder.setValue(Value.newBuilder().setStringVal((String) value)).build(); - default: - throw new IllegalStateException("Unexpected valueType: " + value.getClass()); - } - } - - public static String hash(String input) { - return Hashing.murmur3_32().hashString(input, StandardCharsets.UTF_8).toString(); - } -} diff --git a/storage/connectors/bigquery/pom.xml b/storage/connectors/bigquery/pom.xml deleted file mode 100644 index bb304452ac6..00000000000 --- a/storage/connectors/bigquery/pom.xml +++ /dev/null @@ -1,99 +0,0 @@ - - - - dev.feast - feast-storage-connectors - ${revision} - - - 4.0.0 - feast-storage-connector-bigquery - - Feast Storage Connector for BigQuery - - - - io.pebbletemplates - pebble - 3.1.0 - - - - - com.google.cloud - google-cloud-bigquery - - - - com.google.cloud - google-cloud-storage - - - - org.apache.beam - beam-sdks-java-io-google-cloud-platform - ${org.apache.beam.version} - - - com.google.cloud - google-cloud-spanner - - - com.google.cloud.bigtable - bigtable-client-core - - - - - - io.opencensus - opencensus-contrib-http-util - 0.21.0 - - - - com.google.auto.value - auto-value-annotations - 1.6.6 - - - - junit - junit - 4.12 - test - - - - org.slf4j - slf4j-simple - 1.7.30 - test - - - - org.apache.beam - beam-runners-direct-java - ${org.apache.beam.version} - test - - - - org.hamcrest - hamcrest-core - test - - - org.hamcrest - hamcrest-library - test - - - org.mockito - mockito-core - ${mockito.version} - test - - - diff --git a/storage/connectors/bigquery/src/main/java/feast/storage/connectors/bigquery/common/TypeUtil.java b/storage/connectors/bigquery/src/main/java/feast/storage/connectors/bigquery/common/TypeUtil.java deleted file mode 100644 index 9f6cff20e1d..00000000000 --- a/storage/connectors/bigquery/src/main/java/feast/storage/connectors/bigquery/common/TypeUtil.java +++ /dev/null @@ -1,179 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.storage.connectors.bigquery.common; - -import com.google.cloud.bigquery.StandardSQLTypeName; -import com.google.protobuf.ByteString; -import feast.proto.types.ValueProto; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; - -public class TypeUtil { - - private static final Map - VALUE_TYPE_TO_STANDARD_SQL_TYPE = new HashMap<>(); - - static { - VALUE_TYPE_TO_STANDARD_SQL_TYPE.put(ValueProto.ValueType.Enum.BYTES, StandardSQLTypeName.BYTES); - VALUE_TYPE_TO_STANDARD_SQL_TYPE.put( - ValueProto.ValueType.Enum.STRING, StandardSQLTypeName.STRING); - VALUE_TYPE_TO_STANDARD_SQL_TYPE.put(ValueProto.ValueType.Enum.INT32, StandardSQLTypeName.INT64); - VALUE_TYPE_TO_STANDARD_SQL_TYPE.put(ValueProto.ValueType.Enum.INT64, StandardSQLTypeName.INT64); - VALUE_TYPE_TO_STANDARD_SQL_TYPE.put( - ValueProto.ValueType.Enum.DOUBLE, StandardSQLTypeName.FLOAT64); - VALUE_TYPE_TO_STANDARD_SQL_TYPE.put( - ValueProto.ValueType.Enum.FLOAT, StandardSQLTypeName.FLOAT64); - VALUE_TYPE_TO_STANDARD_SQL_TYPE.put(ValueProto.ValueType.Enum.BOOL, StandardSQLTypeName.BOOL); - VALUE_TYPE_TO_STANDARD_SQL_TYPE.put( - ValueProto.ValueType.Enum.BYTES_LIST, StandardSQLTypeName.BYTES); - VALUE_TYPE_TO_STANDARD_SQL_TYPE.put( - ValueProto.ValueType.Enum.STRING_LIST, StandardSQLTypeName.STRING); - VALUE_TYPE_TO_STANDARD_SQL_TYPE.put( - ValueProto.ValueType.Enum.INT32_LIST, StandardSQLTypeName.INT64); - VALUE_TYPE_TO_STANDARD_SQL_TYPE.put( - ValueProto.ValueType.Enum.INT64_LIST, StandardSQLTypeName.INT64); - VALUE_TYPE_TO_STANDARD_SQL_TYPE.put( - ValueProto.ValueType.Enum.DOUBLE_LIST, StandardSQLTypeName.FLOAT64); - VALUE_TYPE_TO_STANDARD_SQL_TYPE.put( - ValueProto.ValueType.Enum.FLOAT_LIST, StandardSQLTypeName.FLOAT64); - VALUE_TYPE_TO_STANDARD_SQL_TYPE.put( - ValueProto.ValueType.Enum.BOOL_LIST, StandardSQLTypeName.BOOL); - } - - /** - * Converts {@link feast.proto.types.ValueProto.ValueType} to its corresponding {@link - * StandardSQLTypeName} - * - * @param valueType value type to convert - * @return {@link StandardSQLTypeName} - */ - public static StandardSQLTypeName toStandardSqlType(ValueProto.ValueType.Enum valueType) { - return VALUE_TYPE_TO_STANDARD_SQL_TYPE.get(valueType); - } - - public static Object protoValueToObject( - ValueProto.Value value, ValueProto.Value.ValCase valCase) { - switch (valCase) { - case BYTES_VAL: - return value.getBytesVal().toByteArray(); - case STRING_VAL: - return value.getStringVal(); - case INT32_VAL: - return value.getInt32Val(); - case INT64_VAL: - return value.getInt64Val(); - case DOUBLE_VAL: - return value.getDoubleVal(); - case FLOAT_VAL: - return value.getFloatVal(); - case BOOL_VAL: - return value.getBoolVal(); - case BYTES_LIST_VAL: - return value.getBytesListVal().getValList().stream() - .map(ByteString::toByteArray) - .collect(Collectors.toList()); - case STRING_LIST_VAL: - return value.getStringListVal().getValList(); - case INT32_LIST_VAL: - return value.getInt32ListVal().getValList(); - case INT64_LIST_VAL: - return value.getInt64ListVal().getValList(); - case DOUBLE_LIST_VAL: - return value.getDoubleListVal().getValList(); - case FLOAT_LIST_VAL: - return value.getFloatListVal().getValList(); - case BOOL_LIST_VAL: - return value.getBoolListVal().getValList(); - case VAL_NOT_SET: - break; - } - return null; - } - - public static ValueProto.Value objectToProtoValue( - Object value, ValueProto.Value.ValCase valCase) { - ValueProto.Value.Builder builder = ValueProto.Value.newBuilder(); - switch (valCase) { - case BYTES_VAL: - return builder.setBytesVal(ByteString.copyFrom((byte[]) value)).build(); - case STRING_VAL: - return builder.setStringVal((String) value).build(); - case INT32_VAL: - return builder.setInt32Val((Integer) value).build(); - case INT64_VAL: - return builder.setInt64Val((Long) value).build(); - case DOUBLE_VAL: - return builder.setDoubleVal((Double) value).build(); - case FLOAT_VAL: - return builder.setFloatVal((Float) value).build(); - case BOOL_VAL: - return builder.setBoolVal((Boolean) value).build(); - case BYTES_LIST_VAL: - return builder - .setBytesListVal( - ValueProto.BytesList.newBuilder() - .addAllVal( - ((List) value) - .stream().map(ByteString::copyFrom).collect(Collectors.toList())) - .build()) - .build(); - case STRING_LIST_VAL: - return builder - .setStringListVal( - ValueProto.StringList.newBuilder().addAllVal((List) value).build()) - .build(); - case INT32_LIST_VAL: - return builder - .setInt32ListVal( - ValueProto.Int32List.newBuilder().addAllVal((List) value).build()) - .build(); - case INT64_LIST_VAL: - return builder - .setInt64ListVal( - ValueProto.Int64List.newBuilder().addAllVal((List) value).build()) - .build(); - case DOUBLE_LIST_VAL: - return builder - .setDoubleListVal( - ValueProto.DoubleList.newBuilder().addAllVal((List) value).build()) - .build(); - case FLOAT_LIST_VAL: - return builder - .setFloatListVal( - ValueProto.FloatList.newBuilder().addAllVal((List) value).build()) - .build(); - case BOOL_LIST_VAL: - return builder - .setBoolListVal( - ValueProto.BoolList.newBuilder().addAllVal((List) value).build()) - .build(); - case VAL_NOT_SET: - break; - } - return null; - } - - public static Object protoValueToObject(ValueProto.Value value) { - return protoValueToObject(value, value.getValCase()); - } - - public static Object getDefaultProtoValue(ValueProto.Value.ValCase valCase) { - return protoValueToObject(ValueProto.Value.getDefaultInstance(), valCase); - } -} diff --git a/storage/connectors/bigquery/src/main/java/feast/storage/connectors/bigquery/retriever/BigQueryHistoricalRetriever.java b/storage/connectors/bigquery/src/main/java/feast/storage/connectors/bigquery/retriever/BigQueryHistoricalRetriever.java deleted file mode 100644 index f709651beb7..00000000000 --- a/storage/connectors/bigquery/src/main/java/feast/storage/connectors/bigquery/retriever/BigQueryHistoricalRetriever.java +++ /dev/null @@ -1,497 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.storage.connectors.bigquery.retriever; - -import static feast.storage.connectors.bigquery.retriever.QueryTemplater.createEntityTableUUIDQuery; -import static feast.storage.connectors.bigquery.retriever.QueryTemplater.createTimestampLimitQuery; - -import com.google.auto.value.AutoValue; -import com.google.cloud.RetryOption; -import com.google.cloud.bigquery.*; -import com.google.cloud.storage.Blob; -import com.google.cloud.storage.Storage; -import com.google.cloud.storage.StorageOptions; -import feast.proto.core.FeatureSetProto.FeatureSpec; -import feast.proto.serving.ServingAPIProto; -import feast.proto.serving.ServingAPIProto.DataFormat; -import feast.proto.serving.ServingAPIProto.DatasetSource; -import feast.proto.serving.ServingAPIProto.FeatureReference; -import feast.storage.api.retriever.FeatureSetRequest; -import feast.storage.api.retriever.HistoricalRetrievalResult; -import feast.storage.api.retriever.HistoricalRetriever; -import feast.storage.api.statistics.FeatureStatistics; -import feast.storage.connectors.bigquery.statistics.BigQueryStatisticsRetriever; -import feast.storage.connectors.bigquery.statistics.FeatureStatisticsQueryInfo; -import feast.storage.connectors.bigquery.statistics.StatsDataset; -import io.grpc.Status; -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.UUID; -import java.util.concurrent.*; -import java.util.stream.Collectors; -import org.slf4j.Logger; -import org.tensorflow.metadata.v0.DatasetFeatureStatistics; -import org.tensorflow.metadata.v0.DatasetFeatureStatisticsList; -import org.threeten.bp.Duration; - -@AutoValue -public abstract class BigQueryHistoricalRetriever implements HistoricalRetriever { - - private static final Logger log = - org.slf4j.LoggerFactory.getLogger(BigQueryHistoricalRetriever.class); - - public static final long TEMP_TABLE_EXPIRY_DURATION_MS = Duration.ofDays(1).toMillis(); - private static final long SUBQUERY_TIMEOUT_SECS = 900; // 15 minutes - - public static HistoricalRetriever create(Map config) { - - BigQuery bigquery = BigQueryOptions.getDefaultInstance().getService(); - Storage storage = StorageOptions.getDefaultInstance().getService(); - - String jobStagingLocation = config.get("staging_location"); - if (!jobStagingLocation.contains("://")) { - throw new IllegalArgumentException( - String.format("jobStagingLocation is not a valid URI: %s", jobStagingLocation)); - } - if (jobStagingLocation.endsWith("/")) { - jobStagingLocation = jobStagingLocation.substring(0, jobStagingLocation.length() - 1); - } - if (!jobStagingLocation.startsWith("gs://")) { - throw new IllegalArgumentException( - "Store type BIGQUERY requires job staging location to be a valid and existing Google Cloud Storage URI. Invalid staging location: " - + jobStagingLocation); - } - - return builder() - .setBigquery(bigquery) - .setDatasetId(config.get("dataset_id")) - .setProjectId(config.get("project_id")) - .setJobStagingLocation(config.get("staging_location")) - .setInitialRetryDelaySecs(Integer.parseInt(config.get("initial_retry_delay_seconds"))) - .setTotalTimeoutSecs(Integer.parseInt(config.get("total_timeout_seconds"))) - .setStorage(storage) - .build(); - } - - public abstract String projectId(); - - public abstract String datasetId(); - - public abstract BigQuery bigquery(); - - public abstract String jobStagingLocation(); - - public abstract int initialRetryDelaySecs(); - - public abstract int totalTimeoutSecs(); - - public abstract Storage storage(); - - public static Builder builder() { - return new AutoValue_BigQueryHistoricalRetriever.Builder(); - } - - @AutoValue.Builder - public abstract static class Builder { - public abstract Builder setProjectId(String projectId); - - public abstract Builder setDatasetId(String datasetId); - - public abstract Builder setJobStagingLocation(String jobStagingLocation); - - public abstract Builder setBigquery(BigQuery bigquery); - - public abstract Builder setInitialRetryDelaySecs(int initialRetryDelaySecs); - - public abstract Builder setTotalTimeoutSecs(int totalTimeoutSecs); - - public abstract Builder setStorage(Storage storage); - - public abstract BigQueryHistoricalRetriever build(); - } - - @Override - public String getStagingLocation() { - return jobStagingLocation(); - } - - @Override - public HistoricalRetrievalResult getHistoricalFeatures( - String retrievalId, - DatasetSource datasetSource, - List featureSetRequests, - boolean computeStatistics) { - List featureSetQueryInfos = - QueryTemplater.getFeatureSetInfos(featureSetRequests); - - // 1. load entity table - Table entityTable; - String entityTableName; - try { - entityTable = loadEntities(datasetSource); - - TableId entityTableWithUUIDs = generateUUIDs(entityTable); - entityTableName = generateFullTableName(entityTableWithUUIDs); - } catch (Exception e) { - return HistoricalRetrievalResult.error( - retrievalId, - new RuntimeException( - String.format("Unable to load entity table to BigQuery: %s", e.toString()))); - } - - Schema entityTableSchema = entityTable.getDefinition().getSchema(); - List entityTableColumnNames = - entityTableSchema.getFields().stream() - .map(Field::getName) - .filter(name -> !name.equals("event_timestamp")) - .collect(Collectors.toList()); - - // 2. Retrieve the temporal bounds of the entity dataset provided - FieldValueList timestampLimits = getTimestampLimits(entityTableName); - - // 3. Generate the subqueries - List featureSetQueries = - generateQueries(entityTableName, timestampLimits, featureSetQueryInfos); - - QueryJobConfiguration queryConfig; - - try { - // 4. Run the subqueries in parallel then collect the outputs - Job queryJob = - runBatchQuery( - entityTableName, entityTableColumnNames, featureSetQueryInfos, featureSetQueries); - queryConfig = queryJob.getConfiguration(); - String exportTableDestinationUri = - String.format("%s/%s/*.avro", jobStagingLocation(), retrievalId); - - // 5. Export the table - // Hardcode the format to Avro for now - ExtractJobConfiguration extractConfig = - ExtractJobConfiguration.of( - queryConfig.getDestinationTable(), exportTableDestinationUri, "Avro"); - Job extractJob = bigquery().create(JobInfo.of(extractConfig)); - waitForJob(extractJob); - - } catch (BigQueryException | InterruptedException | IOException e) { - return HistoricalRetrievalResult.error(retrievalId, e); - } - - List fileUris = parseOutputFileURIs(retrievalId); - - HistoricalRetrievalResult result = - HistoricalRetrievalResult.success(retrievalId, fileUris, DataFormat.DATA_FORMAT_AVRO); - - // 6. If the user requested to compute statistics, compute them over the output table - if (computeStatistics) { - BigQueryStatisticsRetriever statsRetriever = - BigQueryStatisticsRetriever.newBuilder() - .setProjectId(projectId()) - .setDatasetId(datasetId()) - .setBigquery(bigquery()) - .build(); - - List featureStatisticsQueryInfos = - buildFeatureStatisticsQuery(featureSetRequests); - FeatureStatistics featureStatistics = - statsRetriever.getFeatureStatistics( - featureStatisticsQueryInfos, new StatsDataset(queryConfig.getDestinationTable())); - DatasetFeatureStatisticsList datasetFeatureStatisticsList = - DatasetFeatureStatisticsList.newBuilder() - .addDatasets( - DatasetFeatureStatistics.newBuilder() - .addAllFeatures(featureStatistics.getFeatureNameStatistics()) - .setNumExamples(featureStatistics.getNumExamples())) - .build(); - result = result.withStats(datasetFeatureStatisticsList); - } - return result; - } - - private List buildFeatureStatisticsQuery( - List featureSetRequests) { - List featureStatisticsQueryInfos = new ArrayList<>(); - for (FeatureSetRequest request : featureSetRequests) { - Map refsByName = request.getFeatureRefsByName(); - for (FeatureSpec featureSpec : request.getSpec().getFeaturesList()) { - FeatureReference ref = refsByName.getOrDefault(featureSpec.getName(), null); - if (ref != null) { - if (!ref.getFeatureSet().equals("")) { - featureSpec = - featureSpec - .toBuilder() - .setName(String.format("%s__%s", ref.getFeatureSet(), ref.getName())) - .build(); - } - featureStatisticsQueryInfos.add(FeatureStatisticsQueryInfo.fromProto(featureSpec)); - } - } - } - return featureStatisticsQueryInfos; - } - - private TableId generateUUIDs(Table loadedEntityTable) { - try { - String uuidQuery = - createEntityTableUUIDQuery(generateFullTableName(loadedEntityTable.getTableId())); - QueryJobConfiguration queryJobConfig = - QueryJobConfiguration.newBuilder(uuidQuery) - .setDestinationTable(TableId.of(projectId(), datasetId(), createTempTableName())) - .build(); - Job queryJob = bigquery().create(JobInfo.of(queryJobConfig)); - Job completedJob = waitForJob(queryJob); - TableInfo expiry = - bigquery() - .getTable(queryJobConfig.getDestinationTable()) - .toBuilder() - .setExpirationTime(System.currentTimeMillis() + TEMP_TABLE_EXPIRY_DURATION_MS) - .build(); - bigquery().update(expiry); - queryJobConfig = completedJob.getConfiguration(); - return queryJobConfig.getDestinationTable(); - } catch (InterruptedException | BigQueryException e) { - throw Status.INTERNAL - .withDescription("Failed to load entity dataset into store") - .withCause(e) - .asRuntimeException(); - } - } - - private FieldValueList getTimestampLimits(String entityTableName) { - QueryJobConfiguration getTimestampLimitsQuery = - QueryJobConfiguration.newBuilder(createTimestampLimitQuery(entityTableName)) - .setDefaultDataset(DatasetId.of(projectId(), datasetId())) - .setDestinationTable(TableId.of(projectId(), datasetId(), createTempTableName())) - .build(); - try { - Job job = bigquery().create(JobInfo.of(getTimestampLimitsQuery)); - TableResult getTimestampLimitsQueryResult = waitForJob(job).getQueryResults(); - TableInfo expiry = - bigquery() - .getTable(getTimestampLimitsQuery.getDestinationTable()) - .toBuilder() - .setExpirationTime(System.currentTimeMillis() + TEMP_TABLE_EXPIRY_DURATION_MS) - .build(); - bigquery().update(expiry); - FieldValueList result = null; - for (FieldValueList fields : getTimestampLimitsQueryResult.getValues()) { - result = fields; - } - if (result == null || result.get("min").isNull() || result.get("max").isNull()) { - throw new RuntimeException("query returned insufficient values"); - } - return result; - } catch (InterruptedException e) { - throw Status.INTERNAL - .withDescription("Unable to extract min and max timestamps from query") - .withCause(e) - .asRuntimeException(); - } - } - - private Table loadEntities(ServingAPIProto.DatasetSource datasetSource) { - Table loadedEntityTable; - switch (datasetSource.getDatasetSourceCase()) { - case FILE_SOURCE: - try { - // Currently only AVRO format is supported - if (datasetSource.getFileSource().getDataFormat() - != ServingAPIProto.DataFormat.DATA_FORMAT_AVRO) { - throw Status.INVALID_ARGUMENT - .withDescription("Invalid file format, only AVRO is supported.") - .asRuntimeException(); - } - - TableId tableId = TableId.of(projectId(), datasetId(), createTempTableName()); - log.info( - "Loading entity rows to: {}.{}.{}", projectId(), datasetId(), tableId.getTable()); - - LoadJobConfiguration loadJobConfiguration = - LoadJobConfiguration.of( - tableId, datasetSource.getFileSource().getFileUrisList(), FormatOptions.avro()); - loadJobConfiguration = - loadJobConfiguration.toBuilder().setUseAvroLogicalTypes(true).build(); - Job job = bigquery().create(JobInfo.of(loadJobConfiguration)); - waitForJob(job); - - TableInfo expiry = - bigquery() - .getTable(tableId) - .toBuilder() - .setExpirationTime(System.currentTimeMillis() + TEMP_TABLE_EXPIRY_DURATION_MS) - .build(); - bigquery().update(expiry); - - loadedEntityTable = bigquery().getTable(tableId); - if (!loadedEntityTable.exists()) { - throw new RuntimeException( - "Unable to create entity dataset table, table already exists"); - } - return loadedEntityTable; - } catch (Exception e) { - log.error("Exception has occurred in loadEntities method: ", e); - throw Status.INTERNAL - .withDescription("Failed to load entity dataset into store: " + e.toString()) - .withCause(e) - .asRuntimeException(); - } - case DATASETSOURCE_NOT_SET: - default: - throw Status.INVALID_ARGUMENT - .withDescription("Data source must be set.") - .asRuntimeException(); - } - } - - private List generateQueries( - String entityTableName, - FieldValueList timestampLimits, - List featureSetQueryInfos) { - List featureSetQueries = new ArrayList<>(); - try { - for (FeatureSetQueryInfo featureSetInfo : featureSetQueryInfos) { - String query = - QueryTemplater.createFeatureSetPointInTimeQuery( - featureSetInfo, - projectId(), - datasetId(), - entityTableName, - timestampLimits.get("min").getStringValue(), - timestampLimits.get("max").getStringValue()); - featureSetQueries.add(query); - } - } catch (IOException e) { - throw Status.INTERNAL - .withDescription("Unable to generate query for batch retrieval") - .withCause(e) - .asRuntimeException(); - } - return featureSetQueries; - } - - Job runBatchQuery( - String entityTableName, - List entityTableColumnNames, - List featureSetQueryInfos, - List featureSetQueries) - throws BigQueryException, InterruptedException, IOException { - ExecutorService executorService = Executors.newFixedThreadPool(featureSetQueries.size()); - ExecutorCompletionService executorCompletionService = - new ExecutorCompletionService<>(executorService); - - // For each of the feature sets requested, start an async job joining the features in that - // feature set to the provided entity table - for (int i = 0; i < featureSetQueries.size(); i++) { - QueryJobConfiguration queryJobConfig = - QueryJobConfiguration.newBuilder(featureSetQueries.get(i)) - .setDestinationTable(TableId.of(projectId(), datasetId(), createTempTableName())) - .build(); - Job subqueryJob = bigquery().create(JobInfo.of(queryJobConfig)); - executorCompletionService.submit( - SubqueryCallable.builder() - .setBigquery(bigquery()) - .setFeatureSetInfo(featureSetQueryInfos.get(i)) - .setSubqueryJob(subqueryJob) - .build()); - } - - List completedFeatureSetQueryInfos = new ArrayList<>(); - - for (int i = 0; i < featureSetQueries.size(); i++) { - try { - // Try to retrieve the outputs of all the jobs. The timeout here is a formality; - // a stricter timeout is implemented in the actual SubqueryCallable. - FeatureSetQueryInfo featureSetInfo = - executorCompletionService.take().get(SUBQUERY_TIMEOUT_SECS, TimeUnit.SECONDS); - completedFeatureSetQueryInfos.add(featureSetInfo); - } catch (InterruptedException | ExecutionException | TimeoutException e) { - executorService.shutdownNow(); - throw Status.INTERNAL - .withDescription("Error running batch query") - .withCause(e) - .asRuntimeException(); - } - } - - // Generate and run a join query to collect the outputs of all the - // subqueries into a single table. - String joinQuery = - QueryTemplater.createJoinQuery( - completedFeatureSetQueryInfos, entityTableColumnNames, entityTableName); - QueryJobConfiguration queryJobConfig = - QueryJobConfiguration.newBuilder(joinQuery) - .setDestinationTable(TableId.of(projectId(), datasetId(), createTempTableName())) - .build(); - Job queryJob = bigquery().create(JobInfo.of(queryJobConfig)); - Job completedQueryJob = waitForJob(queryJob); - - TableInfo expiry = - bigquery() - .getTable(queryJobConfig.getDestinationTable()) - .toBuilder() - .setExpirationTime(System.currentTimeMillis() + TEMP_TABLE_EXPIRY_DURATION_MS) - .build(); - bigquery().update(expiry); - - return completedQueryJob; - } - - private List parseOutputFileURIs(String feastJobId) { - String scheme = jobStagingLocation().substring(0, jobStagingLocation().indexOf("://")); - String stagingLocationNoScheme = - jobStagingLocation().substring(jobStagingLocation().indexOf("://") + 3); - String bucket = stagingLocationNoScheme.split("/")[0]; - List prefixParts = new ArrayList<>(); - prefixParts.add( - stagingLocationNoScheme.contains("/") && !stagingLocationNoScheme.endsWith("/") - ? stagingLocationNoScheme.substring(stagingLocationNoScheme.indexOf("/") + 1) - : ""); - prefixParts.add(feastJobId); - String prefix = String.join("/", prefixParts) + "/"; - - List fileUris = new ArrayList<>(); - for (Blob blob : storage().list(bucket, Storage.BlobListOption.prefix(prefix)).iterateAll()) { - fileUris.add(String.format("%s://%s/%s", scheme, blob.getBucket(), blob.getName())); - } - return fileUris; - } - - private Job waitForJob(Job queryJob) throws InterruptedException { - Job completedJob = - queryJob.waitFor( - RetryOption.initialRetryDelay(Duration.ofSeconds(initialRetryDelaySecs())), - RetryOption.totalTimeout(Duration.ofSeconds(totalTimeoutSecs()))); - if (completedJob == null) { - throw Status.INTERNAL.withDescription("Job no longer exists").asRuntimeException(); - } else if (completedJob.getStatus().getError() != null) { - throw Status.INTERNAL - .withDescription("Job failed: " + completedJob.getStatus().getError()) - .asRuntimeException(); - } - return completedJob; - } - - public String generateFullTableName(TableId tableId) { - return String.format( - "%s.%s.%s", tableId.getProject(), tableId.getDataset(), tableId.getTable()); - } - - public String createTempTableName() { - return "_" + UUID.randomUUID().toString().replace("-", ""); - } -} diff --git a/storage/connectors/bigquery/src/main/java/feast/storage/connectors/bigquery/retriever/FeatureSetQueryInfo.java b/storage/connectors/bigquery/src/main/java/feast/storage/connectors/bigquery/retriever/FeatureSetQueryInfo.java deleted file mode 100644 index 74cfb47c9b2..00000000000 --- a/storage/connectors/bigquery/src/main/java/feast/storage/connectors/bigquery/retriever/FeatureSetQueryInfo.java +++ /dev/null @@ -1,79 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2019 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.storage.connectors.bigquery.retriever; - -import feast.proto.serving.ServingAPIProto.FeatureReference; -import java.util.List; - -public class FeatureSetQueryInfo { - - private final String project; - private final String name; - private final long maxAge; - private final List entities; - private final List features; - private final String table; - - public FeatureSetQueryInfo( - String project, - String name, - long maxAge, - List entities, - List features, - String table) { - this.project = project; - this.name = name; - this.maxAge = maxAge; - this.entities = entities; - this.features = features; - this.table = table; - } - - public FeatureSetQueryInfo(FeatureSetQueryInfo featureSetInfo, String table) { - - this.project = featureSetInfo.getProject(); - this.name = featureSetInfo.getName(); - this.maxAge = featureSetInfo.getMaxAge(); - this.entities = featureSetInfo.getEntities(); - this.features = featureSetInfo.getFeatures(); - this.table = table; - } - - public String getProject() { - return project.replace("-", "_"); - } - - public String getName() { - return name.replace("-", "_"); - } - - public long getMaxAge() { - return maxAge; - } - - public List getEntities() { - return entities; - } - - public List getFeatures() { - return features; - } - - public String getTable() { - return table; - } -} diff --git a/storage/connectors/bigquery/src/main/java/feast/storage/connectors/bigquery/retriever/QueryTemplater.java b/storage/connectors/bigquery/src/main/java/feast/storage/connectors/bigquery/retriever/QueryTemplater.java deleted file mode 100644 index 969efb36c38..00000000000 --- a/storage/connectors/bigquery/src/main/java/feast/storage/connectors/bigquery/retriever/QueryTemplater.java +++ /dev/null @@ -1,152 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2019 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.storage.connectors.bigquery.retriever; - -import com.google.cloud.bigquery.TableId; -import com.google.protobuf.Duration; -import com.mitchellbosecke.pebble.PebbleEngine; -import com.mitchellbosecke.pebble.template.PebbleTemplate; -import feast.proto.core.FeatureSetProto.EntitySpec; -import feast.proto.core.FeatureSetProto.FeatureSetSpec; -import feast.proto.serving.ServingAPIProto.FeatureReference; -import feast.storage.api.retriever.FeatureSetRequest; -import java.io.IOException; -import java.io.StringWriter; -import java.io.Writer; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; - -public class QueryTemplater { - - private static final PebbleEngine engine = new PebbleEngine.Builder().build(); - private static final String FEATURESET_TEMPLATE_NAME = "templates/single_featureset_pit_join.sql"; - private static final String JOIN_TEMPLATE_NAME = "templates/join_featuresets.sql"; - - /** - * Get the query for retrieving the earliest and latest timestamps in the entity dataset. - * - * @param leftTableName full entity dataset name - * @return timestamp limit BQ SQL query - */ - public static String createTimestampLimitQuery(String leftTableName) { - return String.format( - "SELECT DATETIME(MAX(event_timestamp)) as max, DATETIME(MIN(event_timestamp)) as min FROM `%s`", - leftTableName); - } - - /** - * Creates a query that generates a UUID for the entity table, for left joins later on. - * - * @param leftTableName full entity dataset name - * @return uuid generation query - */ - public static String createEntityTableUUIDQuery(String leftTableName) { - return String.format( - "SELECT GENERATE_UUID() as uuid, `%s`.* from `%s`", leftTableName, leftTableName); - } - - /** - * Generate the information necessary for the sql templating for point in time correctness join to - * the entity dataset for each feature set requested. - * - * @param featureSetRequests List of {@link FeatureSetRequest} containing a {@link FeatureSetSpec} - * and its corresponding {@link FeatureReference}s provided by the user. - * @return List of FeatureSetInfos - */ - public static List getFeatureSetInfos( - List featureSetRequests) throws IllegalArgumentException { - - List featureSetInfos = new ArrayList<>(); - for (FeatureSetRequest featureSetRequest : featureSetRequests) { - FeatureSetSpec spec = featureSetRequest.getSpec(); - Duration maxAge = spec.getMaxAge(); - List fsEntities = - spec.getEntitiesList().stream().map(EntitySpec::getName).collect(Collectors.toList()); - List features = featureSetRequest.getFeatureReferences().asList(); - featureSetInfos.add( - new FeatureSetQueryInfo( - spec.getProject(), spec.getName(), maxAge.getSeconds(), fsEntities, features, "")); - } - return featureSetInfos; - } - - /** - * Generate the query for point in time correctness join of data for a single feature set to the - * entity dataset. - * - * @param featureSetInfo Information about the feature set necessary for the query templating - * @param projectId google project ID - * @param datasetId feast bigquery dataset ID - * @param leftTableName entity dataset name - * @param minTimestamp earliest allowed timestamp for the historical data in feast - * @param maxTimestamp latest allowed timestamp for the historical data in feast - * @return point in time correctness join BQ SQL query - */ - public static String createFeatureSetPointInTimeQuery( - FeatureSetQueryInfo featureSetInfo, - String projectId, - String datasetId, - String leftTableName, - String minTimestamp, - String maxTimestamp) - throws IOException { - - PebbleTemplate template = engine.getTemplate(FEATURESET_TEMPLATE_NAME); - Map context = new HashMap<>(); - context.put("featureSet", featureSetInfo); - context.put("projectId", projectId); - context.put("datasetId", datasetId); - context.put("minTimestamp", minTimestamp); - context.put("maxTimestamp", maxTimestamp); - context.put("leftTableName", leftTableName); - - Writer writer = new StringWriter(); - template.evaluate(writer, context); - return writer.toString(); - } - - /** - * @param featureSetInfos List of FeatureSetInfos containing information about the feature set - * necessary for the query templating - * @param entityTableColumnNames list of column names in entity table - * @param leftTableName entity dataset name - * @return query to join temporary feature set tables to the entity table - */ - public static String createJoinQuery( - List featureSetInfos, - List entityTableColumnNames, - String leftTableName) - throws IOException { - PebbleTemplate template = engine.getTemplate(JOIN_TEMPLATE_NAME); - Map context = new HashMap<>(); - context.put("entities", entityTableColumnNames); - context.put("featureSets", featureSetInfos); - context.put("leftTableName", leftTableName); - - Writer writer = new StringWriter(); - template.evaluate(writer, context); - return writer.toString(); - } - - public static String generateFullTableName(TableId tableId) { - return String.format( - "%s.%s.%s", tableId.getProject(), tableId.getDataset(), tableId.getTable()); - } -} diff --git a/storage/connectors/bigquery/src/main/java/feast/storage/connectors/bigquery/retriever/SubqueryCallable.java b/storage/connectors/bigquery/src/main/java/feast/storage/connectors/bigquery/retriever/SubqueryCallable.java deleted file mode 100644 index 43a32cef504..00000000000 --- a/storage/connectors/bigquery/src/main/java/feast/storage/connectors/bigquery/retriever/SubqueryCallable.java +++ /dev/null @@ -1,74 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2019 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.storage.connectors.bigquery.retriever; - -import static feast.storage.connectors.bigquery.retriever.BigQueryHistoricalRetriever.TEMP_TABLE_EXPIRY_DURATION_MS; -import static feast.storage.connectors.bigquery.retriever.QueryTemplater.generateFullTableName; - -import com.google.auto.value.AutoValue; -import com.google.cloud.bigquery.*; -import java.util.concurrent.Callable; - -/** - * Waits for a point-in-time correctness join to complete. On completion, returns a featureSetInfo - * updated with the reference to the table containing the results of the query. - */ -@AutoValue -public abstract class SubqueryCallable implements Callable { - - public abstract BigQuery bigquery(); - - public abstract FeatureSetQueryInfo featureSetInfo(); - - public abstract Job subqueryJob(); - - public static Builder builder() { - return new AutoValue_SubqueryCallable.Builder(); - } - - @AutoValue.Builder - public abstract static class Builder { - - public abstract Builder setBigquery(BigQuery bigquery); - - public abstract Builder setFeatureSetInfo(FeatureSetQueryInfo featureSetInfo); - - public abstract Builder setSubqueryJob(Job subqueryJob); - - public abstract SubqueryCallable build(); - } - - @Override - public FeatureSetQueryInfo call() throws BigQueryException, InterruptedException { - QueryJobConfiguration subqueryConfig; - subqueryJob().waitFor(); - subqueryConfig = subqueryJob().getConfiguration(); - TableId destinationTable = subqueryConfig.getDestinationTable(); - - TableInfo expiry = - bigquery() - .getTable(destinationTable) - .toBuilder() - .setExpirationTime(System.currentTimeMillis() + TEMP_TABLE_EXPIRY_DURATION_MS) - .build(); - bigquery().update(expiry); - - String fullTablePath = generateFullTableName(destinationTable); - - return new FeatureSetQueryInfo(featureSetInfo(), fullTablePath); - } -} diff --git a/storage/connectors/bigquery/src/main/java/feast/storage/connectors/bigquery/statistics/BigQueryStatisticsRetriever.java b/storage/connectors/bigquery/src/main/java/feast/storage/connectors/bigquery/statistics/BigQueryStatisticsRetriever.java deleted file mode 100644 index 5835b711905..00000000000 --- a/storage/connectors/bigquery/src/main/java/feast/storage/connectors/bigquery/statistics/BigQueryStatisticsRetriever.java +++ /dev/null @@ -1,161 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.storage.connectors.bigquery.statistics; - -import com.google.auto.value.AutoValue; -import com.google.cloud.bigquery.*; -import com.google.common.collect.Streams; -import com.google.protobuf.Timestamp; -import feast.proto.core.FeatureSetProto.FeatureSetSpec; -import feast.proto.core.StoreProto.Store.BigQueryConfig; -import feast.storage.api.statistics.FeatureStatistics; -import feast.storage.api.statistics.StatisticsRetriever; -import java.io.IOException; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; -import org.tensorflow.metadata.v0.FeatureNameStatistics; - -@AutoValue -public abstract class BigQueryStatisticsRetriever implements StatisticsRetriever { - - public abstract String projectId(); - - public abstract String datasetId(); - - public abstract BigQuery bigquery(); - - public static BigQueryStatisticsRetriever create(BigQueryConfig config) { - BigQuery bigquery = - BigQueryOptions.getDefaultInstance() - .toBuilder() - .setProjectId(config.getProjectId()) - .build() - .getService(); - return newBuilder() - .setBigquery(bigquery) - .setDatasetId(config.getDatasetId()) - .setProjectId(config.getProjectId()) - .build(); - } - - public static Builder newBuilder() { - return new AutoValue_BigQueryStatisticsRetriever.Builder(); - } - - @AutoValue.Builder - public abstract static class Builder { - - public abstract Builder setProjectId(String projectId); - - public abstract Builder setDatasetId(String datasetId); - - public abstract Builder setBigquery(BigQuery bigquery); - - public abstract BigQueryStatisticsRetriever build(); - } - - @Override - public FeatureStatistics getFeatureStatistics( - FeatureSetSpec featureSetSpec, List features, String ingestionId) { - StatsDataset queryDataset = buildStatsDataset(featureSetSpec); - queryDataset.subsetByIngestionId(ingestionId); - List featureStatisticsQueryInfos = - featureSetSpec.getFeaturesList().stream() - .filter(f -> features.contains(f.getName())) - .map(FeatureStatisticsQueryInfo::fromProto) - .collect(Collectors.toList()); - return getFeatureStatistics(featureStatisticsQueryInfos, queryDataset); - } - - @Override - public FeatureStatistics getFeatureStatistics( - FeatureSetSpec featureSetSpec, List features, Timestamp date) { - StatsDataset queryDataset = buildStatsDataset(featureSetSpec); - queryDataset.subsetByDate(date); - List featureStatisticsQueryInfos = - featureSetSpec.getFeaturesList().stream() - .filter(f -> features.contains(f.getName())) - .map(FeatureStatisticsQueryInfo::fromProto) - .collect(Collectors.toList()); - return getFeatureStatistics(featureStatisticsQueryInfos, queryDataset); - } - - public FeatureStatistics getFeatureStatistics( - List features, StatsDataset statsDataset) - throws RuntimeException { - try { - // Generate SQL for and retrieve non-histogram statistics - String getFeatureSetStatsQuery = - StatsQueryTemplater.createGetFeaturesStatsQuery(features, statsDataset); - QueryJobConfiguration queryJobConfiguration = - QueryJobConfiguration.newBuilder(getFeatureSetStatsQuery).build(); - TableResult basicStats = bigquery().query(queryJobConfiguration); - - // Generate SQL for and retrieve histogram statistics - String getFeatureSetHistQuery = - StatsQueryTemplater.createGetFeaturesHistQuery(features, statsDataset); - queryJobConfiguration = QueryJobConfiguration.newBuilder(getFeatureSetHistQuery).build(); - TableResult hist = bigquery().query(queryJobConfiguration); - - // Convert to map of feature_name:row containing the statistics - Map basicStatsValues = getTableResultByFeatureName(basicStats); - Map histValues = getTableResultByFeatureName(hist); - - int totalCountIndex = basicStats.getSchema().getFields().getIndex("total_count"); - String ref = features.get(0).getName(); - FeatureStatistics.Builder featureSetStatisticsBuilder = - FeatureStatistics.newBuilder() - .setNumExamples(basicStatsValues.get(ref).get(totalCountIndex).getLongValue()); - - // Convert BQ rows to FeatureNameStatistics - for (FeatureStatisticsQueryInfo featureInfo : features) { - FeatureNameStatistics featureNameStatistics = - StatsQueryResult.create() - .withBasicStatsResults( - basicStats.getSchema(), basicStatsValues.get(featureInfo.getName())) - .withHistResults(hist.getSchema(), histValues.get(featureInfo.getName())) - .toFeatureNameStatistics(featureInfo); - featureSetStatisticsBuilder.addFeatureNameStatistics(featureNameStatistics); - } - return featureSetStatisticsBuilder.build(); - } catch (IOException | InterruptedException e) { - String featuresList = - features.stream() - .map(FeatureStatisticsQueryInfo::getName) - .collect(Collectors.joining(",")); - throw new RuntimeException( - String.format( - "Unable to retrieve statistics from BigQuery for features %s", featuresList), - e); - } - } - - private Map getTableResultByFeatureName(TableResult basicStats) { - return Streams.stream(basicStats.getValues()) - .collect( - Collectors.toMap( - fieldValueList -> fieldValueList.get(0).getStringValue(), - fieldValueList -> fieldValueList)); - } - - private StatsDataset buildStatsDataset(FeatureSetSpec featureSetSpec) { - String featureSetTableName = - String.format("%s_%s", featureSetSpec.getProject(), featureSetSpec.getName()); - return new StatsDataset(projectId(), datasetId(), featureSetTableName); - } -} diff --git a/storage/connectors/bigquery/src/main/java/feast/storage/connectors/bigquery/statistics/FeatureStatisticsQueryInfo.java b/storage/connectors/bigquery/src/main/java/feast/storage/connectors/bigquery/statistics/FeatureStatisticsQueryInfo.java deleted file mode 100644 index 80612606332..00000000000 --- a/storage/connectors/bigquery/src/main/java/feast/storage/connectors/bigquery/statistics/FeatureStatisticsQueryInfo.java +++ /dev/null @@ -1,85 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.storage.connectors.bigquery.statistics; - -import feast.proto.core.FeatureSetProto.FeatureSpec; -import feast.proto.types.ValueProto.ValueType; -import feast.proto.types.ValueProto.ValueType.Enum; -import java.util.HashMap; -import java.util.Map; -import org.tensorflow.metadata.v0.FeatureNameStatistics; -import org.tensorflow.metadata.v0.FeatureNameStatistics.Type; - -/** - * Value class for Features containing information necessary to template stats-retrieving queries. - */ -public class FeatureStatisticsQueryInfo { - // Map converting Feast type to TFDV type - private static final Map TFDV_TYPE_MAP = new HashMap<>(); - - static { - TFDV_TYPE_MAP.put(ValueType.Enum.INT64, FeatureNameStatistics.Type.INT); - TFDV_TYPE_MAP.put(ValueType.Enum.INT32, FeatureNameStatistics.Type.INT); - TFDV_TYPE_MAP.put(ValueType.Enum.BOOL, FeatureNameStatistics.Type.INT); - TFDV_TYPE_MAP.put(ValueType.Enum.FLOAT, FeatureNameStatistics.Type.FLOAT); - TFDV_TYPE_MAP.put(ValueType.Enum.DOUBLE, FeatureNameStatistics.Type.FLOAT); - TFDV_TYPE_MAP.put(ValueType.Enum.STRING, FeatureNameStatistics.Type.STRING); - TFDV_TYPE_MAP.put(ValueType.Enum.BYTES, FeatureNameStatistics.Type.BYTES); - TFDV_TYPE_MAP.put(ValueType.Enum.BYTES_LIST, FeatureNameStatistics.Type.STRUCT); - TFDV_TYPE_MAP.put(ValueType.Enum.STRING_LIST, FeatureNameStatistics.Type.STRUCT); - TFDV_TYPE_MAP.put(ValueType.Enum.INT32_LIST, FeatureNameStatistics.Type.STRUCT); - TFDV_TYPE_MAP.put(ValueType.Enum.INT64_LIST, FeatureNameStatistics.Type.STRUCT); - TFDV_TYPE_MAP.put(ValueType.Enum.BOOL_LIST, FeatureNameStatistics.Type.STRUCT); - TFDV_TYPE_MAP.put(ValueType.Enum.FLOAT_LIST, FeatureNameStatistics.Type.STRUCT); - TFDV_TYPE_MAP.put(ValueType.Enum.DOUBLE_LIST, FeatureNameStatistics.Type.STRUCT); - } - - // Name of the field - private final String name; - - // Statistics Type to generate for the field - private final String statsType; - - // Value Type of the field - private final String valueType; - - private FeatureStatisticsQueryInfo( - String name, StatsType.Enum statsType, FeatureNameStatistics.Type valueType) { - this.name = name; - this.statsType = statsType.toString(); - this.valueType = valueType.toString(); - } - - public static FeatureStatisticsQueryInfo fromProto(FeatureSpec featureSpec) { - Enum valueType = featureSpec.getValueType(); - StatsType.Enum statsType = StatsType.fromValueType(valueType); - return new FeatureStatisticsQueryInfo( - featureSpec.getName(), statsType, TFDV_TYPE_MAP.get(valueType)); - } - - public String getName() { - return name; - } - - public String getStatsType() { - return statsType; - } - - public String getValueType() { - return valueType; - } -} diff --git a/storage/connectors/bigquery/src/main/java/feast/storage/connectors/bigquery/statistics/StatsDataset.java b/storage/connectors/bigquery/src/main/java/feast/storage/connectors/bigquery/statistics/StatsDataset.java deleted file mode 100644 index 50c76c9ede4..00000000000 --- a/storage/connectors/bigquery/src/main/java/feast/storage/connectors/bigquery/statistics/StatsDataset.java +++ /dev/null @@ -1,66 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.storage.connectors.bigquery.statistics; - -import com.google.cloud.bigquery.TableId; -import com.google.protobuf.Timestamp; -import java.util.HashMap; -import java.util.Map; -import org.joda.time.DateTime; -import org.joda.time.DateTimeZone; -import org.joda.time.format.DateTimeFormat; -import org.joda.time.format.DateTimeFormatter; - -/** - * StatsDataset represents a subset of data within a table to retrieve statistics over. Data can be - * subset by either ingestionId or date. - */ -public class StatsDataset { - private final String table; - private String ingestionId = ""; - private String date = ""; - - public StatsDataset(String project, String bigqueryDataset, String table) { - this.table = generateTableName(project, bigqueryDataset, table); - } - - public StatsDataset(TableId tableId) { - this.table = generateTableName(tableId.getProject(), tableId.getDataset(), tableId.getTable()); - } - - public void subsetByIngestionId(String ingestionId) { - this.ingestionId = ingestionId; - } - - public void subsetByDate(Timestamp date) { - DateTime dateTime = new DateTime(date.getSeconds() * 1000, DateTimeZone.UTC); - DateTimeFormatter fmt = DateTimeFormat.forPattern("yyyy-MM-dd"); - this.date = fmt.print(dateTime); - } - - public Map getMap() { - Map map = new HashMap<>(); - map.put("table", table); - map.put("ingestionId", ingestionId); - map.put("date", date); - return map; - } - - private String generateTableName(String projectId, String datasetId, String tableName) { - return String.format("%s.%s.%s", projectId, datasetId, tableName); - } -} diff --git a/storage/connectors/bigquery/src/main/java/feast/storage/connectors/bigquery/statistics/StatsQueryResult.java b/storage/connectors/bigquery/src/main/java/feast/storage/connectors/bigquery/statistics/StatsQueryResult.java deleted file mode 100644 index c6b20b2f095..00000000000 --- a/storage/connectors/bigquery/src/main/java/feast/storage/connectors/bigquery/statistics/StatsQueryResult.java +++ /dev/null @@ -1,301 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.storage.connectors.bigquery.statistics; - -import com.google.auto.value.AutoValue; -import com.google.cloud.bigquery.*; -import com.google.cloud.bigquery.Schema; -import com.google.common.collect.ComparisonChain; -import com.google.common.collect.Ordering; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; -import javax.annotation.Nullable; -import org.tensorflow.metadata.v0.*; -import org.tensorflow.metadata.v0.Histogram.Bucket; -import org.tensorflow.metadata.v0.Histogram.HistogramType; -import org.tensorflow.metadata.v0.StringStatistics.FreqAndValue; - -@AutoValue -public abstract class StatsQueryResult { - - // Schema of the table returned by the basic stats retrieval query - @Nullable - abstract Schema basicStatsSchema(); - - // Table values returned by the basic stats retrieval query - @Nullable - abstract FieldValueList basicStatsFieldValues(); - - // Schema of the table returned by the histogram retrieval query - @Nullable - abstract Schema histSchema(); - - // Table values returned by the histogram retrieval query - @Nullable - abstract FieldValueList histFieldValues(); - - public static StatsQueryResult create() { - return StatsQueryResult.newBuilder().build(); - } - - private static StatsQueryResult.Builder newBuilder() { - return new AutoValue_StatsQueryResult.Builder(); - } - - abstract Builder toBuilder(); - - /** - * Add basic stats query results to the StatsQueryResult. - * - * @param basicStatsSchema BigQuery {@link Schema} of the retrieved statistics row for the - * non-histogram statistics. Used to retrieve the column names corresponding to each value in - * the row. - * @param basicStatsFieldValues BigQuery {@link FieldValueList} containing a single row of - * non-histogram statistics retrieved from BigQuery - * @return {@link StatsQueryResult} - */ - public StatsQueryResult withBasicStatsResults( - Schema basicStatsSchema, FieldValueList basicStatsFieldValues) { - return toBuilder() - .setBasicStatsSchema(basicStatsSchema) - .setBasicStatsFieldValues(basicStatsFieldValues) - .build(); - } - - /** - * Add histogram stats query results to the StatsQueryResult. - * - * @param histSchema BigQuery {@link Schema} of the retrieved statistics row for the histogram - * statistics. Used to retrieve the column names corresponding to each value in the row. - * @param histFieldValues BigQuery {@link FieldValueList} containing a single row of histogram - * statistics retrieved from BigQuery - * @return {@link StatsQueryResult} - */ - public StatsQueryResult withHistResults(Schema histSchema, FieldValueList histFieldValues) { - return toBuilder().setHistSchema(histSchema).setHistFieldValues(histFieldValues).build(); - } - - @AutoValue.Builder - abstract static class Builder { - abstract Builder setBasicStatsSchema(Schema basicStatsSchema); - - abstract Builder setBasicStatsFieldValues(FieldValueList basicStatsFieldValues); - - abstract Builder setHistSchema(Schema histSchema); - - abstract Builder setHistFieldValues(FieldValueList histFieldValues); - - public abstract StatsQueryResult build(); - } - - /** - * Convert BQ-retrieved statistics to the corresponding TFDV {@link FeatureNameStatistics} - * specific to the feature type. - * - * @param featureInfo {@link FeatureStatisticsQueryInfo} containing information about the feature - * @return {@link FeatureNameStatistics} - */ - public FeatureNameStatistics toFeatureNameStatistics(FeatureStatisticsQueryInfo featureInfo) { - Map valuesMap = new HashMap<>(); - - // Convert the table values to a map of field name : table value for easy retrieval - FieldList basicStatsfields = basicStatsSchema().getFields(); - for (int i = 0; i < basicStatsSchema().getFields().size(); i++) { - valuesMap.put(basicStatsfields.get(i).getName(), basicStatsFieldValues().get(i)); - } - - FieldList histFields = histSchema().getFields(); - for (int i = 0; i < histSchema().getFields().size(); i++) { - valuesMap.put(histFields.get(i).getName(), histFieldValues().get(i)); - } - - FeatureNameStatistics.Builder featureNameStatisticsBuilder = - FeatureNameStatistics.newBuilder() - .setPath(Path.newBuilder().addStep(valuesMap.get("feature_name").getStringValue())) - .setType(FeatureNameStatistics.Type.valueOf(featureInfo.getValueType())); - - switch (StatsType.Enum.valueOf(featureInfo.getStatsType())) { - case NUMERIC: - NumericStatistics numStats = getNumericStatistics(valuesMap); - featureNameStatisticsBuilder.setNumStats(numStats); - break; - case CATEGORICAL: - StringStatistics stringStats = getStringStatistics(valuesMap); - featureNameStatisticsBuilder.setStringStats(stringStats); - break; - case BYTES: - BytesStatistics bytesStats = getBytesStatistics(valuesMap); - featureNameStatisticsBuilder.setBytesStats(bytesStats); - break; - case LIST: - StructStatistics structStats = getStructStatistics(valuesMap); - featureNameStatisticsBuilder.setStructStats(structStats); - break; - default: - throw new IllegalArgumentException( - "Invalid feature type provided. Only statistics for numeric, bytes, string, boolean and list features are supported."); - } - return featureNameStatisticsBuilder.build(); - } - - private BytesStatistics getBytesStatistics(Map valuesMap) { - if (valuesMap.get("total_count").getLongValue() == 0) { - return BytesStatistics.getDefaultInstance(); - } - - return BytesStatistics.newBuilder() - .setCommonStats( - CommonStatistics.newBuilder() - .setNumMissing(valuesMap.get("missing_count").getLongValue()) - .setNumNonMissing(valuesMap.get("feature_count").getLongValue()) - .setMinNumValues(1) - .setMaxNumValues(1) - .setAvgNumValues(1) - .setTotNumValues(valuesMap.get("feature_count").getLongValue())) - .setUnique(valuesMap.get("unique").getLongValue()) - .setMaxNumBytes((float) valuesMap.get("max").getDoubleValue()) - .setMinNumBytes((float) valuesMap.get("min").getDoubleValue()) - .setAvgNumBytes((float) valuesMap.get("mean").getDoubleValue()) - .build(); - } - - private StringStatistics getStringStatistics(Map valuesMap) { - if (valuesMap.get("total_count").getLongValue() == 0) { - return StringStatistics.getDefaultInstance(); - } - - RankHistogram.Builder rankHistogram = RankHistogram.newBuilder(); - valuesMap - .get("cat_hist") - .getRepeatedValue() - .forEach( - v -> { - FieldValueList recordValue = v.getRecordValue(); - rankHistogram.addBuckets( - RankHistogram.Bucket.newBuilder() - .setLabel(recordValue.get(0).getStringValue()) - .setSampleCount(recordValue.get(1).getLongValue())); - }); - - List topCount = - rankHistogram.getBucketsList().stream() - .sorted( - (a, b) -> - ComparisonChain.start() - .compare( - a.getSampleCount(), b.getSampleCount(), Ordering.natural().reverse()) - .result()) - .limit(5) - .map( - bucket -> - FreqAndValue.newBuilder() - .setValue(bucket.getLabel()) - .setFrequency(bucket.getSampleCount()) - .build()) - .collect(Collectors.toList()); - - return StringStatistics.newBuilder() - .setUnique(valuesMap.get("unique").getLongValue()) - .setAvgLength((long) valuesMap.get("mean").getDoubleValue()) - .setCommonStats( - CommonStatistics.newBuilder() - .setNumMissing(valuesMap.get("missing_count").getLongValue()) - .setNumNonMissing(valuesMap.get("feature_count").getLongValue()) - .setMinNumValues(1) - .setMaxNumValues(1) - .setAvgNumValues(1) - .setTotNumValues(valuesMap.get("feature_count").getLongValue())) - .setRankHistogram(rankHistogram) - .addAllTopValues(topCount) - .build(); - } - - private NumericStatistics getNumericStatistics(Map valuesMap) { - if (valuesMap.get("total_count").getLongValue() == 0) { - return NumericStatistics.getDefaultInstance(); - } - - // Build quantiles - long quantileCount = valuesMap.get("feature_count").getLongValue() / 10; - Histogram.Builder quantilesBuilder = Histogram.newBuilder().setType(HistogramType.QUANTILES); - - List quantilesRaw = valuesMap.get("quantiles").getRepeatedValue(); - for (int i = 0; i < quantilesRaw.size() - 1; i++) { - quantilesBuilder.addBuckets( - Bucket.newBuilder() - .setLowValue(quantilesRaw.get(i).getDoubleValue()) - .setHighValue(quantilesRaw.get(i + 1).getDoubleValue()) - .setSampleCount(quantileCount)); - } - // Build histogram - Histogram.Builder histBuilder = Histogram.newBuilder().setType(HistogramType.STANDARD); - - // Order of histogram records is defined in the query hist_stats.sql:L35 - valuesMap - .get("num_hist") - .getRepeatedValue() - .forEach( - v -> { - FieldValueList recordValue = v.getRecordValue(); - histBuilder.addBuckets( - Bucket.newBuilder() - .setHighValue(recordValue.get(2).getDoubleValue()) - .setLowValue(recordValue.get(1).getDoubleValue()) - .setSampleCount(recordValue.get(0).getLongValue())); - }); - - return NumericStatistics.newBuilder() - .setMax(valuesMap.get("max").getDoubleValue()) - .setMin(valuesMap.get("min").getDoubleValue()) - .setMedian(quantilesRaw.get(5).getDoubleValue()) - .setNumZeros(valuesMap.get("zeroes").getLongValue()) - .setStdDev(valuesMap.get("stdev").getDoubleValue()) - .setMean(valuesMap.get("mean").getDoubleValue()) - .setCommonStats( - CommonStatistics.newBuilder() - .setNumMissing(valuesMap.get("missing_count").getLongValue()) - .setNumNonMissing(valuesMap.get("feature_count").getLongValue()) - .setMinNumValues(1) - .setMaxNumValues(1) - .setAvgNumValues(1) - .setTotNumValues(valuesMap.get("feature_count").getLongValue())) - .addHistograms(histBuilder) - .addHistograms(quantilesBuilder) - .build(); - } - - private StructStatistics getStructStatistics(Map valuesMap) { - if (valuesMap.get("total_count").getLongValue() == 0) { - return StructStatistics.getDefaultInstance(); - } - - return StructStatistics.newBuilder() - .setCommonStats( - CommonStatistics.newBuilder() - .setNumMissing(valuesMap.get("missing_count").getLongValue()) - .setNumNonMissing(valuesMap.get("feature_count").getLongValue()) - .setMinNumValues(valuesMap.get("min").getLongValue()) - .setMaxNumValues(valuesMap.get("max").getLongValue()) - .setAvgNumValues(valuesMap.get("mean").getLongValue()) - .setTotNumValues( - valuesMap.get("feature_count").getLongValue() - * valuesMap.get("mean").getLongValue())) - .build(); - } -} diff --git a/storage/connectors/bigquery/src/main/java/feast/storage/connectors/bigquery/statistics/StatsQueryTemplater.java b/storage/connectors/bigquery/src/main/java/feast/storage/connectors/bigquery/statistics/StatsQueryTemplater.java deleted file mode 100644 index 42ed170b560..00000000000 --- a/storage/connectors/bigquery/src/main/java/feast/storage/connectors/bigquery/statistics/StatsQueryTemplater.java +++ /dev/null @@ -1,91 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2019 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.storage.connectors.bigquery.statistics; - -import com.mitchellbosecke.pebble.PebbleEngine; -import com.mitchellbosecke.pebble.template.PebbleTemplate; -import java.io.IOException; -import java.io.StringWriter; -import java.io.Writer; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -public class StatsQueryTemplater { - - private static final PebbleEngine engine = new PebbleEngine.Builder().autoEscaping(false).build(); - private static final String BASIC_STATS_TEMPLATE_NAME = "templates/basic_stats.sql"; - private static final String HIST_STATS_TEMPLATE_NAME = "templates/hist_stats.sql"; - private static final String DATA_SUBSET_TEMPLATE_NAME = "templates/data_subset.sql"; - - /** - * Generate the query for getting basic statistics for a given set of features - * - * @param features Information about the features necessary for the query templating - * @param statsDataset query selecting subset of data to compute statistics over - * @return point in time correctness join BQ SQL query - * @throws IOException - */ - public static String createGetFeaturesStatsQuery( - List features, StatsDataset statsDataset) throws IOException { - - PebbleTemplate template = engine.getTemplate(BASIC_STATS_TEMPLATE_NAME); - Map context = new HashMap<>(); - context.put("features", features); - context.put("dataset", generateDataSubsetQuery(statsDataset)); - - Writer writer = new StringWriter(); - template.evaluate(writer, context); - return writer.toString(); - } - - /** - * Generate the query for getting histograms for given set of features - * - * @param features Information about the features necessary for the query templating - * @param statsDataset query selecting subset of data to compute statistics over - * @return point in time correctness join BQ SQL query - * @throws IOException - */ - public static String createGetFeaturesHistQuery( - List features, StatsDataset statsDataset) throws IOException { - - PebbleTemplate template = engine.getTemplate(HIST_STATS_TEMPLATE_NAME); - Map context = new HashMap<>(); - context.put("features", features); - context.put("dataset", generateDataSubsetQuery(statsDataset)); - - Writer writer = new StringWriter(); - template.evaluate(writer, context); - return writer.toString(); - } - - /** - * generate the query to subset the data to compute statistics over - * - * @param statsDataset {@link StatsDataset} describing the subset of data - * @return BigQuery query selecting the data - * @throws IOException - */ - private static String generateDataSubsetQuery(StatsDataset statsDataset) throws IOException { - PebbleTemplate template = engine.getTemplate(DATA_SUBSET_TEMPLATE_NAME); - - Writer writer = new StringWriter(); - template.evaluate(writer, statsDataset.getMap()); - return writer.toString(); - } -} diff --git a/storage/connectors/bigquery/src/main/java/feast/storage/connectors/bigquery/statistics/StatsType.java b/storage/connectors/bigquery/src/main/java/feast/storage/connectors/bigquery/statistics/StatsType.java deleted file mode 100644 index fd4a4ed93dc..00000000000 --- a/storage/connectors/bigquery/src/main/java/feast/storage/connectors/bigquery/statistics/StatsType.java +++ /dev/null @@ -1,62 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.storage.connectors.bigquery.statistics; - -import feast.proto.types.ValueProto.ValueType; - -public class StatsType { - // Category of statistics a feature falls into. Determines the set of - // statistics to generate for the feature. - enum Enum { - NUMERIC, - CATEGORICAL, - BYTES, - LIST, - } - - /** - * Returns the category of feature statistics to build for the given valueType. - * - * @param valueType {@link ValueType.Enum} of a feature - * @return {@link StatsType.Enum} corresponding to the given valueType. - */ - public static StatsType.Enum fromValueType(ValueType.Enum valueType) { - switch (valueType) { - case FLOAT: - case DOUBLE: - case INT32: - case INT64: - case BOOL: - return Enum.NUMERIC; - case STRING: - return Enum.CATEGORICAL; - case BYTES: - return Enum.BYTES; - case BYTES_LIST: - case BOOL_LIST: - case FLOAT_LIST: - case INT32_LIST: - case INT64_LIST: - case DOUBLE_LIST: - case STRING_LIST: - return Enum.LIST; - default: - throw new IllegalArgumentException( - String.format("Invalid feature type provided: %s", valueType)); - } - } -} diff --git a/storage/connectors/bigquery/src/main/java/feast/storage/connectors/bigquery/writer/BigQueryDeadletterSink.java b/storage/connectors/bigquery/src/main/java/feast/storage/connectors/bigquery/writer/BigQueryDeadletterSink.java deleted file mode 100644 index 96364c96c78..00000000000 --- a/storage/connectors/bigquery/src/main/java/feast/storage/connectors/bigquery/writer/BigQueryDeadletterSink.java +++ /dev/null @@ -1,133 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.storage.connectors.bigquery.writer; - -import com.google.api.services.bigquery.model.TableRow; -import com.google.api.services.bigquery.model.TimePartitioning; -import com.google.auto.value.AutoValue; -import com.google.common.io.Resources; -import feast.storage.api.writer.DeadletterSink; -import feast.storage.api.writer.FailedElement; -import java.nio.charset.StandardCharsets; -import org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO; -import org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO.Write.CreateDisposition; -import org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO.Write.WriteDisposition; -import org.apache.beam.sdk.transforms.DoFn; -import org.apache.beam.sdk.transforms.PTransform; -import org.apache.beam.sdk.transforms.ParDo; -import org.apache.beam.sdk.values.PCollection; -import org.apache.beam.sdk.values.PDone; -import org.slf4j.Logger; - -public class BigQueryDeadletterSink implements DeadletterSink { - - private static final String DEADLETTER_SCHEMA_FILE_PATH = "schemas/deadletter_table_schema.json"; - private static final Logger log = org.slf4j.LoggerFactory.getLogger(BigQueryDeadletterSink.class); - private static final String TIMESTAMP_COLUMN = "timestamp"; - - private final String tableSpec; - private String jsonSchema; - - public BigQueryDeadletterSink(String tableSpec) { - - this.tableSpec = tableSpec; - try { - jsonSchema = - Resources.toString( - Resources.getResource(DEADLETTER_SCHEMA_FILE_PATH), StandardCharsets.UTF_8); - } catch (Exception e) { - log.error( - "Unable to read {} file from the resources folder!", DEADLETTER_SCHEMA_FILE_PATH, e); - } - } - - @Override - public void prepareWrite() {} - - @Override - public PTransform, PDone> write() { - return WriteFailedElement.newBuilder() - .setJsonSchema(jsonSchema) - .setTableSpec(tableSpec) - .build(); - } - - @AutoValue - public abstract static class WriteFailedElement - extends PTransform, PDone> { - - public abstract String getTableSpec(); - - public abstract String getJsonSchema(); - - public static Builder newBuilder() { - return new AutoValue_BigQueryDeadletterSink_WriteFailedElement.Builder(); - } - - @AutoValue.Builder - public abstract static class Builder { - - /** - * @param tableSpec Table spec should follow the format "PROJECT_ID:DATASET_ID.TABLE_ID". - * Table will be created if not exists. - */ - public abstract Builder setTableSpec(String tableSpec); - - /** - * @param jsonSchema JSON string describing the schema - * of the table. - */ - public abstract Builder setJsonSchema(String jsonSchema); - - public abstract WriteFailedElement build(); - } - - @Override - public PDone expand(PCollection input) { - TimePartitioning partition = new TimePartitioning().setType("DAY"); - partition.setField(TIMESTAMP_COLUMN); - input - .apply("FailedElementToTableRow", ParDo.of(new FailedElementToTableRowFn())) - .apply( - "WriteFailedElementsToBigQuery", - BigQueryIO.writeTableRows() - .to(getTableSpec()) - .withJsonSchema(getJsonSchema()) - .withTimePartitioning(partition) - .withCreateDisposition(CreateDisposition.CREATE_IF_NEEDED) - .withWriteDisposition(WriteDisposition.WRITE_APPEND)); - return PDone.in(input.getPipeline()); - } - } - - public static class FailedElementToTableRowFn extends DoFn { - @ProcessElement - public void processElement(ProcessContext context) { - final FailedElement element = context.element(); - final TableRow tableRow = - new TableRow() - .set(TIMESTAMP_COLUMN, element.getTimestamp().toString()) - .set("job_name", element.getJobName()) - .set("transform_name", element.getTransformName()) - .set("payload", element.getPayload()) - .set("error_message", element.getErrorMessage()) - .set("stack_trace", element.getStackTrace()); - context.output(tableRow); - } - } -} diff --git a/storage/connectors/bigquery/src/main/java/feast/storage/connectors/bigquery/writer/BigQueryFeatureSink.java b/storage/connectors/bigquery/src/main/java/feast/storage/connectors/bigquery/writer/BigQueryFeatureSink.java deleted file mode 100644 index 2d55b308dd0..00000000000 --- a/storage/connectors/bigquery/src/main/java/feast/storage/connectors/bigquery/writer/BigQueryFeatureSink.java +++ /dev/null @@ -1,152 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.storage.connectors.bigquery.writer; - -import static com.google.common.base.Preconditions.checkArgument; - -import com.google.api.services.bigquery.model.TableSchema; -import com.google.auto.value.AutoValue; -import com.google.cloud.bigquery.*; -import feast.common.models.FeatureSetReference; -import feast.proto.core.FeatureSetProto; -import feast.proto.core.StoreProto.Store.BigQueryConfig; -import feast.proto.types.FeatureRowProto; -import feast.storage.api.writer.FeatureSink; -import feast.storage.api.writer.WriteResult; -import java.util.Map; -import javax.annotation.Nullable; -import org.apache.beam.sdk.coders.AvroCoder; -import org.apache.beam.sdk.coders.KvCoder; -import org.apache.beam.sdk.io.gcp.bigquery.BigQueryServices; -import org.apache.beam.sdk.options.ValueProvider; -import org.apache.beam.sdk.transforms.*; -import org.apache.beam.sdk.values.KV; -import org.apache.beam.sdk.values.PCollection; -import org.apache.beam.sdk.values.PCollectionView; -import org.joda.time.Duration; - -@AutoValue -public abstract class BigQueryFeatureSink implements FeatureSink { - public abstract String getProjectId(); - - public abstract String getDatasetId(); - - public abstract Duration getTriggeringFrequency(); - - @Nullable - public abstract BigQueryServices getBQTestServices(); - - @Nullable - public abstract ValueProvider getBQClient(); - - private PCollectionView>> schemasView; - - /** - * Initialize a {@link BigQueryFeatureSink.Builder} from a {@link BigQueryConfig}. This method - * initializes a {@link BigQuery} client with default options. Use the builder method to inject - * your own client. - * - * @param config {@link BigQueryConfig} - * @return {@link BigQueryFeatureSink.Builder} - */ - public static FeatureSink fromConfig(BigQueryConfig config) { - checkArgument( - config.getWriteTriggeringFrequencySeconds() > 0, - "Invalid configuration: " - + "write_triggering_frequency_seconds in BigQueryConfig must be positive integer. " - + "Please fix that in your serving configuration."); - - return BigQueryFeatureSink.builder() - .setDatasetId(config.getDatasetId()) - .setProjectId(config.getProjectId()) - .setBQTestServices(null) - .setBQClient( - new ValueProvider() { - @Override - public BigQuery get() { - return BigQueryOptions.getDefaultInstance().getService(); - } - - @Override - public boolean isAccessible() { - return true; - } - }) - .setTriggeringFrequency( - Duration.standardSeconds(config.getWriteTriggeringFrequencySeconds())) - .build(); - } - - public static Builder builder() { - return new AutoValue_BigQueryFeatureSink.Builder(); - } - - @AutoValue.Builder - public abstract static class Builder { - - public abstract Builder setProjectId(String projectId); - - public abstract Builder setDatasetId(String datasetId); - - public abstract Builder setTriggeringFrequency(Duration triggeringFrequency); - - public abstract Builder setBQTestServices(BigQueryServices bigQueryServices); - - public abstract Builder setBQClient(ValueProvider bigQueryOptions); - - public abstract BigQueryFeatureSink build(); - } - - /** @param featureSetSpecs Feature set to be written */ - @Override - public PCollection prepareWrite( - PCollection> featureSetSpecs) { - PCollection> schemas = - featureSetSpecs - .apply( - "GenerateTableSchema", - ParDo.of( - new FeatureSetSpecToTableSchema( - DatasetId.of(getProjectId(), getDatasetId()), getBQClient()))) - .setCoder( - KvCoder.of( - AvroCoder.of(FeatureSetReference.class), - FeatureSetSpecToTableSchema.TableSchemaCoder.of())); - - schemasView = - schemas - .apply("ReferenceString", ParDo.of(new ReferenceToString())) - .apply("View", View.asMultimap()); - - return schemas.apply("Ready", Keys.create()); - } - - @Override - public PTransform, WriteResult> writer() { - return new BigQueryWrite(DatasetId.of(getProjectId(), getDatasetId()), schemasView) - .withTriggeringFrequency(getTriggeringFrequency()) - .withTestServices(getBQTestServices()); - } - - private static class ReferenceToString - extends DoFn, KV> { - @ProcessElement - public void process(ProcessContext c) { - c.output(KV.of(c.element().getKey().getReference(), c.element().getValue())); - } - } -} diff --git a/storage/connectors/bigquery/src/main/java/feast/storage/connectors/bigquery/writer/BigQuerySinkHelpers.java b/storage/connectors/bigquery/src/main/java/feast/storage/connectors/bigquery/writer/BigQuerySinkHelpers.java deleted file mode 100644 index e07cd6c637b..00000000000 --- a/storage/connectors/bigquery/src/main/java/feast/storage/connectors/bigquery/writer/BigQuerySinkHelpers.java +++ /dev/null @@ -1,62 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.storage.connectors.bigquery.writer; - -import com.google.api.services.bigquery.model.TimePartitioning; -import com.google.cloud.bigquery.DatasetId; -import org.apache.beam.sdk.io.gcp.bigquery.TableDestination; - -public class BigQuerySinkHelpers { - - public static final String DEFAULT_PROJECT_NAME = "default"; - - /** - * Generating BQ table destination from dataset reference and featuresSet's project and name. If - * project is undefined "default" would be selected - * - * @param dataset {@link DatasetId} reference to bq project and dataset - * @param featureSetKey Feature Set reference with format <project>/<feature-set-name> - * @return {@link TableDestination} - */ - public static TableDestination getTableDestination(DatasetId dataset, String featureSetKey) { - String[] splitName = featureSetKey.split("/"); - String projectName, setName; - - if (splitName.length == 2) { - projectName = splitName[0]; - setName = splitName[1]; - } else { - projectName = DEFAULT_PROJECT_NAME; - setName = splitName[0]; - } - - TimePartitioning timePartitioning = - new TimePartitioning() - .setType("DAY") - .setField(FeatureRowToTableRow.getEventTimestampColumn()); - - return new TableDestination( - String.format( - "%s:%s.%s_%s", - dataset.getProject(), - dataset.getDataset(), - projectName.replaceAll("-", "_"), - setName.replaceAll("-", "_")), - String.format("Feast table for %s", featureSetKey), - timePartitioning); - } -} diff --git a/storage/connectors/bigquery/src/main/java/feast/storage/connectors/bigquery/writer/BigQueryWrite.java b/storage/connectors/bigquery/src/main/java/feast/storage/connectors/bigquery/writer/BigQueryWrite.java deleted file mode 100644 index 4ed7fd80bdf..00000000000 --- a/storage/connectors/bigquery/src/main/java/feast/storage/connectors/bigquery/writer/BigQueryWrite.java +++ /dev/null @@ -1,267 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.storage.connectors.bigquery.writer; - -import com.google.api.services.bigquery.model.*; -import com.google.cloud.bigquery.DatasetId; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableSet; -import com.google.common.collect.Iterators; -import feast.proto.types.FeatureRowProto.FeatureRow; -import feast.storage.api.writer.FailedElement; -import feast.storage.api.writer.WriteResult; -import java.util.List; -import java.util.Map; -import org.apache.beam.sdk.coders.KvCoder; -import org.apache.beam.sdk.coders.StringUtf8Coder; -import org.apache.beam.sdk.io.gcp.bigquery.*; -import org.apache.beam.sdk.transforms.*; -import org.apache.beam.sdk.transforms.join.CoGbkResult; -import org.apache.beam.sdk.transforms.join.CoGroupByKey; -import org.apache.beam.sdk.transforms.join.KeyedPCollectionTuple; -import org.apache.beam.sdk.transforms.windowing.FixedWindows; -import org.apache.beam.sdk.transforms.windowing.Window; -import org.apache.beam.sdk.values.*; -import org.joda.time.Duration; -import org.slf4j.Logger; - -/** - * A {@link PTransform} that writes {@link FeatureRow FeatureRows} to the specified BigQuery - * dataset, and returns a {@link WriteResult} containing the unsuccessful writes. Since Bigquery - * does not output failed writes, we cannot emit those. - */ -public class BigQueryWrite extends PTransform, WriteResult> { - private static final Logger log = org.slf4j.LoggerFactory.getLogger(BigQueryWrite.class); - - private static final Duration BIGQUERY_DEFAULT_WRITE_TRIGGERING_FREQUENCY = - Duration.standardMinutes(1); - - private static final Duration BIGQUERY_JOB_MAX_EXPECTING_RESULT_TIME = - Duration.standardMinutes(10); - private static final int BIGQUERY_MAX_JOB_RETRIES = 20; - private static final int MAX_SUCCESSFUL_OUTPUTS_PER_DESTINATION = 10000; - - private DatasetId destination; - private PCollectionView>> schemas; - - private Duration triggeringFrequency = BIGQUERY_DEFAULT_WRITE_TRIGGERING_FREQUENCY; - private Duration expectingResultTime = BIGQUERY_JOB_MAX_EXPECTING_RESULT_TIME; - private BigQueryServices testServices; - private int maxSuccessfulOutputs = MAX_SUCCESSFUL_OUTPUTS_PER_DESTINATION; - - public BigQueryWrite( - DatasetId destination, PCollectionView>> schemas) { - this.destination = destination; - this.schemas = schemas; - } - - public BigQueryWrite withTriggeringFrequency(Duration triggeringFrequency) { - this.triggeringFrequency = triggeringFrequency; - return this; - } - - public BigQueryWrite withTestServices(BigQueryServices testServices) { - this.testServices = testServices; - return this; - } - - public BigQueryWrite withExpectingResultTime(Duration expectingResultTime) { - this.expectingResultTime = expectingResultTime; - return this; - } - - public BigQueryWrite withMaxSuccessfulOutputs(int maxSuccessfulOutputs) { - this.maxSuccessfulOutputs = maxSuccessfulOutputs; - return this; - } - - /** - * BigQuery writer 1. choose destination based on featureSetName {@link - * FeatureDynamicDestinations} 2. dynamically pull destination's schema from schemas' view 3. - * convert {@link FeatureRow} into {@link TableRow} 4. group input into fixed windows (configured - * with setTriggeringFrequency) 5. write to bq (via BATCH FILE LOADING) {@link - * BatchLoadsWithResult} 6. join bq job output with input to produce successful inserts - * - * @param input stream of FeatureRows to write - * @return stream of successfully inserted FeatureRows - */ - @Override - public WriteResult expand(PCollection input) { - String jobName = input.getPipeline().getOptions().getJobName(); - - FeatureDynamicDestinations dynamicDestinations = new FeatureDynamicDestinations(); - - BatchLoadsWithResult.Builder writerBuilder = - BatchLoadsWithResult.create() - .setWriteDisposition(BigQueryIO.Write.WriteDisposition.WRITE_APPEND) - .setCreateDisposition(BigQueryIO.Write.CreateDisposition.CREATE_IF_NEEDED) - .setDynamicDestinations(dynamicDestinations) - .setDestinationCoder(StringUtf8Coder.of()) - .setElementCoder(TableRowJsonCoder.of()) - .setTriggeringFrequency(triggeringFrequency) - .setMaxRetryJobs(BIGQUERY_MAX_JOB_RETRIES) - .setSchemaUpdateOptions( - ImmutableSet.of(BigQueryIO.Write.SchemaUpdateOption.ALLOW_FIELD_ADDITION)); - - if (testServices != null) { - writerBuilder.setBigQueryServices(testServices); - } - - PCollection inputInFixedWindow = - input.apply( - "BatchingInput", - Window.into(FixedWindows.of(triggeringFrequency)) - .withAllowedLateness(Duration.ZERO) - .discardingFiredPanes()); - - PCollection> insertionResult = - inputInFixedWindow - .apply( - "PrepareWrite", - new PrepareWrite<>(dynamicDestinations, new FeatureRowToTableRow(jobName))) - .setCoder(KvCoder.of(StringUtf8Coder.of(), TableRowJsonCoder.of())) - .apply("WriteTableRowToBigQuery", writerBuilder.build()); - - PCollection successfulInserts = - mergeInputWithResult(inputInFixedWindow, insertionResult); - - // Since BigQueryIO does not support emitting failure writes, we set failedElements to - // an empty stream - PCollection failedElements = - input - .getPipeline() - .apply(Create.of("")) - .apply( - "dummy", - ParDo.of( - new DoFn() { - @ProcessElement - public void processElement(ProcessContext context) {} - })); - - return WriteResult.in(input.getPipeline(), successfulInserts, failedElements); - } - - /** - * Join input stream of FeatureRows with output stream from write jobs to produce rows that was - * successfully inserted. - * - *

In order to join we expect both streams to have identical windowing strategy. Join key - is - * write destination (table reference), hence as soon as bq load job responsible for specific - * table succeeded - we consider all rows routed to this destination in this window(!) as - * successfully written - * - * @param inputInFixedWindow input stream in fixed window - * @param successful output stream from BatchLoader {@link BatchLoadsWithResult} - * @return collection of successfully inserted rows - */ - private PCollection mergeInputWithResult( - PCollection inputInFixedWindow, - PCollection> successful) { - final TupleTag> inputTag = new TupleTag<>(); - final TupleTag successTag = new TupleTag<>(); - - PCollection>> insertedRows = - inputInFixedWindow - .apply( - "MakeElementKey", - ParDo.of( - new DoFn>() { - @ProcessElement - public void process(ProcessContext c) { - FeatureRow element = c.element(); - c.output( - KV.of( - BigQuerySinkHelpers.getTableDestination( - destination, element.getFeatureSet()) - .getTableSpec(), - element)); - } - })) - .apply(Sample.fixedSizePerKey(this.maxSuccessfulOutputs)); - - PCollection> inputWithResult = - KeyedPCollectionTuple.of(inputTag, insertedRows) - .and( - successTag, - successful.apply( - "MakeResultKey", - ParDo.of( - new DoFn, KV>() { - @ProcessElement - public void process(ProcessContext c) { - c.output( - KV.of(c.element().getKey().getTableSpec(), c.element().getValue())); - } - }))) - .apply(CoGroupByKey.create()); - - return inputWithResult.apply( - "ProduceSuccessfulInserts", - ParDo.of( - new DoFn, FeatureRow>() { - @ProcessElement - public void process(ProcessContext c) { - CoGbkResult result = c.element().getValue(); - boolean ready = result.getAll(successTag).iterator().hasNext(); - if (!ready) { - return; - } - - result.getAll(inputTag).forEach(rows -> rows.forEach(c::output)); - } - })); - } - - /** - * DynamicDestination router allocates incoming {@link FeatureRow} to tables in BigQuery based on - * FeatureSet reference extracted from this row. - * - *

getDestination is called for every item and simply returns grouping key {@link String}. - * - *

On window triggering getTable and getSchema are called once per each group. - * - *

getSchema provides latest table schema calculated from PCollection<FeatureSetSpec>. - * This schema is attached to each BQ Load job to update tables in-flight. - */ - private class FeatureDynamicDestinations extends DynamicDestinations { - @Override - public String getDestination(ValueInSingleWindow element) { - return element.getValue().getFeatureSet(); - } - - @Override - public List> getSideInputs() { - return ImmutableList.of(schemas); - } - - @Override - public TableDestination getTable(String featureSetKey) { - return BigQuerySinkHelpers.getTableDestination(destination, featureSetKey); - } - - @Override - public TableSchema getSchema(String featureSet) { - Map> schemasValue = sideInput(schemas); - Iterable schemasIt = schemasValue.get(featureSet); - if (schemasIt == null) { - return null; - } - return Iterators.getLast(schemasIt.iterator()); - } - } -} diff --git a/storage/connectors/bigquery/src/main/java/feast/storage/connectors/bigquery/writer/FeatureRowToTableRow.java b/storage/connectors/bigquery/src/main/java/feast/storage/connectors/bigquery/writer/FeatureRowToTableRow.java deleted file mode 100644 index f5e5ab1d2e9..00000000000 --- a/storage/connectors/bigquery/src/main/java/feast/storage/connectors/bigquery/writer/FeatureRowToTableRow.java +++ /dev/null @@ -1,113 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2019 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.storage.connectors.bigquery.writer; - -import com.google.api.services.bigquery.model.TableRow; -import com.google.protobuf.util.Timestamps; -import feast.proto.types.FeatureRowProto.FeatureRow; -import feast.proto.types.FieldProto.Field; -import java.util.Base64; -import java.util.stream.Collectors; -import org.apache.beam.sdk.transforms.SerializableFunction; -import org.joda.time.Instant; - -// TODO: Validate FeatureRow against FeatureSetSpec -// i.e. that the value types in FeatureRow matches against those in FeatureSetSpec - -public class FeatureRowToTableRow implements SerializableFunction { - private static final String EVENT_TIMESTAMP_COLUMN = "event_timestamp"; - private static final String CREATED_TIMESTAMP_COLUMN = "created_timestamp"; - private static final String INGESTION_ID_COLUMN = "ingestion_id"; - private static final String JOB_ID_COLUMN = "job_id"; - private final String jobId; - - public FeatureRowToTableRow(String jobId) { - this.jobId = jobId; - } - - public static String getEventTimestampColumn() { - return EVENT_TIMESTAMP_COLUMN; - } - - public TableRow apply(FeatureRow featureRow) { - - TableRow tableRow = new TableRow(); - tableRow.set( - EVENT_TIMESTAMP_COLUMN, - Timestamps.toString(featureRow.getEventTimestamp().toBuilder().setNanos(0).build())); - tableRow.set(CREATED_TIMESTAMP_COLUMN, Instant.now().toString()); - tableRow.set(INGESTION_ID_COLUMN, featureRow.getIngestionId()); - tableRow.set(JOB_ID_COLUMN, jobId); - - for (Field field : featureRow.getFieldsList()) { - switch (field.getValue().getValCase()) { - case BYTES_VAL: - tableRow.set( - field.getName(), - Base64.getEncoder().encodeToString(field.getValue().getBytesVal().toByteArray())); - break; - case STRING_VAL: - tableRow.set(field.getName(), field.getValue().getStringVal()); - break; - case INT32_VAL: - tableRow.set(field.getName(), field.getValue().getInt32Val()); - break; - case INT64_VAL: - tableRow.set(field.getName(), field.getValue().getInt64Val()); - break; - case DOUBLE_VAL: - tableRow.set(field.getName(), field.getValue().getDoubleVal()); - break; - case FLOAT_VAL: - tableRow.set(field.getName(), field.getValue().getFloatVal()); - break; - case BOOL_VAL: - tableRow.set(field.getName(), field.getValue().getBoolVal()); - break; - case BYTES_LIST_VAL: - tableRow.set( - field.getName(), - field.getValue().getBytesListVal().getValList().stream() - .map(x -> Base64.getEncoder().encodeToString(x.toByteArray())) - .collect(Collectors.toList())); - break; - case STRING_LIST_VAL: - tableRow.set(field.getName(), field.getValue().getStringListVal().getValList()); - break; - case INT32_LIST_VAL: - tableRow.set(field.getName(), field.getValue().getInt32ListVal().getValList()); - break; - case INT64_LIST_VAL: - tableRow.set(field.getName(), field.getValue().getInt64ListVal().getValList()); - break; - case DOUBLE_LIST_VAL: - tableRow.set(field.getName(), field.getValue().getDoubleListVal().getValList()); - break; - case FLOAT_LIST_VAL: - tableRow.set(field.getName(), field.getValue().getFloatListVal().getValList()); - break; - case BOOL_LIST_VAL: - tableRow.set(field.getName(), field.getValue().getBytesListVal().getValList()); - break; - case VAL_NOT_SET: - break; - } - } - - return tableRow; - } -} diff --git a/storage/connectors/bigquery/src/main/java/feast/storage/connectors/bigquery/writer/FeatureSetSpecToTableSchema.java b/storage/connectors/bigquery/src/main/java/feast/storage/connectors/bigquery/writer/FeatureSetSpecToTableSchema.java deleted file mode 100644 index cb76c65f8c3..00000000000 --- a/storage/connectors/bigquery/src/main/java/feast/storage/connectors/bigquery/writer/FeatureSetSpecToTableSchema.java +++ /dev/null @@ -1,263 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.storage.connectors.bigquery.writer; - -import com.google.api.services.bigquery.model.TableFieldSchema; -import com.google.api.services.bigquery.model.TableReference; -import com.google.api.services.bigquery.model.TableSchema; -import com.google.cloud.bigquery.*; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; -import feast.common.models.FeatureSetReference; -import feast.proto.core.FeatureSetProto; -import feast.storage.connectors.bigquery.common.TypeUtil; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.util.*; -import java.util.stream.Collectors; -import org.apache.beam.repackaged.core.org.apache.commons.lang3.tuple.Pair; -import org.apache.beam.sdk.coders.Coder; -import org.apache.beam.sdk.coders.CoderException; -import org.apache.beam.sdk.coders.StringUtf8Coder; -import org.apache.beam.sdk.io.gcp.bigquery.BigQueryHelpers; -import org.apache.beam.sdk.io.gcp.bigquery.TableDestination; -import org.apache.beam.sdk.options.ValueProvider; -import org.apache.beam.sdk.transforms.DoFn; -import org.apache.beam.sdk.values.KV; -import org.slf4j.Logger; - -/** - * Converts {@link feast.proto.core.FeatureSetProto.FeatureSetSpec} into BigQuery schema. Serializes - * it into json-like format {@link TableSchema}. Fetches existing schema to merge existing fields - * with new ones. - * - *

As a side effect this Operation may create bq table (if it doesn't exist) to make - * bootstrapping faster - */ -public class FeatureSetSpecToTableSchema - extends DoFn< - KV, - KV> { - private BigQuery bqService; - private DatasetId dataset; - private ValueProvider bqProvider; - - private static final Logger log = - org.slf4j.LoggerFactory.getLogger(FeatureSetSpecToTableSchema.class); - - // Reserved columns - public static final String EVENT_TIMESTAMP_COLUMN = "event_timestamp"; - public static final String CREATED_TIMESTAMP_COLUMN = "created_timestamp"; - public static final String INGESTION_ID_COLUMN = "ingestion_id"; - public static final String JOB_ID_COLUMN = "job_id"; - - // Column description for reserved fields - public static final String BIGQUERY_EVENT_TIMESTAMP_FIELD_DESCRIPTION = - "Event time for the FeatureRow"; - public static final String BIGQUERY_CREATED_TIMESTAMP_FIELD_DESCRIPTION = - "Processing time of the FeatureRow ingestion in Feast\""; - public static final String BIGQUERY_INGESTION_ID_FIELD_DESCRIPTION = - "Unique id identifying groups of rows that have been ingested together"; - public static final String BIGQUERY_JOB_ID_FIELD_DESCRIPTION = - "Feast import job ID for the FeatureRow"; - - public FeatureSetSpecToTableSchema(DatasetId dataset, ValueProvider bqProvider) { - this.dataset = dataset; - this.bqProvider = bqProvider; - } - - @Setup - public void setup() { - this.bqService = bqProvider.get(); - } - - @ProcessElement - public void processElement( - @Element KV element, - OutputReceiver> output, - ProcessContext context) { - String specKey = element.getKey().getReference(); - - Table existingTable = getExistingTable(specKey); - Schema schema = createSchemaFromSpec(element.getValue(), specKey, existingTable); - - if (existingTable == null) { - createTable(specKey, schema); - } - - output.output(KV.of(element.getKey(), serializeSchema(schema))); - } - - private TableId generateTableId(String specKey) { - TableDestination tableDestination = BigQuerySinkHelpers.getTableDestination(dataset, specKey); - TableReference tableReference = BigQueryHelpers.parseTableSpec(tableDestination.getTableSpec()); - return TableId.of( - tableReference.getProjectId(), tableReference.getDatasetId(), tableReference.getTableId()); - } - - private Table getExistingTable(String specKey) { - return bqService.getTable(generateTableId(specKey)); - } - - private void createTable(String specKey, Schema schema) { - TimePartitioning timePartitioning = - TimePartitioning.newBuilder(TimePartitioning.Type.DAY) - .setField(EVENT_TIMESTAMP_COLUMN) - .build(); - - StandardTableDefinition tableDefinition = - StandardTableDefinition.newBuilder() - .setTimePartitioning(timePartitioning) - .setSchema(schema) - .build(); - - TableInfo tableInfo = TableInfo.of(generateTableId(specKey), tableDefinition); - - bqService.create(tableInfo); - } - - /** - * Creates a BigQuery {@link Schema} based on the provided FeatureSetSpec and the existing table, - * if any. If a table already exists, existing fields will be retained, and new fields present in - * the feature set will be appended to the existing FieldsList. - * - * @param spec FeatureSet spec that this table is for - * @param specKey String for retrieving existing table - * @param existingTable Table fetched from BQ. Fields from existing table used to merge with new - * schema - * @return {@link Schema} containing all tombstoned and active fields. - */ - private Schema createSchemaFromSpec( - FeatureSetProto.FeatureSetSpec spec, String specKey, Table existingTable) { - Map fields = new LinkedHashMap<>(); - if (existingTable != null) { - Schema existingSchema = existingTable.getDefinition().getSchema(); - existingSchema.getFields().forEach(f -> fields.put(f.getName(), f)); - } - - for (FeatureSetProto.EntitySpec entitySpec : spec.getEntitiesList()) { - Field.Builder builder = - Field.newBuilder( - entitySpec.getName(), TypeUtil.toStandardSqlType(entitySpec.getValueType())); - if (entitySpec.getValueType().name().toLowerCase().endsWith("_list")) { - builder.setMode(Field.Mode.REPEATED); - } - Field field = builder.build(); - fields.put(field.getName(), field); - } - for (FeatureSetProto.FeatureSpec featureSpec : spec.getFeaturesList()) { - Field.Builder builder = - Field.newBuilder( - featureSpec.getName(), TypeUtil.toStandardSqlType(featureSpec.getValueType())); - if (featureSpec.getValueType().name().toLowerCase().endsWith("_list")) { - builder.setMode(Field.Mode.REPEATED); - } - - Field field = builder.build(); - fields.put(field.getName(), field); - } - - // Refer to protos/feast/core/Store.proto for reserved fields in BigQuery. - Map> - reservedFieldNameToPairOfStandardSQLTypeAndDescription = - ImmutableMap.of( - EVENT_TIMESTAMP_COLUMN, - Pair.of(StandardSQLTypeName.TIMESTAMP, BIGQUERY_EVENT_TIMESTAMP_FIELD_DESCRIPTION), - CREATED_TIMESTAMP_COLUMN, - Pair.of( - StandardSQLTypeName.TIMESTAMP, BIGQUERY_CREATED_TIMESTAMP_FIELD_DESCRIPTION), - INGESTION_ID_COLUMN, - Pair.of(StandardSQLTypeName.STRING, BIGQUERY_INGESTION_ID_FIELD_DESCRIPTION), - JOB_ID_COLUMN, - Pair.of(StandardSQLTypeName.STRING, BIGQUERY_JOB_ID_FIELD_DESCRIPTION)); - for (Map.Entry> entry : - reservedFieldNameToPairOfStandardSQLTypeAndDescription.entrySet()) { - Field field = - Field.newBuilder(entry.getKey(), entry.getValue().getLeft()) - .setDescription(entry.getValue().getRight()) - .build(); - fields.put(field.getName(), field); - } - - log.info("Table {} will have the following fields:", specKey); - fields.values().forEach(f -> log.info("- {}", f.toString())); - - return Schema.of(FieldList.of(fields.values())); - } - - /** - * Convert Table schema into json-like object (prepared for serialization) - * - * @param schema bq table schema - * @return json-like schema - */ - private TableSchema serializeSchema(Schema schema) { - TableSchema tableSchema = new TableSchema(); - FieldList fields = schema.getFields(); - List tableFieldSchemas = - fields.stream() - .map( - field -> { - TableFieldSchema f = - new TableFieldSchema() - .setName(field.getName()) - .setType(field.getType().name()); - - if (field.getMode() != null) { - f.setMode(field.getMode().name()); - } - - if (field.getDescription() != null) { - f.setDescription(field.getDescription()); - } - return f; - }) - .collect(Collectors.toList()); - - tableSchema.setFields(tableFieldSchemas); - return tableSchema; - } - - public static class TableSchemaCoder extends Coder { - private static final StringUtf8Coder stringCoder = StringUtf8Coder.of(); - private static final TableSchemaCoder INSTANCE = new TableSchemaCoder(); - - public static TableSchemaCoder of() { - return INSTANCE; - } - - @Override - public void encode(TableSchema value, OutputStream outStream) - throws CoderException, IOException { - stringCoder.encode(BigQueryHelpers.toJsonString(value), outStream); - } - - @Override - public TableSchema decode(InputStream inStream) throws CoderException, IOException { - return BigQueryHelpers.fromJsonString(stringCoder.decode(inStream), TableSchema.class); - } - - @Override - public List> getCoderArguments() { - return ImmutableList.of(); - } - - @Override - public void verifyDeterministic() throws NonDeterministicException {} - } -} diff --git a/storage/connectors/bigquery/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BatchLoadsWithResult.java b/storage/connectors/bigquery/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BatchLoadsWithResult.java deleted file mode 100644 index 879d518e6a6..00000000000 --- a/storage/connectors/bigquery/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BatchLoadsWithResult.java +++ /dev/null @@ -1,312 +0,0 @@ -package org.apache.beam.sdk.io.gcp.bigquery; - -import static com.google.common.base.Preconditions.checkArgument; -import static org.apache.beam.sdk.io.gcp.bigquery.BigQueryHelpers.resolveTempLocation; - -import com.google.api.services.bigquery.model.TableRow; -import com.google.auto.value.AutoValue; -import java.util.Collections; -import java.util.List; -import java.util.Set; -import javax.annotation.Nullable; -import org.apache.beam.sdk.Pipeline; -import org.apache.beam.sdk.coders.*; -import org.apache.beam.sdk.options.ValueProvider; -import org.apache.beam.sdk.transforms.*; -import org.apache.beam.sdk.transforms.windowing.*; -import org.apache.beam.sdk.values.*; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; -import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; -import org.joda.time.Duration; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -@AutoValue -public abstract class BatchLoadsWithResult - extends PTransform< - PCollection>, PCollection>> { - static final Logger LOG = LoggerFactory.getLogger(BatchLoadsWithResult.class); - - @VisibleForTesting - // Maximum number of files in a single partition. - static final int DEFAULT_MAX_FILES_PER_PARTITION = 10000; - - @VisibleForTesting - // Maximum number of bytes in a single partition -- 11 TiB just under BQ's 12 TiB limit. - static final long DEFAULT_MAX_BYTES_PER_PARTITION = 11 * (1L << 40); - - // The maximum size of a single file - 4TiB, just under the 5 TiB limit. - static final long DEFAULT_MAX_FILE_SIZE = 4 * (1L << 40); - - static final int DEFAULT_MAX_RETRY_JOBS = 3; - - static final int FILE_TRIGGERING_RECORD_COUNT = 500000; - - @Nullable - abstract BigQueryServices getBigQueryServices(); - - abstract boolean getIgnoreUnknownValues(); - - abstract BigQueryIO.Write.WriteDisposition getWriteDisposition(); - - abstract BigQueryIO.Write.CreateDisposition getCreateDisposition(); - - abstract Set getSchemaUpdateOptions(); - - abstract DynamicDestinations getDynamicDestinations(); - - abstract Coder getDestinationCoder(); - - abstract Duration getTriggeringFrequency(); - - @Nullable - abstract ValueProvider getCustomGcsTempLocation(); - - @Nullable - abstract ValueProvider getLoadJobProjectId(); - - abstract Coder getElementCoder(); - - abstract RowWriterFactory getRowWriterFactory(); - - @Nullable - abstract String getKmsKey(); - - abstract int getMaxRetryJobs(); - - @AutoValue.Builder - public abstract static class Builder { - public abstract Builder setBigQueryServices(BigQueryServices bigQueryServices); - - public abstract Builder setIgnoreUnknownValues(boolean ignoreUnknownValues); - - public abstract Builder setWriteDisposition( - BigQueryIO.Write.WriteDisposition writeDisposition); - - public abstract Builder setCreateDisposition( - BigQueryIO.Write.CreateDisposition createDisposition); - - public abstract Builder setSchemaUpdateOptions( - Set schemaUpdateOptions); - - public abstract Builder setDynamicDestinations( - DynamicDestinations dynamicDestinations); - - public abstract Builder setDestinationCoder(Coder destinationCoder); - - public abstract Builder setTriggeringFrequency(Duration triggeringFrequency); - - public abstract Builder setCustomGcsTempLocation( - @Nullable ValueProvider customGcsTempLocation); - - public abstract Builder setLoadJobProjectId( - @Nullable ValueProvider loadJobProjectId); - - public abstract Builder setElementCoder(Coder elementCoder); - - public abstract Builder setRowWriterFactory( - RowWriterFactory rowWriterFactory); - - public abstract Builder setKmsKey(@Nullable String kmsKey); - - public abstract Builder setMaxRetryJobs(int maxRetryJobs); - - public abstract BatchLoadsWithResult build(); - } - - public static Builder create() { - return new AutoValue_BatchLoadsWithResult.Builder() - .setIgnoreUnknownValues(false) - .setBigQueryServices(new BigQueryServicesImpl()) - .setRowWriterFactory(RowWriterFactory.tableRows(SerializableFunctions.identity())) - .setSchemaUpdateOptions(Collections.emptySet()) - .setMaxRetryJobs(DEFAULT_MAX_RETRY_JOBS); - } - - public PCollection> expand( - PCollection> input) { - - // we assume that input must be already windowed and we allow only fixed window - // so our internal generator (JobIdPrefix) would be compatible (hence, joinable) - checkArgument( - input.getWindowingStrategy().getWindowFn() instanceof FixedWindows, - "Input to BQ writer must be windowed in advance"); - - final PCollectionView loadJobIdPrefixView = createLoadJobIdPrefixView(input); - final PCollectionView tempFilePrefixView = - createTempFilePrefixView(input.getPipeline()); - - PCollection> results = - input - .apply( - "WindowWithTrigger", - Window.>configure() - .triggering( - Repeatedly.forever( - AfterPane.elementCountAtLeast(FILE_TRIGGERING_RECORD_COUNT))) - .discardingFiredPanes()) - .apply( - "PutAllRowsInSingleShard", - ParDo.of( - new DoFn, KV, TableRow>>() { - @ProcessElement - public void process(ProcessContext c) { - c.output( - KV.of(ShardedKey.of(c.element().getKey(), 0), c.element().getValue())); - } - })) - .setCoder(KvCoder.of(ShardedKeyCoder.of(getDestinationCoder()), getElementCoder())) - .apply("GroupByDestination", GroupByKey.create()) - .apply( - "WriteGroupedRecords", - ParDo.of( - new WriteGroupedRecordsToFiles<>( - tempFilePrefixView, DEFAULT_MAX_FILE_SIZE, getRowWriterFactory())) - .withSideInputs(tempFilePrefixView)) - .setCoder(WriteBundlesToFiles.ResultCoder.of(getDestinationCoder())); - - TupleTag, List>> multiPartitionsTag = - new TupleTag<>("multiPartitionsTag"); - TupleTag, List>> singlePartitionTag = - new TupleTag<>("singlePartitionTag"); - - // Copied from original BatchLoads - PCollectionTuple partitions = - results - .apply( - Window.>configure() - .triggering(DefaultTrigger.of())) - .apply("AttachSingletonKey", WithKeys.of((Void) null)) - .setCoder( - KvCoder.of( - VoidCoder.of(), WriteBundlesToFiles.ResultCoder.of(getDestinationCoder()))) - .apply("GroupOntoSingleton", GroupByKey.create()) - .apply("ExtractResultValues", Values.create()) - .apply( - "WritePartitionTriggered", - ParDo.of( - new WritePartition<>( - false, - getDynamicDestinations(), - tempFilePrefixView, - DEFAULT_MAX_FILES_PER_PARTITION, - DEFAULT_MAX_BYTES_PER_PARTITION, - multiPartitionsTag, - singlePartitionTag, - getRowWriterFactory())) - .withSideInputs(tempFilePrefixView) - .withOutputTags(multiPartitionsTag, TupleTagList.of(singlePartitionTag))); - - partitions - .get(multiPartitionsTag) - .setCoder( - KvCoder.of( - ShardedKeyCoder.of(NullableCoder.of(getDestinationCoder())), - ListCoder.of(StringUtf8Coder.of()))); - - return writeSinglePartitionWithResult(partitions.get(singlePartitionTag), loadJobIdPrefixView); - } - - /** - * Generates one jobId per window only if any feature row was submitted in this window. We need to - * generate exactly one id per window, otherwise SingletonView will fail. - * - * @param input feature Rows - * @return job id generated once per input's window - */ - private PCollectionView createLoadJobIdPrefixView( - PCollection> input) { - // We generate new JobId per each (input) window - // To keep BQ job's name unique - // Windowing of this generator is expected to be synchronized with input window - // So generated ids can be applied as side input - - String baseName = input.getPipeline().getOptions().getJobName().replaceAll("-", ""); - - return input - .apply( - "EraseKeyAndValue", - ParDo.of( - new DoFn, String>() { - @ProcessElement - public void process(ProcessContext c) { - // we don't need data, only fact of data existing - c.output(""); - } - })) - .apply( - Combine.globally( - (SerializableFunction, String>) - g -> - String.format( - "beam_load_%s_%s", baseName, BigQueryHelpers.randomUUIDString())) - .withoutDefaults()) - .apply("JobIdView", View.asSingleton()); - } - - /** - * Generates one global (per all windows) prefix path to store files before load to BQ - * - * @param p Pipeline - * @return view in global window - */ - private PCollectionView createTempFilePrefixView(final Pipeline p) { - return p.apply("CreateGlobalTempPrefix", Create.of("")) - .apply( - "GetTempFilePrefix", - ParDo.of( - new DoFn() { - @ProcessElement - public void getTempFilePrefix(ProcessContext c) { - String tempLocationRoot; - if (getCustomGcsTempLocation() != null) { - tempLocationRoot = getCustomGcsTempLocation().get(); - } else { - tempLocationRoot = c.getPipelineOptions().getTempLocation(); - } - String tempLocation = - resolveTempLocation( - tempLocationRoot, - "BigQueryWriteTemp", - c.getPipelineOptions().getJobName()); - - c.output(tempLocation); - } - })) - .apply("TempFilePrefixView", View.asSingleton()); - } - - PCollection> writeSinglePartitionWithResult( - PCollection, List>> input, - PCollectionView loadJobIdPrefixView) { - List> sideInputs = Lists.newArrayList(loadJobIdPrefixView); - sideInputs.addAll(getDynamicDestinations().getSideInputs()); - Coder, List>> partitionsCoder = - KvCoder.of( - ShardedKeyCoder.of(NullableCoder.of(getDestinationCoder())), - ListCoder.of(StringUtf8Coder.of())); - // Write single partition to final table - return input - .setCoder(partitionsCoder) - // Reshuffle will distribute this among multiple workers, and also guard against - // reexecution of the WritePartitions step once WriteTables has begun. - .apply("SinglePartitionsReshuffle", Reshuffle.of()) - .apply( - "SinglePartitionWriteTables", - new WriteTables<>( - false, - getBigQueryServices(), - loadJobIdPrefixView, - getWriteDisposition(), - getCreateDisposition(), - sideInputs, - getDynamicDestinations(), - getLoadJobProjectId(), - getMaxRetryJobs(), - getIgnoreUnknownValues(), - getKmsKey(), - getRowWriterFactory().getSourceFormat(), - true, - getSchemaUpdateOptions())); - } -} diff --git a/storage/connectors/bigquery/src/main/resources/schemas/deadletter_table_schema.json b/storage/connectors/bigquery/src/main/resources/schemas/deadletter_table_schema.json deleted file mode 100644 index 92381189073..00000000000 --- a/storage/connectors/bigquery/src/main/resources/schemas/deadletter_table_schema.json +++ /dev/null @@ -1,34 +0,0 @@ -{ - "fields": [ - { - "name": "timestamp", - "type": "TIMESTAMP", - "mode": "REQUIRED" - }, - { - "name": "job_name", - "type": "STRING", - "mode": "NULLABLE" - }, - { - "name": "transform_name", - "type": "STRING", - "mode": "NULLABLE" - }, - { - "name": "payload", - "type": "STRING", - "mode": "NULLABLE" - }, - { - "name": "error_message", - "type": "STRING", - "mode": "NULLABLE" - }, - { - "name": "stack_trace", - "type": "STRING", - "mode": "NULLABLE" - } - ] -} \ No newline at end of file diff --git a/storage/connectors/bigquery/src/main/resources/templates/basic_stats.sql b/storage/connectors/bigquery/src/main/resources/templates/basic_stats.sql deleted file mode 100644 index bf38c1818d5..00000000000 --- a/storage/connectors/bigquery/src/main/resources/templates/basic_stats.sql +++ /dev/null @@ -1,80 +0,0 @@ -WITH subset AS ( -{{ dataset }} -) -{% for feature in features %} -SELECT - "{{ feature.name }}" as feature_name, - -- total count - COUNT(*) AS total_count, - -- count - COUNT({{ feature.name }}) as feature_count, - -- missing - COUNT(*) - COUNT({{ feature.name }}) as missing_count, - {% if feature.statsType equals "NUMERIC" %} - -- mean - AVG({{ feature.name }}) as mean, - -- stdev - STDDEV({{ feature.name }}) as stdev, - -- zeroes - COUNTIF({{ feature.name }} = 0) as zeroes, - -- min - MIN({{ feature.name }}) as min, - -- max - MAX({{ feature.name }}) as max, - -- hist will have to be called separately - -- quantiles - APPROX_QUANTILES(CAST({{ feature.name }} AS FLOAT64), 10) AS quantiles, - -- unique - null as unique - {% elseif feature.statsType equals "CATEGORICAL" %} - -- mean - AVG(LENGTH({{ feature.name }})) as mean, - -- stdev - null as stdev, - -- zeroes - null as zeroes, - -- min - null as min, - -- max - null as max, - -- quantiles - ARRAY[] AS quantiles, - -- unique - COUNT(DISTINCT({{ feature.name }})) as unique - {% elseif feature.statsType equals "BYTES" %} - -- mean - AVG(BIT_COUNT({{ feature.name }})) as mean, - -- stdev - null as stdev, - -- zeroes - null as zeroes, - -- min - MIN(BIT_COUNT({{ feature.name }})) as min, - -- max - MAX(BIT_COUNT({{ feature.name }})) as max, - -- hist will have to be called separately - -- quantiles - ARRAY[] AS quantiles, - -- unique - COUNT(DISTINCT({{ feature.name }})) as unique - {% elseif feature.statsType equals "LIST" %} - -- mean - AVG(ARRAY_LENGTH({{ feature.name }})) as mean, - -- stdev - null as stdev, - -- zeroes - null as zeroes, - -- min - MIN(ARRAY_LENGTH({{ feature.name }})) as min, - -- max - MAX(ARRAY_LENGTH({{ feature.name }})) as max, - -- hist will have to be called separately - -- quantiles - ARRAY[] AS quantiles, - -- unique - null as unique - {% endif %} -FROM subset -{% if loop.last %}{% else %}UNION ALL {% endif %} -{% endfor %} - diff --git a/storage/connectors/bigquery/src/main/resources/templates/data_subset.sql b/storage/connectors/bigquery/src/main/resources/templates/data_subset.sql deleted file mode 100644 index 68fed04ada7..00000000000 --- a/storage/connectors/bigquery/src/main/resources/templates/data_subset.sql +++ /dev/null @@ -1,7 +0,0 @@ -SELECT * FROM `{{ table }}` -{% if ingestionId is not empty %} -WHERE ingestion_id='{{ ingestionId }}' -{% endif %} -{% if date is not empty %} -WHERE DATE(event_timestamp)='{{ date }}' -{% endif %} \ No newline at end of file diff --git a/storage/connectors/bigquery/src/main/resources/templates/hist_stats.sql b/storage/connectors/bigquery/src/main/resources/templates/hist_stats.sql deleted file mode 100644 index b51ad33e71b..00000000000 --- a/storage/connectors/bigquery/src/main/resources/templates/hist_stats.sql +++ /dev/null @@ -1,36 +0,0 @@ -WITH subset AS ( -{{ dataset }} -) -{% for feature in features %} -, {{ feature.name }}_stats AS ( -{% if feature.statsType == 'NUMERIC' %} - WITH stats AS ( - SELECT min+step*i as min, min+step*(i+1) as max - FROM ( - SELECT MIN({{ feature.name }}) as min, MAX({{ feature.name }}) as max, (MAX({{ feature.name }})-MIN({{ feature.name }}))/10 step, GENERATE_ARRAY(0, 10, 1) i - FROM subset - ), UNNEST(i) i - ), counts as ( - SELECT COUNT(*) as count, min, max, - FROM subset - JOIN stats - ON subset.{{ feature.name }} >= stats.min AND subset.{{ feature.name }}>[] as cat_hist FROM counts -{% elseif feature.statsType == 'CATEGORICAL' %} - WITH counts AS ( - SELECT {{ feature.name }}, COUNT({{ feature.name }}) AS count FROM subset GROUP BY {{ feature.name }} - ) - SELECT '{{ feature.name }}' as feature, ARRAY>[] as num_hist, ARRAY_AGG(STRUCT({{ feature.name }} as value, count as count)) as cat_hist FROM counts -{% elseif feature.statsType == 'BYTES' %} - SELECT '{{ feature.name }}' as feature, ARRAY>[] as num_hist, ARRAY>[] as cat_hist -{% elseif feature.statsType == 'LIST' %} - SELECT '{{ feature.name }}' as feature, ARRAY>[] as num_hist, ARRAY>[] as cat_hist -{% endif %} -) -{% endfor %} -{% for feature in features %} -SELECT * FROM {{ feature.name }}_stats -{% if loop.last %}{% else %}UNION ALL {% endif %} -{% endfor %} \ No newline at end of file diff --git a/storage/connectors/bigquery/src/main/resources/templates/join_featuresets.sql b/storage/connectors/bigquery/src/main/resources/templates/join_featuresets.sql deleted file mode 100644 index ddddac8d2cf..00000000000 --- a/storage/connectors/bigquery/src/main/resources/templates/join_featuresets.sql +++ /dev/null @@ -1,24 +0,0 @@ -/* - Joins the outputs of multiple point-in-time-correctness joins to a single table. - */ -WITH joined as ( -SELECT * FROM `{{ leftTableName }}` -{% for featureSet in featureSets %} -LEFT JOIN ( - SELECT - uuid, - {% for feature in featureSet.features %} - {{ featureSet.project }}__{{ featureSet.name }}__{{ feature.name }}{% if loop.last %}{% else %}, {% endif %} - {% endfor %} - FROM `{{ featureSet.table }}` -) USING (uuid) -{% endfor %} -) SELECT - event_timestamp, - {{ entities | join(', ') }} - {% for featureSet in featureSets %} - {% for feature in featureSet.features %} - ,{{ featureSet.project }}__{{ featureSet.name }}__{{ feature.name }} as {% if feature.featureSet != "" %}{{ featureSet.name }}__{% endif %}{{ feature.name }} - {% endfor %} - {% endfor %} -FROM joined diff --git a/storage/connectors/bigquery/src/main/resources/templates/single_featureset_pit_join.sql b/storage/connectors/bigquery/src/main/resources/templates/single_featureset_pit_join.sql deleted file mode 100644 index 24bdab2c29c..00000000000 --- a/storage/connectors/bigquery/src/main/resources/templates/single_featureset_pit_join.sql +++ /dev/null @@ -1,90 +0,0 @@ -/* - This query template performs the point-in-time correctness join for a single feature set table - to the provided entity table. - - 1. Concatenate the timestamp and entities from the feature set table with the entity dataset. - Feature values are joined to this table later for improved efficiency. - featureset_timestamp is equal to null in rows from the entity dataset. - */ -WITH union_features AS ( -SELECT - -- uuid is a unique identifier for each row in the entity dataset. Generated by `QueryTemplater.createEntityTableUUIDQuery` - uuid, - -- event_timestamp contains the timestamps to join onto - event_timestamp, - -- the feature_timestamp, i.e. the latest occurrence of the requested feature relative to the entity_dataset timestamp - NULL as {{ featureSet.project }}_{{ featureSet.name }}_feature_timestamp, - -- created timestamp of the feature at the corresponding feature_timestamp - NULL as created_timestamp, - -- select only entities belonging to this feature set - {{ featureSet.entities | join(', ')}}, - -- boolean for filtering the dataset later - true AS is_entity_table -FROM `{{leftTableName}}` -UNION ALL -SELECT - NULL as uuid, - event_timestamp, - event_timestamp as {{ featureSet.project }}_{{ featureSet.name }}_feature_timestamp, - created_timestamp, - {{ featureSet.entities | join(', ')}}, - false AS is_entity_table -FROM `{{projectId}}.{{datasetId}}.{{ featureSet.project }}_{{ featureSet.name }}` WHERE event_timestamp <= '{{maxTimestamp}}' -{% if featureSet.maxAge == 0 %}{% else %}AND event_timestamp >= Timestamp_sub(TIMESTAMP '{{ minTimestamp }}', interval {{ featureSet.maxAge }} second){% endif %} -), -/* - 2. Window the data in the unioned dataset, partitioning by entity and ordering by event_timestamp, as - well as is_entity_table. - Within each window, back-fill the feature_timestamp - as a result of this, the null feature_timestamps - in the rows from the entity table should now contain the latest timestamps relative to the row's - event_timestamp. - - For rows where event_timestamp(provided datetime) - feature_timestamp > max age, set the - feature_timestamp to null. - */ -joined AS ( -SELECT - uuid, - event_timestamp, - {{ featureSet.entities | join(', ')}}, - {% for feature in featureSet.features %} - IF(event_timestamp >= {{ featureSet.project }}_{{ featureSet.name }}_feature_timestamp {% if featureSet.maxAge == 0 %}{% else %}AND Timestamp_sub(event_timestamp, interval {{ featureSet.maxAge }} second) < {{ featureSet.project }}_{{ featureSet.name }}_feature_timestamp{% endif %}, {{ featureSet.project }}__{{ featureSet.name }}__{{ feature.name }}, NULL) as {{ featureSet.project }}__{{ featureSet.name }}__{{ feature.name }}{% if loop.last %}{% else %}, {% endif %} - {% endfor %} -FROM ( -SELECT - uuid, - event_timestamp, - {{ featureSet.entities | join(', ')}}, - FIRST_VALUE(created_timestamp IGNORE NULLS) over w AS created_timestamp, - FIRST_VALUE({{ featureSet.project }}_{{ featureSet.name }}_feature_timestamp IGNORE NULLS) over w AS {{ featureSet.project }}_{{ featureSet.name }}_feature_timestamp, - is_entity_table -FROM union_features -WINDOW w AS (PARTITION BY {{ featureSet.entities | join(', ') }} ORDER BY event_timestamp DESC, is_entity_table DESC, created_timestamp DESC ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING) -) -/* - 3. Select only the rows from the entity table, and join the features from the original feature set table - to the dataset using the entity values, feature_timestamp, and created_timestamps. - */ -LEFT JOIN ( -SELECT - event_timestamp as {{ featureSet.project }}_{{ featureSet.name }}_feature_timestamp, - created_timestamp, - {{ featureSet.entities | join(', ')}}, - {% for feature in featureSet.features %} - {{ feature.name }} as {{ featureSet.project }}__{{ featureSet.name }}__{{ feature.name }}{% if loop.last %}{% else %}, {% endif %} - {% endfor %} -FROM `{{ projectId }}.{{ datasetId }}.{{ featureSet.project }}_{{ featureSet.name }}` WHERE event_timestamp <= '{{maxTimestamp}}' -{% if featureSet.maxAge == 0 %}{% else %}AND event_timestamp >= Timestamp_sub(TIMESTAMP '{{ minTimestamp }}', interval {{ featureSet.maxAge }} second){% endif %} -) USING ({{ featureSet.project }}_{{ featureSet.name }}_feature_timestamp, created_timestamp, {{ featureSet.entities | join(', ')}}) -WHERE is_entity_table -) -/* - 4. Finally, deduplicate the rows by selecting the first occurrence of each entity table row UUID. - */ -SELECT - k.* -FROM ( - SELECT ARRAY_AGG(row LIMIT 1)[OFFSET(0)] k - FROM joined row - GROUP BY uuid -) \ No newline at end of file diff --git a/storage/connectors/bigquery/src/test/java/com/google/cloud/bigquery/FakeTable.java b/storage/connectors/bigquery/src/test/java/com/google/cloud/bigquery/FakeTable.java deleted file mode 100644 index 7a3eb445602..00000000000 --- a/storage/connectors/bigquery/src/test/java/com/google/cloud/bigquery/FakeTable.java +++ /dev/null @@ -1,37 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.google.cloud.bigquery; - -import java.io.IOException; -import java.io.ObjectInputStream; - -public class FakeTable extends Table { - FakeTable(BigQuery bigquery, BuilderImpl infoBuilder) { - super(bigquery, infoBuilder); - } - - public static FakeTable create(BigQuery bigQuery, TableId tableId, TableDefinition definition) { - BuilderImpl builder = new BuilderImpl(); - builder.setDefinition(definition); - builder.setTableId(tableId); - return new FakeTable(bigQuery, builder); - } - - private void readObject(ObjectInputStream in) throws IOException, ClassNotFoundException { - in.defaultReadObject(); - } -} diff --git a/storage/connectors/bigquery/src/test/java/feast/storage/connectors/bigquery/writer/BigQuerySinkTest.java b/storage/connectors/bigquery/src/test/java/feast/storage/connectors/bigquery/writer/BigQuerySinkTest.java deleted file mode 100644 index 72d01df1796..00000000000 --- a/storage/connectors/bigquery/src/test/java/feast/storage/connectors/bigquery/writer/BigQuerySinkTest.java +++ /dev/null @@ -1,456 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.storage.connectors.bigquery.writer; - -import static feast.storage.common.testing.TestUtil.createRandomValue; -import static feast.storage.common.testing.TestUtil.field; -import static feast.storage.connectors.bigquery.writer.FeatureSetSpecToTableSchema.*; -import static org.hamcrest.CoreMatchers.*; -import static org.junit.Assert.*; -import static org.mockito.Mockito.*; -import static org.mockito.MockitoAnnotations.initMocks; - -import com.google.api.services.bigquery.model.Job; -import com.google.api.services.bigquery.model.JobConfigurationLoad; -import com.google.api.services.bigquery.model.TableFieldSchema; -import com.google.api.services.bigquery.model.TableReference; -import com.google.cloud.bigquery.*; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; -import com.google.common.collect.Iterators; -import feast.common.models.FeatureSetReference; -import feast.proto.core.FeatureSetProto.EntitySpec; -import feast.proto.core.FeatureSetProto.FeatureSetSpec; -import feast.proto.core.FeatureSetProto.FeatureSpec; -import feast.proto.types.FeatureRowProto.FeatureRow; -import feast.proto.types.FieldProto; -import feast.proto.types.ValueProto; -import feast.storage.api.writer.FeatureSink; -import feast.storage.api.writer.WriteResult; -import java.io.IOException; -import java.io.Serializable; -import java.util.*; -import java.util.logging.ConsoleHandler; -import java.util.logging.Logger; -import org.apache.beam.sdk.coders.AvroCoder; -import org.apache.beam.sdk.coders.KvCoder; -import org.apache.beam.sdk.extensions.protobuf.ProtoCoder; -import org.apache.beam.sdk.io.gcp.bigquery.BatchLoadsWithResult; -import org.apache.beam.sdk.io.gcp.testing.FakeBigQueryServices; -import org.apache.beam.sdk.io.gcp.testing.FakeDatasetService; -import org.apache.beam.sdk.io.gcp.testing.FakeJobService; -import org.apache.beam.sdk.options.PipelineOptions; -import org.apache.beam.sdk.options.ValueProvider; -import org.apache.beam.sdk.testing.PAssert; -import org.apache.beam.sdk.testing.TestPipeline; -import org.apache.beam.sdk.testing.TestStream; -import org.apache.beam.sdk.transforms.*; -import org.apache.beam.sdk.transforms.windowing.AfterPane; -import org.apache.beam.sdk.transforms.windowing.GlobalWindows; -import org.apache.beam.sdk.transforms.windowing.Repeatedly; -import org.apache.beam.sdk.transforms.windowing.Window; -import org.apache.beam.sdk.values.KV; -import org.apache.beam.sdk.values.PCollection; -import org.joda.time.Duration; -import org.joda.time.Instant; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.ExpectedException; -import org.mockito.Mock; -import org.mockito.invocation.InvocationOnMock; -import org.mockito.stubbing.Answer; - -public class BigQuerySinkTest { - - @Rule public transient TestPipeline p = TestPipeline.fromOptions(makePipelineOptions()); - @Rule public final ExpectedException exception = ExpectedException.none(); - - @Mock(serializable = true) - private BigQuery bigQuery; - - private FakeJobService jobService = new FakeJobService(); - private FakeDatasetService datasetService = new FakeDatasetService(); - private Random rd = new Random(); - - List commonFields = - Arrays.asList( - new TableFieldSchema() - .setName("event_timestamp") - .setType("TIMESTAMP") - .setDescription(BIGQUERY_EVENT_TIMESTAMP_FIELD_DESCRIPTION), - new TableFieldSchema() - .setName("created_timestamp") - .setType("TIMESTAMP") - .setDescription(BIGQUERY_CREATED_TIMESTAMP_FIELD_DESCRIPTION), - new TableFieldSchema() - .setName("ingestion_id") - .setType("STRING") - .setDescription(BIGQUERY_INGESTION_ID_FIELD_DESCRIPTION), - new TableFieldSchema() - .setName("job_id") - .setType("STRING") - .setDescription(BIGQUERY_JOB_ID_FIELD_DESCRIPTION)); - FeatureSetSpec spec; - - public static PipelineOptions makePipelineOptions() { - PipelineOptions options = TestPipeline.testingPipelineOptions(); - options.setTempLocation("/tmp/feast"); - return options; - } - - private FeatureRow generateRow(String featureSet) { - FeatureRow.Builder row = - FeatureRow.newBuilder() - .setFeatureSet(featureSet) - .setEventTimestamp( - com.google.protobuf.Timestamp.newBuilder() - .setSeconds(System.currentTimeMillis() / 1000) - .build()) - .setIngestionId("ingestion-id") - .addFields(field("entity", rd.nextInt(), ValueProto.ValueType.Enum.INT64)) - .addFields(FieldProto.Field.newBuilder().setName("null_value").build()); - - for (ValueProto.ValueType.Enum type : ValueProto.ValueType.Enum.values()) { - if (type == ValueProto.ValueType.Enum.INVALID - || type == ValueProto.ValueType.Enum.UNRECOGNIZED) { - continue; - } - row.addFields( - FieldProto.Field.newBuilder() - .setName(String.format("feature_%d", type.getNumber())) - .setValue(createRandomValue(type, 5)) - .build()); - } - - return row.build(); - } - - @Before - public void setUp() throws IOException, InterruptedException { - initMocks(this); - - Logger.getLogger(BatchLoadsWithResult.class.getName()).addHandler(new ConsoleHandler()); - - when(bigQuery.getTable(TableId.of("test-project", "test_dataset", "myproject_fs"))) - .thenReturn(null); - - FakeDatasetService.setUp(); - datasetService.createDataset("test-project", "test_dataset", "us-central1", "description", 0L); - - spec = - FeatureSetSpec.newBuilder() - .setName("fs") - .setProject("myproject") - .addEntities( - EntitySpec.newBuilder() - .setName("entity") - .setValueType(ValueProto.ValueType.Enum.INT64) - .build()) - .addFeatures( - FeatureSpec.newBuilder() - .setName("feature") - .setValueType(ValueProto.ValueType.Enum.STRING) - .build()) - .build(); - } - - private FeatureSink makeSink( - ValueProvider bq, PCollection> specs) { - BigQueryFeatureSink sink = - BigQueryFeatureSink.builder() - .setDatasetId("test_dataset") - .setProjectId("test-project") - .setBQTestServices( - new FakeBigQueryServices() - .withJobService(jobService) - .withDatasetService(datasetService)) - .setBQClient(bq) - .setTriggeringFrequency(Duration.standardSeconds(5)) - .build(); - sink.prepareWrite(specs); - return sink; - } - - @Test - public void simpleInsert() { - FeatureRow row1 = generateRow("myproject/fs"); - FeatureRow row2 = generateRow("myproject/fs"); - - TestStream featureRowTestStream = - TestStream.create(ProtoCoder.of(FeatureRow.class)) - .advanceWatermarkTo(Instant.now()) - .addElements(row1, row2) - .advanceWatermarkToInfinity(); - - FeatureSink sink = - makeSink( - ValueProvider.StaticValueProvider.of(bigQuery), - p.apply( - Create.of( - ImmutableMap.of( - FeatureSetReference.of(spec.getProject(), spec.getName(), 1), spec)))); - PCollection successfulInserts = - p.apply(featureRowTestStream).apply(sink.writer()).getSuccessfulInserts(); - - PAssert.that(successfulInserts).containsInAnyOrder(ImmutableList.of(row1, row2)); - p.run(); - - assert jobService.getAllJobs().size() == 1; - Job load = Iterators.getLast(jobService.getAllJobs().iterator()); - JobConfigurationLoad loadConfiguration = load.getConfiguration().getLoad(); - - ArrayList expectedFields = - new ArrayList<>( - Arrays.asList( - new TableFieldSchema().setName("entity").setType("INTEGER"), - new TableFieldSchema().setName("feature").setType("STRING"))); - - expectedFields.addAll(commonFields); - - assertThat(loadConfiguration.getSchema().getFields(), is(expectedFields)); - - assertThat( - loadConfiguration.getDestinationTable(), - is( - new TableReference() - .setDatasetId("test_dataset") - .setProjectId("test-project") - .setTableId("myproject_fs"))); - } - - @Test - public void uniqueJobIdPerWindow() { - TestStream featureRowTestStream = - TestStream.create(ProtoCoder.of(FeatureRow.class)) - .advanceWatermarkTo(Instant.now()) - .addElements(generateRow("myproject/fs")) - .addElements(generateRow("myproject/fs")) - .advanceWatermarkTo(Instant.now().plus(Duration.standardSeconds(10))) - .addElements(generateRow("myproject/fs")) - .addElements(generateRow("myproject/fs")) - .advanceWatermarkToInfinity(); - - FeatureSink sink = - makeSink( - ValueProvider.StaticValueProvider.of(bigQuery), - p.apply( - "StaticSpecs", - Create.of( - ImmutableMap.of( - FeatureSetReference.of(spec.getProject(), spec.getName(), 1), spec)))); - - p.apply(featureRowTestStream).apply(sink.writer()); - p.run(); - - assertThat(jobService.getAllJobs().size(), is(2)); - assertThat( - jobService.getAllJobs().stream() - .map(j -> j.getJobReference().getJobId()) - .distinct() - .count(), - is(2L)); - } - - @Test - public void expectingJobResult() { - FeatureRow featureRow = generateRow("myproject/fs"); - TestStream featureRowTestStream = - TestStream.create(ProtoCoder.of(FeatureRow.class)) - .advanceWatermarkTo(Instant.now()) - .addElements(featureRow) - .advanceWatermarkToInfinity(); - - jobService.setNumFailuresExpected(3); - - FeatureSink sink = - makeSink( - ValueProvider.StaticValueProvider.of(bigQuery), - p.apply( - "StaticSpecs", - Create.of( - ImmutableMap.of( - FeatureSetReference.of(spec.getProject(), spec.getName(), 1), spec)))); - - PTransform, WriteResult> writer = - ((BigQueryWrite) sink.writer()).withExpectingResultTime(Duration.standardSeconds(5)); - PCollection inserts = - p.apply(featureRowTestStream).apply(writer).getSuccessfulInserts(); - - PAssert.that(inserts).containsInAnyOrder(ImmutableList.of(featureRow)); - - p.run(); - } - - @Test - public void updateSchemaWithExistingTable() { - TableId tableId = TableId.of("test-project", "test_dataset", "myproject_fs_2"); - - when(bigQuery.getTable(tableId)) - .thenAnswer( - new TableAnswer( - TableId.of("test-project", "test_dataset", "myproject_fs_2"), - StandardTableDefinition.of( - Schema.of( - Field.newBuilder("old_feature_1", LegacySQLTypeName.FLOAT) - .setDescription("Some old description") - .build(), - Field.of("old_feature_2", LegacySQLTypeName.INTEGER))))); - - FeatureSetSpec spec_fs_2 = - FeatureSetSpec.newBuilder() - .setName("fs_2") - .setProject("myproject") - .addEntities( - EntitySpec.newBuilder() - .setName("entity") - .setValueType(ValueProto.ValueType.Enum.INT64) - .build()) - .addFeatures( - FeatureSpec.newBuilder() - .setName("feature") - .setValueType(ValueProto.ValueType.Enum.STRING) - .build()) - .addFeatures( - FeatureSpec.newBuilder() - .setName("old_feature_1") - .setValueType(ValueProto.ValueType.Enum.FLOAT) - .build()) - .build(); - - FeatureSink sink = - makeSink( - ValueProvider.StaticValueProvider.of(bigQuery), - p.apply( - Create.of( - ImmutableMap.of( - FeatureSetReference.of(spec_fs_2.getProject(), spec_fs_2.getName(), 1), - spec_fs_2)))); - - TestStream featureRowTestStream = - TestStream.create(ProtoCoder.of(FeatureRow.class)) - .advanceWatermarkTo(Instant.now()) - .addElements(generateRow("myproject/fs_2")) - .advanceWatermarkToInfinity(); - - p.apply(featureRowTestStream).apply(sink.writer()); - p.run(); - - assert jobService.getAllJobs().size() == 1; - Job load = Iterators.getLast(jobService.getAllJobs().iterator()); - JobConfigurationLoad loadConfiguration = load.getConfiguration().getLoad(); - - ArrayList expectedFields = - new ArrayList<>( - Arrays.asList( - new TableFieldSchema().setName("old_feature_1").setType("FLOAT"), - new TableFieldSchema().setName("old_feature_2").setType("INTEGER"), - new TableFieldSchema().setName("entity").setType("INTEGER"), - new TableFieldSchema().setName("feature").setType("STRING"))); - expectedFields.addAll(commonFields); - assertThat(loadConfiguration.getSchema().getFields(), is(expectedFields)); - } - - @Test - public void updateSpecInFlight() { - FeatureSetSpec spec_fs_2 = - FeatureSetSpec.newBuilder() - .setName("fs_2") - .setProject("myproject") - .addEntities( - EntitySpec.newBuilder() - .setName("entity") - .setValueType(ValueProto.ValueType.Enum.INT64) - .build()) - .addFeatures( - FeatureSpec.newBuilder() - .setName("feature") - .setValueType(ValueProto.ValueType.Enum.STRING) - .build()) - .addFeatures( - FeatureSpec.newBuilder() - .setName("new_feature") - .setValueType(ValueProto.ValueType.Enum.FLOAT) - .build()) - .build(); - - TestStream> specsStream = - TestStream.create( - KvCoder.of( - AvroCoder.of(FeatureSetReference.class), ProtoCoder.of(FeatureSetSpec.class))) - .advanceWatermarkTo(Instant.now()) - .addElements(KV.of(FeatureSetReference.of("myproject", "fs", 1), spec)) - .addElements(KV.of(FeatureSetReference.of("myproject", "fs", 1), spec_fs_2)) - .advanceWatermarkToInfinity(); - - FeatureSink sink = - makeSink( - ValueProvider.StaticValueProvider.of(bigQuery), - p.apply("SpecsInput", specsStream) - .apply( - Window.>into(new GlobalWindows()) - .triggering(Repeatedly.forever(AfterPane.elementCountAtLeast(1))) - .withAllowedLateness(Duration.millis(0)) - .accumulatingFiredPanes())); - - TestStream featureRowTestStream = - TestStream.create(ProtoCoder.of(FeatureRow.class)) - .advanceWatermarkTo(Instant.now().plus(Duration.standardSeconds(10))) - .advanceProcessingTime(Duration.standardSeconds(10)) - .addElements(generateRow("myproject/fs")) - .advanceWatermarkToInfinity(); - - p.apply("FeaturesInput", featureRowTestStream).apply(sink.writer()); - p.run(); - - assert jobService.getAllJobs().size() == 1; - Job load = Iterators.getLast(jobService.getAllJobs().iterator()); - JobConfigurationLoad loadConfiguration = load.getConfiguration().getLoad(); - - ArrayList expectedFields = - new ArrayList<>( - Arrays.asList( - new TableFieldSchema().setName("entity").setType("INTEGER"), - new TableFieldSchema().setName("feature").setType("STRING"), - new TableFieldSchema().setName("new_feature").setType("FLOAT"))); - expectedFields.addAll(commonFields); - - assertThat(loadConfiguration.getSchema().getFields(), is(expectedFields)); - } - - public static class ExtractKV extends DoFn> { - @ProcessElement - public void process(ProcessContext c) { - c.output(KV.of(c.element().getFeatureSet(), c.element())); - } - } - - public static class TableAnswer implements Answer, Serializable { - TableId tableId; - TableDefinition tableDefinition; - - public TableAnswer(TableId tableId, TableDefinition tableDefinition) { - this.tableId = tableId; - this.tableDefinition = tableDefinition; - } - - @Override - public Table answer(InvocationOnMock invocationOnMock) throws Throwable { - return FakeTable.create(mock(BigQuery.class), tableId, tableDefinition); - } - } -} diff --git a/storage/connectors/pom.xml b/storage/connectors/pom.xml index 219680b1f54..4969364b623 100644 --- a/storage/connectors/pom.xml +++ b/storage/connectors/pom.xml @@ -16,7 +16,6 @@ redis - bigquery diff --git a/storage/connectors/redis/pom.xml b/storage/connectors/redis/pom.xml index ca6e8d42ad6..bbda8dab27f 100644 --- a/storage/connectors/redis/pom.xml +++ b/storage/connectors/redis/pom.xml @@ -56,13 +56,6 @@ test - - org.apache.beam - beam-runners-direct-java - ${org.apache.beam.version} - test - - org.hamcrest hamcrest-core @@ -89,12 +82,6 @@ 4.12 test - - org.apache.beam - beam-sdks-java-extensions-protobuf - ${org.apache.beam.version} - test - org.slf4j slf4j-simple diff --git a/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/common/RedisHashDecoder.java b/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/common/RedisHashDecoder.java index 3a645389426..44f74d3f56b 100644 --- a/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/common/RedisHashDecoder.java +++ b/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/common/RedisHashDecoder.java @@ -36,13 +36,13 @@ public class RedisHashDecoder { * @return List of {@link Feature} * @throws InvalidProtocolBufferException */ - public static List> retrieveFeature( + public static List retrieveFeature( List> redisHashValues, Map byteToFeatureReferenceMap, String timestampPrefix) throws InvalidProtocolBufferException { - List> allFeatures = new ArrayList<>(); - Map> allFeaturesBuilderMap = + List allFeatures = new ArrayList<>(); + Map allFeaturesBuilderMap = new HashMap<>(); Map featureTableTimestampMap = new HashMap<>(); @@ -62,19 +62,19 @@ public static List> retrieveFeature( Feature.Builder featureBuilder = Feature.builder().setFeatureReference(featureReference).setFeatureValue(featureValue); - allFeaturesBuilderMap.put(featureReference, Optional.of(featureBuilder)); + allFeaturesBuilderMap.put(featureReference, featureBuilder); } } } // Add timestamp to features - for (Map.Entry> entry : + for (Map.Entry entry : allFeaturesBuilderMap.entrySet()) { String timestampRedisHashKeyStr = timestampPrefix + ":" + entry.getKey().getFeatureTable(); Timestamp curFeatureTimestamp = featureTableTimestampMap.get(timestampRedisHashKeyStr); - Feature curFeature = entry.getValue().get().setEventTimestamp(curFeatureTimestamp).build(); - allFeatures.add(Optional.of(curFeature)); + Feature curFeature = entry.getValue().setEventTimestamp(curFeatureTimestamp).build(); + allFeatures.add(curFeature); } return allFeatures; diff --git a/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/retriever/FeatureRowDecoder.java b/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/retriever/FeatureRowDecoder.java deleted file mode 100644 index d89e5373669..00000000000 --- a/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/retriever/FeatureRowDecoder.java +++ /dev/null @@ -1,142 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.storage.connectors.redis.retriever; - -import com.google.common.hash.Hashing; -import feast.proto.core.FeatureSetProto.FeatureSetSpec; -import feast.proto.core.FeatureSetProto.FeatureSpec; -import feast.proto.types.FeatureRowProto.FeatureRow; -import feast.proto.types.FieldProto.Field; -import feast.proto.types.ValueProto.Value; -import feast.storage.connectors.redis.writer.RedisCustomIO; -import java.nio.charset.StandardCharsets; -import java.util.Comparator; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; -import java.util.stream.IntStream; - -public class FeatureRowDecoder { - - private final String featureSetRef; - private final FeatureSetSpec spec; - - public FeatureRowDecoder(String featureSetRef, FeatureSetSpec spec) { - this.featureSetRef = featureSetRef; - this.spec = spec; - } - - /** - * Check if encoded feature row can be decoded by v1 Decoder. The v1 Decoder requires that the - * Feature Row to have both it's feature set reference and fields names are not set. The no. of - * fields in the feature row should also match up with the number of fields in the Feature Set - * spec. NOTE: This method is deprecated and will be removed in Feast v0.7. - * - * @param featureRow Feature row - * @return boolean - */ - @Deprecated - private boolean isEncodedV1(FeatureRow featureRow) { - return featureRow.getFeatureSet().isEmpty() - && featureRow.getFieldsList().stream().allMatch(field -> field.getName().isEmpty()) - && featureRow.getFieldsList().size() == spec.getFeaturesList().size(); - } - - /** - * Check if encoded feature row can be decoded by Decoder. The v2 Decoder requires that a Feature - * Row to have both it feature set reference and fields names are set. - * - * @param featureRow Feature row - * @return boolean - */ - private boolean isEncodedV2(FeatureRow featureRow) { - return !featureRow.getFieldsList().stream().anyMatch(field -> field.getName().isEmpty()); - } - - /** - * Decode feature row encoded by {@link RedisCustomIO}. NOTE: The v1 Decoder will be removed in - * Feast 0.7 - * - * @throws IllegalArgumentException if unable to the decode the given feature row - * @param encodedFeatureRow Feature row - * @return boolean - */ - public FeatureRow decode(FeatureRow encodedFeatureRow) { - if (isEncodedV1(encodedFeatureRow)) { - // TODO: remove v1 feature row decoder in Feast 0.7 - // Decode Feature Rows using the v1 Decoder. - final List fieldsWithoutName = encodedFeatureRow.getFieldsList(); - List featureNames = - spec.getFeaturesList().stream() - .sorted(Comparator.comparing(FeatureSpec::getName)) - .map(FeatureSpec::getName) - .collect(Collectors.toList()); - - List fields = - IntStream.range(0, featureNames.size()) - .mapToObj( - featureNameIndex -> { - String featureName = featureNames.get(featureNameIndex); - return fieldsWithoutName - .get(featureNameIndex) - .toBuilder() - .setName(featureName) - .build(); - }) - .collect(Collectors.toList()); - - return encodedFeatureRow - .toBuilder() - .clearFields() - .setFeatureSet(featureSetRef) - .addAllFields(fields) - .build(); - } - if (isEncodedV2(encodedFeatureRow)) { - // Decode Feature Rows using the v2 Decoder. - // v2 Decoder input Feature Rows should use a hashed name as the field name and - // should not have feature set reference set. - // Decoding reverts the field name to a unhashed string and set feature set reference. - Map nameHashValueMap = - encodedFeatureRow.getFieldsList().stream() - .collect(Collectors.toMap(field -> field.getName(), field -> field.getValue())); - - List featureNames = - spec.getFeaturesList().stream().map(FeatureSpec::getName).collect(Collectors.toList()); - - List fields = - featureNames.stream() - .map( - name -> { - String nameHash = - Hashing.murmur3_32().hashString(name, StandardCharsets.UTF_8).toString(); - Value value = - nameHashValueMap.getOrDefault(nameHash, Value.newBuilder().build()); - return Field.newBuilder().setName(name).setValue(value).build(); - }) - .collect(Collectors.toList()); - - return encodedFeatureRow - .toBuilder() - .clearFields() - .setFeatureSet(featureSetRef) - .addAllFields(fields) - .build(); - } - throw new IllegalArgumentException("Failed to decode FeatureRow row: Possible data corruption"); - } -} diff --git a/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/retriever/OnlineRetriever.java b/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/retriever/OnlineRetriever.java index d58fabb9b91..79d00240a00 100644 --- a/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/retriever/OnlineRetriever.java +++ b/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/retriever/OnlineRetriever.java @@ -41,21 +41,21 @@ public OnlineRetriever(RedisClientAdapter redisClientAdapter) { } @Override - public List>> getOnlineFeatures( + public List> getOnlineFeatures( String project, List entityRows, List featureReferences) { List redisKeys = RedisKeyGenerator.buildRedisKeys(project, entityRows); - List>> features = getFeaturesFromRedis(redisKeys, featureReferences); + List> features = getFeaturesFromRedis(redisKeys, featureReferences); return features; } - private List>> getFeaturesFromRedis( + private List> getFeaturesFromRedis( List redisKeys, List featureReferences) { - List>> features = new ArrayList<>(); + List> features = new ArrayList<>(); // To decode bytes back to Feature Reference Map byteToFeatureReferenceMap = new HashMap<>(); @@ -96,7 +96,7 @@ private List>> getFeaturesFromRedis( future -> { try { List> redisValuesList = future.get(); - List> curRedisKeyFeatures = + List curRedisKeyFeatures = RedisHashDecoder.retrieveFeature( redisValuesList, byteToFeatureReferenceMap, timestampPrefix); features.add(curRedisKeyFeatures); diff --git a/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/retriever/RedisClient.java b/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/retriever/RedisClient.java index 15de0e23278..5a7f4b78736 100644 --- a/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/retriever/RedisClient.java +++ b/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/retriever/RedisClient.java @@ -16,6 +16,7 @@ */ package feast.storage.connectors.redis.retriever; +import feast.proto.core.StoreProto; import io.lettuce.core.KeyValue; import io.lettuce.core.RedisFuture; import io.lettuce.core.RedisURI; @@ -23,7 +24,6 @@ import io.lettuce.core.api.async.RedisAsyncCommands; import io.lettuce.core.codec.ByteArrayCodec; import java.util.List; -import java.util.Map; public class RedisClient implements RedisClientAdapter { @@ -46,11 +46,11 @@ private RedisClient(StatefulRedisConnection connection) { this.asyncCommands.setAutoFlushCommands(false); } - public static RedisClientAdapter create(Map config) { + public static RedisClientAdapter create(StoreProto.Store.RedisConfig config) { - RedisURI uri = RedisURI.create(config.get("host"), Integer.parseInt(config.get("port"))); + RedisURI uri = RedisURI.create(config.getHost(), config.getPort()); - if (Boolean.parseBoolean(config.get("ssl"))) { + if (config.getSsl()) { uri.setSsl(true); } StatefulRedisConnection connection = diff --git a/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/retriever/RedisClusterClient.java b/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/retriever/RedisClusterClient.java index ec6a2cff648..aeb8220b0cb 100644 --- a/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/retriever/RedisClusterClient.java +++ b/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/retriever/RedisClusterClient.java @@ -16,9 +16,13 @@ */ package feast.storage.connectors.redis.retriever; +import com.google.common.collect.ImmutableMap; +import feast.proto.core.StoreProto; +import feast.proto.core.StoreProto.Store.RedisClusterConfig; import feast.storage.connectors.redis.serializer.RedisKeyPrefixSerializerV2; import feast.storage.connectors.redis.serializer.RedisKeySerializerV2; import io.lettuce.core.KeyValue; +import io.lettuce.core.ReadFrom; import io.lettuce.core.RedisFuture; import io.lettuce.core.RedisURI; import io.lettuce.core.cluster.api.StatefulRedisClusterConnection; @@ -36,6 +40,13 @@ public class RedisClusterClient implements RedisClientAdapter { private final RedisKeySerializerV2 serializer; @Nullable private final RedisKeySerializerV2 fallbackSerializer; + private static final Map PROTO_TO_LETTUCE_TYPES = + ImmutableMap.of( + RedisClusterConfig.ReadFrom.MASTER, ReadFrom.MASTER, + RedisClusterConfig.ReadFrom.MASTER_PREFERRED, ReadFrom.MASTER_PREFERRED, + RedisClusterConfig.ReadFrom.REPLICA, ReadFrom.REPLICA, + RedisClusterConfig.ReadFrom.REPLICA_PREFERRED, ReadFrom.REPLICA_PREFERRED); + @Override public RedisFuture>> hmget(byte[] key, byte[]... fields) { return asyncCommands.hmget(key, fields); @@ -73,13 +84,16 @@ private RedisClusterClient(Builder builder) { this.serializer = builder.serializer; this.fallbackSerializer = builder.fallbackSerializer; + // allows reading from replicas + this.asyncCommands.readOnly(); + // Disable auto-flushing this.asyncCommands.setAutoFlushCommands(false); } - public static RedisClientAdapter create(Map config) { + public static RedisClientAdapter create(StoreProto.Store.RedisClusterConfig config) { List redisURIList = - Arrays.stream(config.get("connection_string").split(",")) + Arrays.stream(config.getConnectionString().split(",")) .map( hostPort -> { String[] hostPortSplit = hostPort.trim().split(":"); @@ -90,14 +104,15 @@ public static RedisClientAdapter create(Map config) { io.lettuce.core.cluster.RedisClusterClient.create(redisURIList) .connect(new ByteArrayCodec()); - RedisKeySerializerV2 serializer = - new RedisKeyPrefixSerializerV2(config.getOrDefault("key_prefix", "")); + connection.setReadFrom(PROTO_TO_LETTUCE_TYPES.get(config.getReadFrom())); + + RedisKeySerializerV2 serializer = new RedisKeyPrefixSerializerV2(config.getKeyPrefix()); Builder builder = new Builder(connection, serializer); - if (Boolean.parseBoolean(config.getOrDefault("enable_fallback", "false"))) { + if (config.getEnableFallback()) { RedisKeySerializerV2 fallbackSerializer = - new RedisKeyPrefixSerializerV2(config.getOrDefault("fallback_prefix", "")); + new RedisKeyPrefixSerializerV2(config.getKeyPrefix()); builder = builder.withFallbackSerializer(fallbackSerializer); } diff --git a/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/retriever/RedisClusterOnlineRetriever.java b/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/retriever/RedisClusterOnlineRetriever.java deleted file mode 100644 index c49745bbbd1..00000000000 --- a/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/retriever/RedisClusterOnlineRetriever.java +++ /dev/null @@ -1,279 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.storage.connectors.redis.retriever; - -import com.google.protobuf.InvalidProtocolBufferException; -import feast.proto.core.FeatureSetProto.EntitySpec; -import feast.proto.core.FeatureSetProto.FeatureSetSpec; -import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesRequest.EntityRow; -import feast.proto.storage.RedisProto.RedisKey; -import feast.proto.types.FeatureRowProto.FeatureRow; -import feast.proto.types.FieldProto.Field; -import feast.proto.types.ValueProto.Value; -import feast.storage.api.retriever.FeatureSetRequest; -import feast.storage.api.retriever.OnlineRetriever; -import feast.storage.connectors.redis.serializer.RedisKeyPrefixSerializer; -import feast.storage.connectors.redis.serializer.RedisKeySerializer; -import io.grpc.Status; -import io.lettuce.core.RedisURI; -import io.lettuce.core.cluster.RedisClusterClient; -import io.lettuce.core.cluster.api.StatefulRedisClusterConnection; -import io.lettuce.core.cluster.api.sync.RedisAdvancedClusterCommands; -import io.lettuce.core.codec.ByteArrayCodec; -import java.util.*; -import java.util.concurrent.ExecutionException; -import java.util.stream.Collectors; -import java.util.stream.IntStream; -import javax.annotation.Nullable; - -/** Defines a storage retriever */ -public class RedisClusterOnlineRetriever implements OnlineRetriever { - - private final RedisAdvancedClusterCommands syncCommands; - private final RedisKeySerializer serializer; - @Nullable private final RedisKeySerializer fallbackSerializer; - - static class Builder { - private final StatefulRedisClusterConnection connection; - private final RedisKeySerializer serializer; - @Nullable private RedisKeySerializer fallbackSerializer; - - Builder( - StatefulRedisClusterConnection connection, RedisKeySerializer serializer) { - this.connection = connection; - this.serializer = serializer; - } - - Builder withFallbackSerializer(RedisKeySerializer fallbackSerializer) { - this.fallbackSerializer = fallbackSerializer; - return this; - } - - RedisClusterOnlineRetriever build() { - return new RedisClusterOnlineRetriever(this); - } - } - - private RedisClusterOnlineRetriever(Builder builder) { - this.syncCommands = builder.connection.sync(); - this.serializer = builder.serializer; - this.fallbackSerializer = builder.fallbackSerializer; - } - - public static OnlineRetriever create(Map config) { - List redisURIList = - Arrays.stream(config.get("connection_string").split(",")) - .map( - hostPort -> { - String[] hostPortSplit = hostPort.trim().split(":"); - return RedisURI.create(hostPortSplit[0], Integer.parseInt(hostPortSplit[1])); - }) - .collect(Collectors.toList()); - StatefulRedisClusterConnection connection = - RedisClusterClient.create(redisURIList).connect(new ByteArrayCodec()); - - RedisKeySerializer serializer = - new RedisKeyPrefixSerializer(config.getOrDefault("key_prefix", "")); - - Builder builder = new Builder(connection, serializer); - - if (Boolean.parseBoolean(config.getOrDefault("enable_fallback", "false"))) { - RedisKeySerializer fallbackSerializer = - new RedisKeyPrefixSerializer(config.getOrDefault("fallback_prefix", "")); - builder = builder.withFallbackSerializer(fallbackSerializer); - } - - return builder.build(); - } - - /** {@inheritDoc} */ - @Override - public List> getOnlineFeatures( - List entityRows, FeatureSetRequest featureSetRequest) { - - // get features for this features/featureset in featureset request - FeatureSetSpec featureSetSpec = featureSetRequest.getSpec(); - List redisKeys = buildRedisKeys(entityRows, featureSetSpec); - FeatureRowDecoder decoder = - new FeatureRowDecoder(generateFeatureSetStringRef(featureSetSpec), featureSetSpec); - List> featureRows = new ArrayList<>(); - try { - featureRows = getFeaturesFromRedis(redisKeys, decoder); - } catch (InvalidProtocolBufferException | ExecutionException e) { - throw Status.INTERNAL - .withDescription("Unable to parse protobuf while retrieving feature") - .withCause(e) - .asRuntimeException(); - } - return featureRows; - } - - private List buildRedisKeys(List entityRows, FeatureSetSpec featureSetSpec) { - String featureSetRef = generateFeatureSetStringRef(featureSetSpec); - List featureSetEntityNames = - featureSetSpec.getEntitiesList().stream() - .map(EntitySpec::getName) - .collect(Collectors.toList()); - return entityRows.stream() - .map(row -> makeRedisKey(featureSetRef, featureSetEntityNames, row)) - .collect(Collectors.toList()); - } - - /** - * Create {@link RedisKey} - * - * @param featureSet featureSet reference of the feature. E.g. feature_set_1:1 - * @param featureSetEntityNames entity names that belong to the featureSet - * @param entityRow entityRow to build the key from - * @return {@link RedisKey} - */ - private RedisKey makeRedisKey( - String featureSet, List featureSetEntityNames, EntityRow entityRow) { - RedisKey.Builder builder = RedisKey.newBuilder().setFeatureSet(featureSet); - Map fieldsMap = entityRow.getFieldsMap(); - featureSetEntityNames.sort(String::compareTo); - for (String entityName : featureSetEntityNames) { - if (!fieldsMap.containsKey(entityName)) { - throw Status.INVALID_ARGUMENT - .withDescription( - String.format( - "Entity row fields \"%s\" does not contain required entity field \"%s\"", - fieldsMap.keySet().toString(), entityName)) - .asRuntimeException(); - } - - builder.addEntities( - Field.newBuilder().setName(entityName).setValue(fieldsMap.get(entityName))); - } - return builder.build(); - } - - /** - * Get features from data pulled from the Redis for a specific featureset. - * - * @param redisKeys keys used to retrieve data from Redis for a specific featureset. - * @param decoder used to decode the data retrieved from Redis for a specific featureset. - * @return List of {@link FeatureRow} optionals - */ - private List> getFeaturesFromRedis( - List redisKeys, FeatureRowDecoder decoder) - throws InvalidProtocolBufferException, ExecutionException { - // pull feature row data bytes from redis using given redis keys - List featureRowsBytes = sendMultiGet(redisKeys); - List> featureRows = new ArrayList<>(); - - for (byte[] featureRowBytes : featureRowsBytes) { - if (featureRowBytes == null) { - featureRows.add(Optional.empty()); - continue; - } - - // decode feature rows from data bytes using decoder. - FeatureRow featureRow = FeatureRow.parseFrom(featureRowBytes); - try { - featureRow = decoder.decode(featureRow); - } catch (IllegalArgumentException e) { - // decoding feature row failed: data corruption could have occurred - throw Status.DATA_LOSS.withCause(e).withDescription(e.getMessage()).asRuntimeException(); - } - featureRows.add(Optional.of(featureRow)); - } - return featureRows; - } - - /** - * Pull the data stored in Redis at the given keys as bytes using the mget command. If no data is - * stored at a given key in Redis, will subsitute the data with null. - * - * @param keys list of {@link RedisKey} to pull from redis. - * @return list of data bytes or null pulled from redis for each given key. - */ - private List sendMultiGet(List keys) { - try { - byte[][] binaryKeys = - keys.stream() - .map(serializer::serialize) - .collect(Collectors.toList()) - .toArray(new byte[0][0]); - List redisValues = - syncCommands.mget(binaryKeys).stream() - .map( - keyValue -> { - if (keyValue == null) { - return null; - } - return keyValue.getValueOrElse(null); - }) - .collect(Collectors.toList()); - - List redisValuesWithFallback = redisValues; - if (fallbackSerializer != null) { - List indexMissingValue = - IntStream.range(0, keys.size()) - .filter(i -> redisValues.get(i) == null) - .boxed() - .collect(Collectors.toList()); - - if (indexMissingValue.isEmpty()) { - return redisValues; - } - - byte[][] fallbackBinaryKeys = - indexMissingValue.stream() - .map(i -> fallbackSerializer.serialize(keys.get(i))) - .collect(Collectors.toList()) - .toArray(new byte[0][0]); - - List fallBackValues = - syncCommands.mget(fallbackBinaryKeys).stream() - .map( - keyValue -> { - if (keyValue == null) { - return null; - } - return keyValue.getValueOrElse(null); - }) - .collect(Collectors.toList()); - - redisValuesWithFallback = - IntStream.range(0, keys.size()) - .mapToObj( - i -> { - if (indexMissingValue.contains(i)) { - return fallBackValues.get(indexMissingValue.indexOf(i)); - } else { - return redisValues.get(i); - } - }) - .collect(Collectors.toList()); - } - - return redisValuesWithFallback; - } catch (Exception e) { - throw Status.NOT_FOUND - .withDescription("Unable to retrieve feature from Redis") - .withCause(e) - .asRuntimeException(); - } - } - - // TODO: Refactor this out to common package? - private static String generateFeatureSetStringRef(FeatureSetSpec featureSetSpec) { - String ref = String.format("%s/%s", featureSetSpec.getProject(), featureSetSpec.getName()); - return ref; - } -} diff --git a/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/retriever/RedisOnlineRetriever.java b/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/retriever/RedisOnlineRetriever.java deleted file mode 100644 index 049175879de..00000000000 --- a/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/retriever/RedisOnlineRetriever.java +++ /dev/null @@ -1,201 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.storage.connectors.redis.retriever; - -import com.google.protobuf.AbstractMessageLite; -import com.google.protobuf.InvalidProtocolBufferException; -import feast.proto.core.FeatureSetProto.EntitySpec; -import feast.proto.core.FeatureSetProto.FeatureSetSpec; -import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesRequest.EntityRow; -import feast.proto.storage.RedisProto.RedisKey; -import feast.proto.types.FeatureRowProto.FeatureRow; -import feast.proto.types.FieldProto.Field; -import feast.proto.types.ValueProto.Value; -import feast.storage.api.retriever.FeatureSetRequest; -import feast.storage.api.retriever.OnlineRetriever; -import io.grpc.Status; -import io.lettuce.core.RedisClient; -import io.lettuce.core.RedisURI; -import io.lettuce.core.api.StatefulRedisConnection; -import io.lettuce.core.api.sync.RedisCommands; -import io.lettuce.core.codec.ByteArrayCodec; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.concurrent.ExecutionException; -import java.util.stream.Collectors; - -public class RedisOnlineRetriever implements OnlineRetriever { - - private final RedisCommands syncCommands; - - private RedisOnlineRetriever(StatefulRedisConnection connection) { - this.syncCommands = connection.sync(); - } - - public static OnlineRetriever create(Map config) { - - StatefulRedisConnection connection = - RedisClient.create( - RedisURI.create(config.get("host"), Integer.parseInt(config.get("port")))) - .connect(new ByteArrayCodec()); - - return new RedisOnlineRetriever(connection); - } - - public static OnlineRetriever create(StatefulRedisConnection connection) { - return new RedisOnlineRetriever(connection); - } - - /** {@inheritDoc} */ - @Override - public List> getOnlineFeatures( - List entityRows, FeatureSetRequest featureSetRequest) { - - // get features for this features/featureset in featureset request - FeatureSetSpec featureSetSpec = featureSetRequest.getSpec(); - List redisKeys = buildRedisKeys(entityRows, featureSetSpec); - FeatureRowDecoder decoder = - new FeatureRowDecoder(generateFeatureSetStringRef(featureSetSpec), featureSetSpec); - List> featureRows = new ArrayList<>(); - try { - featureRows = getFeaturesFromRedis(redisKeys, decoder); - } catch (InvalidProtocolBufferException | ExecutionException e) { - throw Status.INTERNAL - .withDescription("Unable to parse protobuf while retrieving feature") - .withCause(e) - .asRuntimeException(); - } - - return featureRows; - } - - private List buildRedisKeys(List entityRows, FeatureSetSpec featureSetSpec) { - String featureSetRef = generateFeatureSetStringRef(featureSetSpec); - List featureSetEntityNames = - featureSetSpec.getEntitiesList().stream() - .map(EntitySpec::getName) - .collect(Collectors.toList()); - List redisKeys = - entityRows.stream() - .map(row -> makeRedisKey(featureSetRef, featureSetEntityNames, row)) - .collect(Collectors.toList()); - return redisKeys; - } - - /** - * Create {@link RedisKey} - * - * @param featureSet featureSet reference of the feature. E.g. feature_set_1 - * @param featureSetEntityNames entity names that belong to the featureSet - * @param entityRow entityRow to build the key from - * @return {@link RedisKey} - */ - private RedisKey makeRedisKey( - String featureSet, List featureSetEntityNames, EntityRow entityRow) { - RedisKey.Builder builder = RedisKey.newBuilder().setFeatureSet(featureSet); - Map fieldsMap = entityRow.getFieldsMap(); - featureSetEntityNames.sort(String::compareTo); - for (int i = 0; i < featureSetEntityNames.size(); i++) { - String entityName = featureSetEntityNames.get(i); - - if (!fieldsMap.containsKey(entityName)) { - throw Status.INVALID_ARGUMENT - .withDescription( - String.format( - "Entity row fields \"%s\" does not contain required entity field \"%s\"", - fieldsMap.keySet().toString(), entityName)) - .asRuntimeException(); - } - - builder.addEntities( - Field.newBuilder().setName(entityName).setValue(fieldsMap.get(entityName))); - } - return builder.build(); - } - - /** - * Get features from data pulled from the Redis for a specific featureset. - * - * @param redisKeys keys used to retrieve data from Redis for a specific featureset. - * @param decoder used to decode the data retrieved from Redis for a specific featureset. - * @return List of {@link FeatureRow} optionals - */ - private List> getFeaturesFromRedis( - List redisKeys, FeatureRowDecoder decoder) - throws InvalidProtocolBufferException, ExecutionException { - // pull feature row data bytes from redis using given redis keys - List featureRowsBytes = sendMultiGet(redisKeys); - List> featureRows = new ArrayList<>(); - - for (byte[] featureRowBytes : featureRowsBytes) { - if (featureRowBytes == null) { - featureRows.add(Optional.empty()); - continue; - } - - // decode feature rows from data bytes using decoder. - FeatureRow featureRow = FeatureRow.parseFrom(featureRowBytes); - try { - featureRow = decoder.decode(featureRow); - } catch (IllegalArgumentException e) { - // decoding feature row failed: data corruption could have occurred - throw Status.DATA_LOSS.withCause(e).withDescription(e.getMessage()).asRuntimeException(); - } - featureRows.add(Optional.of(featureRow)); - } - return featureRows; - } - - /** - * Pull the data stored in Redis at the given keys as bytes using the mget command. If no data is - * stored at a given key in Redis, will subsitute the data with null. - * - * @param keys list of {@link RedisKey} to pull from redis. - * @return list of data bytes or null pulled from redis for each given key. - */ - private List sendMultiGet(List keys) { - try { - byte[][] binaryKeys = - keys.stream() - .map(AbstractMessageLite::toByteArray) - .collect(Collectors.toList()) - .toArray(new byte[0][0]); - return syncCommands.mget(binaryKeys).stream() - .map( - keyValue -> { - if (keyValue == null) { - return null; - } - return keyValue.getValueOrElse(null); - }) - .collect(Collectors.toList()); - } catch (Exception e) { - throw Status.UNKNOWN - .withDescription("Unexpected error when pulling data from from Redis.") - .withCause(e) - .asRuntimeException(); - } - } - - // TODO: Refactor this out to common package - private static String generateFeatureSetStringRef(FeatureSetSpec featureSetSpec) { - String ref = String.format("%s/%s", featureSetSpec.getProject(), featureSetSpec.getName()); - return ref; - } -} diff --git a/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/serializer/RedisKeyPrefixSerializer.java b/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/serializer/RedisKeyPrefixSerializer.java deleted file mode 100644 index b6764796928..00000000000 --- a/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/serializer/RedisKeyPrefixSerializer.java +++ /dev/null @@ -1,41 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.storage.connectors.redis.serializer; - -import feast.proto.storage.RedisProto.RedisKey; - -public class RedisKeyPrefixSerializer implements RedisKeySerializer { - - private final byte[] prefixBytes; - - public RedisKeyPrefixSerializer(String prefix) { - this.prefixBytes = prefix.getBytes(); - } - - public byte[] serialize(RedisKey redisKey) { - byte[] key = redisKey.toByteArray(); - - if (prefixBytes.length == 0) { - return key; - } - - byte[] keyWithPrefix = new byte[prefixBytes.length + key.length]; - System.arraycopy(prefixBytes, 0, keyWithPrefix, 0, prefixBytes.length); - System.arraycopy(key, 0, keyWithPrefix, prefixBytes.length, key.length); - return keyWithPrefix; - } -} diff --git a/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/serializer/RedisKeyProtoSerializer.java b/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/serializer/RedisKeyProtoSerializer.java deleted file mode 100644 index 62c3cfd30b5..00000000000 --- a/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/serializer/RedisKeyProtoSerializer.java +++ /dev/null @@ -1,26 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.storage.connectors.redis.serializer; - -import feast.proto.storage.RedisProto.RedisKey; - -public class RedisKeyProtoSerializer implements RedisKeySerializer { - - public byte[] serialize(RedisKey redisKey) { - return redisKey.toByteArray(); - } -} diff --git a/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/serializer/RedisKeySerializer.java b/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/serializer/RedisKeySerializer.java deleted file mode 100644 index cac3d10f7d1..00000000000 --- a/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/serializer/RedisKeySerializer.java +++ /dev/null @@ -1,25 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.storage.connectors.redis.serializer; - -import feast.proto.storage.RedisProto.RedisKey; -import java.io.Serializable; - -public interface RedisKeySerializer extends Serializable { - - byte[] serialize(RedisKey key); -} diff --git a/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/writer/BatchDoFnWithRedis.java b/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/writer/BatchDoFnWithRedis.java deleted file mode 100644 index d6c83c3a540..00000000000 --- a/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/writer/BatchDoFnWithRedis.java +++ /dev/null @@ -1,88 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.storage.connectors.redis.writer; - -import feast.storage.common.retry.Retriable; -import io.lettuce.core.RedisException; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.Future; -import java.util.function.Function; -import org.apache.beam.sdk.transforms.DoFn; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * Base class for redis-related DoFns. Assumes that operations will be batched. Prepares redisClient - * on DoFn.Setup stage and close it on DoFn.Teardown stage. - * - * @param - * @param - */ -public class BatchDoFnWithRedis extends DoFn { - private static final Logger log = LoggerFactory.getLogger(BatchDoFnWithRedis.class); - - private final RedisIngestionClient redisIngestionClient; - - BatchDoFnWithRedis(RedisIngestionClient redisIngestionClient) { - this.redisIngestionClient = redisIngestionClient; - } - - @Setup - public void setup() { - this.redisIngestionClient.setup(); - } - - @StartBundle - public void startBundle() { - try { - redisIngestionClient.connect(); - } catch (RedisException e) { - log.error("Connection to redis cannot be established: %s", e); - } - } - - void executeBatch(Function>> executor) - throws Exception { - this.redisIngestionClient - .getBackOffExecutor() - .execute( - new Retriable() { - @Override - public void execute() throws ExecutionException, InterruptedException { - if (!redisIngestionClient.isConnected()) { - redisIngestionClient.connect(); - } - - Iterable> futures = executor.apply(redisIngestionClient); - redisIngestionClient.sync(futures); - } - - @Override - public Boolean isExceptionRetriable(Exception e) { - return e instanceof RedisException; - } - - @Override - public void cleanUpAfterFailure() {} - }); - } - - @Teardown - public void teardown() { - redisIngestionClient.shutdown(); - } -} diff --git a/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/writer/RedisClusterIngestionClient.java b/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/writer/RedisClusterIngestionClient.java deleted file mode 100644 index f36d70563e1..00000000000 --- a/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/writer/RedisClusterIngestionClient.java +++ /dev/null @@ -1,109 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.storage.connectors.redis.writer; - -import com.google.common.collect.Lists; -import feast.proto.core.StoreProto; -import feast.storage.common.retry.BackOffExecutor; -import io.lettuce.core.LettuceFutures; -import io.lettuce.core.RedisURI; -import io.lettuce.core.cluster.RedisClusterClient; -import io.lettuce.core.cluster.api.StatefulRedisClusterConnection; -import io.lettuce.core.cluster.api.async.RedisAdvancedClusterAsyncCommands; -import io.lettuce.core.codec.ByteArrayCodec; -import java.util.Arrays; -import java.util.List; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.Future; -import java.util.concurrent.TimeUnit; -import java.util.stream.Collectors; -import org.joda.time.Duration; - -public class RedisClusterIngestionClient implements RedisIngestionClient { - - private final BackOffExecutor backOffExecutor; - private final List uriList; - private transient RedisClusterClient clusterClient; - private StatefulRedisClusterConnection connection; - private RedisAdvancedClusterAsyncCommands commands; - - public RedisClusterIngestionClient(StoreProto.Store.RedisClusterConfig redisClusterConfig) { - this.uriList = - Arrays.stream(redisClusterConfig.getConnectionString().split(",")) - .map( - hostPort -> { - String[] hostPortSplit = hostPort.trim().split(":"); - return RedisURI.create(hostPortSplit[0], Integer.parseInt(hostPortSplit[1])); - }) - .collect(Collectors.toList()); - - long backoffMs = - redisClusterConfig.getInitialBackoffMs() > 0 ? redisClusterConfig.getInitialBackoffMs() : 1; - this.backOffExecutor = - new BackOffExecutor(redisClusterConfig.getMaxRetries(), Duration.millis(backoffMs)); - } - - @Override - public void setup() { - this.clusterClient = RedisClusterClient.create(this.uriList); - } - - @Override - public BackOffExecutor getBackOffExecutor() { - return this.backOffExecutor; - } - - @Override - public void shutdown() { - this.clusterClient.shutdown(); - } - - @Override - public void connect() { - if (!isConnected()) { - this.connection = clusterClient.connect(new ByteArrayCodec()); - this.commands = connection.async(); - - // despite we're using async API client still flushes after each command by default - // which we don't want since we produce all commands in batches - this.commands.setAutoFlushCommands(false); - } - } - - @Override - public boolean isConnected() { - return this.connection != null; - } - - @Override - public void sync(Iterable> futures) { - this.connection.flushCommands(); - - LettuceFutures.awaitAll( - 60, TimeUnit.SECONDS, Lists.newArrayList(futures).toArray(new Future[0])); - } - - @Override - public CompletableFuture set(byte[] key, byte[] value) { - return commands.set(key, value).toCompletableFuture(); - } - - @Override - public CompletableFuture get(byte[] key) { - return commands.get(key).toCompletableFuture(); - } -} diff --git a/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/writer/RedisCustomIO.java b/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/writer/RedisCustomIO.java deleted file mode 100644 index 9952adc8855..00000000000 --- a/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/writer/RedisCustomIO.java +++ /dev/null @@ -1,321 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2019 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.storage.connectors.redis.writer; - -import com.google.common.collect.Iterators; -import com.google.common.collect.Streams; -import com.google.common.hash.Hashing; -import com.google.protobuf.InvalidProtocolBufferException; -import feast.proto.core.FeatureSetProto.EntitySpec; -import feast.proto.core.FeatureSetProto.FeatureSetSpec; -import feast.proto.core.FeatureSetProto.FeatureSpec; -import feast.proto.storage.RedisProto.RedisKey; -import feast.proto.storage.RedisProto.RedisKey.Builder; -import feast.proto.types.FeatureRowProto.FeatureRow; -import feast.proto.types.FieldProto.Field; -import feast.proto.types.ValueProto; -import feast.storage.api.writer.FailedElement; -import feast.storage.api.writer.WriteResult; -import feast.storage.connectors.redis.retriever.FeatureRowDecoder; -import feast.storage.connectors.redis.serializer.RedisKeySerializer; -import java.nio.charset.StandardCharsets; -import java.util.*; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.function.BinaryOperator; -import java.util.stream.Collectors; -import org.apache.beam.sdk.transforms.*; -import org.apache.beam.sdk.transforms.windowing.*; -import org.apache.beam.sdk.values.*; -import org.apache.commons.lang3.exception.ExceptionUtils; -import org.apache.commons.lang3.tuple.ImmutablePair; -import org.joda.time.DateTime; -import org.joda.time.Duration; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class RedisCustomIO { - - private static TupleTag successfulInsertsTag = - new TupleTag("successfulInserts") {}; - private static TupleTag failedInsertsTupleTag = - new TupleTag("failedInserts") {}; - - private static final Logger log = LoggerFactory.getLogger(RedisCustomIO.class); - - private RedisCustomIO() {} - - public static Write write( - RedisIngestionClient redisIngestionClient, - PCollectionView>> featureSetSpecs, - RedisKeySerializer serializer) { - return new Write(redisIngestionClient, featureSetSpecs, serializer); - } - - /** ServingStoreWrite data to a Redis server. */ - public static class Write extends PTransform, WriteResult> { - - private PCollectionView>> featureSetSpecs; - private RedisIngestionClient redisIngestionClient; - private RedisKeySerializer serializer; - private int batchSize; - private Duration flushFrequency; - - public Write( - RedisIngestionClient redisIngestionClient, - PCollectionView>> featureSetSpecs, - RedisKeySerializer serializer) { - this.redisIngestionClient = redisIngestionClient; - this.featureSetSpecs = featureSetSpecs; - this.serializer = serializer; - } - - public Write withBatchSize(int batchSize) { - this.batchSize = batchSize; - return this; - } - - public Write withFlushFrequency(Duration frequency) { - this.flushFrequency = frequency; - return this; - } - - @Override - public WriteResult expand(PCollection input) { - PCollectionTuple redisWrite = - input - .apply("FixedFlushWindow", Window.into(FixedWindows.of(flushFrequency))) - .apply( - "AttachFeatureReferenceKey", - ParDo.of( - new DoFn>() { - @ProcessElement - public void process(ProcessContext c) { - c.output(KV.of(c.element().getFeatureSet(), c.element())); - } - })) - .apply("IntoBatches", GroupIntoBatches.ofSize(batchSize)) - .apply("ExtractResultValues", Values.create()) - .apply("GlobalWindow", Window.>into(new GlobalWindows())) - .apply( - ParDo.of(new WriteDoFn(redisIngestionClient, featureSetSpecs, serializer)) - .withOutputTags(successfulInsertsTag, TupleTagList.of(failedInsertsTupleTag)) - .withSideInputs(featureSetSpecs)); - return WriteResult.in( - input.getPipeline(), - redisWrite.get(successfulInsertsTag), - redisWrite.get(failedInsertsTupleTag)); - } - - /** - * Writes batch of {@link FeatureRow} to Redis. Only latest values should be written. In order - * to guarantee that we first fetch all existing values (first batch operation), compare with - * current batch by eventTimestamp, and send to redis values (second batch operation) that were - * confirmed to be most recent. - */ - public static class WriteDoFn extends BatchDoFnWithRedis, FeatureRow> { - private final PCollectionView>> featureSetSpecsView; - private final RedisKeySerializer serializer; - - WriteDoFn( - RedisIngestionClient redisIngestionClient, - PCollectionView>> featureSetSpecsView, - RedisKeySerializer serializer) { - - super(redisIngestionClient); - this.featureSetSpecsView = featureSetSpecsView; - this.serializer = serializer; - } - - private FailedElement toFailedElement( - FeatureRow featureRow, Exception exception, String jobName) { - return FailedElement.newBuilder() - .setJobName(jobName) - .setTransformName("RedisCustomIO") - .setPayload(featureRow.toString()) - .setErrorMessage(exception.getMessage()) - .setStackTrace(ExceptionUtils.getStackTrace(exception)) - .build(); - } - - private RedisKey getKey(FeatureRow featureRow, FeatureSetSpec spec) { - List entityNames = - spec.getEntitiesList().stream() - .map(EntitySpec::getName) - .sorted() - .collect(Collectors.toList()); - - Map entityFields = new HashMap<>(); - Builder redisKeyBuilder = RedisKey.newBuilder().setFeatureSet(featureRow.getFeatureSet()); - for (Field field : featureRow.getFieldsList()) { - if (entityNames.contains(field.getName())) { - entityFields.putIfAbsent( - field.getName(), - Field.newBuilder().setName(field.getName()).setValue(field.getValue()).build()); - } - } - for (String entityName : entityNames) { - redisKeyBuilder.addEntities(entityFields.get(entityName)); - } - return redisKeyBuilder.build(); - } - - /** - * Encode the Feature Row as bytes to store in Redis in encoded Feature Row encoding. To - * reduce storage space consumption in redis, feature rows are "encoded" by hashing the fields - * names and not unsetting the feature set reference. {@link FeatureRowDecoder} is - * rensponsible for reversing this "encoding" step. - */ - private FeatureRow getValue(FeatureRow featureRow, FeatureSetSpec spec) { - List featureNames = - spec.getFeaturesList().stream().map(FeatureSpec::getName).collect(Collectors.toList()); - - Map fieldValueOnlyMap = - featureRow.getFieldsList().stream() - .filter(field -> featureNames.contains(field.getName())) - .distinct() - .collect( - Collectors.toMap( - Field::getName, field -> Field.newBuilder().setValue(field.getValue()))); - - List values = - featureNames.stream() - .sorted() - .map( - featureName -> { - Field.Builder field = - fieldValueOnlyMap.getOrDefault( - featureName, - Field.newBuilder().setValue(ValueProto.Value.getDefaultInstance())); - - // Encode the name of the as the hash of the field name. - // Use hash of name instead of the name of to reduce redis storage consumption - // per feature row stored. - String nameHash = - Hashing.murmur3_32() - .hashString(featureName, StandardCharsets.UTF_8) - .toString(); - field.setName(nameHash); - - return field.build(); - }) - .collect(Collectors.toList()); - - return FeatureRow.newBuilder() - .setEventTimestamp(featureRow.getEventTimestamp()) - .addAllFields(values) - .build(); - } - - @ProcessElement - public void processElement(ProcessContext context) { - List filteredFeatureRows = Collections.synchronizedList(new ArrayList<>()); - Map latestSpecs = - getLatestSpecs(context.sideInput(featureSetSpecsView)); - - Map deduplicatedRows = - deduplicateRows(context.element(), latestSpecs); - - try { - executeBatch( - (redisIngestionClient) -> - deduplicatedRows.entrySet().stream() - .map( - entry -> - redisIngestionClient - .get(serializer.serialize(entry.getKey())) - .thenAccept( - currentValue -> { - FeatureRow newRow = entry.getValue(); - if (rowShouldBeWritten(newRow, currentValue)) { - filteredFeatureRows.add(newRow); - } - })) - .collect(Collectors.toList())); - - executeBatch( - redisIngestionClient -> - filteredFeatureRows.stream() - .map( - row -> - redisIngestionClient.set( - serializer.serialize( - getKey(row, latestSpecs.get(row.getFeatureSet()))), - getValue(row, latestSpecs.get(row.getFeatureSet())) - .toByteArray())) - .collect(Collectors.toList())); - - filteredFeatureRows.forEach(row -> context.output(successfulInsertsTag, row)); - } catch (Exception e) { - deduplicatedRows - .values() - .forEach( - failedMutation -> { - FailedElement failedElement = - toFailedElement( - failedMutation, e, context.getPipelineOptions().getJobName()); - context.output(failedInsertsTupleTag, failedElement); - }); - } - } - - boolean rowShouldBeWritten(FeatureRow newRow, byte[] currentValue) { - if (currentValue == null) { - // nothing to compare with - return true; - } - FeatureRow currentRow; - try { - currentRow = FeatureRow.parseFrom(currentValue); - } catch (InvalidProtocolBufferException e) { - // definitely need to replace current value - return true; - } - - // check whether new row has later eventTimestamp - return new DateTime(currentRow.getEventTimestamp().getSeconds() * 1000L) - .isBefore(new DateTime(newRow.getEventTimestamp().getSeconds() * 1000L)); - } - - /** Deduplicate rows by key within batch. Keep only latest eventTimestamp */ - Map deduplicateRows( - Iterable rows, Map latestSpecs) { - Comparator byEventTimestamp = - Comparator.comparing(r -> r.getEventTimestamp().getSeconds()); - - FeatureRow identity = - FeatureRow.newBuilder() - .setEventTimestamp( - com.google.protobuf.Timestamp.newBuilder().setSeconds(-1).build()) - .build(); - - return Streams.stream(rows) - .collect( - Collectors.groupingBy( - row -> getKey(row, latestSpecs.get(row.getFeatureSet())), - Collectors.reducing(identity, BinaryOperator.maxBy(byEventTimestamp)))); - } - - Map getLatestSpecs(Map> specs) { - return specs.entrySet().stream() - .map(e -> ImmutablePair.of(e.getKey(), Iterators.getLast(e.getValue().iterator()))) - .collect(Collectors.toMap(ImmutablePair::getLeft, ImmutablePair::getRight)); - } - } - } -} diff --git a/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/writer/RedisFeatureSink.java b/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/writer/RedisFeatureSink.java deleted file mode 100644 index fed7f678d78..00000000000 --- a/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/writer/RedisFeatureSink.java +++ /dev/null @@ -1,172 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.storage.connectors.redis.writer; - -import com.google.auto.value.AutoValue; -import feast.common.models.FeatureSetReference; -import feast.proto.core.FeatureSetProto; -import feast.proto.core.FeatureSetProto.FeatureSetSpec; -import feast.proto.core.StoreProto; -import feast.proto.core.StoreProto.Store.RedisClusterConfig; -import feast.proto.core.StoreProto.Store.RedisConfig; -import feast.proto.types.FeatureRowProto.FeatureRow; -import feast.storage.api.writer.FeatureSink; -import feast.storage.api.writer.WriteResult; -import feast.storage.connectors.redis.serializer.RedisKeyPrefixSerializer; -import feast.storage.connectors.redis.serializer.RedisKeyProtoSerializer; -import feast.storage.connectors.redis.serializer.RedisKeySerializer; -import io.lettuce.core.RedisClient; -import io.lettuce.core.RedisConnectionException; -import io.lettuce.core.RedisURI; -import java.util.Map; -import javax.annotation.Nullable; -import org.apache.beam.sdk.transforms.*; -import org.apache.beam.sdk.values.KV; -import org.apache.beam.sdk.values.PCollection; -import org.apache.beam.sdk.values.PCollectionView; -import org.joda.time.Duration; - -@AutoValue -public abstract class RedisFeatureSink implements FeatureSink { - private static final int DEFAULT_BATCH_SIZE = 10000; - private static final int DEFAULT_FREQUENCY_SECONDS = 30; - - /** - * Initialize a {@link RedisFeatureSink.Builder} from a {@link StoreProto.Store.RedisConfig}. - * - * @param redisConfig {@link RedisConfig} - * @return {@link RedisFeatureSink.Builder} - */ - public static FeatureSink fromConfig(RedisConfig redisConfig) { - return builder().setRedisConfig(redisConfig).build(); - } - - public static FeatureSink fromConfig(RedisClusterConfig redisConfig) { - return builder().setRedisClusterConfig(redisConfig).build(); - } - - @Nullable - public abstract RedisConfig getRedisConfig(); - - @Nullable - public abstract RedisClusterConfig getRedisClusterConfig(); - - public abstract Builder toBuilder(); - - public static Builder builder() { - return new AutoValue_RedisFeatureSink.Builder(); - } - - @AutoValue.Builder - public abstract static class Builder { - public abstract Builder setRedisConfig(RedisConfig redisConfig); - - public abstract Builder setRedisClusterConfig(RedisClusterConfig redisConfig); - - public abstract RedisFeatureSink build(); - } - - PCollectionView>> specsView; - - public RedisFeatureSink withSpecsView( - PCollectionView>> specsView) { - this.specsView = specsView; - return this; - } - - PCollectionView>> getSpecsView() { - return specsView; - } - - @Override - public PCollection prepareWrite( - PCollection> featureSetSpecs) { - if (getRedisConfig() != null) { - RedisClient redisClient = - RedisClient.create( - RedisURI.create(getRedisConfig().getHost(), getRedisConfig().getPort())); - try { - redisClient.connect(); - } catch (RedisConnectionException e) { - throw new RuntimeException( - String.format( - "Failed to connect to Redis at host: '%s' port: '%d'. Please check that your Redis is running and accessible from Feast.", - getRedisConfig().getHost(), getRedisConfig().getPort())); - } - redisClient.shutdown(); - } else if (getRedisClusterConfig() == null) { - throw new RuntimeException( - "At least one RedisConfig or RedisClusterConfig must be provided to Redis Sink"); - } - specsView = featureSetSpecs.apply(ParDo.of(new ReferenceToString())).apply(View.asMultimap()); - return featureSetSpecs - .apply( - "DummyDelay", - ParDo.of( - new DoFn< - KV, - KV>() { - @ProcessElement - public void process(ProcessContext c) throws InterruptedException { - Thread.sleep(1000); - c.output(c.element()); - } - })) - .apply(Keys.create()); - } - - @Override - public PTransform, WriteResult> writer() { - int flushFrequencySeconds = DEFAULT_FREQUENCY_SECONDS; - - if (getRedisClusterConfig() != null) { - - if (getRedisClusterConfig().getFlushFrequencySeconds() > 0) { - flushFrequencySeconds = getRedisClusterConfig().getFlushFrequencySeconds(); - } - - RedisKeySerializer serializer = - new RedisKeyPrefixSerializer(getRedisClusterConfig().getKeyPrefix()); - return new RedisCustomIO.Write( - new RedisClusterIngestionClient(getRedisClusterConfig()), getSpecsView(), serializer) - .withFlushFrequency(Duration.standardSeconds(flushFrequencySeconds)) - .withBatchSize(DEFAULT_BATCH_SIZE); - - } else if (getRedisConfig() != null) { - if (getRedisConfig().getFlushFrequencySeconds() > 0) { - flushFrequencySeconds = getRedisConfig().getFlushFrequencySeconds(); - } - - RedisKeySerializer serializer = new RedisKeyProtoSerializer(); - return new RedisCustomIO.Write( - new RedisStandaloneIngestionClient(getRedisConfig()), getSpecsView(), serializer) - .withFlushFrequency(Duration.standardSeconds(flushFrequencySeconds)) - .withBatchSize(DEFAULT_BATCH_SIZE); - } else { - throw new RuntimeException( - "At least one RedisConfig or RedisClusterConfig must be provided to Redis Sink"); - } - } - - private static class ReferenceToString - extends DoFn, KV> { - @ProcessElement - public void process(ProcessContext c) { - c.output(KV.of(c.element().getKey().getReference(), c.element().getValue())); - } - } -} diff --git a/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/writer/RedisIngestionClient.java b/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/writer/RedisIngestionClient.java deleted file mode 100644 index e9b1a5dc445..00000000000 --- a/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/writer/RedisIngestionClient.java +++ /dev/null @@ -1,41 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.storage.connectors.redis.writer; - -import feast.storage.common.retry.BackOffExecutor; -import java.io.Serializable; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.Future; - -public interface RedisIngestionClient extends Serializable { - - void setup(); - - BackOffExecutor getBackOffExecutor(); - - void shutdown(); - - void connect(); - - boolean isConnected(); - - void sync(Iterable> futures); - - CompletableFuture set(byte[] key, byte[] value); - - CompletableFuture get(byte[] key); -} diff --git a/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/writer/RedisStandaloneIngestionClient.java b/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/writer/RedisStandaloneIngestionClient.java deleted file mode 100644 index f0a2054b9bd..00000000000 --- a/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/writer/RedisStandaloneIngestionClient.java +++ /dev/null @@ -1,99 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.storage.connectors.redis.writer; - -import com.google.common.collect.Lists; -import feast.proto.core.StoreProto; -import feast.storage.common.retry.BackOffExecutor; -import io.lettuce.core.LettuceFutures; -import io.lettuce.core.RedisClient; -import io.lettuce.core.RedisURI; -import io.lettuce.core.api.StatefulRedisConnection; -import io.lettuce.core.api.async.RedisAsyncCommands; -import io.lettuce.core.codec.ByteArrayCodec; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.Future; -import java.util.concurrent.TimeUnit; -import org.joda.time.Duration; - -public class RedisStandaloneIngestionClient implements RedisIngestionClient { - private final String host; - private final int port; - private final BackOffExecutor backOffExecutor; - private RedisClient redisclient; - private static final int DEFAULT_TIMEOUT = 2000; - private StatefulRedisConnection connection; - private RedisAsyncCommands commands; - - public RedisStandaloneIngestionClient(StoreProto.Store.RedisConfig redisConfig) { - this.host = redisConfig.getHost(); - this.port = redisConfig.getPort(); - long backoffMs = redisConfig.getInitialBackoffMs() > 0 ? redisConfig.getInitialBackoffMs() : 1; - this.backOffExecutor = - new BackOffExecutor(redisConfig.getMaxRetries(), Duration.millis(backoffMs)); - } - - @Override - public void setup() { - this.redisclient = - RedisClient.create(new RedisURI(host, port, java.time.Duration.ofMillis(DEFAULT_TIMEOUT))); - } - - @Override - public BackOffExecutor getBackOffExecutor() { - return this.backOffExecutor; - } - - @Override - public void shutdown() { - this.redisclient.shutdown(); - } - - @Override - public void connect() { - if (!isConnected()) { - this.connection = this.redisclient.connect(new ByteArrayCodec()); - this.commands = connection.async(); - - // enable pipelining of commands - this.commands.setAutoFlushCommands(false); - } - } - - @Override - public boolean isConnected() { - return connection != null; - } - - @Override - public void sync(Iterable> futures) { - this.connection.flushCommands(); - - LettuceFutures.awaitAll( - 60, TimeUnit.SECONDS, Lists.newArrayList(futures).toArray(new Future[0])); - } - - @Override - public CompletableFuture set(byte[] key, byte[] value) { - return commands.set(key, value).toCompletableFuture(); - } - - @Override - public CompletableFuture get(byte[] key) { - return commands.get(key).toCompletableFuture(); - } -} diff --git a/storage/connectors/redis/src/test/java/feast/storage/connectors/redis/retriever/FeatureRowDecoderTest.java b/storage/connectors/redis/src/test/java/feast/storage/connectors/redis/retriever/FeatureRowDecoderTest.java deleted file mode 100644 index c843d311274..00000000000 --- a/storage/connectors/redis/src/test/java/feast/storage/connectors/redis/retriever/FeatureRowDecoderTest.java +++ /dev/null @@ -1,191 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.storage.connectors.redis.retriever; - -import static org.junit.Assert.*; - -import com.google.common.hash.Hashing; -import com.google.protobuf.Timestamp; -import feast.proto.core.FeatureSetProto; -import feast.proto.core.FeatureSetProto.FeatureSetSpec; -import feast.proto.types.FeatureRowProto; -import feast.proto.types.FieldProto.Field; -import feast.proto.types.ValueProto.Value; -import feast.proto.types.ValueProto.ValueType; -import java.nio.charset.StandardCharsets; -import java.util.Collections; -import org.junit.Test; - -public class FeatureRowDecoderTest { - - private FeatureSetProto.EntitySpec entity = - FeatureSetProto.EntitySpec.newBuilder().setName("entity1").build(); - - private FeatureSetSpec spec = - FeatureSetSpec.newBuilder() - .addAllEntities(Collections.singletonList(entity)) - .addFeatures( - FeatureSetProto.FeatureSpec.newBuilder() - .setName("feature1") - .setValueType(ValueType.Enum.FLOAT)) - .addFeatures( - FeatureSetProto.FeatureSpec.newBuilder() - .setName("feature2") - .setValueType(ValueType.Enum.INT32)) - .setName("feature_set_name") - .build(); - - @Test - public void shouldDecodeValidEncodedFeatureRowV2() { - FeatureRowDecoder decoder = new FeatureRowDecoder("feature_set_ref", spec); - - FeatureRowProto.FeatureRow encodedFeatureRow = - FeatureRowProto.FeatureRow.newBuilder() - .setEventTimestamp(Timestamp.newBuilder().setNanos(1000)) - .addFields( - Field.newBuilder() - .setName( - Hashing.murmur3_32() - .hashString("feature1", StandardCharsets.UTF_8) - .toString()) - .setValue(Value.newBuilder().setInt32Val(2))) - .addFields( - Field.newBuilder() - .setName( - Hashing.murmur3_32() - .hashString("feature2", StandardCharsets.UTF_8) - .toString()) - .setValue(Value.newBuilder().setFloatVal(1.0f))) - .build(); - - FeatureRowProto.FeatureRow expectedFeatureRow = - FeatureRowProto.FeatureRow.newBuilder() - .setFeatureSet("feature_set_ref") - .setEventTimestamp(Timestamp.newBuilder().setNanos(1000)) - .addFields( - Field.newBuilder().setName("feature1").setValue(Value.newBuilder().setInt32Val(2))) - .addFields( - Field.newBuilder() - .setName("feature2") - .setValue(Value.newBuilder().setFloatVal(1.0f))) - .build(); - - assertEquals(expectedFeatureRow, decoder.decode(encodedFeatureRow)); - } - - @Test - public void shouldDecodeValidFeatureRowV2WithIncompleteFields() { - FeatureRowDecoder decoder = new FeatureRowDecoder("feature_set_ref", spec); - - FeatureRowProto.FeatureRow encodedFeatureRow = - FeatureRowProto.FeatureRow.newBuilder() - .setEventTimestamp(Timestamp.newBuilder().setNanos(1000)) - .addFields( - Field.newBuilder() - .setName( - Hashing.murmur3_32() - .hashString("feature1", StandardCharsets.UTF_8) - .toString()) - .setValue(Value.newBuilder().setInt32Val(2))) - .build(); - - // should decode missing fields as fields with unset value. - FeatureRowProto.FeatureRow expectedFeatureRow = - FeatureRowProto.FeatureRow.newBuilder() - .setFeatureSet("feature_set_ref") - .setEventTimestamp(Timestamp.newBuilder().setNanos(1000)) - .addFields( - Field.newBuilder().setName("feature1").setValue(Value.newBuilder().setInt32Val(2))) - .addFields(Field.newBuilder().setName("feature2").setValue(Value.newBuilder().build())) - .build(); - - assertEquals(expectedFeatureRow, decoder.decode(encodedFeatureRow)); - } - - @Test - public void shouldDecodeValidFeatureRowV2AndIgnoreExtraFields() { - FeatureRowDecoder decoder = new FeatureRowDecoder("feature_set_ref", spec); - - FeatureRowProto.FeatureRow encodedFeatureRow = - FeatureRowProto.FeatureRow.newBuilder() - .setEventTimestamp(Timestamp.newBuilder().setNanos(1000)) - .addFields( - Field.newBuilder() - .setName( - Hashing.murmur3_32() - .hashString("feature1", StandardCharsets.UTF_8) - .toString()) - .setValue(Value.newBuilder().setInt32Val(2))) - .addFields( - Field.newBuilder() - .setName( - Hashing.murmur3_32() - .hashString("feature2", StandardCharsets.UTF_8) - .toString()) - .setValue(Value.newBuilder().setFloatVal(1.0f))) - .addFields( - Field.newBuilder() - .setName( - Hashing.murmur3_32() - .hashString("feature3", StandardCharsets.UTF_8) - .toString()) - .setValue(Value.newBuilder().setStringVal("data"))) - .build(); - - // should decode missing fields as fields with unset value. - FeatureRowProto.FeatureRow expectedFeatureRow = - FeatureRowProto.FeatureRow.newBuilder() - .setFeatureSet("feature_set_ref") - .setEventTimestamp(Timestamp.newBuilder().setNanos(1000)) - .addFields( - Field.newBuilder().setName("feature1").setValue(Value.newBuilder().setInt32Val(2))) - .addFields( - Field.newBuilder() - .setName("feature2") - .setValue(Value.newBuilder().setFloatVal(1.0f))) - .build(); - - assertEquals(expectedFeatureRow, decoder.decode(encodedFeatureRow)); - } - - // TODO: remove this test in Feast 0.7 when support for Feature Row v1 encoding is removed - @Test - public void shouldDecodeValidEncodedFeatureRowV1() { - FeatureRowDecoder decoder = new FeatureRowDecoder("feature_set_ref", spec); - - FeatureRowProto.FeatureRow encodedFeatureRow = - FeatureRowProto.FeatureRow.newBuilder() - .setEventTimestamp(Timestamp.newBuilder().setNanos(1000)) - .addFields(Field.newBuilder().setValue(Value.newBuilder().setInt32Val(2))) - .addFields(Field.newBuilder().setValue(Value.newBuilder().setFloatVal(1.0f))) - .build(); - - FeatureRowProto.FeatureRow expectedFeatureRow = - FeatureRowProto.FeatureRow.newBuilder() - .setFeatureSet("feature_set_ref") - .setEventTimestamp(Timestamp.newBuilder().setNanos(1000)) - .addFields( - Field.newBuilder().setName("feature1").setValue(Value.newBuilder().setInt32Val(2))) - .addFields( - Field.newBuilder() - .setName("feature2") - .setValue(Value.newBuilder().setFloatVal(1.0f))) - .build(); - - assertEquals(expectedFeatureRow, decoder.decode(encodedFeatureRow)); - } -} diff --git a/storage/connectors/redis/src/test/java/feast/storage/connectors/redis/retriever/RedisClusterOnlineRetrieverTest.java b/storage/connectors/redis/src/test/java/feast/storage/connectors/redis/retriever/RedisClusterOnlineRetrieverTest.java deleted file mode 100644 index 419ce8e0a95..00000000000 --- a/storage/connectors/redis/src/test/java/feast/storage/connectors/redis/retriever/RedisClusterOnlineRetrieverTest.java +++ /dev/null @@ -1,273 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.storage.connectors.redis.retriever; - -import static org.hamcrest.MatcherAssert.assertThat; -import static org.hamcrest.Matchers.equalTo; -import static org.mockito.Mockito.*; -import static org.mockito.MockitoAnnotations.initMocks; - -import com.google.common.collect.ImmutableList; -import com.google.common.collect.Lists; -import com.google.protobuf.Duration; -import com.google.protobuf.Timestamp; -import feast.proto.core.FeatureSetProto.EntitySpec; -import feast.proto.core.FeatureSetProto.FeatureSetSpec; -import feast.proto.core.FeatureSetProto.FeatureSpec; -import feast.proto.serving.ServingAPIProto.FeatureReference; -import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesRequest.EntityRow; -import feast.proto.storage.RedisProto.RedisKey; -import feast.proto.types.FeatureRowProto.FeatureRow; -import feast.proto.types.FieldProto.Field; -import feast.proto.types.ValueProto.Value; -import feast.storage.api.retriever.FeatureSetRequest; -import feast.storage.api.retriever.OnlineRetriever; -import feast.storage.connectors.redis.serializer.RedisKeyPrefixSerializer; -import feast.storage.connectors.redis.serializer.RedisKeySerializer; -import io.lettuce.core.KeyValue; -import io.lettuce.core.cluster.api.StatefulRedisClusterConnection; -import io.lettuce.core.cluster.api.sync.RedisAdvancedClusterCommands; -import java.util.List; -import java.util.Optional; -import org.junit.Before; -import org.junit.Test; -import org.mockito.Mock; - -public class RedisClusterOnlineRetrieverTest { - - @Mock StatefulRedisClusterConnection connection; - - @Mock RedisAdvancedClusterCommands syncCommands; - - RedisKeySerializer serializer = new RedisKeyPrefixSerializer("test:"); - - RedisKeySerializer fallbackSerializer = new RedisKeyPrefixSerializer(""); - - private List redisKeys; - private FeatureSetRequest featureSetRequest; - private List entityRows; - private List featureRows; - - @Before - public void setUp() { - initMocks(this); - when(connection.sync()).thenReturn(syncCommands); - redisKeys = - Lists.newArrayList( - RedisKey.newBuilder() - .setFeatureSet("project/featureSet") - .addAllEntities( - Lists.newArrayList( - Field.newBuilder().setName("entity1").setValue(intValue(1)).build(), - Field.newBuilder().setName("entity2").setValue(strValue("a")).build())) - .build(), - RedisKey.newBuilder() - .setFeatureSet("project/featureSet") - .addAllEntities( - Lists.newArrayList( - Field.newBuilder().setName("entity1").setValue(intValue(2)).build(), - Field.newBuilder().setName("entity2").setValue(strValue("b")).build())) - .build()); - - FeatureSetSpec featureSetSpec = - FeatureSetSpec.newBuilder() - .setProject("project") - .setName("featureSet") - .addEntities(EntitySpec.newBuilder().setName("entity1")) - .addEntities(EntitySpec.newBuilder().setName("entity2")) - .addFeatures(FeatureSpec.newBuilder().setName("feature1")) - .addFeatures(FeatureSpec.newBuilder().setName("feature2")) - .setMaxAge(Duration.newBuilder().setSeconds(30)) // default - .build(); - - featureSetRequest = - FeatureSetRequest.newBuilder() - .setSpec(featureSetSpec) - .addFeatureReference( - FeatureReference.newBuilder().setName("feature1").setProject("project").build()) - .addFeatureReference( - FeatureReference.newBuilder().setName("feature2").setProject("project").build()) - .build(); - - entityRows = - ImmutableList.of( - EntityRow.newBuilder() - .setEntityTimestamp(Timestamp.newBuilder().setSeconds(100)) - .putFields("entity1", intValue(1)) - .putFields("entity2", strValue("a")) - .build(), - EntityRow.newBuilder() - .setEntityTimestamp(Timestamp.newBuilder().setSeconds(100)) - .putFields("entity1", intValue(2)) - .putFields("entity2", strValue("b")) - .build()); - - featureRows = - Lists.newArrayList( - FeatureRow.newBuilder() - .setEventTimestamp(Timestamp.newBuilder().setSeconds(100)) - .addAllFields( - Lists.newArrayList( - Field.newBuilder().setValue(intValue(1)).build(), - Field.newBuilder().setValue(intValue(1)).build())) - .build(), - FeatureRow.newBuilder() - .setEventTimestamp(Timestamp.newBuilder().setSeconds(100)) - .addAllFields( - Lists.newArrayList( - Field.newBuilder().setValue(intValue(2)).build(), - Field.newBuilder().setValue(intValue(2)).build())) - .build()); - } - - @Test - public void shouldReturnResponseWithValuesIfKeysPresent() { - byte[] serializedKey1 = serializer.serialize(redisKeys.get(0)); - byte[] serializedKey2 = serializer.serialize(redisKeys.get(1)); - - KeyValue keyValue1 = - KeyValue.from(serializedKey1, Optional.of(featureRows.get(0).toByteArray())); - KeyValue keyValue2 = - KeyValue.from(serializedKey2, Optional.of(featureRows.get(1).toByteArray())); - - List> featureRowBytes = Lists.newArrayList(keyValue1, keyValue2); - - OnlineRetriever redisClusterOnlineRetriever = - new RedisClusterOnlineRetriever.Builder(connection, serializer) - .withFallbackSerializer(fallbackSerializer) - .build(); - when(syncCommands.mget(serializedKey1, serializedKey2)).thenReturn(featureRowBytes); - - List> expected = - Lists.newArrayList( - Optional.of( - FeatureRow.newBuilder() - .setEventTimestamp(Timestamp.newBuilder().setSeconds(100)) - .setFeatureSet("project/featureSet") - .addAllFields( - Lists.newArrayList( - Field.newBuilder().setName("feature1").setValue(intValue(1)).build(), - Field.newBuilder().setName("feature2").setValue(intValue(1)).build())) - .build()), - Optional.of( - FeatureRow.newBuilder() - .setEventTimestamp(Timestamp.newBuilder().setSeconds(100)) - .setFeatureSet("project/featureSet") - .addAllFields( - Lists.newArrayList( - Field.newBuilder().setName("feature1").setValue(intValue(2)).build(), - Field.newBuilder().setName("feature2").setValue(intValue(2)).build())) - .build())); - - List> actual = - redisClusterOnlineRetriever.getOnlineFeatures(entityRows, featureSetRequest); - assertThat(actual, equalTo(expected)); - - // check that fallback is used only when there's something to fallback - verify(syncCommands, never()).mget(); - } - - @Test - public void shouldReturnNullIfKeysNotPresent() { - byte[] serializedKey1 = serializer.serialize(redisKeys.get(0)); - byte[] serializedKey2 = serializer.serialize(redisKeys.get(1)); - - KeyValue keyValue1 = - KeyValue.from(serializedKey1, Optional.of(featureRows.get(0).toByteArray())); - KeyValue keyValue2 = KeyValue.empty(serializedKey2); - - List> featureRowBytes = Lists.newArrayList(keyValue1, keyValue2); - - OnlineRetriever redisClusterOnlineRetriever = - new RedisClusterOnlineRetriever.Builder(connection, serializer).build(); - when(syncCommands.mget(serializedKey1, serializedKey2)).thenReturn(featureRowBytes); - - List> expected = - Lists.newArrayList( - Optional.of( - FeatureRow.newBuilder() - .setEventTimestamp(Timestamp.newBuilder().setSeconds(100)) - .setFeatureSet("project/featureSet") - .addAllFields( - Lists.newArrayList( - Field.newBuilder().setName("feature1").setValue(intValue(1)).build(), - Field.newBuilder().setName("feature2").setValue(intValue(1)).build())) - .build()), - Optional.empty()); - - List> actual = - redisClusterOnlineRetriever.getOnlineFeatures(entityRows, featureSetRequest); - assertThat(actual, equalTo(expected)); - } - - @Test - public void shouldUseFallbackIfAvailable() { - byte[] serializedKey1 = serializer.serialize(redisKeys.get(0)); - byte[] serializedKey2 = serializer.serialize(redisKeys.get(1)); - byte[] fallbackSerializedKey2 = fallbackSerializer.serialize(redisKeys.get(1)); - - KeyValue keyValue1 = - KeyValue.from(serializedKey1, Optional.of(featureRows.get(0).toByteArray())); - KeyValue keyValue2 = KeyValue.empty(serializedKey2); - KeyValue fallbackKeyValue2 = - KeyValue.from(serializedKey2, Optional.of(featureRows.get(1).toByteArray())); - - List> featureRowBytes = Lists.newArrayList(keyValue1, keyValue2); - List> fallbackFeatureRowBytes = Lists.newArrayList(fallbackKeyValue2); - - OnlineRetriever redisClusterOnlineRetriever = - new RedisClusterOnlineRetriever.Builder(connection, serializer) - .withFallbackSerializer(fallbackSerializer) - .build(); - - when(syncCommands.mget(serializedKey1, serializedKey2)).thenReturn(featureRowBytes); - when(syncCommands.mget(fallbackSerializedKey2)).thenReturn(fallbackFeatureRowBytes); - - List> expected = - Lists.newArrayList( - Optional.of( - FeatureRow.newBuilder() - .setEventTimestamp(Timestamp.newBuilder().setSeconds(100)) - .setFeatureSet("project/featureSet") - .addAllFields( - Lists.newArrayList( - Field.newBuilder().setName("feature1").setValue(intValue(1)).build(), - Field.newBuilder().setName("feature2").setValue(intValue(1)).build())) - .build()), - Optional.of( - FeatureRow.newBuilder() - .setEventTimestamp(Timestamp.newBuilder().setSeconds(100)) - .setFeatureSet("project/featureSet") - .addAllFields( - Lists.newArrayList( - Field.newBuilder().setName("feature1").setValue(intValue(2)).build(), - Field.newBuilder().setName("feature2").setValue(intValue(2)).build())) - .build())); - - List> actual = - redisClusterOnlineRetriever.getOnlineFeatures(entityRows, featureSetRequest); - assertThat(actual, equalTo(expected)); - } - - private Value intValue(int val) { - return Value.newBuilder().setInt64Val(val).build(); - } - - private Value strValue(String val) { - return Value.newBuilder().setStringVal(val).build(); - } -} diff --git a/storage/connectors/redis/src/test/java/feast/storage/connectors/redis/retriever/RedisOnlineRetrieverTest.java b/storage/connectors/redis/src/test/java/feast/storage/connectors/redis/retriever/RedisOnlineRetrieverTest.java deleted file mode 100644 index 1292f4ab0dc..00000000000 --- a/storage/connectors/redis/src/test/java/feast/storage/connectors/redis/retriever/RedisOnlineRetrieverTest.java +++ /dev/null @@ -1,240 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2020 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.storage.connectors.redis.retriever; - -import static org.hamcrest.MatcherAssert.assertThat; -import static org.hamcrest.Matchers.equalTo; -import static org.mockito.Mockito.when; -import static org.mockito.MockitoAnnotations.initMocks; - -import com.google.common.collect.ImmutableList; -import com.google.common.collect.Lists; -import com.google.protobuf.AbstractMessageLite; -import com.google.protobuf.Duration; -import com.google.protobuf.Timestamp; -import feast.proto.core.FeatureSetProto.EntitySpec; -import feast.proto.core.FeatureSetProto.FeatureSetSpec; -import feast.proto.core.FeatureSetProto.FeatureSpec; -import feast.proto.serving.ServingAPIProto.FeatureReference; -import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesRequest.EntityRow; -import feast.proto.storage.RedisProto.RedisKey; -import feast.proto.types.FeatureRowProto.FeatureRow; -import feast.proto.types.FieldProto.Field; -import feast.proto.types.ValueProto.Value; -import feast.storage.api.retriever.FeatureSetRequest; -import feast.storage.api.retriever.OnlineRetriever; -import io.lettuce.core.KeyValue; -import io.lettuce.core.api.StatefulRedisConnection; -import io.lettuce.core.api.sync.RedisCommands; -import java.util.List; -import java.util.Optional; -import java.util.stream.Collectors; -import org.junit.Before; -import org.junit.Test; -import org.mockito.Mock; - -public class RedisOnlineRetrieverTest { - - @Mock StatefulRedisConnection connection; - - @Mock RedisCommands syncCommands; - - private OnlineRetriever redisOnlineRetriever; - private byte[][] redisKeyList; - - @Before - public void setUp() { - initMocks(this); - when(connection.sync()).thenReturn(syncCommands); - redisOnlineRetriever = RedisOnlineRetriever.create(connection); - redisKeyList = - Lists.newArrayList( - RedisKey.newBuilder() - .setFeatureSet("project/featureSet") - .addAllEntities( - Lists.newArrayList( - Field.newBuilder().setName("entity1").setValue(intValue(1)).build(), - Field.newBuilder().setName("entity2").setValue(strValue("a")).build())) - .build(), - RedisKey.newBuilder() - .setFeatureSet("project/featureSet") - .addAllEntities( - Lists.newArrayList( - Field.newBuilder().setName("entity1").setValue(intValue(2)).build(), - Field.newBuilder().setName("entity2").setValue(strValue("b")).build())) - .build()) - .stream() - .map(AbstractMessageLite::toByteArray) - .collect(Collectors.toList()) - .toArray(new byte[0][0]); - } - - @Test - public void shouldReturnResponseWithValuesIfKeysPresent() { - FeatureSetRequest featureSetRequest = - FeatureSetRequest.newBuilder() - .setSpec(getFeatureSetSpec()) - .addFeatureReference( - FeatureReference.newBuilder().setName("feature1").setProject("project").build()) - .addFeatureReference( - FeatureReference.newBuilder().setName("feature2").setProject("project").build()) - .build(); - List entityRows = - ImmutableList.of( - EntityRow.newBuilder() - .setEntityTimestamp(Timestamp.newBuilder().setSeconds(100)) - .putFields("entity1", intValue(1)) - .putFields("entity2", strValue("a")) - .build(), - EntityRow.newBuilder() - .setEntityTimestamp(Timestamp.newBuilder().setSeconds(100)) - .putFields("entity1", intValue(2)) - .putFields("entity2", strValue("b")) - .build()); - - List featureRows = - Lists.newArrayList( - FeatureRow.newBuilder() - .setEventTimestamp(Timestamp.newBuilder().setSeconds(100)) - .addAllFields( - Lists.newArrayList( - Field.newBuilder().setValue(intValue(1)).build(), - Field.newBuilder().setValue(intValue(1)).build())) - .build(), - FeatureRow.newBuilder() - .setEventTimestamp(Timestamp.newBuilder().setSeconds(100)) - .addAllFields( - Lists.newArrayList( - Field.newBuilder().setValue(intValue(2)).build(), - Field.newBuilder().setValue(intValue(2)).build())) - .build()); - - List> featureRowBytes = - featureRows.stream() - .map(x -> KeyValue.from(new byte[1], Optional.of(x.toByteArray()))) - .collect(Collectors.toList()); - - redisOnlineRetriever = RedisOnlineRetriever.create(connection); - when(connection.sync()).thenReturn(syncCommands); - when(syncCommands.mget(redisKeyList)).thenReturn(featureRowBytes); - - List> expected = - Lists.newArrayList( - Optional.of( - FeatureRow.newBuilder() - .setEventTimestamp(Timestamp.newBuilder().setSeconds(100)) - .setFeatureSet("project/featureSet") - .addAllFields( - Lists.newArrayList( - Field.newBuilder().setName("feature1").setValue(intValue(1)).build(), - Field.newBuilder().setName("feature2").setValue(intValue(1)).build())) - .build()), - Optional.of( - FeatureRow.newBuilder() - .setEventTimestamp(Timestamp.newBuilder().setSeconds(100)) - .setFeatureSet("project/featureSet") - .addAllFields( - Lists.newArrayList( - Field.newBuilder().setName("feature1").setValue(intValue(2)).build(), - Field.newBuilder().setName("feature2").setValue(intValue(2)).build())) - .build())); - - List> actual = - redisOnlineRetriever.getOnlineFeatures(entityRows, featureSetRequest); - assertThat(actual, equalTo(expected)); - } - - @Test - public void shouldReturnNullIfKeysNotPresent() { - FeatureSetRequest featureSetRequest = - FeatureSetRequest.newBuilder() - .setSpec(getFeatureSetSpec()) - .addFeatureReference( - FeatureReference.newBuilder().setName("feature1").setProject("project").build()) - .addFeatureReference( - FeatureReference.newBuilder().setName("feature2").setProject("project").build()) - .build(); - List entityRows = - ImmutableList.of( - EntityRow.newBuilder() - .setEntityTimestamp(Timestamp.newBuilder().setSeconds(100)) - .putFields("entity1", intValue(1)) - .putFields("entity2", strValue("a")) - .build(), - EntityRow.newBuilder() - .setEntityTimestamp(Timestamp.newBuilder().setSeconds(100)) - .putFields("entity1", intValue(2)) - .putFields("entity2", strValue("b")) - .build()); - - List featureRows = - Lists.newArrayList( - FeatureRow.newBuilder() - .setEventTimestamp(Timestamp.newBuilder().setSeconds(100)) - .addAllFields( - Lists.newArrayList( - Field.newBuilder().setValue(intValue(1)).build(), - Field.newBuilder().setValue(intValue(1)).build())) - .build()); - - List> featureRowBytes = - featureRows.stream() - .map(row -> KeyValue.from(new byte[1], Optional.of(row.toByteArray()))) - .collect(Collectors.toList()); - featureRowBytes.add(null); - - redisOnlineRetriever = RedisOnlineRetriever.create(connection); - when(connection.sync()).thenReturn(syncCommands); - when(syncCommands.mget(redisKeyList)).thenReturn(featureRowBytes); - - List> expected = - Lists.newArrayList( - Optional.of( - FeatureRow.newBuilder() - .setEventTimestamp(Timestamp.newBuilder().setSeconds(100)) - .setFeatureSet("project/featureSet") - .addAllFields( - Lists.newArrayList( - Field.newBuilder().setName("feature1").setValue(intValue(1)).build(), - Field.newBuilder().setName("feature2").setValue(intValue(1)).build())) - .build()), - Optional.empty()); - List> actual = - redisOnlineRetriever.getOnlineFeatures(entityRows, featureSetRequest); - assertThat(actual, equalTo(expected)); - } - - private Value intValue(int val) { - return Value.newBuilder().setInt64Val(val).build(); - } - - private Value strValue(String val) { - return Value.newBuilder().setStringVal(val).build(); - } - - private FeatureSetSpec getFeatureSetSpec() { - return FeatureSetSpec.newBuilder() - .setProject("project") - .setName("featureSet") - .addEntities(EntitySpec.newBuilder().setName("entity1")) - .addEntities(EntitySpec.newBuilder().setName("entity2")) - .addFeatures(FeatureSpec.newBuilder().setName("feature1")) - .addFeatures(FeatureSpec.newBuilder().setName("feature2")) - .setMaxAge(Duration.newBuilder().setSeconds(30)) // default - .build(); - } -} diff --git a/storage/connectors/redis/src/test/java/feast/storage/connectors/redis/serializer/RedisKeyPrefixSerializerTest.java b/storage/connectors/redis/src/test/java/feast/storage/connectors/redis/serializer/RedisKeyPrefixSerializerTest.java index c73108e1188..e663cf81ee4 100644 --- a/storage/connectors/redis/src/test/java/feast/storage/connectors/redis/serializer/RedisKeyPrefixSerializerTest.java +++ b/storage/connectors/redis/src/test/java/feast/storage/connectors/redis/serializer/RedisKeyPrefixSerializerTest.java @@ -18,35 +18,28 @@ import static org.junit.Assert.*; -import com.google.common.collect.Lists; -import feast.proto.storage.RedisProto.RedisKey; -import feast.proto.types.FieldProto; +import feast.proto.storage.RedisProto.RedisKeyV2; import feast.proto.types.ValueProto; import org.junit.Test; public class RedisKeyPrefixSerializerTest { - private RedisKey key = - RedisKey.newBuilder() - .setFeatureSet("project/featureSet") - .addAllEntities( - Lists.newArrayList( - FieldProto.Field.newBuilder() - .setName("entity1") - .setValue(ValueProto.Value.newBuilder().setInt64Val(1)) - .build())) + private RedisKeyV2 key = + RedisKeyV2.newBuilder() + .addEntityNames("entity1") + .addEntityValues(ValueProto.Value.newBuilder().setInt64Val(1)) .build(); @Test public void shouldPrependKey() { - RedisKeyPrefixSerializer serializer = new RedisKeyPrefixSerializer("namespace:"); + RedisKeyPrefixSerializerV2 serializer = new RedisKeyPrefixSerializerV2("namespace:"); String keyWithPrefix = new String(serializer.serialize(key)); assertEquals(String.format("namespace:%s", new String(key.toByteArray())), keyWithPrefix); } @Test public void shouldNotPrependKeyIfEmptyString() { - RedisKeyPrefixSerializer serializer = new RedisKeyPrefixSerializer(""); + RedisKeyPrefixSerializerV2 serializer = new RedisKeyPrefixSerializerV2(""); assertArrayEquals(key.toByteArray(), serializer.serialize(key)); } } diff --git a/storage/connectors/redis/src/test/java/feast/storage/connectors/redis/writer/RedisFeatureSinkTest.java b/storage/connectors/redis/src/test/java/feast/storage/connectors/redis/writer/RedisFeatureSinkTest.java deleted file mode 100644 index 12377fd1d1b..00000000000 --- a/storage/connectors/redis/src/test/java/feast/storage/connectors/redis/writer/RedisFeatureSinkTest.java +++ /dev/null @@ -1,665 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * Copyright 2018-2019 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package feast.storage.connectors.redis.writer; - -import static feast.storage.common.testing.TestUtil.field; -import static feast.storage.common.testing.TestUtil.hash; -import static org.hamcrest.CoreMatchers.equalTo; -import static org.hamcrest.MatcherAssert.assertThat; -import static org.hamcrest.collection.IsCollectionWithSize.hasSize; - -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; -import com.google.common.collect.Lists; -import com.google.protobuf.Message; -import com.google.protobuf.Timestamp; -import feast.common.models.FeatureSetReference; -import feast.proto.core.FeatureSetProto.EntitySpec; -import feast.proto.core.FeatureSetProto.FeatureSetSpec; -import feast.proto.core.FeatureSetProto.FeatureSpec; -import feast.proto.core.StoreProto; -import feast.proto.core.StoreProto.Store.RedisClusterConfig; -import feast.proto.core.StoreProto.Store.RedisConfig; -import feast.proto.storage.RedisProto.RedisKey; -import feast.proto.types.FeatureRowProto.FeatureRow; -import feast.proto.types.FieldProto.Field; -import feast.proto.types.ValueProto.Value; -import feast.proto.types.ValueProto.ValueType.Enum; -import io.lettuce.core.AbstractRedisClient; -import io.lettuce.core.RedisClient; -import io.lettuce.core.RedisURI; -import io.lettuce.core.api.sync.RedisStringCommands; -import io.lettuce.core.cluster.RedisClusterClient; -import io.lettuce.core.codec.ByteArrayCodec; -import java.util.*; -import java.util.concurrent.ScheduledFuture; -import java.util.concurrent.ScheduledThreadPoolExecutor; -import java.util.concurrent.TimeUnit; -import java.util.stream.Collectors; -import java.util.stream.IntStream; -import net.ishiis.redis.unit.Redis; -import net.ishiis.redis.unit.RedisCluster; -import net.ishiis.redis.unit.RedisServer; -import org.apache.beam.sdk.extensions.protobuf.ProtoCoder; -import org.apache.beam.sdk.testing.PAssert; -import org.apache.beam.sdk.testing.TestPipeline; -import org.apache.beam.sdk.testing.TestStream; -import org.apache.beam.sdk.transforms.Count; -import org.apache.beam.sdk.transforms.Create; -import org.apache.beam.sdk.values.PCollection; -import org.junit.*; -import org.junit.runner.RunWith; -import org.junit.runners.Parameterized; - -@RunWith(Parameterized.class) -public class RedisFeatureSinkTest { - @Rule public transient TestPipeline p = TestPipeline.create(); - - private static String REDIS_HOST = "localhost"; - private static int REDIS_PORT = 51233; - private static Integer[] REDIS_CLUSTER_PORTS = {6380, 6381, 6382}; - - private RedisStringCommands sync; - private RedisFeatureSink redisFeatureSink; - private Map specMap; - - @Parameterized.Parameters - public static Iterable backends() { - Redis redis = new RedisServer(REDIS_PORT); - RedisClient client = - RedisClient.create(new RedisURI(REDIS_HOST, REDIS_PORT, java.time.Duration.ofMillis(2000))); - - Redis redisCluster = new RedisCluster(REDIS_CLUSTER_PORTS); - RedisClusterClient clientCluster = - RedisClusterClient.create( - Lists.newArrayList(REDIS_CLUSTER_PORTS).stream() - .map(port -> RedisURI.create(REDIS_HOST, port)) - .collect(Collectors.toList())); - - StoreProto.Store.RedisConfig redisConfig = - StoreProto.Store.RedisConfig.newBuilder().setHost(REDIS_HOST).setPort(REDIS_PORT).build(); - - StoreProto.Store.RedisClusterConfig redisClusterConfig = - StoreProto.Store.RedisClusterConfig.newBuilder() - .setConnectionString( - Lists.newArrayList(REDIS_CLUSTER_PORTS).stream() - .map(port -> String.format("%s:%d", REDIS_HOST, port)) - .collect(Collectors.joining(","))) - .setInitialBackoffMs(2000) - .setMaxRetries(4) - .build(); - - return Arrays.asList( - new Object[] {redis, client, redisConfig}, - new Object[] {redisCluster, clientCluster, redisClusterConfig}); - } - - @Parameterized.Parameter(0) - public Redis redisServer; - - @Parameterized.Parameter(1) - public AbstractRedisClient redisClient; - - @Parameterized.Parameter(2) - public Message redisConfig; - - @Before - public void setUp() { - redisServer.start(); - - if (redisClient instanceof RedisClient) { - sync = ((RedisClient) redisClient).connect(new ByteArrayCodec()).sync(); - } else { - sync = ((RedisClusterClient) redisClient).connect(new ByteArrayCodec()).sync(); - } - - FeatureSetSpec spec1 = - FeatureSetSpec.newBuilder() - .setName("fs") - .setProject("myproject") - .addEntities(EntitySpec.newBuilder().setName("entity").setValueType(Enum.INT64).build()) - .addFeatures( - FeatureSpec.newBuilder().setName("feature").setValueType(Enum.STRING).build()) - .build(); - - FeatureSetSpec spec2 = - FeatureSetSpec.newBuilder() - .setName("feature_set") - .setProject("myproject") - .addEntities( - EntitySpec.newBuilder() - .setName("entity_id_primary") - .setValueType(Enum.INT32) - .build()) - .addEntities( - EntitySpec.newBuilder() - .setName("entity_id_secondary") - .setValueType(Enum.STRING) - .build()) - .addFeatures( - FeatureSpec.newBuilder().setName("feature_1").setValueType(Enum.STRING).build()) - .addFeatures( - FeatureSpec.newBuilder().setName("feature_2").setValueType(Enum.INT64).build()) - .build(); - - specMap = - ImmutableMap.of( - FeatureSetReference.of("myproject", "fs", 1), spec1, - FeatureSetReference.of("myproject", "feature_set", 1), spec2); - - RedisFeatureSink.Builder builder = RedisFeatureSink.builder(); - if (redisConfig instanceof RedisConfig) { - builder = builder.setRedisConfig((RedisConfig) redisConfig); - } else { - builder = builder.setRedisClusterConfig((RedisClusterConfig) redisConfig); - } - redisFeatureSink = builder.build(); - redisFeatureSink.prepareWrite(p.apply("Specs-1", Create.of(specMap))); - } - - @After - public void tearDown() { - if (redisServer.isActive()) { - redisServer.stop(); - } - } - - private RedisKey createRedisKey(String featureSetRef, Field... fields) { - return RedisKey.newBuilder() - .setFeatureSet(featureSetRef) - .addAllEntities(Lists.newArrayList(fields)) - .build(); - } - - private FeatureRow createFeatureRow(String featureSetRef, Timestamp timestamp, Field... fields) { - FeatureRow.Builder builder = FeatureRow.newBuilder(); - if (featureSetRef != null) { - builder.setFeatureSet(featureSetRef); - } - - if (timestamp != null) { - builder.setEventTimestamp(timestamp); - } - - return builder.addAllFields(Lists.newArrayList(fields)).build(); - } - - @Test - public void shouldWriteToRedis() { - - HashMap kvs = new LinkedHashMap<>(); - kvs.put( - createRedisKey("myproject/fs", field("entity", 1, Enum.INT64)), - createFeatureRow( - null, Timestamp.getDefaultInstance(), field(hash("feature"), "one", Enum.STRING))); - kvs.put( - createRedisKey("myproject/fs", field("entity", 2, Enum.INT64)), - createFeatureRow( - null, Timestamp.getDefaultInstance(), field(hash("feature"), "two", Enum.STRING))); - - List featureRows = - ImmutableList.of( - createFeatureRow( - "myproject/fs", - null, - field("entity", 1, Enum.INT64), - field("feature", "one", Enum.STRING)), - createFeatureRow( - "myproject/fs", - null, - field("entity", 2, Enum.INT64), - field("feature", "two", Enum.STRING))); - - p.apply(Create.of(featureRows)).apply(redisFeatureSink.writer()); - p.run(); - - kvs.forEach( - (key, value) -> { - byte[] actual = sync.get(key.toByteArray()); - assertThat(actual, equalTo(value.toByteArray())); - }); - } - - @Test(timeout = 30000) - public void shouldRetryFailConnection() throws InterruptedException { - RedisConfig redisConfig = - RedisConfig.newBuilder() - .setHost(REDIS_HOST) - .setPort(REDIS_PORT) - .setMaxRetries(4) - .setInitialBackoffMs(2000) - .build(); - redisFeatureSink = - redisFeatureSink - .toBuilder() - .setRedisConfig(redisConfig) - .build() - .withSpecsView(redisFeatureSink.getSpecsView()); - - HashMap kvs = new LinkedHashMap<>(); - kvs.put( - createRedisKey("myproject/fs", field("entity", 1, Enum.INT64)), - createFeatureRow( - "", Timestamp.getDefaultInstance(), field(hash("feature"), "one", Enum.STRING))); - - List featureRows = - ImmutableList.of( - createFeatureRow( - "myproject/fs", - null, - field("entity", 1, Enum.INT64), - field("feature", "one", Enum.STRING))); - - PCollection failedElementCount = - p.apply(Create.of(featureRows)) - .apply(redisFeatureSink.writer()) - .getFailedInserts() - .apply(Count.globally()); - - redisServer.stop(); - final ScheduledThreadPoolExecutor redisRestartExecutor = new ScheduledThreadPoolExecutor(1); - ScheduledFuture scheduledRedisRestart = - redisRestartExecutor.schedule( - () -> { - redisServer.start(); - }, - 3, - TimeUnit.SECONDS); - - PAssert.that(failedElementCount).containsInAnyOrder(0L); - p.run(); - scheduledRedisRestart.cancel(true); - - kvs.forEach( - (key, value) -> { - byte[] actual = sync.get(key.toByteArray()); - assertThat(actual, equalTo(value.toByteArray())); - }); - } - - @Test - public void shouldProduceFailedElementIfRetryExceeded() { - - RedisConfig redisConfig = - RedisConfig.newBuilder().setHost(REDIS_HOST).setPort(REDIS_PORT + 1).build(); - redisFeatureSink = - redisFeatureSink - .toBuilder() - .setRedisConfig(redisConfig) - .build() - .withSpecsView(redisFeatureSink.getSpecsView()); - - HashMap kvs = new LinkedHashMap<>(); - kvs.put( - createRedisKey("myproject/fs", field("entity", 1, Enum.INT64)), - createFeatureRow( - "", Timestamp.getDefaultInstance(), field(hash("feature"), "one", Enum.STRING))); - - List featureRows = - ImmutableList.of( - FeatureRow.newBuilder() - .setFeatureSet("myproject/fs") - .addFields(field("entity", 1, Enum.INT64)) - .addFields(field("feature", "one", Enum.STRING)) - .build()); - - PCollection failedElementCount = - p.apply(Create.of(featureRows)) - .apply(redisFeatureSink.writer()) - .getFailedInserts() - .apply(Count.globally()); - - redisServer.stop(); - PAssert.that(failedElementCount).containsInAnyOrder(1L); - p.run(); - } - - @Test - public void shouldConvertRowWithDuplicateEntitiesToValidKey() { - - FeatureRow offendingRow = - createFeatureRow( - "myproject/feature_set", - Timestamp.newBuilder().setSeconds(10).build(), - field("entity_id_primary", 1, Enum.INT32), - field("entity_id_primary", 2, Enum.INT32), - field("entity_id_secondary", "a", Enum.STRING), - field("feature_1", "strValue1", Enum.STRING), - field("feature_2", 1001, Enum.INT64)); - - RedisKey expectedKey = - createRedisKey( - "myproject/feature_set", - field("entity_id_primary", 1, Enum.INT32), - field("entity_id_secondary", "a", Enum.STRING)); - - FeatureRow expectedValue = - createFeatureRow( - "", - Timestamp.newBuilder().setSeconds(10).build(), - field(hash("feature_1"), "strValue1", Enum.STRING), - field(hash("feature_2"), 1001, Enum.INT64)); - - p.apply(Create.of(offendingRow)).apply(redisFeatureSink.writer()); - - p.run(); - - byte[] actual = sync.get(expectedKey.toByteArray()); - assertThat(actual, equalTo(expectedValue.toByteArray())); - } - - @Test - public void shouldConvertRowWithOutOfOrderFieldsToValidKey() { - FeatureRow offendingRow = - createFeatureRow( - "myproject/feature_set", - Timestamp.newBuilder().setSeconds(10).build(), - field("entity_id_secondary", "a", Enum.STRING), - field("entity_id_primary", 1, Enum.INT32), - field("feature_2", 1001, Enum.INT64), - field("feature_1", "strValue1", Enum.STRING)); - - RedisKey expectedKey = - createRedisKey( - "myproject/feature_set", - field("entity_id_primary", 1, Enum.INT32), - field("entity_id_secondary", "a", Enum.STRING)); - - FeatureRow expectedValue = - createFeatureRow( - "", - Timestamp.newBuilder().setSeconds(10).build(), - field(hash("feature_1"), "strValue1", Enum.STRING), - field(hash("feature_2"), 1001, Enum.INT64)); - - p.apply(Create.of(offendingRow)).apply(redisFeatureSink.writer()); - - p.run(); - - byte[] actual = sync.get(expectedKey.toByteArray()); - assertThat(actual, equalTo(expectedValue.toByteArray())); - } - - @Test - public void shouldMergeDuplicateFeatureFields() { - FeatureRow featureRowWithDuplicatedFeatureFields = - createFeatureRow( - "myproject/feature_set", - Timestamp.newBuilder().setSeconds(10).build(), - field("entity_id_primary", 1, Enum.INT32), - field("entity_id_secondary", "a", Enum.STRING), - field("feature_2", 1001, Enum.INT64), - field("feature_1", "strValue1", Enum.STRING), - field("feature_1", "strValue1", Enum.STRING)); - - RedisKey expectedKey = - createRedisKey( - "myproject/feature_set", - field("entity_id_primary", 1, Enum.INT32), - field("entity_id_secondary", "a", Enum.STRING)); - - FeatureRow expectedValue = - createFeatureRow( - "", - Timestamp.newBuilder().setSeconds(10).build(), - field(hash("feature_1"), "strValue1", Enum.STRING), - field(hash("feature_2"), 1001, Enum.INT64)); - - p.apply(Create.of(featureRowWithDuplicatedFeatureFields)).apply(redisFeatureSink.writer()); - - p.run(); - - byte[] actual = sync.get(expectedKey.toByteArray()); - assertThat(actual, equalTo(expectedValue.toByteArray())); - } - - @Test - public void shouldPopulateMissingFeatureValuesWithDefaultInstance() { - FeatureRow featureRowWithDuplicatedFeatureFields = - createFeatureRow( - "myproject/feature_set", - Timestamp.newBuilder().setSeconds(10).build(), - field("entity_id_primary", 1, Enum.INT32), - field("entity_id_secondary", "a", Enum.STRING), - field("feature_1", "strValue1", Enum.STRING)); - - RedisKey expectedKey = - createRedisKey( - "myproject/feature_set", - field("entity_id_primary", 1, Enum.INT32), - field("entity_id_secondary", "a", Enum.STRING)); - - FeatureRow expectedValue = - createFeatureRow( - "", - Timestamp.newBuilder().setSeconds(10).build(), - field(hash("feature_1"), "strValue1", Enum.STRING), - Field.newBuilder() - .setName(hash("feature_2")) - .setValue(Value.getDefaultInstance()) - .build()); - - p.apply(Create.of(featureRowWithDuplicatedFeatureFields)).apply(redisFeatureSink.writer()); - - p.run(); - - byte[] actual = sync.get(expectedKey.toByteArray()); - assertThat(actual, equalTo(expectedValue.toByteArray())); - } - - @Test - public void shouldDeduplicateRowsWithinBatch() { - TestStream featureRowTestStream = - TestStream.create(ProtoCoder.of(FeatureRow.class)) - .addElements( - createFeatureRow( - "myproject/feature_set", - Timestamp.newBuilder().setSeconds(20).build(), - field("entity_id_primary", 1, Enum.INT32), - field("entity_id_secondary", "a", Enum.STRING), - field("feature_2", 111, Enum.INT32))) - .addElements( - createFeatureRow( - "myproject/feature_set", - Timestamp.newBuilder().setSeconds(10).build(), - field("entity_id_primary", 1, Enum.INT32), - field("entity_id_secondary", "a", Enum.STRING), - field("feature_2", 222, Enum.INT32))) - .addElements( - createFeatureRow( - "myproject/feature_set", - Timestamp.getDefaultInstance(), - field("entity_id_primary", 1, Enum.INT32), - field("entity_id_secondary", "a", Enum.STRING), - field("feature_2", 333, Enum.INT32))) - .advanceWatermarkToInfinity(); - - p.apply(featureRowTestStream).apply(redisFeatureSink.writer()); - p.run(); - - RedisKey expectedKey = - createRedisKey( - "myproject/feature_set", - field("entity_id_primary", 1, Enum.INT32), - field("entity_id_secondary", "a", Enum.STRING)); - - FeatureRow expectedValue = - createFeatureRow( - "", - Timestamp.newBuilder().setSeconds(20).build(), - Field.newBuilder() - .setName(hash("feature_1")) - .setValue(Value.getDefaultInstance()) - .build(), - field(hash("feature_2"), 111, Enum.INT32)); - - byte[] actual = sync.get(expectedKey.toByteArray()); - assertThat(actual, equalTo(expectedValue.toByteArray())); - } - - @Test - public void shouldWriteWithLatterTimestamp() { - TestStream featureRowTestStream = - TestStream.create(ProtoCoder.of(FeatureRow.class)) - .addElements( - createFeatureRow( - "myproject/feature_set", - Timestamp.newBuilder().setSeconds(20).build(), - field("entity_id_primary", 1, Enum.INT32), - field("entity_id_secondary", "a", Enum.STRING), - field("feature_2", 111, Enum.INT32))) - .addElements( - createFeatureRow( - "myproject/feature_set", - Timestamp.newBuilder().setSeconds(20).build(), - field("entity_id_primary", 2, Enum.INT32), - field("entity_id_secondary", "b", Enum.STRING), - field("feature_2", 222, Enum.INT32))) - .addElements( - createFeatureRow( - "myproject/feature_set", - Timestamp.newBuilder().setSeconds(10).build(), - field("entity_id_primary", 3, Enum.INT32), - field("entity_id_secondary", "c", Enum.STRING), - field("feature_2", 333, Enum.INT32))) - .advanceWatermarkToInfinity(); - - RedisKey keyA = - createRedisKey( - "myproject/feature_set", - field("entity_id_primary", 1, Enum.INT32), - field("entity_id_secondary", "a", Enum.STRING)); - - RedisKey keyB = - createRedisKey( - "myproject/feature_set", - field("entity_id_primary", 2, Enum.INT32), - field("entity_id_secondary", "b", Enum.STRING)); - - RedisKey keyC = - createRedisKey( - "myproject/feature_set", - field("entity_id_primary", 3, Enum.INT32), - field("entity_id_secondary", "c", Enum.STRING)); - - sync.set( - keyA.toByteArray(), - createFeatureRow("", Timestamp.newBuilder().setSeconds(30).build()).toByteArray()); - - sync.set( - keyB.toByteArray(), - createFeatureRow("", Timestamp.newBuilder().setSeconds(10).build()).toByteArray()); - - sync.set( - keyC.toByteArray(), - createFeatureRow("", Timestamp.newBuilder().setSeconds(10).build()).toByteArray()); - - p.apply(featureRowTestStream).apply(redisFeatureSink.writer()); - p.run(); - - assertThat( - sync.get(keyA.toByteArray()), - equalTo(createFeatureRow("", Timestamp.newBuilder().setSeconds(30).build()).toByteArray())); - - assertThat( - sync.get(keyB.toByteArray()), - equalTo( - createFeatureRow( - "", - Timestamp.newBuilder().setSeconds(20).build(), - Field.newBuilder() - .setName(hash("feature_1")) - .setValue(Value.getDefaultInstance()) - .build(), - field(hash("feature_2"), 222, Enum.INT32)) - .toByteArray())); - - assertThat( - sync.get(keyC.toByteArray()), - equalTo(createFeatureRow("", Timestamp.newBuilder().setSeconds(10).build()).toByteArray())); - } - - @Test - public void shouldOverwriteInvalidRows() { - TestStream featureRowTestStream = - TestStream.create(ProtoCoder.of(FeatureRow.class)) - .addElements( - createFeatureRow( - "myproject/feature_set", - Timestamp.newBuilder().setSeconds(20).build(), - field("entity_id_primary", 1, Enum.INT32), - field("entity_id_secondary", "a", Enum.STRING), - field("feature_1", "text", Enum.STRING), - field("feature_2", 111, Enum.INT32))) - .advanceWatermarkToInfinity(); - - RedisKey expectedKey = - createRedisKey( - "myproject/feature_set", - field("entity_id_primary", 1, Enum.INT32), - field("entity_id_secondary", "a", Enum.STRING)); - - sync.set(expectedKey.toByteArray(), "some-invalid-data".getBytes()); - - p.apply(featureRowTestStream).apply(redisFeatureSink.writer()); - p.run(); - - FeatureRow expectedValue = - createFeatureRow( - "", - Timestamp.newBuilder().setSeconds(20).build(), - field(hash("feature_1"), "text", Enum.STRING), - field(hash("feature_2"), 111, Enum.INT32)); - - byte[] actual = sync.get(expectedKey.toByteArray()); - assertThat(actual, equalTo(expectedValue.toByteArray())); - } - - @Test - public void loadTest() { - List rows = - IntStream.range(0, 10000) - .mapToObj( - i -> - createFeatureRow( - "myproject/feature_set", - Timestamp.newBuilder().setSeconds(20).build(), - field("entity_id_primary", i, Enum.INT32), - field("entity_id_secondary", "a", Enum.STRING), - field("feature_1", "text", Enum.STRING), - field("feature_2", 111, Enum.INT32))) - .collect(Collectors.toList()); - - p.apply(Create.of(rows)).apply(redisFeatureSink.writer()); - p.run(); - - List outcome = - IntStream.range(0, 10000) - .mapToObj( - i -> - createRedisKey( - "myproject/feature_set", - field("entity_id_primary", i, Enum.INT32), - field("entity_id_secondary", "a", Enum.STRING)) - .toByteArray()) - .map(sync::get) - .collect(Collectors.toList()); - - assertThat(outcome, hasSize(10000)); - assertThat("All rows were stored", outcome.stream().allMatch(Objects::nonNull)); - } -} diff --git a/tests/README.md b/tests/README.md new file mode 100644 index 00000000000..fcbbc2f02d2 --- /dev/null +++ b/tests/README.md @@ -0,0 +1,90 @@ +# Running full e2e test suite in Minikube + +This doc describes how to to run the entire suite of e2e tests locally, with no external (cloud) dependencies. It will use minikube, spark k8s operator and minio for storage. + +The tests will be run against your local copy of the Python SDK and ingestion jar. For other components like core and serving this setup will use docker images from the public GCR repo, built from the latest master. + +## Prerequisites: +* Docker (highly recommend increasing default disk image size on OSX in docker settings) +* awscli +* kubectl +* minikube (tested using docker driver on OSX) +* helm 3 +* bash 5.0+ (you'll need to brew install it on OSX) +* Java 11 toolchain with maven +* make + +## Steps + +1. Start minikube. We'll need more memory and disk than default. +```bash +minikube start --disk-size='40000mb' --memory 4096 +``` + +2. Install minio. +```bash +helm repo add minio https://helm.min.io/ +helm install --namespace minio --create-namespace minio minio/minio --set resources.requests.memory=2Gi +``` + +3. Create k8s namespace to run tests in +```bash +kubectl create namespace sparkop +``` + +4. Install spark operator. +```bash +helm repo add spark-operator https://googlecloudplatform.github.io/spark-on-k8s-operator +helm install spark-op spark-operator/spark-operator \ + --namespace spark-operator \ + --create-namespace \ + --set "image.tag=v1beta2-1.1.2-2.4.5" \ + --set "sparkJobNamespace=sparkop" \ + --set "serviceAccounts.spark.name=spark" +``` + +5. Copy secret from minio into new sparkop namespace. +```bash +kubectl get secret minio --namespace=minio -oyaml | grep -v '^\s*namespace:\s' | kubectl apply --namespace=sparkop -f - +``` + +6. Install Feast. That may fail due to timeout, rerun again in that case +```bash +# +# NB!! make sure to use bash 5.0+ for the next step. You'd need to brew install it on OSX +DOCKER_REPOSITORY=gcr.io/kf-feast GIT_TAG=develop bash ./infra/scripts/setup-e2e-local.sh +``` + +7. Build the ingestion jar locally +```bash +make build-java-no-tests REVISION=develop +``` + +8. Create staging bucket using awscli. First, create a port-forward for minio: +```bash +# port forward minio in a separate terminal +export POD_NAME=$(kubectl get pods --namespace minio -l "release=minio" -o jsonpath="{.items[0].metadata.name}") + +kubectl port-forward $POD_NAME 9000 --namespace minio +``` +Then create the bucket +```bash + +export AWS_ACCESS_KEY_ID=$(kubectl get secret minio -o jsonpath="{.data.accesskey}" -n minio | base64 --decode) +export AWS_SECRET_ACCESS_KEY=$(kubectl get secret minio -o jsonpath="{.data.secretkey}" -n minio | base64 --decode) +export AWS_DEFAULT_REGION=us-east-1 + +# Finally, create staging bucket +aws --endpoint-url http://localhost:9000 s3 mb s3://feast-staging +``` + +9. Use minikube docker to build a docker image with tests from your working copy. +```bash +eval $(minikube docker-env) +make build-local-test-docker +``` + +10. Finally, run tests: +```bash +./infra/scripts/run-minikube-test.sh +``` diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/e2e/conftest.py b/tests/e2e/conftest.py index 17524573dd3..e9c500b7734 100644 --- a/tests/e2e/conftest.py +++ b/tests/e2e/conftest.py @@ -9,7 +9,9 @@ def pytest_addoption(parser): parser.addoption("--job-service-url", action="store", default="localhost:6568") parser.addoption("--kafka-brokers", action="store", default="localhost:9092") - parser.addoption("--env", action="store", help="local|aws|gcloud", default="local") + parser.addoption( + "--env", action="store", help="local|aws|gcloud|k8s", default="local" + ) parser.addoption("--with-job-service", action="store_true") parser.addoption("--staging-path", action="store") parser.addoption("--dataproc-cluster-name", action="store") @@ -17,12 +19,23 @@ def pytest_addoption(parser): parser.addoption("--emr-cluster-id", action="store") parser.addoption("--emr-region", action="store") parser.addoption("--dataproc-project", action="store") + parser.addoption("--dataproc-executor-instances", action="store", default="2") + parser.addoption("--dataproc-executor-cores", action="store", default="2") + parser.addoption("--dataproc-executor-memory", action="store", default="2g") parser.addoption("--ingestion-jar", action="store") parser.addoption("--redis-url", action="store", default="localhost:6379") parser.addoption("--redis-cluster", action="store_true") parser.addoption("--feast-version", action="store") parser.addoption("--bq-project", action="store") parser.addoption("--feast-project", action="store", default="default") + parser.addoption("--statsd-url", action="store", default="localhost:8125") + parser.addoption("--prometheus-url", action="store", default="localhost:9102") + parser.addoption( + "--scheduled-streaming-job", + action="store_true", + help="When set tests won't manually start streaming jobs," + " instead jobservice's loop is responsible for that", + ) def pytest_runtest_setup(item): @@ -38,6 +51,7 @@ def pytest_runtest_setup(item): global_staging_path, ingestion_job_jar, local_staging_path, + tfrecord_feast_client, ) if not os.environ.get("DISABLE_SERVICE_FIXTURES"): @@ -45,10 +59,15 @@ def pytest_runtest_setup(item): kafka_port, kafka_server, redis_server, + statsd_server, zookeeper_server, ) else: - from .fixtures.external_services import kafka_server, redis_server # noqa + from .fixtures.external_services import ( # type: ignore # noqa + kafka_server, + redis_server, + statsd_server, + ) if not os.environ.get("DISABLE_FEAST_SERVICE_FIXTURES"): from .fixtures.feast_services import * # type: ignore # noqa diff --git a/tests/e2e/fixtures/base.py b/tests/e2e/fixtures/base.py index 68b9be5a4da..a68a45c91fd 100644 --- a/tests/e2e/fixtures/base.py +++ b/tests/e2e/fixtures/base.py @@ -1,3 +1,4 @@ +import xml.etree.ElementTree as ET from pathlib import Path import pytest @@ -9,8 +10,10 @@ def project_root(): @pytest.fixture(scope="session") -def project_version(pytestconfig): +def project_version(pytestconfig, project_root): if pytestconfig.getoption("feast_version"): return pytestconfig.getoption("feast_version") - return "0.8-SNAPSHOT" + pom_xml = ET.parse(project_root / "pom.xml") + root = pom_xml.getroot() + return root.find(".properties/revision").text diff --git a/tests/e2e/fixtures/client.py b/tests/e2e/fixtures/client.py index 264d5dc6874..e3e21229124 100644 --- a/tests/e2e/fixtures/client.py +++ b/tests/e2e/fixtures/client.py @@ -7,6 +7,7 @@ from pytest_redis.executor import RedisExecutor from feast import Client +from tests.e2e.fixtures.statsd_stub import StatsDServer @pytest.fixture @@ -14,6 +15,7 @@ def feast_client( pytestconfig, ingestion_job_jar, redis_server: RedisExecutor, + statsd_server: StatsDServer, feast_core: Tuple[str, int], feast_serving: Tuple[str, int], local_staging_path, @@ -43,6 +45,10 @@ def feast_client( historical_feature_output_location=os.path.join( local_staging_path, "historical_output" ), + ingestion_drop_invalid_rows=True, + statsd_enabled=True, + statsd_host=statsd_server.host, + statsd_port=statsd_server.port, **job_service_env, ) @@ -61,6 +67,8 @@ def feast_client( historical_feature_output_location=os.path.join( local_staging_path, "historical_output" ), + ingestion_drop_invalid_rows=True, + grpc_connection_timeout=30, **job_service_env, ) elif pytestconfig.getoption("env") == "aws": @@ -78,6 +86,96 @@ def feast_client( historical_feature_output_location=os.path.join( local_staging_path, "historical_output" ), + ingestion_drop_invalid_rows=True, + ) + elif pytestconfig.getoption("env") == "k8s": + return Client( + core_url=f"{feast_core[0]}:{feast_core[1]}", + serving_url=f"{feast_serving[0]}:{feast_serving[1]}", + spark_launcher="k8s", + spark_staging_location=os.path.join(local_staging_path, "k8s"), + spark_ingestion_jar=ingestion_job_jar, + redis_host=pytestconfig.getoption("redis_url").split(":")[0], + redis_port=pytestconfig.getoption("redis_url").split(":")[1], + historical_feature_output_location=os.path.join( + local_staging_path, "historical_output" + ), + ) + else: + raise KeyError(f"Unknown environment {pytestconfig.getoption('env')}") + + c.set_project(pytestconfig.getoption("feast_project")) + return c + + +@pytest.fixture +def tfrecord_feast_client( + pytestconfig, + feast_core: Tuple[str, int], + local_staging_path, + feast_jobservice: Optional[Tuple[str, int]], + enable_auth, +): + if feast_jobservice is None: + job_service_env = dict() + else: + job_service_env = dict( + job_service_url=f"{feast_jobservice[0]}:{feast_jobservice[1]}" + ) + + if pytestconfig.getoption("env") == "local": + import pyspark + + return Client( + core_url=f"{feast_core[0]}:{feast_core[1]}", + spark_launcher="standalone", + spark_standalone_master="local", + spark_home=os.getenv("SPARK_HOME") or os.path.dirname(pyspark.__file__), + spark_staging_location=os.path.join(local_staging_path, "spark"), + historical_feature_output_format="tfrecord", + historical_feature_output_location=os.path.join( + local_staging_path, "historical_output" + ), + **job_service_env, + ) + + elif pytestconfig.getoption("env") == "gcloud": + c = Client( + core_url=f"{feast_core[0]}:{feast_core[1]}", + spark_launcher="dataproc", + dataproc_cluster_name=pytestconfig.getoption("dataproc_cluster_name"), + dataproc_project=pytestconfig.getoption("dataproc_project"), + dataproc_region=pytestconfig.getoption("dataproc_region"), + spark_staging_location=os.path.join(local_staging_path, "dataproc"), + historical_feature_output_format="tfrecord", + historical_feature_output_location=os.path.join( + local_staging_path, "historical_output" + ), + ingestion_drop_invalid_rows=True, + **job_service_env, + ) + elif pytestconfig.getoption("env") == "aws": + return Client( + core_url=f"{feast_core[0]}:{feast_core[1]}", + spark_launcher="emr", + emr_cluster_id=pytestconfig.getoption("emr_cluster_id"), + emr_region=pytestconfig.getoption("emr_region"), + spark_staging_location=os.path.join(local_staging_path, "emr"), + emr_log_location=os.path.join(local_staging_path, "emr_logs"), + historical_feature_output_format="tfrecord", + historical_feature_output_location=os.path.join( + local_staging_path, "historical_output" + ), + ) + elif pytestconfig.getoption("env") == "k8s": + return Client( + core_url=f"{feast_core[0]}:{feast_core[1]}", + spark_launcher="k8s", + spark_staging_location=os.path.join(local_staging_path, "k8s"), + historical_feature_output_format="tfrecord", + historical_feature_output_location=os.path.join( + local_staging_path, "historical_output" + ), ) else: raise KeyError(f"Unknown environment {pytestconfig.getoption('env')}") diff --git a/tests/e2e/fixtures/external_services.py b/tests/e2e/fixtures/external_services.py index 8bf1c4b4b76..4edb68ac4e4 100644 --- a/tests/e2e/fixtures/external_services.py +++ b/tests/e2e/fixtures/external_services.py @@ -1,6 +1,8 @@ import pytest from pytest_redis.executor import NoopRedis +from tests.e2e.fixtures.statsd_stub import PrometheusStatsDServer + __all__ = ( "feast_core", "feast_serving", @@ -8,6 +10,7 @@ "kafka_server", "enable_auth", "feast_jobservice", + "statsd_server", ) @@ -44,3 +47,12 @@ def enable_auth(): def feast_jobservice(pytestconfig): host, port = pytestconfig.getoption("job_service_url").split(":") return host, port + + +@pytest.fixture(scope="session") +def statsd_server(pytestconfig): + host, port = pytestconfig.getoption("statsd_url").split(":") + prometheus_host, prometheus_port = pytestconfig.getoption("prometheus_url").split( + ":" + ) + return PrometheusStatsDServer(host, port, prometheus_host, prometheus_port) diff --git a/tests/e2e/fixtures/feast_services.py b/tests/e2e/fixtures/feast_services.py index 441864ba503..f2afd2bf6d5 100644 --- a/tests/e2e/fixtures/feast_services.py +++ b/tests/e2e/fixtures/feast_services.py @@ -47,7 +47,9 @@ def _wait_port_open(port, max_wait=60): return -@pytest.fixture(scope="session", params=[True, False]) +@pytest.fixture( + scope="session", params=[False], +) def enable_auth(request): return request.param @@ -185,6 +187,15 @@ def feast_jobservice( ) env["FEAST_DATAPROC_PROJECT"] = pytestconfig.getoption("dataproc_project") env["FEAST_DATAPROC_REGION"] = pytestconfig.getoption("dataproc_region") + env["FEAST_DATAPROC_EXECUTOR_INSTANCES"] = pytestconfig.getoption( + "dataproc_executor_instances" + ) + env["FEAST_DATAPROC_EXECUTOR_CORES"] = pytestconfig.getoption( + "dataproc_executor_cores" + ) + env["FEAST_DATAPROC_EXECUTOR_MEMORY"] = pytestconfig.getoption( + "dataproc_executor_memory" + ) env["FEAST_SPARK_STAGING_LOCATION"] = os.path.join( global_staging_path, "dataproc" ) diff --git a/tests/e2e/fixtures/services.py b/tests/e2e/fixtures/services.py index 38927a07299..aa242288781 100644 --- a/tests/e2e/fixtures/services.py +++ b/tests/e2e/fixtures/services.py @@ -1,7 +1,8 @@ +import os import pathlib import shutil -import tempfile +import port_for import pytest import requests from pytest_kafka import make_kafka_server, make_zookeeper_process @@ -14,16 +15,23 @@ "zookeeper_server", "postgres_server", "redis_server", + "statsd_server", ) +from tests.e2e.fixtures.statsd_stub import StatsDStub + def download_kafka(version="2.12-2.6.0"): - r = requests.get(f"https://downloads.apache.org/kafka/2.6.0/kafka_{version}.tgz") - temp_dir = pathlib.Path(tempfile.mkdtemp()) - local_path = temp_dir / "kafka.tgz" + temp_dir = pathlib.Path("/tmp") + local_path = temp_dir / f"kafka_{version}.tgz" + + if not os.path.isfile(local_path): + r = requests.get( + f"https://downloads.apache.org/kafka/2.6.0/kafka_{version}.tgz" + ) - with open(local_path, "wb") as f: - f.write(r.content) + with open(local_path, "wb") as f: + f.write(r.content) shutil.unpack_archive(str(local_path), str(temp_dir)) return temp_dir / f"kafka_{version}" / "bin" @@ -35,8 +43,19 @@ def kafka_server(kafka_port): return "localhost", port +@pytest.fixture +def statsd_server(): + port = port_for.select_random(None) + server = StatsDStub(port=port) + server.start() + yield server + server.stop() + + postgres_server = pg_factories.postgresql_proc(password="password") -redis_server = redis_factories.redis_proc(executable=shutil.which("redis-server")) +redis_server = redis_factories.redis_proc( + executable=shutil.which("redis-server"), timeout=3600 +) KAFKA_BIN = download_kafka() zookeeper_server = make_zookeeper_process( diff --git a/tests/e2e/fixtures/statsd_stub.py b/tests/e2e/fixtures/statsd_stub.py new file mode 100644 index 00000000000..ff85cb6edbd --- /dev/null +++ b/tests/e2e/fixtures/statsd_stub.py @@ -0,0 +1,115 @@ +import select +import socket +import threading +from collections import defaultdict, namedtuple +from typing import Dict + +import requests + +MetricLine = namedtuple("MetricLine", ["name", "value", "type"]) + + +class StatsDServer: + host: str + port: int + metrics: Dict[str, int] + + +class PrometheusStatsDServer(StatsDServer): + def __init__( + self, + statsd_host: str, + statsd_port: int, + prometheus_host: str, + prometheus_port: int, + ): + self.host = statsd_host + self.port = statsd_port + + self.prometheus_host = prometheus_host + self.prometheus_port = prometheus_port + + @property + def metrics(self): + """ Parse Prometheus response into metrics dict """ + + data = requests.get( + f"http://{self.prometheus_host}:{self.prometheus_port}/metrics" + ).content.decode() + lines = [line for line in data.split("\n") if not line.startswith("#")] + metrics = {} + for line in lines: + if not line: + continue + + name, value = line.split(" ") + + try: + value = int(value) # type: ignore + except ValueError: + value = float(value) # type: ignore + + if "{" in name and "}" in name: + base = name[: name.index("{")] + tags = name[name.index("{") + 1 : -1] + tags = [tag.split("=") for tag in tags.split(",")] + tags = [(key, val.replace('"', "")) for key, val in tags] + + name = base + "#" + ",".join(f"{k}:{v}" for k, v in sorted(tags)) + + metrics[name] = value + + return metrics + + +def parse_metric_line(line: str) -> MetricLine: + parts = line.split("|") + name, value = parts[0].split(":") + type_ = parts[1] + + try: + value = int(value) # type: ignore + except ValueError: + value = float(value) # type: ignore + + if len(parts) == 3 and parts[2].startswith("#"): + # Add tags to name + tags = sorted(parts[2][1:].split(",")) + name = name + "#" + ",".join(tags) + + return MetricLine(name, value, type_) + + +class StatsDStub(StatsDServer): + def __init__(self, port: int): + self.host = "localhost" + self.port = port + + self._stop_event = threading.Event() + self.metrics = defaultdict(lambda: 0) + + def serve(self): + sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + sock.bind((self.host, self.port)) + sock.setblocking(False) + + while True: + ready = select.select([sock], [], [], 1) + if ready[0]: + data = sock.recv(65535) + + lines = data.decode("utf-8").split("\n") + for line in lines: + print("Metric received:", line) + m = parse_metric_line(line) + self.metrics[m.name] += m.value + + if self._stop_event.wait(0.01): + break + + def start(self): + t = threading.Thread(target=self.serve) + t.start() + + def stop(self): + self._stop_event.set() diff --git a/tests/e2e/test_historical_features.py b/tests/e2e/test_historical_features.py index ac65cbde863..38d6ecb47c6 100644 --- a/tests/e2e/test_historical_features.py +++ b/tests/e2e/test_historical_features.py @@ -1,6 +1,7 @@ +import os from datetime import datetime, timedelta from typing import Union -from urllib.parse import urlparse +from urllib.parse import urlparse, urlunparse import gcsfs import numpy as np @@ -10,12 +11,14 @@ from pyarrow import parquet from feast import Client, Entity, Feature, FeatureTable, ValueType +from feast.constants import ConfigOptions as opt from feast.data_source import BigQuerySource, FileSource +from feast.pyspark.abc import SparkJobStatus np.random.seed(0) -def read_parquet(uri): +def read_parquet(uri, azure_account_name=None, azure_account_key=None): parsed_uri = urlparse(uri) if parsed_uri.scheme == "file": return pd.read_parquet(parsed_uri.path) @@ -24,11 +27,30 @@ def read_parquet(uri): files = ["gs://" + path for path in fs.glob(uri + "/part-*")] ds = parquet.ParquetDataset(files, filesystem=fs) return ds.read().to_pandas() - elif parsed_uri.scheme == "s3": + elif parsed_uri.scheme == "s3" or parsed_uri.scheme == "s3a": + + s3uri = urlunparse(parsed_uri._replace(scheme="s3")) + import s3fs - fs = s3fs.S3FileSystem() - files = ["s3://" + path for path in fs.glob(uri + "/part-*")] + # AWS_S3_ENDPOINT_URL needs to be set when using minio + if "AWS_S3_ENDPOINT_URL" in os.environ: + fs = s3fs.S3FileSystem( + client_kwargs={"endpoint_url": os.getenv("AWS_S3_ENDPOINT_URL")} + ) + else: + fs = s3fs.S3FileSystem() + files = ["s3://" + path for path in fs.glob(s3uri + "/part-*")] + ds = parquet.ParquetDataset(files, filesystem=fs) + return ds.read().to_pandas() + elif parsed_uri.scheme == "wasbs": + import adlfs + + fs = adlfs.AzureBlobFileSystem( + account_name=azure_account_name, account_key=azure_account_key + ) + uripath = parsed_uri.username + parsed_uri.path + files = fs.glob(uripath + "/part-*") ds = parquet.ParquetDataset(files, filesystem=fs) return ds.read().to_pandas() else: @@ -36,11 +58,7 @@ def read_parquet(uri): def generate_data(): - retrieval_date = ( - datetime.utcnow() - .replace(hour=0, minute=0, second=0, microsecond=0) - .replace(tzinfo=None) - ) + retrieval_date = datetime.utcnow().replace(tzinfo=None) retrieval_outside_max_age_date = retrieval_date + timedelta(1) event_date = retrieval_date - timedelta(2) creation_date = retrieval_date - timedelta(1) @@ -68,13 +86,22 @@ def generate_data(): return transactions_df, customer_df +def _get_azure_creds(feast_client: Client): + return ( + feast_client._config.get(opt.AZURE_BLOB_ACCOUNT_NAME, None), + feast_client._config.get(opt.AZURE_BLOB_ACCOUNT_ACCESS_KEY, None), + ) + + def test_historical_features( - feast_client: Client, batch_source: Union[BigQuerySource, FileSource] + feast_client: Client, + tfrecord_feast_client: Client, + batch_source: Union[BigQuerySource, FileSource], ): customer_entity = Entity( name="user_id", description="Customer", value_type=ValueType.INT64 ) - feast_client.apply_entity(customer_entity) + feast_client.apply(customer_entity) max_age = Duration() max_age.FromSeconds(2 * 86400) @@ -90,16 +117,27 @@ def test_historical_features( max_age=max_age, ) - feast_client.apply_feature_table(transactions_feature_table) + feast_client.apply(transactions_feature_table) transactions_df, customers_df = generate_data() feast_client.ingest(transactions_feature_table, transactions_df) feature_refs = ["transactions:daily_transactions"] + # remove microseconds because job.get_start_time() does not contain microseconds + job_submission_time = datetime.utcnow().replace(microsecond=0) job = feast_client.get_historical_features(feature_refs, customers_df) + assert job.get_start_time() >= job_submission_time + assert job.get_start_time() <= job_submission_time + timedelta(hours=1) + output_dir = job.get_output_file_uri() - joined_df = read_parquet(output_dir) + + # will both be None if not using Azure blob storage + account_name, account_key = _get_azure_creds(feast_client) + + joined_df = read_parquet( + output_dir, azure_account_name=account_name, azure_account_key=account_key + ) expected_joined_df = pd.DataFrame( { @@ -116,3 +154,7 @@ def test_historical_features( drop=True ), ) + + job = tfrecord_feast_client.get_historical_features(feature_refs, customers_df) + job.get_output_file_uri() + assert job.get_status() == SparkJobStatus.COMPLETED diff --git a/tests/e2e/test_online_features.py b/tests/e2e/test_online_features.py index 9e927d5c59a..09d77e145f9 100644 --- a/tests/e2e/test_online_features.py +++ b/tests/e2e/test_online_features.py @@ -1,4 +1,3 @@ -import io import json import os import time @@ -6,15 +5,10 @@ from datetime import datetime, timedelta from typing import Union -import avro.schema import numpy as np import pandas as pd import pytest -import pytz -from avro.io import BinaryEncoder, DatumWriter from google.cloud import bigquery -from kafka.admin import KafkaAdminClient -from kafka.producer import KafkaProducer from feast import ( BigQuerySource, @@ -29,6 +23,7 @@ from feast.data_format import AvroFormat, ParquetFormat from feast.pyspark.abc import SparkJobStatus from feast.wait import wait_retry_backoff +from tests.e2e.utils.kafka import check_consumer_exist, ingest_and_retrieve def generate_data(): @@ -55,8 +50,8 @@ def test_offline_ingestion( batch_source=batch_source, ) - feast_client.apply_entity(entity) - feast_client.apply_feature_table(feature_table) + feast_client.apply(entity) + feast_client.apply(feature_table) original = generate_data() feast_client.ingest(feature_table, original) # write to batch (offline) storage @@ -95,14 +90,14 @@ def test_offline_ingestion_from_bq_view(pytestconfig, bq_dataset, feast_client: ), ) - feast_client.apply_entity(entity) - feast_client.apply_feature_table(feature_table) + feast_client.apply(entity) + feast_client.apply(feature_table) ingest_and_verify(feast_client, feature_table, original) def test_streaming_ingestion( - feast_client: Client, local_staging_path: str, kafka_server + feast_client: Client, local_staging_path: str, kafka_server, pytestconfig ): entity = Entity(name="s2id", description="S2id", value_type=ValueType.INT64,) kafka_broker = f"{kafka_server[0]}:{kafka_server[1]}" @@ -126,48 +121,43 @@ def test_streaming_ingestion( ), ) - feast_client.apply_entity(entity) - feast_client.apply_feature_table(feature_table) + feast_client.apply(entity) + feast_client.apply(feature_table) - job = feast_client.start_stream_to_online_ingestion(feature_table) + if not pytestconfig.getoption("scheduled_streaming_job"): + job = feast_client.start_stream_to_online_ingestion(feature_table) + assert job.get_feature_table() == feature_table.name + wait_retry_backoff( + lambda: (None, job.get_status() == SparkJobStatus.IN_PROGRESS), 180 + ) + else: + job = None wait_retry_backoff( - lambda: (None, job.get_status() == SparkJobStatus.IN_PROGRESS), 120 + lambda: (None, check_consumer_exist(kafka_broker, topic_name)), 300 ) - wait_retry_backoff( - lambda: (None, check_consumer_exist(kafka_broker, topic_name)), 120 - ) + test_data = generate_data()[["s2id", "unique_drivers", "event_timestamp"]] try: - original = generate_data()[["s2id", "unique_drivers", "event_timestamp"]] - for record in original.to_dict("records"): - record["event_timestamp"] = ( - record["event_timestamp"].to_pydatetime().replace(tzinfo=pytz.utc) - ) - - send_avro_record_to_kafka( - topic_name, - record, - bootstrap_servers=kafka_broker, - avro_schema_json=avro_schema(), - ) - - def get_online_features(): - features = feast_client.get_online_features( - ["drivers_stream:unique_drivers"], - entity_rows=[{"s2id": s2_id} for s2_id in original["s2id"].tolist()], - ).to_dict() - df = pd.DataFrame.from_dict(features) - return df, not df["drivers_stream:unique_drivers"].isna().any() - - ingested = wait_retry_backoff(get_online_features, 60) + ingested = ingest_and_retrieve( + feast_client, + test_data, + avro_schema_json=avro_schema(), + topic_name=topic_name, + kafka_broker=kafka_broker, + entity_rows=[{"s2id": s2_id} for s2_id in test_data["s2id"].tolist()], + feature_names=["drivers_stream:unique_drivers"], + ) finally: - job.cancel() + if job: + job.cancel() + else: + feast_client.delete_feature_table(feature_table.name) pd.testing.assert_frame_equal( ingested[["s2id", "drivers_stream:unique_drivers"]], - original[["s2id", "unique_drivers"]].rename( + test_data[["s2id", "unique_drivers"]].rename( columns={"unique_drivers": "drivers_stream:unique_drivers"} ), ) @@ -181,6 +171,7 @@ def ingest_and_verify( original.event_timestamp.min().to_pydatetime(), original.event_timestamp.max().to_pydatetime() + timedelta(seconds=1), ) + assert job.get_feature_table() == feature_table.name wait_retry_backoff( lambda: (None, job.get_status() == SparkJobStatus.COMPLETED), 180 @@ -200,6 +191,42 @@ def ingest_and_verify( ) +def test_list_jobs_long_table_name( + feast_client: Client, batch_source: Union[BigQuerySource, FileSource] +): + entity = Entity(name="s2id", description="S2id", value_type=ValueType.INT64,) + + feature_table = FeatureTable( + name="just1a2featuretable3with4a5really6really7really8really9really10really11really12long13name", + entities=["s2id"], + features=[Feature("unique_drivers", ValueType.INT64)], + batch_source=batch_source, + ) + + feast_client.apply(entity) + feast_client.apply(feature_table) + + data_sample = generate_data() + feast_client.ingest(feature_table, data_sample) + + job = feast_client.start_offline_to_online_ingestion( + feature_table, + data_sample.event_timestamp.min().to_pydatetime(), + data_sample.event_timestamp.max().to_pydatetime() + timedelta(seconds=1), + ) + + wait_retry_backoff( + lambda: (None, job.get_status() == SparkJobStatus.COMPLETED), 180 + ) + all_job_ids = [ + job.get_id() + for job in feast_client.list_jobs( + include_terminated=True, table_name=feature_table.name + ) + ] + assert job.get_id() in all_job_ids + + def avro_schema(): return json.dumps( { @@ -215,45 +242,3 @@ def avro_schema(): ], } ) - - -def send_avro_record_to_kafka(topic, value, bootstrap_servers, avro_schema_json): - value_schema = avro.schema.parse(avro_schema_json) - - producer = KafkaProducer(bootstrap_servers=bootstrap_servers) - - writer = DatumWriter(value_schema) - bytes_writer = io.BytesIO() - encoder = BinaryEncoder(bytes_writer) - - writer.write(value, encoder) - - try: - producer.send(topic=topic, value=bytes_writer.getvalue()) - except Exception as e: - print( - f"Exception while producing record value - {value} to topic - {topic}: {e}" - ) - else: - print(f"Successfully producing record value - {value} to topic - {topic}") - - producer.flush() - - -def check_consumer_exist(bootstrap_servers, topic_name): - admin = KafkaAdminClient(bootstrap_servers=bootstrap_servers) - consumer_groups = admin.describe_consumer_groups( - group_ids=[ - group_id - for group_id, _ in admin.list_consumer_groups() - if group_id.startswith("spark-kafka-source") - ] - ) - subscriptions = { - subscription - for group in consumer_groups - for member in group.members - if not isinstance(member.member_metadata, bytes) - for subscription in member.member_metadata.subscription - } - return topic_name in subscriptions diff --git a/tests/e2e/test_register.py b/tests/e2e/test_register.py index b2faa91b065..23a2adfc42e 100644 --- a/tests/e2e/test_register.py +++ b/tests/e2e/test_register.py @@ -143,8 +143,8 @@ def test_get_list_basic( ): # ApplyEntity - feast_client.apply_entity(customer_entity) - feast_client.apply_entity(driver_entity) + feast_client.apply(customer_entity) + feast_client.apply(driver_entity) # GetEntity Check assert feast_client.get_entity(name="customer_id") == customer_entity @@ -162,7 +162,7 @@ def test_get_list_basic( assert len(actual_matchmaking_entities) == 1 # ApplyFeatureTable - feast_client.apply_feature_table(basic_featuretable) + feast_client.apply(basic_featuretable) # GetFeatureTable Check actual_get_feature_table = feast_client.get_feature_table(name="basic_featuretable") @@ -181,7 +181,7 @@ def test_get_list_alltypes( feast_client: Client, alltypes_entity: Entity, alltypes_featuretable: FeatureTable ): # ApplyEntity - feast_client.apply_entity(alltypes_entity) + feast_client.apply(alltypes_entity) # GetEntity Check assert feast_client.get_entity(name="alltypes_id") == alltypes_entity @@ -194,7 +194,7 @@ def test_get_list_alltypes( assert len(actual_alltypes_entities) == 1 # ApplyFeatureTable - feast_client.apply_feature_table(alltypes_featuretable) + feast_client.apply(alltypes_featuretable) # GetFeatureTable Check actual_get_feature_table = feast_client.get_feature_table(name="alltypes") @@ -234,11 +234,11 @@ def test_ingest_into_bq( ) # ApplyEntity - feast_client.apply_entity(customer_entity) - feast_client.apply_entity(driver_entity) + feast_client.apply(customer_entity) + feast_client.apply(driver_entity) # ApplyFeatureTable - feast_client.apply_feature_table(ft) + feast_client.apply(ft) feast_client.ingest(ft, bq_dataframe, timeout=120) bq_client = bigquery.Client(project=bq_project) diff --git a/tests/e2e/test_validation.py b/tests/e2e/test_validation.py new file mode 100644 index 00000000000..48bc5611b86 --- /dev/null +++ b/tests/e2e/test_validation.py @@ -0,0 +1,180 @@ +import time +import uuid + +import numpy as np +import pandas as pd +import pytest +from great_expectations.dataset import PandasDataset + +from feast import Client +from feast.contrib.validation.ge import apply_validation, create_validation_udf +from feast.wait import wait_retry_backoff +from tests.e2e.fixtures.statsd_stub import StatsDServer +from tests.e2e.utils.common import avro_schema, create_schema, start_job, stop_job +from tests.e2e.utils.kafka import check_consumer_exist, ingest_and_retrieve + + +def generate_train_data(): + df = pd.DataFrame(columns=["key", "num", "set", "event_timestamp"]) + df["key"] = np.random.choice(999999, size=100, replace=False) + df["num"] = np.random.randint(0, 100, 100) + df["set"] = np.random.choice(["a", "b", "c"], size=100) + df["event_timestamp"] = pd.to_datetime(int(time.time()), unit="s") + + return df + + +def generate_test_data(): + df = pd.DataFrame(columns=["key", "num", "set", "event_timestamp"]) + df["key"] = np.random.choice(999999, size=100, replace=False) + df["num"] = np.random.randint(0, 150, 100) + df["set"] = np.random.choice(["a", "b", "c", "d"], size=100) + df["event_timestamp"] = pd.to_datetime(int(time.time()), unit="s") + + return df + + +def test_validation_with_ge(feast_client: Client, kafka_server, pytestconfig): + kafka_broker = f"{kafka_server[0]}:{kafka_server[1]}" + topic_name = f"avro-{uuid.uuid4()}" + + entity, feature_table = create_schema(kafka_broker, topic_name, "validation_ge") + feast_client.apply_entity(entity) + feast_client.apply_feature_table(feature_table) + + train_data = generate_train_data() + ge_ds = PandasDataset(train_data) + ge_ds.expect_column_values_to_be_between("num", 0, 100) + ge_ds.expect_column_values_to_be_in_set("set", ["a", "b", "c"]) + expectations = ge_ds.get_expectation_suite() + + udf = create_validation_udf("testUDF", expectations, feature_table) + apply_validation(feast_client, feature_table, udf, validation_window_secs=1) + + job = start_job(feast_client, feature_table, pytestconfig) + + wait_retry_backoff( + lambda: (None, check_consumer_exist(kafka_broker, topic_name)), 300 + ) + + test_data = generate_test_data() + ge_ds = PandasDataset(test_data) + validation_result = ge_ds.validate(expectations, result_format="COMPLETE") + invalid_idx = list( + { + idx + for check in validation_result.results + for idx in check.result["unexpected_index_list"] + } + ) + + entity_rows = [{"key": key} for key in test_data["key"].tolist()] + + try: + ingested = ingest_and_retrieve( + feast_client, + test_data, + avro_schema_json=avro_schema(), + topic_name=topic_name, + kafka_broker=kafka_broker, + entity_rows=entity_rows, + feature_names=["validation_ge:num", "validation_ge:set"], + expected_ingested_count=test_data.shape[0] - len(invalid_idx), + ) + finally: + stop_job(job, feast_client, feature_table) + + test_data["num"] = test_data["num"].astype(np.float64) + test_data["num"].iloc[invalid_idx] = np.nan + test_data["set"].iloc[invalid_idx] = None + + pd.testing.assert_frame_equal( + ingested[["key", "validation_ge:num", "validation_ge:set"]], + test_data[["key", "num", "set"]].rename( + columns={"num": "validation_ge:num", "set": "validation_ge:set"} + ), + ) + + +@pytest.mark.env("local") +def test_validation_reports_metrics( + feast_client: Client, kafka_server, statsd_server: StatsDServer, pytestconfig +): + kafka_broker = f"{kafka_server[0]}:{kafka_server[1]}" + topic_name = f"avro-{uuid.uuid4()}" + + entity, feature_table = create_schema( + kafka_broker, topic_name, "validation_ge_metrics" + ) + feast_client.apply_entity(entity) + feast_client.apply_feature_table(feature_table) + + train_data = generate_train_data() + ge_ds = PandasDataset(train_data) + ge_ds.expect_column_values_to_be_between("num", 0, 100) + ge_ds.expect_column_values_to_be_in_set("set", ["a", "b", "c"]) + expectations = ge_ds.get_expectation_suite() + + udf = create_validation_udf("testUDF", expectations, feature_table) + apply_validation(feast_client, feature_table, udf, validation_window_secs=10) + + job = start_job(feast_client, feature_table, pytestconfig) + + wait_retry_backoff( + lambda: (None, check_consumer_exist(kafka_broker, topic_name)), 300 + ) + + test_data = generate_test_data() + ge_ds = PandasDataset(test_data) + validation_result = ge_ds.validate(expectations, result_format="COMPLETE") + unexpected_counts = { + "expect_column_values_to_be_between_num_0_100": validation_result.results[ + 0 + ].result["unexpected_count"], + "expect_column_values_to_be_in_set_set": validation_result.results[1].result[ + "unexpected_count" + ], + } + invalid_idx = list( + { + idx + for check in validation_result.results + for idx in check.result["unexpected_index_list"] + } + ) + + entity_rows = [{"key": key} for key in test_data["key"].tolist()] + + try: + ingest_and_retrieve( + feast_client, + test_data, + avro_schema_json=avro_schema(), + topic_name=topic_name, + kafka_broker=kafka_broker, + entity_rows=entity_rows, + feature_names=["validation_ge_metrics:num", "validation_ge_metrics:set"], + expected_ingested_count=test_data.shape[0] - len(invalid_idx), + ) + finally: + stop_job(job, feast_client, feature_table) + + expected_metrics = [ + ( + f"feast_feature_validation_check_failed#check:{check_name}," + f"feature_table:{feature_table.name},project:{feast_client.project}", + value, + ) + for check_name, value in unexpected_counts.items() + ] + wait_retry_backoff( + lambda: ( + None, + all(statsd_server.metrics.get(m) == v for m, v in expected_metrics), + ), + timeout_secs=30, + timeout_msg="Expected metrics were not received: " + + str(expected_metrics) + + "\n" + "Actual received metrics" + str(statsd_server.metrics), + ) diff --git a/tests/e2e/utils/__init__.py b/tests/e2e/utils/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/e2e/utils/common.py b/tests/e2e/utils/common.py new file mode 100644 index 00000000000..4663a610fde --- /dev/null +++ b/tests/e2e/utils/common.py @@ -0,0 +1,71 @@ +import json + +from feast import ( + Client, + Entity, + Feature, + FeatureTable, + FileSource, + KafkaSource, + ValueType, +) +from feast.data_format import AvroFormat, ParquetFormat +from feast.pyspark.abc import SparkJobStatus +from feast.wait import wait_retry_backoff + + +def create_schema(kafka_broker, topic_name, feature_table_name): + entity = Entity(name="key", description="Key", value_type=ValueType.INT64) + feature_table = FeatureTable( + name=feature_table_name, + entities=["key"], + features=[Feature("num", ValueType.INT64), Feature("set", ValueType.STRING)], + batch_source=FileSource( + event_timestamp_column="event_timestamp", + file_format=ParquetFormat(), + file_url="/dev/null", + ), + stream_source=KafkaSource( + event_timestamp_column="event_timestamp", + bootstrap_servers=kafka_broker, + message_format=AvroFormat(avro_schema()), + topic=topic_name, + ), + ) + return entity, feature_table + + +def start_job(feast_client: Client, feature_table: FeatureTable, pytestconfig): + if pytestconfig.getoption("scheduled_streaming_job"): + return + + job = feast_client.start_stream_to_online_ingestion(feature_table) + wait_retry_backoff( + lambda: (None, job.get_status() == SparkJobStatus.IN_PROGRESS), 180 + ) + return job + + +def stop_job(job, feast_client: Client, feature_table: FeatureTable): + if job: + job.cancel() + else: + feast_client.delete_feature_table(feature_table.name) + + +def avro_schema(): + return json.dumps( + { + "type": "record", + "name": "TestMessage", + "fields": [ + {"name": "key", "type": "long"}, + {"name": "num", "type": "long"}, + {"name": "set", "type": "string"}, + { + "name": "event_timestamp", + "type": {"type": "long", "logicalType": "timestamp-micros"}, + }, + ], + } + ) diff --git a/tests/e2e/utils/kafka.py b/tests/e2e/utils/kafka.py new file mode 100644 index 00000000000..130a59d50ad --- /dev/null +++ b/tests/e2e/utils/kafka.py @@ -0,0 +1,88 @@ +import io +from typing import Any, Dict, List, Optional + +import avro.schema +import pandas as pd +import pytz +from avro.io import BinaryEncoder, DatumWriter +from kafka import KafkaAdminClient, KafkaProducer + +from feast import Client +from feast.wait import wait_retry_backoff + + +def send_avro_record_to_kafka(topic, value, bootstrap_servers, avro_schema_json): + value_schema = avro.schema.parse(avro_schema_json) + + producer = KafkaProducer(bootstrap_servers=bootstrap_servers) + + writer = DatumWriter(value_schema) + bytes_writer = io.BytesIO() + encoder = BinaryEncoder(bytes_writer) + + writer.write(value, encoder) + + try: + producer.send(topic=topic, value=bytes_writer.getvalue()) + except Exception as e: + print( + f"Exception while producing record value - {value} to topic - {topic}: {e}" + ) + else: + print(f"Successfully producing record value - {value} to topic - {topic}") + + producer.flush() + + +def check_consumer_exist(bootstrap_servers, topic_name): + admin = KafkaAdminClient(bootstrap_servers=bootstrap_servers) + consumer_groups = admin.describe_consumer_groups( + group_ids=[ + group_id + for group_id, _ in admin.list_consumer_groups() + if group_id.startswith("spark-kafka-source") + ] + ) + subscriptions = { + subscription + for group in consumer_groups + for member in group.members + if not isinstance(member.member_metadata, bytes) + for subscription in member.member_metadata.subscription + } + return topic_name in subscriptions + + +def ingest_and_retrieve( + feast_client: Client, + df: pd.DataFrame, + topic_name: str, + kafka_broker: str, + avro_schema_json: str, + entity_rows: List[Dict[str, Any]], + feature_names: List[Any], + expected_ingested_count: Optional[int] = None, +): + expected_ingested_count = expected_ingested_count or df.shape[0] + + for record in df.to_dict("records"): + record["event_timestamp"] = ( + record["event_timestamp"].to_pydatetime().replace(tzinfo=pytz.utc) + ) + + send_avro_record_to_kafka( + topic_name, + record, + bootstrap_servers=kafka_broker, + avro_schema_json=avro_schema_json, + ) + + def get_online_features(): + features = feast_client.get_online_features( + feature_names, entity_rows=entity_rows, + ).to_dict() + out_df = pd.DataFrame.from_dict(features) + return out_df, out_df[feature_names].count().min() >= expected_ingested_count + + ingested = wait_retry_backoff(get_online_features, 180) + return ingested diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 88ab8743a50..a80e6948295 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -3,5 +3,8 @@ def pytest_addoption(parser): parser.addoption("--dataproc-region", action="store") parser.addoption("--dataproc-project", action="store") parser.addoption("--dataproc-staging-location", action="store") + parser.addoption("--dataproc-executor-instances", action="store", default="2") + parser.addoption("--dataproc-executor-cores", action="store", default="2") + parser.addoption("--dataproc-executor-memory", action="store", default="2g") parser.addoption("--redis-url", action="store") parser.addoption("--redis-cluster", action="store_true") diff --git a/tests/integration/fixtures/job_parameters.py b/tests/integration/fixtures/job_parameters.py index e1024354ae8..bf7e222ad87 100644 --- a/tests/integration/fixtures/job_parameters.py +++ b/tests/integration/fixtures/job_parameters.py @@ -17,7 +17,7 @@ def customer_entity() -> pd.DataFrame: return pd.DataFrame( np.array([[1001, datetime(year=2020, month=9, day=1, tzinfo=utc)]]), - columns=["customer_id", "event_timestamp"], + columns=["customer", "event_timestamp"], ) @@ -35,7 +35,7 @@ def customer_feature() -> pd.DataFrame: ] ), columns=[ - "customer_id", + "customer", "total_transactions", "event_timestamp", "created_timestamp", @@ -59,11 +59,14 @@ def upload_dataframe_to_gcs_as_parquet(df: pd.DataFrame, staging_location: str): def new_retrieval_job_params( - entity_source_uri: str, feature_source_uri: str, destination_uri: str + entity_source_uri: str, + feature_source_uri: str, + destination_uri: str, + output_format: str, ) -> RetrievalJobParameters: entity_source = { "file": { - "format": "parquet", + "format": {"json_class": "ParquetFormat"}, "path": entity_source_uri, "event_timestamp_column": "event_timestamp", } @@ -72,7 +75,7 @@ def new_retrieval_job_params( feature_tables_sources = [ { "file": { - "format": "parquet", + "format": {"json_class": "ParquetFormat"}, "path": feature_source_uri, "event_timestamp_column": "event_timestamp", "created_timestamp_column": "created_timestamp", @@ -83,17 +86,19 @@ def new_retrieval_job_params( feature_tables = [ { "name": "customer_transactions", - "entities": [{"name": "customer", "type": "int32"}], + "entities": [{"name": "customer", "type": "int64"}], + "features": [{"name": "total_transactions", "type": "double"}], } ] - destination = {"format": "parquet", "path": destination_uri} + destination = {"format": output_format, "path": destination_uri} return RetrievalJobParameters( feature_tables=feature_tables, feature_tables_sources=feature_tables_sources, entity_source=entity_source, destination=destination, + extra_packages=["com.linkedin.sparktfrecord:spark-tfrecord_2.12:0.3.0"], ) @@ -111,5 +116,23 @@ def dataproc_retrieval_job_params( destination_uri = path.join(staging_location, str(uuid.uuid4())) return new_retrieval_job_params( - entity_source_uri, feature_source_uri, destination_uri + entity_source_uri, feature_source_uri, destination_uri, "parquet" + ) + + +@pytest.fixture(scope="module") +def dataproc_retrieval_job_params_with_tfrecord_output( + pytestconfig, customer_entity, customer_feature +) -> RetrievalJobParameters: + staging_location = pytestconfig.getoption("--dataproc-staging-location") + entity_source_uri = upload_dataframe_to_gcs_as_parquet( + customer_entity, staging_location + ) + feature_source_uri = upload_dataframe_to_gcs_as_parquet( + customer_feature, staging_location + ) + destination_uri = path.join(staging_location, str(uuid.uuid4())) + + return new_retrieval_job_params( + entity_source_uri, feature_source_uri, destination_uri, "tfrecord" ) diff --git a/tests/integration/fixtures/launchers.py b/tests/integration/fixtures/launchers.py index ebe93172d1e..d289d974ac4 100644 --- a/tests/integration/fixtures/launchers.py +++ b/tests/integration/fixtures/launchers.py @@ -9,9 +9,15 @@ def dataproc_launcher(pytestconfig) -> DataprocClusterLauncher: region = pytestconfig.getoption("--dataproc-region") project_id = pytestconfig.getoption("--dataproc-project") staging_location = pytestconfig.getoption("--dataproc-staging-location") + executor_instances = pytestconfig.getoption("dataproc_executor_instances") + executor_cores = pytestconfig.getoption("dataproc_executor_cores") + executor_memory = pytestconfig.getoption("dataproc_executor_memory") return DataprocClusterLauncher( cluster_name=cluster_name, staging_location=staging_location, region=region, project_id=project_id, + executor_instances=executor_instances, + executor_cores=executor_cores, + executor_memory=executor_memory, ) diff --git a/tests/integration/test_launchers.py b/tests/integration/test_launchers.py index 4b80afe054d..44a6fd22ed2 100644 --- a/tests/integration/test_launchers.py +++ b/tests/integration/test_launchers.py @@ -6,16 +6,25 @@ from .fixtures.job_parameters import customer_entity # noqa: F401 from .fixtures.job_parameters import customer_feature # noqa: F401 from .fixtures.job_parameters import dataproc_retrieval_job_params # noqa: F401 +from .fixtures.job_parameters import ( # noqa: F401 + dataproc_retrieval_job_params_with_tfrecord_output +) from .fixtures.launchers import dataproc_launcher # noqa: F401 -def wait_for_job_status(job: SparkJob, expected_status: SparkJobStatus, max_retry: int = 4, retry_interval: int = 5): +def wait_for_job_status( + job: SparkJob, + expected_status: SparkJobStatus, + max_retry: int = 4, + retry_interval: int = 5, +): for i in range(max_retry): if job.get_status() == expected_status: return time.sleep(retry_interval) raise ValueError(f"Timeout waiting for job status to become {expected_status.name}") + def test_dataproc_job_api( dataproc_launcher: DataprocClusterLauncher, # noqa: F811 dataproc_retrieval_job_params: RetrievalJobParameters, # noqa: F811 @@ -23,6 +32,7 @@ def test_dataproc_job_api( job = dataproc_launcher.historical_feature_retrieval(dataproc_retrieval_job_params) job_id = job.get_id() retrieved_job = dataproc_launcher.get_job_by_id(job_id) + assert retrieved_job.get_log_uri is not None assert retrieved_job.get_id() == job_id status = retrieved_job.get_status() assert status in [ @@ -37,3 +47,14 @@ def test_dataproc_job_api( wait_for_job_status(retrieved_job, SparkJobStatus.IN_PROGRESS) retrieved_job.cancel() assert retrieved_job.get_status() == SparkJobStatus.FAILED + + +def test_dataproc_job_tfrecord_output( + dataproc_launcher: DataprocClusterLauncher, # noqa: F811 + dataproc_retrieval_job_params_with_tfrecord_output: RetrievalJobParameters, # noqa: F811 +): + job = dataproc_launcher.historical_feature_retrieval( + dataproc_retrieval_job_params_with_tfrecord_output + ) + job.get_output_file_uri() + assert job.get_status() == SparkJobStatus.COMPLETED