diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml index 8e52ba12c9e..a538970714f 100644 --- a/.github/workflows/build_wheels.yml +++ b/.github/workflows/build_wheels.yml @@ -5,51 +5,25 @@ name: build_wheels # Devs should check out their fork, add a tag to the last master commit on their fork, and run the release off of their fork on the added tag to ensure wheels will be built correctly. on: workflow_dispatch: - tags: - - 'v*.*.*' + tags: + - 'v*.*.*' workflow_call: + inputs: + release_version: + description: 'The release version to use (e.g., v1.2.3)' + required: true + type: string + highest_semver_tag: + description: 'The highest semantic version tag without the "v" prefix (e.g., 1.2.3)' + required: true + type: string jobs: get-version: - runs-on: ubuntu-latest - outputs: - release_version: ${{ steps.get_release_version.outputs.release_version }} - version_without_prefix: ${{ steps.get_release_version_without_prefix.outputs.version_without_prefix }} - highest_semver_tag: ${{ steps.get_highest_semver.outputs.highest_semver_tag }} - steps: - - name: Checkout - uses: actions/checkout@v4 - with: - persist-credentials: false - - name: Get release version - id: get_release_version - run: echo ::set-output name=release_version::${GITHUB_REF#refs/*/} - - name: Get release version without prefix - id: get_release_version_without_prefix - env: - RELEASE_VERSION: ${{ steps.get_release_version.outputs.release_version }} - run: | - echo ::set-output name=version_without_prefix::${RELEASE_VERSION:1} - - name: Get highest semver - id: get_highest_semver - env: - RELEASE_VERSION: ${{ steps.get_release_version.outputs.release_version }} - run: | - source infra/scripts/setup-common-functions.sh - SEMVER_REGEX='^v[0-9]+\.[0-9]+\.[0-9]+(-([0-9A-Za-z-]+(\.[0-9A-Za-z-]+)*))?$' - if echo "${RELEASE_VERSION}" | grep -P "$SEMVER_REGEX" &>/dev/null ; then - echo ::set-output name=highest_semver_tag::$(get_tag_release -m) - fi - - name: Check output - id: check_output - env: - RELEASE_VERSION: ${{ steps.get_release_version.outputs.release_version }} - VERSION_WITHOUT_PREFIX: ${{ steps.get_release_version_without_prefix.outputs.version_without_prefix }} - HIGHEST_SEMVER_TAG: ${{ steps.get_highest_semver.outputs.highest_semver_tag }} - run: | - echo $RELEASE_VERSION - echo $VERSION_WITHOUT_PREFIX - echo $HIGHEST_SEMVER_TAG + uses: ./.github/workflows/get_semantic_release_version.yaml + with: + custom_version: ${{ github.event.inputs.custom_version }} + token: ${{ github.event.inputs.token }} build-python-wheel: name: Build wheels @@ -70,11 +44,13 @@ jobs: run: make build-ui - name: Build wheels run: | + git fetch --tags + git checkout v0.42.0 python -m pip install build python -m build --wheel --outdir wheelhouse/ - - uses: actions/upload-artifact@v3 + - uses: actions/upload-artifact@v4 with: - name: wheels + name: python-wheels path: ./wheelhouse/*.whl build-source-distribution: @@ -96,6 +72,8 @@ jobs: - name: Build and install dependencies # There's a `git restore` in here because `make install-go-ci-dependencies` is actually messing up go.mod & go.sum. run: | + git fetch --tags + git checkout v0.42.0 pip install -U pip setuptools wheel twine make build-ui git status @@ -104,9 +82,9 @@ jobs: - name: Build run: | python3 setup.py sdist - - uses: actions/upload-artifact@v3 + - uses: actions/upload-artifact@v4 with: - name: wheels + name: source-distribution path: dist/* # We add this step so the docker images can be built as part of the pre-release verification steps. @@ -115,7 +93,7 @@ jobs: needs: get-version strategy: matrix: - component: [feature-server, feature-server-java, feature-transformation-server] + component: [ feature-server, feature-server-java, feature-transformation-server, feast-operator ] env: REGISTRY: feastdev steps: @@ -132,11 +110,11 @@ jobs: verify-python-wheels: runs-on: ${{ matrix.os }} - needs: [build-python-wheel, build-source-distribution, get-version] + needs: [ build-python-wheel, build-source-distribution, get-version ] strategy: matrix: - os: [ubuntu-latest, macos-13 ] - python-version: ["3.9", "3.10", "3.11"] + os: [ ubuntu-latest, macos-13 ] + python-version: [ "3.9", "3.10", "3.11" ] from-source: [ True, False ] env: # this script is for testing servers @@ -161,7 +139,11 @@ jobs: architecture: x64 - uses: actions/download-artifact@v4.1.7 with: - name: wheels + name: python-wheels + path: dist + - uses: actions/download-artifact@v4.1.7 + with: + name: source-distribution path: dist - name: Install OS X dependencies if: matrix.os == 'macos-13' @@ -180,11 +162,16 @@ jobs: # Validate that the feast version installed is not development and is the correct version of the tag we ran it off of. - name: Validate Feast Version run: | + feast version + if ! VERSION_OUTPUT=$(feast version); then + echo "Error: Failed to get Feast version." + exit 1 + fi VERSION_REGEX='[0-9]+\.[0-9]+\.[0-9]+' OUTPUT_REGEX='^Feast SDK Version: "$VERSION_REGEX"$' - VERSION_OUTPUT=$(feast version) VERSION=$(echo $VERSION_OUTPUT | grep -oE "$VERSION_REGEX") OUTPUT=$(echo $VERSION_OUTPUT | grep -E "$REGEX") + echo "Installed Feast Version: $VERSION and using Feast Version: $VERSION_WITHOUT_PREFIX" if [ -n "$OUTPUT" ] && [ "$VERSION" = "$VERSION_WITHOUT_PREFIX" ]; then echo "Correct Feast Version Installed" else diff --git a/.github/workflows/get_semantic_release_version.yml b/.github/workflows/get_semantic_release_version.yml new file mode 100644 index 00000000000..c800810dced --- /dev/null +++ b/.github/workflows/get_semantic_release_version.yml @@ -0,0 +1,84 @@ +name: Get semantic release version + +on: + workflow_dispatch: # Allows manual trigger of the workflow + inputs: + custom_version: # Optional input for a custom version + description: 'Custom version to publish (e.g., v1.2.3) -- only edit if you know what you are doing' + required: false + token: + description: 'Personal Access Token' + required: true + default: "" + type: string + +jobs: + get-version: + if: github.repository == 'feast-dev/feast' + runs-on: ubuntu-latest + env: + GITHUB_TOKEN: ${{ github.event.inputs.token }} + GIT_AUTHOR_NAME: feast-ci-bot + GIT_AUTHOR_EMAIL: feast-ci-bot@willem.co + GIT_COMMITTER_NAME: feast-ci-bot + GIT_COMMITTER_EMAIL: feast-ci-bot@willem.co + outputs: + release_version: ${{ steps.get_release_version.outputs.release_version }} + version_without_prefix: ${{ steps.get_release_version_without_prefix.outputs.version_without_prefix }} + highest_semver_tag: ${{ steps.get_highest_semver.outputs.highest_semver_tag }} + steps: + - uses: actions/checkout@v4 + - name: Get release version + id: get_release_version + run: | + if [[ -n "${{ github.event.inputs.custom_version }}" ]]; then + VERSION_REGEX="^v[0-9]+\.[0-9]+\.[0-9]+$" + echo "Using custom version: ${{ github.event.inputs.custom_version }}" + if [[ ! "${{ github.event.inputs.custom_version }}" =~ $VERSION_REGEX ]]; then + echo "Error: custom_version must match semantic versioning (e.g., v1.2.3)." + exit 1 + fi + echo "::set-output name=release_version::${{ github.event.inputs.custom_version }}" + elif [[ "${GITHUB_REF}" == refs/tags/* ]]; then + echo "Using tag reference: ${GITHUB_REF#refs/tags/}" + echo "::set-output name=release_version::${GITHUB_REF#refs/tags/}" + else + echo "Defaulting to branch name: ${GITHUB_REF#refs/heads/}" + echo "::set-output name=release_version::${GITHUB_REF#refs/heads/}" + fi + - name: Get release version without prefix + id: get_release_version_without_prefix + env: + RELEASE_VERSION: ${{ steps.get_release_version.outputs.release_version }} + run: | + if [[ "${RELEASE_VERSION}" == v* ]]; then + echo "::set-output name=version_without_prefix::${RELEASE_VERSION:1}" + else + echo "::set-output name=version_without_prefix::${RELEASE_VERSION}" + fi + - name: Get highest semver + id: get_highest_semver + env: + RELEASE_VERSION: ${{ steps.get_release_version.outputs.release_version }} + run: | + if [[ -n "${{ github.event.inputs.custom_version }}" ]]; then + HIGHEST_SEMVER_TAG="${{ github.event.inputs.custom_version }}" + echo "::set-output name=highest_semver_tag::$HIGHEST_SEMVER_TAG" + echo "Using custom version as highest semantic version: $HIGHEST_SEMVER_TAG" + else + source infra/scripts/setup-common-functions.sh + SEMVER_REGEX='^v[0-9]+\.[0-9]+\.[0-9]+(-([0-9A-Za-z-]+(\.[0-9A-Za-z-]+)*))?$' + if echo "${RELEASE_VERSION}" | grep -P "$SEMVER_REGEX" &>/dev/null ; then + echo ::set-output name=highest_semver_tag::$(get_tag_release -m) + echo "Using infra/scripts/setup-common-functions.sh to generate highest semantic version: $HIGHEST_SEMVER_TAG" + fi + fi + - name: Check output + env: + RELEASE_VERSION: ${{ steps.get_release_version.outputs.release_version }} + VERSION_WITHOUT_PREFIX: ${{ steps.get_release_version_without_prefix.outputs.version_without_prefix }} + HIGHEST_SEMVER_TAG: ${{ steps.get_highest_semver.outputs.highest_semver_tag }} + run: | + echo $RELEASE_VERSION + echo $VERSION_WITHOUT_PREFIX + echo $HIGHEST_SEMVER_TAG \ No newline at end of file diff --git a/.github/workflows/java_master_only.yml b/.github/workflows/java_master_only.yml index 127b59c5437..0307034bdb1 100644 --- a/.github/workflows/java_master_only.yml +++ b/.github/workflows/java_master_only.yml @@ -16,7 +16,7 @@ jobs: component: [feature-server-java] env: MAVEN_CACHE: gs://feast-templocation-kf-feast/.m2.2020-08-19.tar - REGISTRY: gcr.io/kf-feast + REGISTRY: quay.io/feastdev-ci steps: - uses: actions/checkout@v4 with: @@ -40,6 +40,12 @@ jobs: run: echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV - name: Build image run: make build-${{ matrix.component }}-docker REGISTRY=${REGISTRY} VERSION=${GITHUB_SHA} + - name: Login to Quay.io + uses: docker/login-action@v3 + with: + registry: quay.io + username: ${{ secrets.QUAYIO_CI_USERNAME }} + password: ${{ secrets.QUAYIO_CI_TOKEN }} - name: Push image run: make push-${{ matrix.component }}-docker REGISTRY=${REGISTRY} VERSION=${GITHUB_SHA} - name: Push development Docker image @@ -72,13 +78,13 @@ jobs: java-version: '11' java-package: jdk architecture: x64 - - uses: actions/cache@v2 + - uses: actions/cache@v4 with: path: ~/.m2/repository key: ${{ runner.os }}-it-maven-${{ hashFiles('**/pom.xml') }} restore-keys: | ${{ runner.os }}-it-maven- - - uses: actions/cache@v2 + - uses: actions/cache@v4 with: path: ~/.m2/repository key: ${{ runner.os }}-ut-maven-${{ hashFiles('**/pom.xml') }} @@ -86,7 +92,7 @@ jobs: ${{ runner.os }}-ut-maven- - name: Test java run: make test-java-with-coverage - - uses: actions/upload-artifact@v3 + - uses: actions/upload-artifact@v4 with: name: java-coverage-report path: ${{ github.workspace }}/docs/coverage/java/target/site/jacoco-aggregate/ diff --git a/.github/workflows/java_pr.yml b/.github/workflows/java_pr.yml index 8b83646c11f..40a2a7a7ec9 100644 --- a/.github/workflows/java_pr.yml +++ b/.github/workflows/java_pr.yml @@ -53,13 +53,13 @@ jobs: java-version: '11' java-package: jdk architecture: x64 - - uses: actions/cache@v2 + - uses: actions/cache@v4 with: path: ~/.m2/repository key: ${{ runner.os }}-it-maven-${{ hashFiles('**/pom.xml') }} restore-keys: | ${{ runner.os }}-it-maven- - - uses: actions/cache@v2 + - uses: actions/cache@v4 with: path: ~/.m2/repository key: ${{ runner.os }}-ut-maven-${{ hashFiles('**/pom.xml') }} @@ -67,7 +67,7 @@ jobs: ${{ runner.os }}-ut-maven- - name: Test java run: make test-java-with-coverage - - uses: actions/upload-artifact@v3 + - uses: actions/upload-artifact@v4 with: name: java-coverage-report path: ${{ github.workspace }}/docs/coverage/java/target/site/jacoco-aggregate/ @@ -84,7 +84,7 @@ jobs: component: [ feature-server-java ] env: MAVEN_CACHE: gs://feast-templocation-kf-feast/.m2.2020-08-19.tar - REGISTRY: gcr.io/kf-feast + REGISTRY: quay.io/feastdev-ci steps: - uses: actions/checkout@v4 with: @@ -97,11 +97,11 @@ jobs: python-version: "3.11" architecture: x64 - name: Authenticate to Google Cloud - uses: 'google-github-actions/auth@v1' + uses: google-github-actions/auth@v2 with: credentials_json: '${{ secrets.GCP_SA_KEY }}' - name: Set up gcloud SDK - uses: google-github-actions/setup-gcloud@v1 + uses: google-github-actions/setup-gcloud@v2 with: project_id: ${{ secrets.GCP_PROJECT_ID }} - run: gcloud auth configure-docker --quiet @@ -137,18 +137,18 @@ jobs: with: python-version: '3.11' architecture: 'x64' - - uses: actions/cache@v2 + - uses: actions/cache@v4 with: path: ~/.m2/repository key: ${{ runner.os }}-it-maven-${{ hashFiles('**/pom.xml') }} restore-keys: | ${{ runner.os }}-it-maven- - name: Authenticate to Google Cloud - uses: 'google-github-actions/auth@v1' + uses: google-github-actions/auth@v2 with: credentials_json: '${{ secrets.GCP_SA_KEY }}' - name: Set up gcloud SDK - uses: google-github-actions/setup-gcloud@v1 + uses: google-github-actions/setup-gcloud@v2 with: project_id: ${{ secrets.GCP_PROJECT_ID }} - name: Use gcloud CLI @@ -184,7 +184,7 @@ jobs: - name: Run integration tests run: make test-java-integration - name: Save report - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 if: failure() with: name: it-report diff --git a/.github/workflows/lint_pr.yml b/.github/workflows/lint_pr.yml index 81732258455..33fafdcd23d 100644 --- a/.github/workflows/lint_pr.yml +++ b/.github/workflows/lint_pr.yml @@ -14,7 +14,7 @@ jobs: name: Validate PR title runs-on: ubuntu-latest steps: - - uses: amannn/action-semantic-pull-request@v4 + - uses: amannn/action-semantic-pull-request@v5 with: # Must use uppercase subjectPattern: ^(?=[A-Z]).+$ diff --git a/.github/workflows/master_only.yml b/.github/workflows/master_only.yml index 446c3b1f3be..840a8007236 100644 --- a/.github/workflows/master_only.yml +++ b/.github/workflows/master_only.yml @@ -94,40 +94,32 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - component: [ feature-server-java, feature-transformation-server ] + component: [ feature-server-dev, feature-transformation-server, feast-operator ] env: - MAVEN_CACHE: gs://feast-templocation-kf-feast/.m2.2020-08-19.tar - REGISTRY: gcr.io/kf-feast + REGISTRY: quay.io/feastdev-ci steps: - uses: actions/checkout@v4 - name: Set up QEMU - uses: docker/setup-qemu-action@v1 + uses: docker/setup-qemu-action@v3 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 + uses: docker/setup-buildx-action@v3 with: install: true - - name: Login to DockerHub - uses: docker/login-action@v1 + - name: Login to Quay.io + uses: docker/login-action@v3 with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - name: Authenticate to Google Cloud - uses: 'google-github-actions/auth@v1' - with: - credentials_json: '${{ secrets.GCP_SA_KEY }}' - - name: Set up gcloud SDK - uses: google-github-actions/setup-gcloud@v1 - with: - project_id: ${{ secrets.GCP_PROJECT_ID }} - - name: Use gcloud CLI - run: gcloud info - - run: gcloud auth configure-docker --quiet + registry: quay.io + username: ${{ secrets.QUAYIO_CI_USERNAME }} + password: ${{ secrets.QUAYIO_CI_TOKEN }} - name: Build image run: | make build-${{ matrix.component }}-docker REGISTRY=${REGISTRY} VERSION=${GITHUB_SHA} - name: Push image run: | - make push-${{ matrix.component }}-docker REGISTRY=${REGISTRY} VERSION=${GITHUB_SHA} - - docker tag ${REGISTRY}/${{ matrix.component }}:${GITHUB_SHA} ${REGISTRY}/${{ matrix.component }}:develop - docker push ${REGISTRY}/${{ matrix.component }}:develop + if [[ "${{ matrix.component }}" == "feature-server-dev" ]]; then + docker tag ${REGISTRY}/feature-server:${GITHUB_SHA} ${REGISTRY}/feature-server:develop + docker push ${REGISTRY}/feature-server --all-tags + else + docker tag ${REGISTRY}/${{ matrix.component }}:${GITHUB_SHA} ${REGISTRY}/${{ matrix.component }}:develop + docker push ${REGISTRY}/${{ matrix.component }} --all-tags + fi diff --git a/.github/workflows/operator-e2e-integration-tests.yml b/.github/workflows/operator-e2e-integration-tests.yml index 23c250cc535..83b38e52be3 100644 --- a/.github/workflows/operator-e2e-integration-tests.yml +++ b/.github/workflows/operator-e2e-integration-tests.yml @@ -10,9 +10,14 @@ on: - opened - synchronize - labeled + paths-ignore: + - 'community/**' + - 'docs/**' + - 'examples/**' jobs: operator-e2e-tests: + timeout-minutes: 40 if: ((github.event.action == 'labeled' && (github.event.label.name == 'approved' || github.event.label.name == 'lgtm' || github.event.label.name == 'ok-to-test')) || (github.event.action != 'labeled' && (contains(github.event.pull_request.labels.*.name, 'ok-to-test') || contains(github.event.pull_request.labels.*.name, 'approved') || contains(github.event.pull_request.labels.*.name, 'lgtm')))) && @@ -34,11 +39,11 @@ jobs: - name: Set up Go uses: actions/setup-go@v5 with: - go-version: '1.21.0' + go-version: 1.22.9 - name: Create KIND cluster run: | - kind create cluster --name $KIND_CLUSTER --wait 5m + kind create cluster --name $KIND_CLUSTER --wait 10m - name: Set up kubernetes context run: | @@ -51,8 +56,16 @@ jobs: cd infra/feast-operator/ make test-e2e + - name: Debug KIND Cluster when there is a failure + if: failure() + run: | + kubectl get pods --all-namespaces + kubectl describe nodes + - name: Clean up if: always() run: | # Delete the KIND cluster after tests kind delete cluster --name kind-$KIND_CLUSTER + + diff --git a/.github/workflows/operator_pr.yml b/.github/workflows/operator_pr.yml index e4d371b9454..2feed8dbf32 100644 --- a/.github/workflows/operator_pr.yml +++ b/.github/workflows/operator_pr.yml @@ -7,9 +7,9 @@ jobs: steps: - uses: actions/checkout@v4 - name: Install Go - uses: actions/setup-go@v2 + uses: actions/setup-go@v5 with: - go-version: 1.21.x + go-version: 1.22.9 - name: Operator tests run: | cd infra/feast-operator/ diff --git a/.github/workflows/pr_integration_tests.yml b/.github/workflows/pr_integration_tests.yml index 5a1b483b39e..923c0b0335b 100644 --- a/.github/workflows/pr_integration_tests.yml +++ b/.github/workflows/pr_integration_tests.yml @@ -6,6 +6,10 @@ on: - opened - synchronize - labeled + paths-ignore: + - 'community/**' + - 'docs/**' + - 'examples/**' # concurrency is currently broken, see details https://github.com/actions/runner/issues/1532 #concurrency: diff --git a/.github/workflows/pr_local_integration_tests.yml b/.github/workflows/pr_local_integration_tests.yml index 8b2f8c13d2e..d3488cd08c3 100644 --- a/.github/workflows/pr_local_integration_tests.yml +++ b/.github/workflows/pr_local_integration_tests.yml @@ -7,6 +7,10 @@ on: - opened - synchronize - labeled + paths-ignore: + - 'community/**' + - 'docs/**' + - 'examples/**' jobs: integration-test-python-local: @@ -41,12 +45,12 @@ jobs: - name: Get uv cache dir id: uv-cache run: | - echo "::set-output name=dir::$(uv cache dir)" + echo "dir=$(uv cache dir)" >> $GITHUB_OUTPUT - name: uv cache uses: actions/cache@v4 with: path: ${{ steps.uv-cache.outputs.dir }} - key: ${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-uv-${{ hashFiles(format('**/py{0}-ci-requirements.txt', env.PYTHON)) }} + key: ${{ runner.os }}-${{ matrix.python-version }}-uv-${{ hashFiles(format('**/py{0}-ci-requirements.txt', matrix.python-version)) }} - name: Install dependencies run: make install-python-dependencies-ci - name: Test local integration tests diff --git a/.github/workflows/pr_remote_rbac_integration_tests.yml b/.github/workflows/pr_remote_rbac_integration_tests.yml new file mode 100644 index 00000000000..98fa5a52c58 --- /dev/null +++ b/.github/workflows/pr_remote_rbac_integration_tests.yml @@ -0,0 +1,58 @@ +name: pr-remote-rbac-integration-tests +# This runs the integration tests related to rbac functionality and remote registry and online features. + +on: + pull_request: + types: + - opened + - synchronize + - labeled + paths-ignore: + - 'community/**' + - 'docs/**' + - 'examples/**' + +jobs: + remote-rbac-integration-tests-python: + if: + ((github.event.action == 'labeled' && (github.event.label.name == 'approved' || github.event.label.name == 'lgtm' || github.event.label.name == 'ok-to-test')) || + (github.event.action != 'labeled' && (contains(github.event.pull_request.labels.*.name, 'ok-to-test') || contains(github.event.pull_request.labels.*.name, 'approved') || contains(github.event.pull_request.labels.*.name, 'lgtm')))) && + github.event.pull_request.base.repo.full_name == 'feast-dev/feast' + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + python-version: [ "3.11" ] + os: [ ubuntu-latest ] + env: + OS: ${{ matrix.os }} + PYTHON: ${{ matrix.python-version }} + steps: + - uses: actions/checkout@v4 + with: + repository: ${{ github.event.repository.full_name }} # Uses the full repository name + ref: ${{ github.ref }} # Uses the ref from the event + token: ${{ secrets.GITHUB_TOKEN }} # Automatically provided token + submodules: recursive + - name: Setup Python + uses: actions/setup-python@v5 + id: setup-python + with: + python-version: ${{ matrix.python-version }} + architecture: x64 + - name: Install uv + run: curl -LsSf https://astral.sh/uv/install.sh | sh + - name: Get uv cache dir + id: uv-cache + run: | + echo "dir=$(uv cache dir)" >> $GITHUB_OUTPUT + - name: uv cache + uses: actions/cache@v4 + with: + path: ${{ steps.uv-cache.outputs.dir }} + key: ${{ runner.os }}-${{ matrix.python-version }}-uv-${{ hashFiles(format('**/py{0}-ci-requirements.txt', matrix.python-version)) }} + - name: Install dependencies + run: make install-python-dependencies-ci + - name: Test rbac and remote feature integration tests + if: ${{ always() }} # this will guarantee that step won't be canceled and resources won't leak + run: make test-python-integration-rbac-remote diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 7d5dca8e08b..eb81ca193cd 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -4,179 +4,47 @@ on: push: tags: - 'v*.*.*' + workflow_dispatch: # Allows manual trigger of the workflow + inputs: + custom_version: # Optional input for a custom version + description: 'Custom version to publish (e.g., v1.2.3) -- only edit if you know what you are doing' + required: false + token: + description: 'Personal Access Token' + required: true + default: "" + type: string jobs: get-version: - if: github.repository == 'feast-dev/feast' - runs-on: ubuntu-latest - outputs: - release_version: ${{ steps.get_release_version.outputs.release_version }} - version_without_prefix: ${{ steps.get_release_version_without_prefix.outputs.version_without_prefix }} - highest_semver_tag: ${{ steps.get_highest_semver.outputs.highest_semver_tag }} - steps: - - uses: actions/checkout@v4 - - name: Get release version - id: get_release_version - run: echo ::set-output name=release_version::${GITHUB_REF#refs/*/} - - name: Get release version without prefix - id: get_release_version_without_prefix - env: - RELEASE_VERSION: ${{ steps.get_release_version.outputs.release_version }} - run: | - echo ::set-output name=version_without_prefix::${RELEASE_VERSION:1} - - name: Get highest semver - id: get_highest_semver - env: - RELEASE_VERSION: ${{ steps.get_release_version.outputs.release_version }} - run: | - source infra/scripts/setup-common-functions.sh - SEMVER_REGEX='^v[0-9]+\.[0-9]+\.[0-9]+(-([0-9A-Za-z-]+(\.[0-9A-Za-z-]+)*))?$' - if echo "${RELEASE_VERSION}" | grep -P "$SEMVER_REGEX" &>/dev/null ; then - echo ::set-output name=highest_semver_tag::$(get_tag_release -m) - fi - - name: Check output - env: - RELEASE_VERSION: ${{ steps.get_release_version.outputs.release_version }} - VERSION_WITHOUT_PREFIX: ${{ steps.get_release_version_without_prefix.outputs.version_without_prefix }} - HIGHEST_SEMVER_TAG: ${{ steps.get_highest_semver.outputs.highest_semver_tag }} - run: | - echo $RELEASE_VERSION - echo $VERSION_WITHOUT_PREFIX - echo $HIGHEST_SEMVER_TAG + uses: ./.github/workflows/get_semantic_release_version.yaml + with: + custom_version: ${{ github.event.inputs.custom_version }} + token: ${{ github.event.inputs.token }} - build-publish-docker-images: - runs-on: ubuntu-latest - needs: [get-version, publish-python-sdk] - strategy: - matrix: - component: [feature-server, feature-server-java, feature-transformation-server, feast-helm-operator, feast-operator] - env: - MAVEN_CACHE: gs://feast-templocation-kf-feast/.m2.2020-08-19.tar - REGISTRY: feastdev - steps: - - uses: actions/checkout@v4 - - name: Set up QEMU - uses: docker/setup-qemu-action@v1 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 - - name: Login to DockerHub - uses: docker/login-action@v1 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - name: Authenticate to Google Cloud - uses: 'google-github-actions/auth@v1' - with: - credentials_json: '${{ secrets.GCP_SA_KEY }}' - - name: Set up gcloud SDK - uses: google-github-actions/setup-gcloud@v1 - with: - project_id: ${{ secrets.GCP_PROJECT_ID }} - - name: Use gcloud CLI - run: gcloud info - - run: gcloud auth configure-docker --quiet - - name: Build image - run: | - make build-${{ matrix.component }}-docker REGISTRY=${REGISTRY} VERSION=${VERSION_WITHOUT_PREFIX} - env: - RELEASE_VERSION: ${{ needs.get-version.outputs.release_version }} - VERSION_WITHOUT_PREFIX: ${{ needs.get-version.outputs.version_without_prefix }} - HIGHEST_SEMVER_TAG: ${{ needs.get-version.outputs.highest_semver_tag }} - - name: Push versioned images - env: - RELEASE_VERSION: ${{ needs.get-version.outputs.release_version }} - VERSION_WITHOUT_PREFIX: ${{ needs.get-version.outputs.version_without_prefix }} - HIGHEST_SEMVER_TAG: ${{ needs.get-version.outputs.highest_semver_tag }} - run: | - make push-${{ matrix.component }}-docker REGISTRY=${REGISTRY} VERSION=${VERSION_WITHOUT_PREFIX} + publish-python-sdk: + uses: ./.github/workflows/publish_python_sdk.yaml + with: + custom_version: ${{ github.event.inputs.custom_version }} + token: ${{ github.event.inputs.token }} - echo "Only push to latest tag if tag is the highest semver version $HIGHEST_SEMVER_TAG" - if [ "${VERSION_WITHOUT_PREFIX}" = "${HIGHEST_SEMVER_TAG:1}" ] - then - docker tag feastdev/${{ matrix.component }}:${VERSION_WITHOUT_PREFIX} feastdev/${{ matrix.component }}:latest - docker push feastdev/${{ matrix.component }}:latest - fi + build-publish-docker-images: + uses: ./.github/workflows/publish_images.yaml + needs: [ get-version, publish-python-sdk ] + with: + custom_version: ${{ github.event.inputs.custom_version }} + token: ${{ github.event.inputs.token }} publish-helm-charts: - if: github.repository == 'feast-dev/feast' - runs-on: ubuntu-latest - needs: get-version - env: - HELM_VERSION: v3.8.0 - VERSION_WITHOUT_PREFIX: ${{ needs.get-version.outputs.version_without_prefix }} - steps: - - uses: actions/checkout@v4 - - name: Authenticate to Google Cloud - uses: 'google-github-actions/auth@v1' - with: - credentials_json: '${{ secrets.GCP_SA_KEY }}' - - name: Set up gcloud SDK - uses: google-github-actions/setup-gcloud@v1 - with: - project_id: ${{ secrets.GCP_PROJECT_ID }} - - run: gcloud auth configure-docker --quiet - - name: Remove previous Helm - run: sudo rm -rf $(which helm) - - name: Install Helm - run: ./infra/scripts/helm/install-helm.sh - - name: Validate Helm chart prior to publishing - run: ./infra/scripts/helm/validate-helm-chart-publish.sh - - name: Validate all version consistency - run: ./infra/scripts/helm/validate-helm-chart-versions.sh $VERSION_WITHOUT_PREFIX - - name: Publish Helm charts - run: ./infra/scripts/helm/push-helm-charts.sh $VERSION_WITHOUT_PREFIX - - build_wheels: - uses: ./.github/workflows/build_wheels.yml - - publish-python-sdk: - if: github.repository == 'feast-dev/feast' - runs-on: ubuntu-latest - needs: [build_wheels] - steps: - - uses: actions/download-artifact@v4.1.7 - with: - name: wheels - path: dist - - uses: pypa/gh-action-pypi-publish@v1.4.2 - with: - user: __token__ - password: ${{ secrets.PYPI_PASSWORD }} + uses: ./.github/workflows/publish_helm_charts.yml + needs: [ get-version, publish-python-sdk ] + with: + custom_version: ${{ github.event.inputs.custom_version }} + token: ${{ github.event.inputs.token }} publish-java-sdk: - if: github.repository == 'feast-dev/feast' - container: maven:3.6-jdk-11 - runs-on: ubuntu-latest - needs: get-version - steps: - - uses: actions/checkout@v4 - with: - submodules: 'true' - - name: Set up JDK 11 - uses: actions/setup-java@v1 - with: - java-version: '11' - java-package: jdk - architecture: x64 - - uses: actions/setup-python@v5 - with: - python-version: '3.11' - architecture: 'x64' - - uses: actions/cache@v2 - with: - path: ~/.m2/repository - key: ${{ runner.os }}-it-maven-${{ hashFiles('**/pom.xml') }} - restore-keys: | - ${{ runner.os }}-it-maven- - - name: Publish java sdk - env: - VERSION_WITHOUT_PREFIX: ${{ needs.get-version.outputs.version_without_prefix }} - GPG_PUBLIC_KEY: ${{ secrets.GPG_PUBLIC_KEY }} - GPG_PRIVATE_KEY: ${{ secrets.GPG_PRIVATE_KEY }} - MAVEN_SETTINGS: ${{ secrets.MAVEN_SETTINGS }} - run: | - echo -n "$GPG_PUBLIC_KEY" > /root/public-key - echo -n "$GPG_PRIVATE_KEY" > /root/private-key - mkdir -p /root/.m2/ - echo -n "$MAVEN_SETTINGS" > /root/.m2/settings.xml - infra/scripts/publish-java-sdk.sh --revision ${VERSION_WITHOUT_PREFIX} --gpg-key-import-dir /root + uses: ./.github/workflows/publish_java_sdk.yml + needs: [ get-version, publish-python-sdk ] + with: + custom_version: ${{ github.event.inputs.custom_version }} + token: ${{ github.event.inputs.token }} diff --git a/.github/workflows/publish_helm_charts.yml b/.github/workflows/publish_helm_charts.yml new file mode 100644 index 00000000000..060bdb05d4e --- /dev/null +++ b/.github/workflows/publish_helm_charts.yml @@ -0,0 +1,50 @@ +name: publish images + +on: + workflow_dispatch: # Allows manual trigger of the workflow + inputs: + custom_version: # Optional input for a custom version + description: 'Custom version to publish (e.g., v1.2.3) -- only edit if you know what you are doing' + required: false + token: + description: 'Personal Access Token' + required: true + default: "" + type: string + +jobs: + get-version: + uses: ./.github/workflows/get_semantic_release_version.yaml + with: + custom_version: ${{ github.event.inputs.custom_version }} + token: ${{ github.event.inputs.token }} + + publish-helm-charts: + if: github.repository == 'feast-dev/feast' + runs-on: ubuntu-latest + needs: get-version + env: + HELM_VERSION: v3.8.0 + VERSION_WITHOUT_PREFIX: ${{ needs.get-version.outputs.version_without_prefix }} + steps: + - uses: actions/checkout@v4 + - name: Authenticate to Google Cloud + uses: 'google-github-actions/auth@v1' + with: + credentials_json: '${{ secrets.GCP_SA_KEY }}' + - name: Set up gcloud SDK + uses: google-github-actions/setup-gcloud@v1 + with: + project_id: ${{ secrets.GCP_PROJECT_ID }} + - run: gcloud auth configure-docker --quiet + - name: Remove previous Helm + run: sudo rm -rf $(which helm) + - name: Install Helm + run: ./infra/scripts/helm/install-helm.sh + - name: Validate Helm chart prior to publishing + run: ./infra/scripts/helm/validate-helm-chart-publish.sh + - name: Validate all version consistency + run: ./infra/scripts/helm/validate-helm-chart-versions.sh $VERSION_WITHOUT_PREFIX + - name: Publish Helm charts + run: ./infra/scripts/helm/push-helm-charts.sh $VERSION_WITHOUT_PREFIX + diff --git a/.github/workflows/publish_images.yml b/.github/workflows/publish_images.yml new file mode 100644 index 00000000000..26201aaa5c2 --- /dev/null +++ b/.github/workflows/publish_images.yml @@ -0,0 +1,75 @@ +name: build and publish docker images + +on: + workflow_dispatch: # Allows manual trigger of the workflow + inputs: + custom_version: # Optional input for a custom version + description: 'Custom version to publish (e.g., v1.2.3) -- only edit if you know what you are doing' + required: false + token: + description: 'Personal Access Token' + required: true + default: "" + type: string + +jobs: + get-version: + uses: ./.github/workflows/get_semantic_release_version.yaml + with: + custom_version: ${{ github.event.inputs.custom_version }} + token: ${{ github.event.inputs.token }} + + build-publish-docker-images: + if: github.repository == 'feast-dev/feast' + runs-on: ubuntu-latest + needs: [ get-version ] + strategy: + matrix: + component: [ feature-server, feature-server-java, feature-transformation-server, feast-helm-operator, feast-operator ] + env: + MAVEN_CACHE: gs://feast-templocation-kf-feast/.m2.2020-08-19.tar + REGISTRY: feastdev + steps: + - uses: actions/checkout@v4 + - name: Set up QEMU + uses: docker/setup-qemu-action@v1 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + - name: Login to DockerHub + uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + - name: Authenticate to Google Cloud + uses: 'google-github-actions/auth@v1' + with: + credentials_json: '${{ secrets.GCP_SA_KEY }}' + - name: Set up gcloud SDK + uses: google-github-actions/setup-gcloud@v1 + with: + project_id: ${{ secrets.GCP_PROJECT_ID }} + - name: Use gcloud CLI + run: gcloud info + - run: gcloud auth configure-docker --quiet + - name: Build image + run: | + make build-${{ matrix.component }}-docker REGISTRY=${REGISTRY} VERSION=${VERSION_WITHOUT_PREFIX} + env: + RELEASE_VERSION: ${{ needs.get-version.outputs.release_version }} + VERSION_WITHOUT_PREFIX: ${{ needs.get-version.outputs.version_without_prefix }} + HIGHEST_SEMVER_TAG: ${{ needs.get-version.outputs.highest_semver_tag }} + - name: Push versioned images + env: + RELEASE_VERSION: ${{ needs.get-version.outputs.release_version }} + VERSION_WITHOUT_PREFIX: ${{ needs.get-version.outputs.version_without_prefix }} + HIGHEST_SEMVER_TAG: ${{ needs.get-version.outputs.highest_semver_tag }} + run: | + make push-${{ matrix.component }}-docker REGISTRY=${REGISTRY} VERSION=${VERSION_WITHOUT_PREFIX} + + echo "Only push to latest tag if tag is the highest semver version $HIGHEST_SEMVER_TAG" + if [ "${VERSION_WITHOUT_PREFIX}" = "${HIGHEST_SEMVER_TAG:1}" ] + then + docker tag feastdev/${{ matrix.component }}:${VERSION_WITHOUT_PREFIX} feastdev/${{ matrix.component }}:latest + docker push feastdev/${{ matrix.component }}:latest + fi + diff --git a/.github/workflows/publish_java_sdk.yml b/.github/workflows/publish_java_sdk.yml new file mode 100644 index 00000000000..c158010995d --- /dev/null +++ b/.github/workflows/publish_java_sdk.yml @@ -0,0 +1,58 @@ +name: publish java sdk + +on: + workflow_dispatch: # Allows manual trigger of the workflow + inputs: + custom_version: # Optional input for a custom version + description: 'Custom version to publish (e.g., v1.2.3) -- only edit if you know what you are doing' + required: false + token: + description: 'Personal Access Token' + required: true + default: "" + type: string + +jobs: + get-version: + uses: ./.github/workflows/get_semantic_release_version.yaml + with: + custom_version: ${{ github.event.inputs.custom_version }} + token: ${{ github.event.inputs.token }} + + publish-java-sdk: + if: github.repository == 'feast-dev/feast' + container: maven:3.6-jdk-11 + runs-on: ubuntu-latest + needs: [ get-version, publish-python-sdk ] + steps: + - uses: actions/checkout@v4 + with: + submodules: 'true' + - name: Set up JDK 11 + uses: actions/setup-java@v1 + with: + java-version: '11' + java-package: jdk + architecture: x64 + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + architecture: 'x64' + - uses: actions/cache@v2 + with: + path: ~/.m2/repository + key: ${{ runner.os }}-it-maven-${{ hashFiles('**/pom.xml') }} + restore-keys: | + ${{ runner.os }}-it-maven- + - name: Publish java sdk + env: + VERSION_WITHOUT_PREFIX: ${{ needs.get-version.outputs.version_without_prefix }} + GPG_PUBLIC_KEY: ${{ secrets.GPG_PUBLIC_KEY }} + GPG_PRIVATE_KEY: ${{ secrets.GPG_PRIVATE_KEY }} + MAVEN_SETTINGS: ${{ secrets.MAVEN_SETTINGS }} + run: | + echo -n "$GPG_PUBLIC_KEY" > /root/public-key + echo -n "$GPG_PRIVATE_KEY" > /root/private-key + mkdir -p /root/.m2/ + echo -n "$MAVEN_SETTINGS" > /root/.m2/settings.xml + infra/scripts/publish-java-sdk.sh --revision ${VERSION_WITHOUT_PREFIX} --gpg-key-import-dir /root diff --git a/.github/workflows/publish_python_sdk.yml b/.github/workflows/publish_python_sdk.yml new file mode 100644 index 00000000000..1a9c111de74 --- /dev/null +++ b/.github/workflows/publish_python_sdk.yml @@ -0,0 +1,41 @@ +name: publish python sdk + +on: + workflow_dispatch: # Allows manual trigger of the workflow + inputs: + custom_version: # Optional input for a custom version + description: 'Custom version to publish (e.g., v1.2.3) -- only edit if you know what you are doing' + required: false + token: + description: 'Personal Access Token' + required: true + default: "" + type: string + +jobs: + get-version: + uses: ./.github/workflows/get_semantic_release_version.yaml + with: + custom_version: ${{ github.event.inputs.custom_version }} + token: ${{ github.event.inputs.token }} + + build_wheels: + uses: ./.github/workflows/build_wheels.yml + needs: get-version + with: + release_version: ${{ needs.get-version.outputs.release_version }} + highest_semver_tag: ${{ needs.get-version.outputs.highest_semver_tag }} + + publish-python-sdk: + if: github.repository == 'feast-dev/feast' + runs-on: ubuntu-latest + needs: [ get-version, build_wheels ] + steps: + - uses: actions/download-artifact@v4.1.7 + with: + name: python-wheels + path: dist + - uses: pypa/gh-action-pypi-publish@v1.4.2 + with: + user: __token__ + password: ${{ secrets.PYPI_PASSWORD }} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index ec7ffb29eda..00f65929265 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -20,8 +20,8 @@ on: type: boolean jobs: - get_dry_release_versions: + if: github.repository == 'feast-dev/feast' runs-on: ubuntu-latest env: GITHUB_TOKEN: ${{ github.event.inputs.token }} @@ -91,14 +91,14 @@ jobs: - name: Install Go uses: actions/setup-go@v2 with: - go-version: 1.21.x + go-version: 1.22.9 - name: Build & version operator-specific release files run: | cd infra/feast-operator/ make build-installer bundle publish-web-ui-npm: - needs: [validate_version_bumps, get_dry_release_versions] + needs: [ validate_version_bumps, get_dry_release_versions ] runs-on: ubuntu-latest env: # This publish is working using an NPM automation token to bypass 2FA @@ -121,7 +121,7 @@ jobs: run: yarn build:lib - name: Publish UI package working-directory: ./ui - if: github.event.inputs.dry_run == 'false' && github.event.inputs.publish_ui == 'true' + if: github.event.inputs.dry_run == 'false' && github.event.inputs.publish_ui == 'true' run: npm publish env: # This publish is working using an NPM automation token to bypass 2FA @@ -138,25 +138,25 @@ jobs: GIT_COMMITTER_NAME: feast-ci-bot GIT_COMMITTER_EMAIL: feast-ci-bot@willem.co steps: - - name: Checkout - uses: actions/checkout@v4 - with: - persist-credentials: false - - name: Setup Node.js - uses: actions/setup-node@v3 - with: - node-version-file: './ui/.nvmrc' - - name: Set up Homebrew - id: set-up-homebrew - uses: Homebrew/actions/setup-homebrew@master - - name: Setup Helm-docs - run: | - brew install norwoodj/tap/helm-docs - - name: Release (Dry Run) - if: github.event.inputs.dry_run == 'true' - run: | - npx -p @semantic-release/changelog -p @semantic-release/git -p @semantic-release/exec -p semantic-release semantic-release --dry-run - - name: Release - if: github.event.inputs.dry_run == 'false' - run: | - npx -p @semantic-release/changelog -p @semantic-release/git -p @semantic-release/exec -p semantic-release semantic-release + - name: Checkout + uses: actions/checkout@v4 + with: + persist-credentials: false + - name: Setup Node.js + uses: actions/setup-node@v3 + with: + node-version-file: './ui/.nvmrc' + - name: Set up Homebrew + id: set-up-homebrew + uses: Homebrew/actions/setup-homebrew@master + - name: Setup Helm-docs + run: | + brew install norwoodj/tap/helm-docs + - name: Release (Dry Run) + if: github.event.inputs.dry_run == 'true' + run: | + npx -p @semantic-release/changelog -p @semantic-release/git -p @semantic-release/exec -p semantic-release semantic-release --dry-run + - name: Release + if: github.event.inputs.dry_run == 'false' + run: | + npx -p @semantic-release/changelog -p @semantic-release/git -p @semantic-release/exec -p semantic-release semantic-release diff --git a/.github/workflows/show_semantic_release.yml b/.github/workflows/show_semantic_release.yml deleted file mode 100644 index f4aef6be54b..00000000000 --- a/.github/workflows/show_semantic_release.yml +++ /dev/null @@ -1,38 +0,0 @@ -name: show semantic release versions - -on: - workflow_dispatch: - inputs: - token: - description: 'Personal Access Token' - required: true - default: "" - type: string - -jobs: - - get_dry_release_versions: - runs-on: ubuntu-latest - env: - GITHUB_TOKEN: ${{ github.event.inputs.token }} - outputs: - current_version: ${{ steps.get_versions.outputs.current_version }} - next_version: ${{ steps.get_versions.outputs.next_version }} - steps: - - name: Checkout - uses: actions/checkout@v4 - with: - persist-credentials: false - - name: Setup Node.js - uses: actions/setup-node@v3 - with: - node-version: "lts/*" - - name: Release (Dry Run) - id: get_versions - run: | - CURRENT_VERSION=$(npx -p @semantic-release/changelog -p @semantic-release/git -p @semantic-release/exec -p semantic-release semantic-release --dry-run | grep "associated with version " | sed -E 's/.* version//' | sed -E 's/ on.*//') - NEXT_VERSION=$(npx -p @semantic-release/changelog -p @semantic-release/git -p @semantic-release/exec -p semantic-release semantic-release --dry-run | grep 'The next release version is' | sed -E 's/.* ([[:digit:].]+)$/\1/') - echo ::set-output name=current_version::$CURRENT_VERSION - echo ::set-output name=next_version::$NEXT_VERSION - echo "Current version is ${CURRENT_VERSION}" - echo "Next version is ${NEXT_VERSION}" diff --git a/.github/workflows/smoke_tests.yml b/.github/workflows/smoke_tests.yml index 774d58d22b4..a7eb1966269 100644 --- a/.github/workflows/smoke_tests.yml +++ b/.github/workflows/smoke_tests.yml @@ -1,6 +1,11 @@ name: smoke-tests -on: [pull_request] +on: + pull_request: + paths-ignore: + - 'community/**' + - 'docs/**' + - 'examples/**' jobs: unit-test-python: runs-on: ${{ matrix.os }} @@ -26,7 +31,7 @@ jobs: - name: Get uv cache dir id: uv-cache run: | - echo "::set-output name=dir::$(uv cache dir)" + echo "dir=$(uv cache dir)" >> $GITHUB_OUTPUT - name: uv cache uses: actions/cache@v4 with: diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index a8ddd397e30..3ece863de3b 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -1,6 +1,11 @@ name: unit-tests -on: [pull_request] +on: + pull_request: + paths-ignore: + - 'community/**' + - 'docs/**' + - 'examples/**' jobs: unit-test-python: runs-on: ${{ matrix.os }} @@ -8,10 +13,15 @@ jobs: fail-fast: false matrix: python-version: [ "3.9", "3.10", "3.11"] - os: [ ubuntu-latest, macos-13 ] + os: [ ubuntu-latest, macos-13, macos-14 ] exclude: - os: macos-13 python-version: "3.9" + - os: macos-14 + python-version: "3.9" + - os: macos-14 + python-version: "3.10" + env: OS: ${{ matrix.os }} PYTHON: ${{ matrix.python-version }} @@ -28,8 +38,8 @@ jobs: curl -LsSf https://astral.sh/uv/install.sh | sh - name: Get uv cache dir id: uv-cache - run: | - echo "::set-output name=dir::$(uv cache dir)" + run: | + echo "dir=$(uv cache dir)" >> $GITHUB_OUTPUT - name: uv cache uses: actions/cache@v4 with: @@ -47,7 +57,7 @@ jobs: NPM_TOKEN: ${{ secrets.NPM_TOKEN }} steps: - uses: actions/checkout@v4 - - uses: actions/setup-node@v3 + - uses: actions/setup-node@v4 with: node-version-file: './ui/.nvmrc' registry-url: 'https://registry.npmjs.org' diff --git a/.gitignore b/.gitignore index d558463c657..6823221aed4 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,7 @@ scratch* ### Local Environment ### *local*.env +tools ### Secret ### **/service_account.json @@ -101,6 +102,7 @@ htmlcov/ .cache nosetests.xml coverage.xml +coverage.out *.cover .hypothesis/ .pytest_cache/ @@ -222,4 +224,4 @@ ui/.vercel **/yarn-error.log* # Go subprocess binaries (built during feast pip package building) -sdk/python/feast/binaries/ \ No newline at end of file +sdk/python/feast/binaries/ diff --git a/CHANGELOG.md b/CHANGELOG.md index 5d97c33480c..93b7ea5edf1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,59 @@ # Changelog +# [0.43.0](https://github.com/feast-dev/feast/compare/v0.42.0...v0.43.0) (2025-01-20) + + +### Bug Fixes + +* Add k8s module to feature-server image ([#4839](https://github.com/feast-dev/feast/issues/4839)) ([f565565](https://github.com/feast-dev/feast/commit/f565565e0132ea5170221dc6af2e93a5dc3e750d)) +* Adding input to workflow ([e3e8c97](https://github.com/feast-dev/feast/commit/e3e8c975b4b9891913d0be8d50df909d4d243191)) +* Change image push to use --all-tags option ([#4926](https://github.com/feast-dev/feast/issues/4926)) ([02458fd](https://github.com/feast-dev/feast/commit/02458fd7aad49d5daa5b9836f5abdc4dd81d07bb)) +* Fix integration build/push for images ([#4923](https://github.com/feast-dev/feast/issues/4923)) ([695e49b](https://github.com/feast-dev/feast/commit/695e49bd93a4c8af2ce5839586295b5e74e1b98e)) +* Fix integration operator push ([#4924](https://github.com/feast-dev/feast/issues/4924)) ([13c7267](https://github.com/feast-dev/feast/commit/13c7267b555cca4f3361f34fb384a6fd9f27dedf)) +* Fix release.yml ([#4845](https://github.com/feast-dev/feast/issues/4845)) ([b4768a8](https://github.com/feast-dev/feast/commit/b4768a81b94352de037dc305df309fcf06fd2973)) +* Fixing some of the warnings with the github actions ([#4763](https://github.com/feast-dev/feast/issues/4763)) ([1119439](https://github.com/feast-dev/feast/commit/1119439c49bc90e62f02da078901509c1d740236)) +* Improve status.applied updates & add offline pvc unit test ([#4871](https://github.com/feast-dev/feast/issues/4871)) ([3f49517](https://github.com/feast-dev/feast/commit/3f49517dfeabea5ffbd3f6b589cc0f2280ee4018)) +* Made fixes to Go Operator DB persistence ([#4830](https://github.com/feast-dev/feast/issues/4830)) ([cdc0753](https://github.com/feast-dev/feast/commit/cdc075360242bfdf3812d394a3c9c550f81b0f98)) +* Make transformation_service_endpoint configuration optional ([#4880](https://github.com/feast-dev/feast/issues/4880)) ([c62377b](https://github.com/feast-dev/feast/commit/c62377bc095a83022d13e5a8a3a9413d7e0f3e2c)) +* Move pre-release image builds to quay.io, retire gcr.io pushes ([#4922](https://github.com/feast-dev/feast/issues/4922)) ([40b975b](https://github.com/feast-dev/feast/commit/40b975b8468de2678af8b191e93495e51af0b6aa)) +* Performance regression in /get-online-features ([#4892](https://github.com/feast-dev/feast/issues/4892)) ([0db56a2](https://github.com/feast-dev/feast/commit/0db56a2cb5888bc21dbdb331e2b5fc3d33508424)) +* Refactor Operator to deploy all feast services to the same Deployment/Pod ([#4863](https://github.com/feast-dev/feast/issues/4863)) ([88854dd](https://github.com/feast-dev/feast/commit/88854dd56fd0becf4a5d5293735a1c9ba394d53d)) +* Remove unnecessary google cloud steps & upgrade docker action versions ([#4925](https://github.com/feast-dev/feast/issues/4925)) ([32aaf9a](https://github.com/feast-dev/feast/commit/32aaf9aba96c53e1c69577312982472182e99659)) +* Remove verifyClient TLS offlineStore option from the Operator ([#4847](https://github.com/feast-dev/feast/issues/4847)) ([79fa247](https://github.com/feast-dev/feast/commit/79fa247026dd95e75a19308d437997310d061b35)) +* Resolving syntax error while querying a feature view with column name starting with a number and BigQuery as data source ([#4908](https://github.com/feast-dev/feast/issues/4908)) ([d3495a0](https://github.com/feast-dev/feast/commit/d3495a09083b1e6a746fff8444f0bbb887d6ac8b)) +* Updated python-helm-demo example to use MinIO instead of GS ([#4691](https://github.com/feast-dev/feast/issues/4691)) ([31afd99](https://github.com/feast-dev/feast/commit/31afd99c0969002fe04982e40cf7a857960f7abf)) + + +### Features + +* Add date field support to spark ([#4913](https://github.com/feast-dev/feast/issues/4913)) ([a8aeb79](https://github.com/feast-dev/feast/commit/a8aeb79830f12358c2355be44fca68e61992cb46)) +* Add date support when converting from python to feast types ([#4918](https://github.com/feast-dev/feast/issues/4918)) ([bd9f071](https://github.com/feast-dev/feast/commit/bd9f071017756e205fbabe6af0d38dfaa9be3d7b)) +* Add duckdb extra to multicloud release image ([#4862](https://github.com/feast-dev/feast/issues/4862)) ([b539eba](https://github.com/feast-dev/feast/commit/b539ebaad5ec2c1a199fe08ceccd206754ce82f0)) +* Add milvus package to release image & option to Operator ([#4870](https://github.com/feast-dev/feast/issues/4870)) ([ef724b6](https://github.com/feast-dev/feast/commit/ef724b66bd4d5f355b055d6d81525c4a17ce94c1)) +* Add Milvus Vector Database Implementation ([#4751](https://github.com/feast-dev/feast/issues/4751)) ([22c7b58](https://github.com/feast-dev/feast/commit/22c7b58f9590a357eaa57c77d5ed351f1fa07501)) +* Add online/offline replica support ([#4812](https://github.com/feast-dev/feast/issues/4812)) ([b97da6c](https://github.com/feast-dev/feast/commit/b97da6ca3a08e3f0fc35552dd7f0bd3b59083f35)) +* Added pvc accessModes support ([#4851](https://github.com/feast-dev/feast/issues/4851)) ([a73514c](https://github.com/feast-dev/feast/commit/a73514cd4f7fecbc89679566e0f8a0af16b6b06d)) +* Adding EnvFrom support for the OptionalConfigs type to the Go Operator ([#4909](https://github.com/feast-dev/feast/issues/4909)) ([e01e510](https://github.com/feast-dev/feast/commit/e01e51076f5d8fe5be459037bd254e6f94e0cb0f)) +* Adding Feature Server to components docs ([#4868](https://github.com/feast-dev/feast/issues/4868)) ([f95e54b](https://github.com/feast-dev/feast/commit/f95e54bdbee80be6b0e290a02e56f92daac2cf64)) +* Adding features field to retrieve_online_features to return mor… ([#4869](https://github.com/feast-dev/feast/issues/4869)) ([7df287e](https://github.com/feast-dev/feast/commit/7df287e8c0f5ec3ab3fa88fd5576f636053a3769)) +* Adding packages for Milvus Online Store ([#4854](https://github.com/feast-dev/feast/issues/4854)) ([49171bd](https://github.com/feast-dev/feast/commit/49171bd53fb8bfc325eb7167cac8cae18a28bd63)) +* Adding vector_search parameter to fields ([#4855](https://github.com/feast-dev/feast/issues/4855)) ([739eaa7](https://github.com/feast-dev/feast/commit/739eaa78e6d995ee0750292d2f8d81886a3f9829)) +* Feast Operator support log level configuration for services ([#4808](https://github.com/feast-dev/feast/issues/4808)) ([19424bc](https://github.com/feast-dev/feast/commit/19424bcc975d90d922791b5bd0da6ac13955c0c5)) +* Go Operator - Parsing the output to go structs ([#4832](https://github.com/feast-dev/feast/issues/4832)) ([732865f](https://github.com/feast-dev/feast/commit/732865f20e7fae7a46f54be7bc469ce2b3bc44e2)) +* Implement `date_partition_column` for `SparkSource` ([#4844](https://github.com/feast-dev/feast/issues/4844)) ([c5ffa03](https://github.com/feast-dev/feast/commit/c5ffa037cb030c64d6e25995199cf762cc0e9b2a)) +* Loading the CA trusted store certificate into Feast to verify the public certificate. ([#4852](https://github.com/feast-dev/feast/issues/4852)) ([132ce2a](https://github.com/feast-dev/feast/commit/132ce2a6c9e3ff8544680d5237e9e1523d988d7e)) +* Operator E2E test to validate FeatureStore custom resource using remote registry ([#4822](https://github.com/feast-dev/feast/issues/4822)) ([d558ef7](https://github.com/feast-dev/feast/commit/d558ef7e19aa561c37c38d4d0da2b8c1467414f5)) +* Operator improvements ([#4928](https://github.com/feast-dev/feast/issues/4928)) ([7a1f4dd](https://github.com/feast-dev/feast/commit/7a1f4dd8b96a40d055467e1e5f72c91167e40484)) +* Removing the tls_verify_client flag from feast cli for offline server. ([#4842](https://github.com/feast-dev/feast/issues/4842)) ([8320e23](https://github.com/feast-dev/feast/commit/8320e23eb85cc419ef8aa0fdc07efa81857e0345)) +* Separating the RBAC and Remote related integration tests. ([#4905](https://github.com/feast-dev/feast/issues/4905)) ([76e1e21](https://github.com/feast-dev/feast/commit/76e1e2178c285886136e8f2fc4436302e4291715)) +* Snyk vulnerability issues fix. ([#4867](https://github.com/feast-dev/feast/issues/4867)) ([dbc9207](https://github.com/feast-dev/feast/commit/dbc92070c8ef6b9e4e53d89ec03090bf30bd0f60)), closes [#6](https://github.com/feast-dev/feast/issues/6) [#3](https://github.com/feast-dev/feast/issues/3) [#4](https://github.com/feast-dev/feast/issues/4) +* Use ASOF JOIN in Snowflake offline store query ([#4850](https://github.com/feast-dev/feast/issues/4850)) ([8f591a2](https://github.com/feast-dev/feast/commit/8f591a235ba5bd9d1bc598195f46c7e12e437a2c)) + + +### Reverts + +* Revert "chore: Add Milvus to pr_integration_tests.yml" ([#4900](https://github.com/feast-dev/feast/issues/4900)) ([07958f7](https://github.com/feast-dev/feast/commit/07958f71cd89984325ec3ca2006b17fe5d333d02)), closes [#4891](https://github.com/feast-dev/feast/issues/4891) + # [0.42.0](https://github.com/feast-dev/feast/compare/v0.41.0...v0.42.0) (2024-12-05) diff --git a/Makefile b/Makefile index d7374d347c8..c199eb3a5ee 100644 --- a/Makefile +++ b/Makefile @@ -14,7 +14,13 @@ # limitations under the License. # -ROOT_DIR := $(shell dirname $(realpath $(firstword $(MAKEFILE_LIST)))) +ROOT_DIR := $(shell dirname $(realpath $(firstword $(MAKEFILE_LIST)))) + +# install tools in project (tool) dir to not pollute the system +TOOL_DIR := $(ROOT_DIR)/tools +export GOBIN=$(TOOL_DIR)/bin +export PATH := $(TOOL_DIR)/bin:$(PATH) + MVN := mvn -f java/pom.xml ${MAVEN_EXTRA_OPTS} OS := linux ifeq ($(shell uname -s), Darwin) @@ -24,12 +30,15 @@ TRINO_VERSION ?= 376 PYTHON_VERSION = ${shell python --version | grep -Eo '[0-9]\.[0-9]+'} PYTHON_VERSIONS := 3.9 3.10 3.11 + define get_env_name $(subst .,,py$(1)) endef # General +$(TOOL_DIR): + mkdir -p $@/bin format: format-python format-java @@ -96,15 +105,28 @@ test-python-unit: python -m pytest -n 8 --color=yes sdk/python/tests test-python-integration: - python -m pytest -n 8 --integration --color=yes --durations=10 --timeout=1200 --timeout_method=thread --dist loadgroup \ + python -m pytest --tb=short -v -n 8 --integration --color=yes --durations=10 --timeout=1200 --timeout_method=thread --dist loadgroup \ -k "(not snowflake or not test_historical_features_main)" \ + -m "not rbac_remote_integration_test" \ + --log-cli-level=INFO -s \ sdk/python/tests test-python-integration-local: FEAST_IS_LOCAL_TEST=True \ FEAST_LOCAL_ONLINE_CONTAINER=True \ - python -m pytest -n 8 --color=yes --integration --durations=10 --timeout=1200 --timeout_method=thread --dist loadgroup \ + python -m pytest --tb=short -v -n 8 --color=yes --integration --durations=10 --timeout=1200 --timeout_method=thread --dist loadgroup \ + -k "not test_lambda_materialization and not test_snowflake_materialization" \ + -m "not rbac_remote_integration_test" \ + --log-cli-level=INFO -s \ + sdk/python/tests + +test-python-integration-rbac-remote: + FEAST_IS_LOCAL_TEST=True \ + FEAST_LOCAL_ONLINE_CONTAINER=True \ + python -m pytest --tb=short -v -n 8 --color=yes --integration --durations=10 --timeout=1200 --timeout_method=thread --dist loadgroup \ -k "not test_lambda_materialization and not test_snowflake_materialization" \ + -m "rbac_remote_integration_test" \ + --log-cli-level=INFO -s \ sdk/python/tests test-python-integration-container: @@ -268,7 +290,7 @@ test-python-universal-postgres-online: not test_snowflake" \ sdk/python/tests - test-python-universal-mysql-online: +test-python-universal-mysql-online: PYTHONPATH='.' \ FULL_REPO_CONFIGS_MODULE=sdk.python.feast.infra.online_stores.mysql_online_store.mysql_repo_configuration \ PYTEST_PLUGINS=sdk.python.tests.integration.feature_repos.universal.online_store.mysql \ @@ -292,7 +314,11 @@ test-python-universal-cassandra: FULL_REPO_CONFIGS_MODULE=sdk.python.feast.infra.online_stores.cassandra_online_store.cassandra_repo_configuration \ PYTEST_PLUGINS=sdk.python.tests.integration.feature_repos.universal.online_store.cassandra \ python -m pytest -x --integration \ - sdk/python/tests + sdk/python/tests/integration/offline_store/test_feature_logging.py \ + --ignore=sdk/python/tests/integration/offline_store/test_validation.py \ + -k "not test_snowflake and \ + not test_spark_materialization_consistency and \ + not test_universal_materialization" test-python-universal-hazelcast: PYTHONPATH='.' \ @@ -330,7 +356,7 @@ test-python-universal-cassandra-no-cloud-providers: not test_snowflake" \ sdk/python/tests - test-python-universal-elasticsearch-online: +test-python-universal-elasticsearch-online: PYTHONPATH='.' \ FULL_REPO_CONFIGS_MODULE=sdk.python.feast.infra.online_stores.elasticsearch_online_store.elasticsearch_repo_configuration \ PYTEST_PLUGINS=sdk.python.tests.integration.feature_repos.universal.online_store.elasticsearch \ @@ -349,6 +375,14 @@ test-python-universal-cassandra-no-cloud-providers: not test_snowflake" \ sdk/python/tests +test-python-universal-milvus-online: + PYTHONPATH='.' \ + FULL_REPO_CONFIGS_MODULE=sdk.python.feast.infra.online_stores.milvus_online_store.milvus_repo_configuration \ + PYTEST_PLUGINS=sdk.python.tests.integration.feature_repos.universal.online_store.milvus \ + python -m pytest -n 8 --integration \ + -k "test_retrieve_online_milvus_documents" \ + sdk/python/tests --ignore=sdk/python/tests/integration/offline_store/test_dqm_validation.py + test-python-universal-singlestore-online: PYTHONPATH='.' \ FULL_REPO_CONFIGS_MODULE=sdk.python.feast.infra.online_stores.singlestore_repo_configuration \ @@ -437,17 +471,17 @@ kill-trino-locally: # Docker -build-docker: build-feature-server-python-aws-docker build-feature-transformation-server-docker build-feature-server-java-docker +build-docker: build-feature-server-docker build-feature-transformation-server-docker build-feature-server-java-docker build-feast-operator-docker push-ci-docker: docker push $(REGISTRY)/feast-ci:$(VERSION) push-feature-server-docker: - docker push $(REGISTRY)/feature-server:$$VERSION + docker push $(REGISTRY)/feature-server:$(VERSION) build-feature-server-docker: - docker buildx build --build-arg VERSION=$$VERSION \ - -t $(REGISTRY)/feature-server:$$VERSION \ + docker buildx build --build-arg VERSION=$(VERSION) \ + -t $(REGISTRY)/feature-server:$(VERSION) \ -f sdk/python/feast/infra/feature_servers/multicloud/Dockerfile --load . push-feature-transformation-server-docker: @@ -493,10 +527,18 @@ build-feast-operator-docker: # Dev images build-feature-server-dev: - docker buildx build --build-arg VERSION=dev \ + docker buildx build \ -t feastdev/feature-server:dev \ -f sdk/python/feast/infra/feature_servers/multicloud/Dockerfile.dev --load . +build-feature-server-dev-docker: + docker buildx build \ + -t $(REGISTRY)/feature-server:$(VERSION) \ + -f sdk/python/feast/infra/feature_servers/multicloud/Dockerfile.dev --load . + +push-feature-server-dev-docker: + docker push $(REGISTRY)/feature-server:$(VERSION) + build-java-docker-dev: make build-java-no-tests REVISION=dev docker buildx build --build-arg VERSION=dev \ @@ -549,43 +591,60 @@ build-ui: # Go SDK & embedded -install-protoc-dependencies: - pip install "protobuf>=4.24.0,<5.0.0" "grpcio-tools>=1.56.2,<2" "mypy-protobuf>=3.1" +PB_REL = https://github.com/protocolbuffers/protobuf/releases +PB_VERSION = 3.11.2 +PB_ARCH := $(shell uname -m) +ifeq ($(PB_ARCH), arm64) + PB_ARCH=aarch_64 +endif +PB_PROTO_FOLDERS=core registry serving types storage -install-go-proto-dependencies: +$(TOOL_DIR)/protoc-$(PB_VERSION)-$(OS)-$(PB_ARCH).zip: $(TOOL_DIR) + cd $(TOOL_DIR) && \ + curl -LO $(PB_REL)/download/v$(PB_VERSION)/protoc-$(PB_VERSION)-$(OS)-$(PB_ARCH).zip + +.PHONY: install-go-proto-dependencies +install-go-proto-dependencies: $(TOOL_DIR)/protoc-$(PB_VERSION)-$(OS)-$(PB_ARCH).zip + unzip -u $(TOOL_DIR)/protoc-$(PB_VERSION)-$(OS)-$(PB_ARCH).zip -d $(TOOL_DIR) go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.31.0 go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@v1.3.0 +.PHONY: compile-protos-go +compile-protos-go: install-go-proto-dependencies + $(foreach folder,$(PB_PROTO_FOLDERS), \ + protoc --proto_path=$(ROOT_DIR)/protos \ + --go_out=$(ROOT_DIR)/go/protos \ + --go_opt=module=github.com/feast-dev/feast/go/protos \ + --go-grpc_out=$(ROOT_DIR)/go/protos \ + --go-grpc_opt=module=github.com/feast-dev/feast/go/protos $(ROOT_DIR)/protos/feast/$(folder)/*.proto; ) true + #install-go-ci-dependencies: # go install golang.org/x/tools/cmd/goimports # python -m pip install "pybindgen==0.22.1" "grpcio-tools>=1.56.2,<2" "mypy-protobuf>=3.1" -build-go: - compile-protos-go +.PHONY: build-go +build-go: compile-protos-go go build -o feast ./go/main.go +.PHONY: install-feast-ci-locally install-feast-ci-locally: - pip install -e ".[ci]" + uv pip install -e ".[ci]" -test-go: - compile-protos-go - compile-protos-python - install-feast-ci-locally +.PHONY: test-go +test-go: compile-protos-go install-feast-ci-locally compile-protos-python CGO_ENABLED=1 go test -coverprofile=coverage.out ./... && go tool cover -html=coverage.out -o coverage.html +.PHONY: format-go format-go: gofmt -s -w go/ -lint-go: - compile-protos-go +.PHONY: lint-go +lint-go: compile-protos-go go vet ./go/internal/feast +.PHONY: build-go-docker-dev build-go-docker-dev: docker buildx build --build-arg VERSION=dev \ -t feastdev/feature-server-go:dev \ -f go/infra/docker/feature-server/Dockerfile --load . -compile-protos-go: - install-go-proto-dependencies - install-protoc-dependencies - python setup.py build_go_protos \ No newline at end of file diff --git a/docs/README.md b/docs/README.md index 5e36e1ce40a..73c6860137e 100644 --- a/docs/README.md +++ b/docs/README.md @@ -11,12 +11,12 @@ for historical feature extraction used in model training and an (2) [online stor for serving features at low-latency in production systems and applications. Feast is a configurable operational data system that re-uses existing infrastructure to manage and serve machine learning -features to realtime models. For more details please review our [architecture](getting-started/architecture/overview.md). +features to realtime models. For more details, please review our [architecture](getting-started/architecture/overview.md). Concretely, Feast provides: -* A python SDK for programtically defining features, entities, sources, and (optionally) transformations -* A python SDK for for reading and writing features to configured offline and online data stores +* A Python SDK for programmatically defining features, entities, sources, and (optionally) transformations +* A Python SDK for reading and writing features to configured offline and online data stores * An [optional feature server](reference/feature-servers/README.md) for reading and writing features (useful for non-python languages) * A [UI](reference/alpha-web-ui.md) for viewing and exploring information about features defined in the project * A [CLI tool](reference/feast-cli-commands.md) for viewing and updating feature information @@ -24,8 +24,8 @@ Concretely, Feast provides: Feast allows ML platform teams to: * **Make features consistently available for training and low-latency serving** by managing an _offline store_ (to process historical data for scale-out batch scoring or model training), a low-latency _online store_ (to power real-time prediction)_,_ and a battle-tested _feature server_ (to serve pre-computed features online). -* **Avoid data leakage** by generating point-in-time correct feature sets so data scientists can focus on feature engineering rather than debugging error-prone dataset joining logic. This ensure that future feature values do not leak to models during training. -* **Decouple ML from data infrastructure** by providing a single data access layer that abstracts feature storage from feature retrieval, ensuring models remain portable as you move from training models to serving models, from batch models to realtime models, and from one data infra system to another. +* **Avoid data leakage** by generating point-in-time correct feature sets so data scientists can focus on feature engineering rather than debugging error-prone dataset joining logic. This ensures that future feature values do not leak to models during training. +* **Decouple ML from data infrastructure** by providing a single data access layer that abstracts feature storage from feature retrieval, ensuring models remain portable as you move from training models to serving models, from batch models to real-time models, and from one data infra system to another. {% hint style="info" %} **Note:** Feast today primarily addresses _timestamped structured data_. @@ -42,25 +42,28 @@ serving system must make a request to the feature store to retrieve feature valu ## Who is Feast for? -Feast helps ML platform/MLOps teams with DevOps experience productionize real-time models. Feast also helps these teams -build a feature platform that improves collaboration between data engineers, software engineers, machine learning -engineers, and data scientists. +Feast helps ML platform/MLOps teams with DevOps experience productionize real-time models. Feast also helps these teams build a feature platform that improves collaboration between data engineers, software engineers, machine learning engineers, and data scientists. -Feast is likely **not** the right tool if you -* are in an organization that’s just getting started with ML and is not yet sure what the business impact of ML is +* *For Data Scientists*: Feast is a tool where you can easily define, store, and retrieve your features for both model development and model deployment. By using Feast, you can focus on what you do best: build features that power your AI/ML models and maximize the value of your data. +    +* *For MLOps Engineers*: Feast is a library that allows you to connect your existing infrastructure (e.g., online database, application server, microservice, analytical database, and orchestration tooling) that enables your Data Scientists to ship features for their models to production using a friendly SDK without having to be concerned with software engineering challenges that occur from serving real-time production systems. By using Feast, you can focus on maintaining a resilient system, instead of implementing features for Data Scientists. +    +* *For Data Engineers*: Feast provides a centralized catalog for storing feature definitions, allowing one to maintain a single source of truth for feature data. It provides the abstraction for reading and writing to many different types of offline and online data stores. Using either the provided Python SDK or the feature server service, users can write data to the online and/or offline stores and then read that data out again in either low-latency online scenarios for model inference, or in batch scenarios for model training. + +* *For AI Engineers*: Feast provides a platform designed to scale your AI applications by enabling seamless integration of richer data and facilitating fine-tuning. With Feast, you can optimize the performance of your AI models while ensuring a scalable and efficient data pipeline. ## What Feast is not? ### Feast is not -* **an** [**ETL**](https://en.wikipedia.org/wiki/Extract,\_transform,\_load) / [**ELT**](https://en.wikipedia.org/wiki/Extract,\_load,\_transform) **system.** Feast is not a general purpose data pipelining system. Users often leverage tools like [dbt](https://www.getdbt.com) to manage upstream data transformations. Feast does support some [transformations](getting-started/architecture/feature-transformetion.md). -* **a data orchestration tool:** Feast does not manage or orchestrate complex workflow DAGs. It relies on upstream data pipelines to produce feature values and integrations with tools like [Airflow](https://airflow.apache.org) to make features consistently available. -* **a data warehouse:** Feast is not a replacement for your data warehouse or the source of truth for all transformed data in your organization. Rather, Feast is a light-weight downstream layer that can serve data from an existing data warehouse (or other data sources) to models in production. -* **a database:** Feast is not a database, but helps manage data stored in other systems (e.g. BigQuery, Snowflake, DynamoDB, Redis) to make features consistently available at training / serving time +* **An** [**ETL**](https://en.wikipedia.org/wiki/Extract,\_transform,\_load) / [**ELT**](https://en.wikipedia.org/wiki/Extract,\_load,\_transform) **system.** Feast is not a general purpose data pipelining system. Users often leverage tools like [dbt](https://www.getdbt.com) to manage upstream data transformations. Feast does support some [transformations](getting-started/architecture/feature-transformetion.md). +* **A data orchestration tool:** Feast does not manage or orchestrate complex workflow DAGs. It relies on upstream data pipelines to produce feature values and integrations with tools like [Airflow](https://airflow.apache.org) to make features consistently available. +* **A data warehouse:** Feast is not a replacement for your data warehouse or the source of truth for all transformed data in your organization. Rather, Feast is a lightweight downstream layer that can serve data from an existing data warehouse (or other data sources) to models in production. +* **A database:** Feast is not a database, but helps manage data stored in other systems (e.g. BigQuery, Snowflake, DynamoDB, Redis) to make features consistently available at training / serving time ### Feast does not _fully_ solve * **reproducible model training / model backtesting / experiment management**: Feast captures feature and model metadata, but does not version-control datasets / labels or manage train / test splits. Other tools like [DVC](https://dvc.org/), [MLflow](https://www.mlflow.org/), and [Kubeflow](https://www.kubeflow.org/) are better suited for this. -* **batch feature engineering**: Feast supports on demand and streaming transformations. Feast is also investing in supporting batch transformations. +* **batch feature engineering**: Feast supports on-demand and streaming transformations. Feast is also investing in supporting batch transformations. * **native streaming feature integration:** Feast enables users to push streaming features, but does not pull from streaming sources or manage streaming pipelines. * **lineage:** Feast helps tie feature values to model versions, but is not a complete solution for capturing end-to-end lineage from raw data sources to model versions. Feast also has community contributed plugins with [DataHub](https://datahubproject.io/docs/generated/ingestion/sources/feast/) and [Amundsen](https://github.com/amundsen-io/amundsen/blob/4a9d60176767c4d68d1cad5b093320ea22e26a49/databuilder/databuilder/extractor/feast\_extractor.py). * **data quality / drift detection**: Feast has experimental integrations with [Great Expectations](https://greatexpectations.io/), but is not purpose built to solve data drift / data quality issues. This requires more sophisticated monitoring across data pipelines, served feature values, labels, and model versions. @@ -72,7 +75,7 @@ Many companies have used Feast to power real-world ML use cases such as: * Personalizing online recommendations by leveraging pre-computed historical user or item features. * Online fraud detection, using features that compare against (pre-computed) historical transaction patterns * Churn prediction (an offline model), generating feature values for all users at a fixed cadence in batch -* Credit scoring, using pre-computed historical features to compute probability of default +* Credit scoring, using pre-computed historical features to compute the probability of default ## How can I get started? diff --git a/docs/SUMMARY.md b/docs/SUMMARY.md index 7aad7a94428..e24e15fb5cb 100644 --- a/docs/SUMMARY.md +++ b/docs/SUMMARY.md @@ -32,6 +32,7 @@ * [Registry](getting-started/components/registry.md) * [Offline store](getting-started/components/offline-store.md) * [Online store](getting-started/components/online-store.md) + * [Feature server](getting-started/components/feature-server.md) * [Batch Materialization Engine](getting-started/components/batch-materialization-engine.md) * [Provider](getting-started/components/provider.md) * [Authorization Manager](getting-started/components/authz_manager.md) diff --git a/docs/getting-started/components/README.md b/docs/getting-started/components/README.md index e1c000abced..4c6f3a54dfc 100644 --- a/docs/getting-started/components/README.md +++ b/docs/getting-started/components/README.md @@ -12,6 +12,10 @@ [online-store.md](online-store.md) {% endcontent-ref %} +{% content-ref url="feature-server.md" %} +[feature-server.md](feature-server.md) +{% endcontent-ref %} + {% content-ref url="batch-materialization-engine.md" %} [batch-materialization-engine.md](batch-materialization-engine.md) {% endcontent-ref %} diff --git a/docs/getting-started/components/feature-server.md b/docs/getting-started/components/feature-server.md new file mode 100644 index 00000000000..90e6d25e5a2 --- /dev/null +++ b/docs/getting-started/components/feature-server.md @@ -0,0 +1,40 @@ +# Feature Server + +The Feature Server is a core architectural component in Feast, designed to provide low-latency feature retrieval and updates for machine learning applications. + +It is a REST API server built using [FastAPI](https://fastapi.tiangolo.com/) and exposes a limited set of endpoints to serve features, push data, and support materialization operations. The server is scalable, flexible, and designed to work seamlessly with various deployment environments, including local setups and cloud-based systems. + +## Motivation + +In machine learning workflows, real-time access to feature values is critical for enabling low-latency predictions. The Feature Server simplifies this requirement by: + +1. **Serving Features:** Allowing clients to retrieve feature values for specific entities in real-time, reducing the complexity of direct interactions with the online store. +2. **Data Integration:** Providing endpoints to push feature data directly into the online or offline store, ensuring data freshness and consistency. +3. **Scalability:** Supporting horizontal scaling to handle high request volumes efficiently. +4. **Standardized API:** Exposing HTTP/JSON endpoints that integrate seamlessly with various programming languages and ML pipelines. +5. **Secure Communication:** Supporting TLS (SSL) for secure data transmission in production environments. + +## Architecture + +The Feature Server operates as a stateless service backed by two key components: + +- **[Online Store](./online-store.md):** The primary data store used for low-latency feature retrieval. +- **[Registry](./registry.md):** The metadata store that defines feature sets, feature views, and their relationships to entities. + +## Key Features + +1. **RESTful API:** Provides standardized endpoints for feature retrieval and data pushing. +2. **CLI Integration:** Easily managed through the Feast CLI with commands like `feast serve`. +3. **Flexible Deployment:** Can be deployed locally, via Docker, or on Kubernetes using Helm charts. +4. **Scalability:** Designed for distributed deployments to handle large-scale workloads. +5. **TLS Support:** Ensures secure communication in production setups. + +## Endpoints Overview + +| Endpoint | Description | +| -------------------------- | ----------------------------------------------------------------------- | +| `/get-online-features` | Retrieves feature values for specified entities and feature references. | +| `/push` | Pushes feature data to the online and/or offline store. | +| `/materialize` | Materializes features within a specific time range to the online store. | +| `/materialize-incremental` | Incrementally materializes features up to the current timestamp. | + diff --git a/docs/getting-started/components/overview.md b/docs/getting-started/components/overview.md index ac0b99de8ab..05c7503d842 100644 --- a/docs/getting-started/components/overview.md +++ b/docs/getting-started/components/overview.md @@ -13,6 +13,7 @@ * **Deploy Model:** The trained model binary (and list of features) are deployed into a model serving system. This step is not executed by Feast. * **Prediction:** A backend system makes a request for a prediction from the model serving service. * **Get Online Features:** The model serving service makes a request to the Feast Online Serving service for online features using a Feast SDK. +* **Feature Retrieval:** The online serving service retrieves the latest feature values from the online store and returns them to the model serving service. ## Components @@ -24,6 +25,7 @@ A complete Feast deployment contains the following components: * Materialize (load) feature values into the online store. * Build and retrieve training datasets from the offline store. * Retrieve online features. +* **Feature Server:** The Feature Server is a REST API server that serves feature values for a given entity key and feature reference. The Feature Server is designed to be horizontally scalable and can be deployed in a distributed manner. * **Stream Processor:** The Stream Processor can be used to ingest feature data from streams and write it into the online or offline stores. Currently, there's an experimental Spark processor that's able to consume data from Kafka. * **Batch Materialization Engine:** The [Batch Materialization Engine](batch-materialization-engine.md) component launches a process which loads data into the online store from the offline store. By default, Feast uses a local in-process engine implementation to materialize data. However, additional infrastructure can be used for a more scalable materialization process. * **Online Store:** The online store is a database that stores only the latest feature values for each entity. The online store is either populated through materialization jobs or through [stream ingestion](../../reference/data-sources/push.md). diff --git a/docs/getting-started/concepts/permission.md b/docs/getting-started/concepts/permission.md index a6353579687..8db67032878 100644 --- a/docs/getting-started/concepts/permission.md +++ b/docs/getting-started/concepts/permission.md @@ -69,7 +69,7 @@ Permission( name="feature-reader", types=[FeatureView, FeatureService], policy=RoleBasedPolicy(roles=["super-reader"]), - actions=[AuthzedAction.DESCRIBE, READ], + actions=[AuthzedAction.DESCRIBE, *READ], ) ``` diff --git a/docs/getting-started/quickstart.md b/docs/getting-started/quickstart.md index d35446ce7f0..a83897005fd 100644 --- a/docs/getting-started/quickstart.md +++ b/docs/getting-started/quickstart.md @@ -10,6 +10,9 @@ Feast (Feature Store) is an open-source feature store designed to facilitate the * *For Data Engineers*: Feast provides a centralized catalog for storing feature definitions allowing one to maintain a single source of truth for feature data. It provides the abstraction for reading and writing to many different types of offline and online data stores. Using either the provided python SDK or the feature server service, users can write data to the online and/or offline stores and then read that data out again in either low-latency online scenarios for model inference, or in batch scenarios for model training. +* *For AI Engineers*: Feast provides a platform designed to scale your AI applications by enabling seamless integration of richer data and facilitating fine-tuning. With Feast, you can optimize the performance of your AI models while ensuring a scalable and efficient data pipeline. + + For more info refer to [Introduction to feast](../README.md) ## Prerequisites diff --git a/docs/how-to-guides/customizing-feast/adding-support-for-a-new-online-store.md b/docs/how-to-guides/customizing-feast/adding-support-for-a-new-online-store.md index 5e26f133cef..ee75aa6b74f 100644 --- a/docs/how-to-guides/customizing-feast/adding-support-for-a-new-online-store.md +++ b/docs/how-to-guides/customizing-feast/adding-support-for-a-new-online-store.md @@ -25,7 +25,7 @@ OnlineStore class names must end with the OnlineStore suffix! ### Contrib online stores -New online stores go in `sdk/python/feast/infra/online_stores/contrib/`. +New online stores go in `sdk/python/feast/infra/online_stores/`. #### What is a contrib plugin? diff --git a/docs/how-to-guides/starting-feast-servers-tls-mode.md b/docs/how-to-guides/starting-feast-servers-tls-mode.md index e1ddbc08be5..a868e17cf96 100644 --- a/docs/how-to-guides/starting-feast-servers-tls-mode.md +++ b/docs/how-to-guides/starting-feast-servers-tls-mode.md @@ -189,3 +189,8 @@ INFO: Waiting for application startup. INFO: Application startup complete. INFO: Uvicorn running on https://0.0.0.0:8888 (Press CTRL+C to quit) ``` + + +## Adding public key to CA trust store and configuring the feast to use the trust store. +You can pass the public key for SSL verification using the `cert` parameter, however, it is sometimes difficult to maintain individual certificates and pass them individually. +The alternative recommendation is to add the public certificate to CA trust store and set the path as an environment variable (e.g., `FEAST_CA_CERT_FILE_PATH`). Feast will use the trust store path in the `FEAST_CA_CERT_FILE_PATH` environment variable. \ No newline at end of file diff --git a/docs/reference/feature-servers/README.md b/docs/reference/feature-servers/README.md index 2ceaf5807f3..156e60c7431 100644 --- a/docs/reference/feature-servers/README.md +++ b/docs/reference/feature-servers/README.md @@ -1,4 +1,4 @@ -# Feature servers +# Feast servers Feast users can choose to retrieve features from a feature server, as opposed to through the Python SDK. @@ -12,4 +12,8 @@ Feast users can choose to retrieve features from a feature server, as opposed to {% content-ref url="offline-feature-server.md" %} [offline-feature-server.md](offline-feature-server.md) +{% endcontent-ref %} + +{% content-ref url="registry-server.md" %} +[registry-server.md](registry-server.md) {% endcontent-ref %} \ No newline at end of file diff --git a/docs/reference/feature-servers/registry-server.md b/docs/reference/feature-servers/registry-server.md new file mode 100644 index 00000000000..9707a597035 --- /dev/null +++ b/docs/reference/feature-servers/registry-server.md @@ -0,0 +1,26 @@ +# Registry server + +## Description + +The Registry server uses the gRPC communication protocol to exchange data. +This enables users to communicate with the server using any programming language that can make gRPC requests. + +## How to configure the server + +## CLI + +There is a CLI command that starts the Registry server: `feast serve_registry`. By default, remote Registry Server uses port 6570, the port can be overridden with a `--port` flag. +To start the Registry Server in TLS mode, you need to provide the private and public keys using the `--key` and `--cert` arguments. +More info about TLS mode can be found in [feast-client-connecting-to-remote-registry-sever-started-in-tls-mode](../../how-to-guides/starting-feast-servers-tls-mode.md#starting-feast-registry-server-in-tls-mode) + +## How to configure the client + +Please see the detail how to configure Remote Registry client [remote.md](../registries/remote.md) + +# Registry Server Permissions and Access Control + +Please refer the [page](./../registry/registry-permissions.md) for more details on API Endpoints and Permissions. + +## How to configure Authentication and Authorization ? + +Please refer the [page](./../../../docs/getting-started/concepts/permission.md) for more details on how to configure authentication and authorization. \ No newline at end of file diff --git a/docs/reference/registries/remote.md b/docs/reference/registries/remote.md new file mode 100644 index 00000000000..3651aeb71ea --- /dev/null +++ b/docs/reference/registries/remote.md @@ -0,0 +1,28 @@ +# Remote Registry + +## Description + +The Remote Registry is a gRPC client for the registry that implements the `RemoteRegistry` class using the existing `BaseRegistry` interface. + +## How to configure the client + +User needs to create a client side `feature_store.yaml` file, set the `registry_type` to `remote` and provide the server connection configuration. +The `path` parameter is a URL with a port (default is 6570) used by the client to connect with the Remote Registry server. + +{% code title="feature_store.yaml" %} +```yaml +registry: + registry_type: remote + path: http://localhost:6570 +``` +{% endcode %} + +The optional `cert` parameter can be configured as well, it should point to the public certificate path when the Registry Server starts in SSL mode. This may be needed if the Registry Server is started with a self-signed certificate, typically this file ends with *.crt, *.cer, or *.pem. +More info about the `cert` parameter can be found in [feast-client-connecting-to-remote-registry-sever-started-in-tls-mode](../../how-to-guides/starting-feast-servers-tls-mode.md#feast-client-connecting-to-remote-registry-sever-started-in-tls-mode) + +## How to configure the server + +Please see the detail how to configure registry server [registry-server.md](../feature-servers/registry-server.md) + +## How to configure Authentication and Authorization +Please refer the [page](./../../../docs/getting-started/concepts/permission.md) for more details on how to configure authentication and authorization. diff --git a/examples/credit-risk-end-to-end/01_Credit_Risk_Data_Prep.ipynb b/examples/credit-risk-end-to-end/01_Credit_Risk_Data_Prep.ipynb new file mode 100644 index 00000000000..a345ec8ca46 --- /dev/null +++ b/examples/credit-risk-end-to-end/01_Credit_Risk_Data_Prep.ipynb @@ -0,0 +1,757 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "a52c80c4-1ea2-4d1e-b582-fac51081e76d", + "metadata": {}, + "source": [ + "
" + ] + }, + { + "cell_type": "markdown", + "id": "576a8e30-fe4c-4eda-bc56-9edd7fde3385", + "metadata": {}, + "source": [ + "# Credit Risk Data Preparation" + ] + }, + { + "cell_type": "markdown", + "id": "1f3fbd5a-1587-4b4e-9263-a57490657337", + "metadata": {}, + "source": [ + "Predicting credit risk is an important task for financial institutions. If a bank can accurately determine the probability that a borrower will pay back a future loan, then they can make better decisions on loan terms and approvals. Getting credit risk right is critical to offering good financial services, and getting credit risk wrong could mean going out of business.\n", + "\n", + "AI models have played a central role in modern credit risk assessment systems. In this example, we develop a credit risk model to predict whether a future loan will be good or bad, given some context data (presumably supplied from the loan application). We use the modeling process to demonstrate how Feast can be used to facilitate the serving of data for training and inference use-cases.\n", + "\n", + "In this notebook, we prepare the data." + ] + }, + { + "cell_type": "markdown", + "id": "4d05715f-ddb8-42de-8f0c-212dcbad9e0e", + "metadata": {}, + "source": [ + "### Setup" + ] + }, + { + "cell_type": "markdown", + "id": "6fba29f9-db1f-4ceb-b066-5b2df2c95d33", + "metadata": {}, + "source": [ + "*The following code assumes that you have read the example README.md file, and that you have setup an environment where the code can be run. Please make sure you have addressed the prerequisite needs.*" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "8a897b19-6f82-4631-ae51-8a23182ff267", + "metadata": {}, + "outputs": [], + "source": [ + "# Import Python libraries\n", + "import os\n", + "import warnings\n", + "import datetime as dt\n", + "import pandas as pd\n", + "import numpy as np\n", + "from sklearn.datasets import fetch_openml" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "b944ed48-54b3-43fa-8373-ce788d7e71af", + "metadata": {}, + "outputs": [], + "source": [ + "# suppress warning messages for example flow (don't run if you want to see warnings)\n", + "warnings.filterwarnings('ignore')" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "70788c73-144f-4ecf-b370-c5669c538d93", + "metadata": {}, + "outputs": [], + "source": [ + "# Seed for reproducibility\n", + "SEED = 142" + ] + }, + { + "cell_type": "markdown", + "id": "cfb4dfd0-f583-4aa0-bd39-3ff9fbb80db0", + "metadata": {}, + "source": [ + "### Pull the Data" + ] + }, + { + "cell_type": "markdown", + "id": "3c206dfc-d551-4002-ae63-ccbb981768fa", + "metadata": {}, + "source": [ + "The data we will use to train the model is from the [OpenML](https://www.openml.org/) dataset [credit-g](https://www.openml.org/search?type=data&sort=runs&status=active&id=31), obtained from a 1994 German study. More details on the data can be found in the `DESC` attribute and `details` map (see below)." + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "31a9e964-bdb3-4ae4-b2b4-64bbe0ab93a3", + "metadata": {}, + "outputs": [], + "source": [ + "data = fetch_openml(name=\"credit-g\", version=1, parser='auto')" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "58dbf7c2-f40b-4965-baac-6903a27ef622", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "**Author**: Dr. Hans Hofmann \n", + "**Source**: [UCI](https://archive.ics.uci.edu/ml/datasets/statlog+(german+credit+data)) - 1994 \n", + "**Please cite**: [UCI](https://archive.ics.uci.edu/ml/citation_policy.html)\n", + "\n", + "**German Credit dataset** \n", + "This dataset classifies people described by a set of attributes as good or bad credit risks.\n", + "\n", + "This dataset comes with a cost matrix: \n", + "``` \n", + "Good Bad (predicted) \n", + "Good 0 1 (actual) \n", + "Bad 5 0 \n", + "```\n", + "\n", + "It is worse to class a customer as good when they are bad (5), than it is to class a customer as bad when they are good (1). \n", + "\n", + "### Attribute description \n", + "\n", + "1. Status of existing checking account, in Deutsche Mark. \n", + "2. Duration in months \n", + "3. Credit history (credits taken, paid back duly, delays, critical accounts) \n", + "4. Purpose of the credit (car, television,...) \n", + "5. Credit amount \n", + "6. Status of savings account/bonds, in Deutsche Mark. \n", + "7. Present employment, in number of years. \n", + "8. Installment rate in percentage of disposable income \n", + "9. Personal status (married, single,...) and sex \n", + "10. Other debtors / guarantors \n", + "11. Present residence since X years \n", + "12. Property (e.g. real estate) \n", + "13. Age in years \n", + "14. Other installment plans (banks, stores) \n", + "15. Housing (rent, own,...) \n", + "16. Number of existing credits at this bank \n", + "17. Job \n", + "18. Number of people being liable to provide maintenance for \n", + "19. Telephone (yes,no) \n", + "20. Foreign worker (yes,no)\n", + "\n", + "Downloaded from openml.org.\n" + ] + } + ], + "source": [ + "print(data.DESCR)" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "53de57ec-0fb6-4b51-9c27-696b059a1847", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Original data url: https://archive.ics.uci.edu/ml/datasets/statlog+(german+credit+data)\n", + "Paper url: https://dl.acm.org/doi/abs/10.1145/967900.968104\n" + ] + } + ], + "source": [ + "print(\"Original data url: \".ljust(20), data.details[\"original_data_url\"])\n", + "print(\"Paper url: \".ljust(20), data.details[\"paper_url\"])" + ] + }, + { + "cell_type": "markdown", + "id": "6b2c2514-484e-46cb-aedc-89a301266f44", + "metadata": {}, + "source": [ + "### High-Level Data Inspection" + ] + }, + { + "cell_type": "markdown", + "id": "a76af306-caba-403d-a9cb-b5de12573075", + "metadata": {}, + "source": [ + "Let's inspect the data to see high level details like data types and size. We also want to make sure there are no glaring issues (like a large number of null values)." + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "20fb82c4-ed8d-42f8-b386-c7ebdc9bf786", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "RangeIndex: 1000 entries, 0 to 999\n", + "Data columns (total 21 columns):\n", + " # Column Non-Null Count Dtype \n", + "--- ------ -------------- ----- \n", + " 0 checking_status 1000 non-null category\n", + " 1 duration 1000 non-null int64 \n", + " 2 credit_history 1000 non-null category\n", + " 3 purpose 1000 non-null category\n", + " 4 credit_amount 1000 non-null int64 \n", + " 5 savings_status 1000 non-null category\n", + " 6 employment 1000 non-null category\n", + " 7 installment_commitment 1000 non-null int64 \n", + " 8 personal_status 1000 non-null category\n", + " 9 other_parties 1000 non-null category\n", + " 10 residence_since 1000 non-null int64 \n", + " 11 property_magnitude 1000 non-null category\n", + " 12 age 1000 non-null int64 \n", + " 13 other_payment_plans 1000 non-null category\n", + " 14 housing 1000 non-null category\n", + " 15 existing_credits 1000 non-null int64 \n", + " 16 job 1000 non-null category\n", + " 17 num_dependents 1000 non-null int64 \n", + " 18 own_telephone 1000 non-null category\n", + " 19 foreign_worker 1000 non-null category\n", + " 20 class 1000 non-null category\n", + "dtypes: category(14), int64(7)\n", + "memory usage: 71.0 KB\n" + ] + } + ], + "source": [ + "df = data.frame\n", + "df.info()" + ] + }, + { + "cell_type": "markdown", + "id": "a384932a-40df-45f6-bfbc-a9cf6c708f1b", + "metadata": {}, + "source": [ + "We see that there are 21 columns, each with 1000 non-null values. The first 20 columns are contextual fields with `Dtype` of `category` or `int64`, while the last field is actually the target variable, `class`, which we wish to predict. \n", + "\n", + "From the description (above), the `class` tells us whether a loan to a customer was \"good\" or \"bad\". We are anticipating that patterns in the contextual data, as well as their relationship to the class outcomes, can give insight into loan classification. In the following notebooks, we will build a loan classification model that seeks to encode these patterns and relationships in its weights, such that given a new loan application (context data), the model can predict whether the loan (if approved) will be good or bad in the future." + ] + }, + { + "cell_type": "markdown", + "id": "a451c9a3-0390-4d5a-b687-c59f52445eb1", + "metadata": {}, + "source": [ + "### Data Preparation For Demonstrating Feast" + ] + }, + { + "cell_type": "markdown", + "id": "dc4e7653-b118-44c3-ade3-f1b217b112fc", + "metadata": {}, + "source": [ + "At this point, it's important to bring up that Feast was developed primarily to work with production data. Feast requires datasets to have entities (in our case, IDs) and timestamps, which it uses in joins. Feast can support joining data on multiple entities (like primary keys in SQL), as well as \"created\" timestamps and \"event\" timestamps. However, in this example, we'll keep things more simple.\n", + "\n", + "In a real loan application scenario, the application fields (in a database) would be associated with a timestamp, while the actual loan outcome (label) would be determined much later and recorded separately with a different timestamp.\n", + "\n", + "In order to demonstrate Feast capabilities, such as point-in-time joins, we will mock IDs and timestamps for this data. For IDs, we will use the original dataframe index values. For the timestamps, we will generate random values between \"Tue Sep 24 12:00:00 2023\" and \"Wed Oct 9 12:00:00 2023\"." + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "9d6ec4f6-9410-4858-a440-45dccaa0896b", + "metadata": {}, + "outputs": [], + "source": [ + "# Make index into \"ID\" column\n", + "df = df.reset_index(names=[\"ID\"])" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "055f2cb7-3abf-4d01-be60-e4c7b8ad1988", + "metadata": {}, + "outputs": [], + "source": [ + "# Add mock timestamps\n", + "time_format = \"%a %b %d %H:%M:%S %Y\"\n", + "date = dt.datetime.strptime(\"Wed Oct 9 12:00:00 2023\", time_format)\n", + "end = int(date.timestamp())\n", + "start = int((date - dt.timedelta(days=15)).timestamp()) # 'Tue Sep 24 12:00:00 2023'\n", + "\n", + "def make_tstamp(date):\n", + " dtime = dt.datetime.fromtimestamp(date).ctime()\n", + " return dtime\n", + " \n", + "# (seed set for reproducibility)\n", + "np.random.seed(SEED)\n", + "df[\"application_timestamp\"] = pd.to_datetime([\n", + " make_tstamp(d) for d in np.random.randint(start, end, len(df))\n", + "])" + ] + }, + { + "cell_type": "markdown", + "id": "f7800ea9-de9a-4aab-9d77-c4276e7db5f9", + "metadata": {}, + "source": [ + "Verify that the newly created \"ID\" and \"application_timestamp\" fields were added to the data as expected." + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "9516fc5c-7c25-4e60-acba-7400ab6bab42", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
012
ID012
checking_status<00<=X<200no checking
duration64812
credit_historycritical/other existing creditexisting paidcritical/other existing credit
purposeradio/tvradio/tveducation
credit_amount116959512096
savings_statusno known savings<100<100
employment>=71<=X<44<=X<7
installment_commitment422
personal_statusmale singlefemale div/dep/marmale single
other_partiesnonenonenone
residence_since423
property_magnitudereal estatereal estatereal estate
age672249
other_payment_plansnonenonenone
housingownownown
existing_credits211
jobskilledskilledunskilled resident
num_dependents112
own_telephoneyesnonenone
foreign_workeryesyesyes
classgoodbadgood
application_timestamp2023-10-04 17:50:132023-09-28 18:10:132023-10-03 23:06:03
\n", + "
" + ], + "text/plain": [ + " 0 1 \\\n", + "ID 0 1 \n", + "checking_status <0 0<=X<200 \n", + "duration 6 48 \n", + "credit_history critical/other existing credit existing paid \n", + "purpose radio/tv radio/tv \n", + "credit_amount 1169 5951 \n", + "savings_status no known savings <100 \n", + "employment >=7 1<=X<4 \n", + "installment_commitment 4 2 \n", + "personal_status male single female div/dep/mar \n", + "other_parties none none \n", + "residence_since 4 2 \n", + "property_magnitude real estate real estate \n", + "age 67 22 \n", + "other_payment_plans none none \n", + "housing own own \n", + "existing_credits 2 1 \n", + "job skilled skilled \n", + "num_dependents 1 1 \n", + "own_telephone yes none \n", + "foreign_worker yes yes \n", + "class good bad \n", + "application_timestamp 2023-10-04 17:50:13 2023-09-28 18:10:13 \n", + "\n", + " 2 \n", + "ID 2 \n", + "checking_status no checking \n", + "duration 12 \n", + "credit_history critical/other existing credit \n", + "purpose education \n", + "credit_amount 2096 \n", + "savings_status <100 \n", + "employment 4<=X<7 \n", + "installment_commitment 2 \n", + "personal_status male single \n", + "other_parties none \n", + "residence_since 3 \n", + "property_magnitude real estate \n", + "age 49 \n", + "other_payment_plans none \n", + "housing own \n", + "existing_credits 1 \n", + "job unskilled resident \n", + "num_dependents 2 \n", + "own_telephone none \n", + "foreign_worker yes \n", + "class good \n", + "application_timestamp 2023-10-03 23:06:03 " + ] + }, + "execution_count": 10, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# Check data (first few records, transposed for readability)\n", + "df.head(3).T" + ] + }, + { + "cell_type": "markdown", + "id": "72b2105a-b459-4715-aa53-6fe69fc4a210", + "metadata": {}, + "source": [ + "We'll also generate counterpart IDs and timestamps on the label data. In a real-life scenario, the label data would come separate and later relative to the loan application data. To mimic this, let's create a labels dataset with an \"outcome_timestamp\" column with a variable lag from the application timestamp of 30 to 90 days." + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "e214478b-ed9b-4354-ba6f-4117813c56c3", + "metadata": {}, + "outputs": [], + "source": [ + "# Add (lagged) label timestamps (30 to 90 days)\n", + "def lag_delta(data, seed):\n", + " np.random.seed(seed)\n", + " delta_days = np.random.randint(30, 90, len(data))\n", + " delta_hours = np.random.randint(0, 24, len(data))\n", + " delta = np.array([dt.timedelta(days=int(delta_days[i]), hours=int(delta_hours[i])) for i in range(len(data))])\n", + " return delta\n", + "\n", + "labels = df[[\"ID\", \"class\"]]\n", + "labels[\"outcome_timestamp\"] = pd.to_datetime(df.application_timestamp + lag_delta(df, SEED))" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "356a7225-db20-4c15-87a3-4a0eb3127475", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
IDclassoutcome_timestamp
00good2023-11-24 22:50:13
11bad2023-11-03 12:10:13
22good2023-11-30 22:06:03
\n", + "
" + ], + "text/plain": [ + " ID class outcome_timestamp\n", + "0 0 good 2023-11-24 22:50:13\n", + "1 1 bad 2023-11-03 12:10:13\n", + "2 2 good 2023-11-30 22:06:03" + ] + }, + "execution_count": 12, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# Check labels\n", + "labels.head(3)" + ] + }, + { + "cell_type": "markdown", + "id": "4a29f754-f758-402b-ac42-2dcfcee3b7fc", + "metadata": {}, + "source": [ + "You can verify that the `outcome timestamp` has a difference of 30 to 90 days from the \"application_timestamp\" (above)." + ] + }, + { + "cell_type": "markdown", + "id": "e720ce24-e092-4fcd-be3e-68bb18f4d2a7", + "metadata": {}, + "source": [ + "### Save Data" + ] + }, + { + "cell_type": "markdown", + "id": "5cae0578-8431-46c7-8d64-e52146f47d46", + "metadata": {}, + "source": [ + "Now that we have our data prepared, let's save it to local parquet files in the `data` directory (parquet is one of the file formats supported by Feast).\n", + "\n", + "One more step we will add is splitting the context data column-wise and saving it in two files. This step is contrived--we don't usually split data when we don't need to--but it will allow us to demonstrate later how Feast can easily join datasets (a common need in Data Science projects)." + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "cebef56c-1f54-4d31-a545-75d708d38579", + "metadata": {}, + "outputs": [], + "source": [ + "# Create the data directory if it doesn't exist\n", + "os.makedirs(\"Feature_Store/data\", exist_ok=True)\n", + "\n", + "# Split columns and save context data\n", + "a_cols = [\n", + " 'ID', 'checking_status', 'duration', 'credit_history', 'purpose',\n", + " 'credit_amount', 'savings_status', 'employment', 'application_timestamp',\n", + " 'installment_commitment', 'personal_status', 'other_parties',\n", + "]\n", + "b_cols = [\n", + " 'ID', 'residence_since', 'property_magnitude', 'age', 'other_payment_plans',\n", + " 'housing', 'existing_credits', 'job', 'num_dependents', 'own_telephone',\n", + " 'foreign_worker', 'application_timestamp'\n", + "]\n", + "\n", + "df[a_cols].to_parquet(\"Feature_Store/data/data_a.parquet\", engine=\"pyarrow\")\n", + "df[b_cols].to_parquet(\"Feature_Store/data/data_b.parquet\", engine=\"pyarrow\")\n", + "\n", + "# Save label data\n", + "labels.to_parquet(\"Feature_Store/data/labels.parquet\", engine=\"pyarrow\")" + ] + }, + { + "cell_type": "markdown", + "id": "d8d5de9f-bd27-4e95-802c-b121743dd1b0", + "metadata": {}, + "source": [ + "We have saved the following files to the `Feature_Store/data` directory: \n", + "- `data_a.parquet` (training data, a columns)\n", + "- `data_b.parquet` (training data, b columns)\n", + "- `labels.parquet` (label outcomes)" + ] + }, + { + "cell_type": "markdown", + "id": "af6355dc-ff5b-4b3f-b0bd-3c4020ef67e8", + "metadata": {}, + "source": [ + "With the feature data prepared, we are ready to setup and deploy the feature store. \n", + "\n", + "Continue with the [02_Deploying_the_Feature_Store.ipynb](02_Deploying_the_Feature_Store.ipynb) notebook." + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.9" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/examples/credit-risk-end-to-end/02_Deploying_the_Feature_Store.ipynb b/examples/credit-risk-end-to-end/02_Deploying_the_Feature_Store.ipynb new file mode 100644 index 00000000000..f736cdaed93 --- /dev/null +++ b/examples/credit-risk-end-to-end/02_Deploying_the_Feature_Store.ipynb @@ -0,0 +1,801 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "08d9e060-d455-43e2-b1ec-51e2a53e3169", + "metadata": {}, + "source": [ + "
" + ] + }, + { + "cell_type": "markdown", + "id": "93095241-3886-44a2-83b1-2a9537c21bc8", + "metadata": {}, + "source": [ + "# Deploying the Feature Store" + ] + }, + { + "cell_type": "markdown", + "id": "465783da-18eb-4945-98e7-bb1058a7af1b", + "metadata": {}, + "source": [ + "### Introduction" + ] + }, + { + "cell_type": "markdown", + "id": "11961d1b-72db-48dc-a07d-dcea9ba223b4", + "metadata": {}, + "source": [ + "Feast enables AI/ML teams to serve (and consume) features via feature stores. In this notebook, we will configure the feature stores and feature definitions, and deploy a Feast feature store server. We will also materialize (move) data from the offline store to the online store.\n", + "\n", + "In Feast, offline stores support pulling large amounts of data for model training using tools like Redshift, Snowflake, Bigquery, and Spark. In contrast, the focus of Feast online stores is feature serving in support of model inference, using tools like Redis, Snowflake, PostgreSQL, and SQLite.\n", + "\n", + "In this notebook, we will setup a file-based (Dask) offline store and SQLite online store. The online store will be made available through the Feast server." + ] + }, + { + "cell_type": "markdown", + "id": "dfed8ccf-0d7d-46a1-82f0-5765f8796088", + "metadata": {}, + "source": [ + "This notebook assumes that you have prepared the data by running the notebook [01_Credit_Risk_Data_Prep.ipynb](01_Credit_Risk_Data_Prep.ipynb). " + ] + }, + { + "cell_type": "markdown", + "id": "e66b7a08-5d15-4804-a82a-8bc571777496", + "metadata": {}, + "source": [ + "### Setup" + ] + }, + { + "cell_type": "markdown", + "id": "1c1e87a4-900b-48f3-a400-ce6608046ce3", + "metadata": {}, + "source": [ + "*The following code assumes that you have read the example README.md file, and that you have setup an environment where the code can be run. Please make sure you have addressed the prerequisite needs.*" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "8bd21689-4a8e-4b0c-937d-0911df9db1d3", + "metadata": {}, + "outputs": [], + "source": [ + "# Imports\n", + "import re\n", + "import sys\n", + "import time\n", + "import signal\n", + "import sqlite3\n", + "import subprocess\n", + "import datetime as dt\n", + "from feast import FeatureStore" + ] + }, + { + "cell_type": "markdown", + "id": "471db4b0-ea93-47a1-9d55-a80e4d2bdc1e", + "metadata": {}, + "source": [ + "### Feast Feature Store Configuration" + ] + }, + { + "cell_type": "markdown", + "id": "0a307490-4121-4bf3-a5c4-77a8885a4f6a", + "metadata": {}, + "source": [ + "For model training, we usually don't need (or want) a constantly running feature server. All we need is the ability to efficiently query and pull all of the training data at training time. In contrast, during model serving we need servers that are always ready to supply feature records in response to application requests. \n", + "\n", + "This training-serving dichotomy is reflected in Feast using \"offline\" and \"online\" stores. Offline stores are configured to work with database technologies typically used for training, while online stores are configured to use storage and streaming technologies that are popular for feature serving.\n", + "\n", + "We need to create a `feature_store.yaml` config file to tell feast the structure we want in our offline and online feature stores. Below, we write the configuration for a local \"Dask\" offline store and local SQLite online store. We give the feature store a project name of `loan_applications`, and provider `local`. The registry is where the feature store will keep track of feature definitions and online store updates; we choose a file location in this case.\n", + "\n", + "See the [feature_store.yaml](https://docs.feast.dev/reference/feature-repository/feature-store-yaml) documentation for further details. " + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "b3757221-2037-49eb-867f-b9529fec06e2", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Writing Feature_Store/feature_store.yaml\n" + ] + } + ], + "source": [ + "%%writefile Feature_Store/feature_store.yaml\n", + "\n", + "project: loan_applications\n", + "registry: data/registry.db\n", + "provider: local\n", + "offline_store:\n", + " type: dask\n", + "online_store:\n", + " type: sqlite\n", + " path: data/online_store.db\n", + "entity_key_serialization_version: 2" + ] + }, + { + "cell_type": "markdown", + "id": "180038f3-e5ce-4cce-bdf0-118eee7a822d", + "metadata": {}, + "source": [ + "### Feature Definitions" + ] + }, + { + "cell_type": "markdown", + "id": "dd44b206-1f5c-4f55-bbab-41ba2d3f5202", + "metadata": {}, + "source": [ + "We also need to create feature definitions and other feature constructs in a python file, which we name `feature_definitions.py`. For our purposes, we define the following:\n", + "\n", + "- Data Source: connections to data storage or data-producing endpoints\n", + "- Entity: primary key fields which can be used for joining data\n", + "- FeatureView: collections of features from a data source\n", + "\n", + "For more information on these, see the [Concepts](https://docs.feast.dev/getting-started/concepts) section of the Feast documentation." + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "d3e8fd80-0bee-463c-b3fb-bd0d1ee83a9c", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Writing Feature_Store/feature_definitions.py\n" + ] + } + ], + "source": [ + "%%writefile Feature_Store/feature_definitions.py\n", + "\n", + "# Imports\n", + "import os\n", + "from pathlib import Path\n", + "from feast import (\n", + " FileSource,\n", + " Entity,\n", + " FeatureView,\n", + " Field,\n", + " FeatureService\n", + ")\n", + "from feast.types import Float32, String\n", + "from feast.data_format import ParquetFormat\n", + "\n", + "CURRENT_DIR = os.path.abspath(os.curdir)\n", + "\n", + "# Data Sources\n", + "# A data source tells Feast where the data lives\n", + "data_a = FileSource(\n", + " file_format=ParquetFormat(),\n", + " path=Path(CURRENT_DIR,\"data/data_a.parquet\").as_uri()\n", + ")\n", + "data_b = FileSource(\n", + " file_format=ParquetFormat(),\n", + " path=Path(CURRENT_DIR,\"data/data_b.parquet\").as_uri()\n", + ")\n", + "\n", + "# Entity\n", + "# An entity tells Feast the column it can use to join tables\n", + "loan_id = Entity(\n", + " name = \"loan_id\",\n", + " join_keys = [\"ID\"]\n", + ")\n", + "\n", + "# Feature views\n", + "# A feature view is how Feast groups features\n", + "features_a = FeatureView(\n", + " name=\"data_a\",\n", + " entities=[loan_id],\n", + " schema=[\n", + " Field(name=\"checking_status\", dtype=String),\n", + " Field(name=\"duration\", dtype=Float32),\n", + " Field(name=\"credit_history\", dtype=String),\n", + " Field(name=\"purpose\", dtype=String),\n", + " Field(name=\"credit_amount\", dtype=Float32),\n", + " Field(name=\"savings_status\", dtype=String),\n", + " Field(name=\"employment\", dtype=String),\n", + " Field(name=\"installment_commitment\", dtype=Float32),\n", + " Field(name=\"personal_status\", dtype=String),\n", + " Field(name=\"other_parties\", dtype=String),\n", + " ],\n", + " source=data_a\n", + ")\n", + "features_b = FeatureView(\n", + " name=\"data_b\",\n", + " entities=[loan_id],\n", + " schema=[\n", + " Field(name=\"residence_since\", dtype=Float32),\n", + " Field(name=\"property_magnitude\", dtype=String),\n", + " Field(name=\"age\", dtype=Float32),\n", + " Field(name=\"other_payment_plans\", dtype=String),\n", + " Field(name=\"housing\", dtype=String),\n", + " Field(name=\"existing_credits\", dtype=Float32),\n", + " Field(name=\"job\", dtype=String),\n", + " Field(name=\"num_dependents\", dtype=Float32),\n", + " Field(name=\"own_telephone\", dtype=String),\n", + " Field(name=\"foreign_worker\", dtype=String),\n", + " ],\n", + " source=data_b\n", + ")\n", + "\n", + "# Feature Service\n", + "# a feature service in Feast represents a logical group of features\n", + "loan_fs = FeatureService(\n", + " name=\"loan_fs\",\n", + " features=[features_a, features_b]\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "4b47c1b5-849e-43f3-8043-60466aaed69f", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "markdown", + "id": "be9723eb-8fa0-4338-b50c-f9f1ff6bb13a", + "metadata": {}, + "source": [ + "### Applying the Configuration and Definitions" + ] + }, + { + "cell_type": "markdown", + "id": "c796d45f-28c0-4875-bbb1-71e5a15dcb96", + "metadata": {}, + "source": [ + "Now that we have our feature store configuration (`feature_store.yaml`) and feature definitions (`feature_definitions.py`), we are ready to \"apply\" them. The `feast apply` command creates a registry file (`Feature_Store/data/registry.db`) and sets up data connections; in this case, it creates a SQLite database (`Feature_Store/data/online_store.db`)." + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "394467f3-4ced-492a-9379-105aea9d4a6d", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "10/27/2024 02:19:03 PM root WARNING: Cannot use sqlite_vec for vector search\n", + "10/27/2024 02:19:03 PM root WARNING: Cannot use sqlite_vec for vector search\n", + "10/27/2024 02:19:03 PM root WARNING: Cannot use sqlite_vec for vector search\n", + "10/27/2024 02:19:03 PM root WARNING: Cannot use sqlite_vec for vector search\n", + "Created entity \u001b[1m\u001b[32mloan_id\u001b[0m\n", + "Created feature view \u001b[1m\u001b[32mdata_a\u001b[0m\n", + "Created feature view \u001b[1m\u001b[32mdata_b\u001b[0m\n", + "Created feature service \u001b[1m\u001b[32mloan_fs\u001b[0m\n", + "\n", + "10/27/2024 02:19:03 PM root WARNING: Cannot use sqlite_vec for vector search\n", + "10/27/2024 02:19:03 PM root WARNING: Cannot use sqlite_vec for vector search\n", + "Created sqlite table \u001b[1m\u001b[32mloan_applications_data_a\u001b[0m\n", + "Created sqlite table \u001b[1m\u001b[32mloan_applications_data_b\u001b[0m\n", + "\n" + ] + } + ], + "source": [ + "# Run 'feast apply' in the Feature_Store directory\n", + "!feast --chdir ./Feature_Store apply" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "e32f40eb-a31a-4877-8f40-2d8515302f39", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "total 232\n", + "-rw-r--r-- 1 501 20 33K Oct 27 14:17 data_a.parquet\n", + "-rw-r--r-- 1 501 20 27K Oct 27 14:17 data_b.parquet\n", + "-rw-r--r-- 1 501 20 17K Oct 27 14:17 labels.parquet\n", + "-rw-r--r-- 1 501 20 28K Oct 27 14:19 online_store.db\n", + "-rw-r--r-- 1 501 20 2.8K Oct 27 14:19 registry.db\n" + ] + } + ], + "source": [ + "# List the Feature_Store/data/ directory to see newly created files\n", + "!ls -nlh Feature_Store/data/" + ] + }, + { + "cell_type": "markdown", + "id": "31014885-ce6a-4007-8bdb-d74d3b44781b", + "metadata": {}, + "source": [ + "Note that while `feast apply` set up the `sqlite` online database, `online_store.db`, no data has been added to the online database as of yet. We can verify this by connecting with the `sqlite3` library." + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "107ca856-af06-40c4-8339-70daf59cdf37", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Online Store Tables: [('loan_applications_data_a',), ('loan_applications_data_b',)]\n", + "loan_applications_data_a data: []\n", + "loan_applications_data_b data: []\n" + ] + } + ], + "source": [ + "# Connect to sqlite database\n", + "conn = sqlite3.connect(\"Feature_Store/data/online_store.db\")\n", + "cursor = conn.cursor()\n", + "# Query table data (3 tables)\n", + "print(\n", + " \"Online Store Tables: \",\n", + " cursor.execute(\"SELECT name FROM sqlite_master WHERE type='table';\").fetchall()\n", + ")\n", + "print(\n", + " \"loan_applications_data_a data: \",\n", + " cursor.execute(\"SELECT * FROM loan_applications_data_a\").fetchall()\n", + ")\n", + "print(\n", + " \"loan_applications_data_b data: \",\n", + " cursor.execute(\"SELECT * FROM loan_applications_data_b\").fetchall()\n", + ")\n", + "conn.close()" + ] + }, + { + "cell_type": "markdown", + "id": "03b927ee-7913-4a8a-b17b-9bee361d8d94", + "metadata": {}, + "source": [ + "Since we have used `feast apply` to create the registry, we can now use the Feast Python SDK to interact with our new feature store. To see other possible commands see the [Feast Python SDK documentation](https://rtd.feast.dev/en/master/)." + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "c764a60a-b911-41a8-ba8f-7ef0a0bc7257", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "RepoConfig(project='loan_applications', provider='local', registry_config='data/registry.db', online_config={'type': 'sqlite', 'path': 'data/online_store.db'}, offline_config={'type': 'dask'}, batch_engine_config='local', feature_server=None, flags=None, repo_path=PosixPath('Feature_Store'), entity_key_serialization_version=2, coerce_tz_aware=True)" + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# Get feature store config\n", + "store = FeatureStore(repo_path=\"./Feature_Store\")\n", + "store.config" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "fc572976-6ce9-44f6-8b67-28ee6157e29c", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Feature view: data_a | Features: [checking_status-String, duration-Float32, credit_history-String, purpose-String, credit_amount-Float32, savings_status-String, employment-String, installment_commitment-Float32, personal_status-String, other_parties-String]\n", + "Feature view: data_b | Features: [residence_since-Float32, property_magnitude-String, age-Float32, other_payment_plans-String, housing-String, existing_credits-Float32, job-String, num_dependents-Float32, own_telephone-String, foreign_worker-String]\n" + ] + } + ], + "source": [ + "# List feature views\n", + "feature_views = store.list_batch_feature_views()\n", + "for fv in feature_views:\n", + " print(f\"Feature view: {fv.name} | Features: {fv.features}\")" + ] + }, + { + "cell_type": "markdown", + "id": "027edcfe-58d7-4dcb-92e2-5a5514c0f1f0", + "metadata": {}, + "source": [ + "### Deploying the Feature Store Servers" + ] + }, + { + "cell_type": "markdown", + "id": "c9aab68d-395f-421e-ba11-ad8c4acc9d6f", + "metadata": {}, + "source": [ + "If you wish to share a feature store with your team, Feast provides feature servers. To spin up an offline feature server process, we can use the `feast serve_offline` command, while to spin up a Feast online feature server, we use the `feast serve` command.\n", + "\n", + "Let's spin up an offline and an online server that we can use in the subsequent notebooks to get features during model training and model serving. We will run both servers as background processes, that we can communicate with in the other notebooks.\n", + "\n", + "First, we write a helper function to extract the first few printed log lines (so we can print it in the notebook cell output)." + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "568f81b8-df34-4b06-8a3f-1a6bdc2e6cff", + "metadata": {}, + "outputs": [], + "source": [ + "# TimeoutError class\n", + "class TimeoutError(Exception):\n", + " pass\n", + "\n", + "# TimeoutError raise function\n", + "def timeout():\n", + " raise TimeoutError(\"timeout\")\n", + "\n", + "# Get first few log lines function\n", + "def print_first_proc_lines(proc, wait):\n", + " '''Given a process, `proc`, read and print output lines until they stop \n", + " comming (waiting up to `wait` seconds for new lines to appear)'''\n", + " lines = \"\"\n", + " while True:\n", + " signal.signal(signal.SIGALRM, timeout)\n", + " signal.alarm(wait)\n", + " try:\n", + " lines += proc.stderr.readline()\n", + " except:\n", + " break\n", + " if lines:\n", + " print(lines, file=sys.stderr)" + ] + }, + { + "cell_type": "markdown", + "id": "88d25a87-241a-46c6-9ca7-d035959c5f74", + "metadata": {}, + "source": [ + "Launch the offline server with the command `feast --chdir ./Feature_Store serve_offline`." + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "ce965dd4-652b-4c36-a064-fd0fd97d3ef7", + "metadata": {}, + "outputs": [], + "source": [ + "# Feast offline server process\n", + "offline_server_proc = subprocess.Popen(\n", + " \"feast --chdir ./Feature_Store serve_offline 2>&2 & echo $! > server_proc.txt\",\n", + " shell=True,\n", + " text=True,\n", + " stdout=subprocess.PIPE,\n", + " stderr=subprocess.PIPE,\n", + " bufsize=0\n", + ")\n", + "print_first_proc_lines(offline_server_proc, 2)" + ] + }, + { + "cell_type": "markdown", + "id": "59958d64-8e68-45ff-9549-556cbf46908c", + "metadata": {}, + "source": [ + "The tail end of the command above, `2>&2 & echo $! > server_proc.txt`, captures log messages (in the offline case there are none), and writes the process PID to the file `server_proc.txt` (we will use this in the cleanup notebook, [05_Credit_Risk_Cleanup.ipynb](05_Credit_Risk_Cleanup.ipynb))." + ] + }, + { + "cell_type": "markdown", + "id": "cfed4334-9e62-4f3f-be96-3f7db2f06ada", + "metadata": {}, + "source": [ + "Next, launch the online server with the command `feast --chdir ./Feature_Store serve`." + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "a581fbe2-13ba-433e-8e76-dc82cc22af74", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/Users/ddowler/Code/Feast/feast/examples/credit-risk-end-to-end/venv-py3.11/lib/python3.11/site-packages/uvicorn/workers.py:16: DeprecationWarning: The `uvicorn.workers` module is deprecated. Please use `uvicorn-worker` package instead.\n", + "For more details, see https://github.com/Kludex/uvicorn-worker.\n", + " warnings.warn(\n", + "[2024-10-27 14:19:07 -0600] [44621] [INFO] Starting gunicorn 23.0.0\n", + "[2024-10-27 14:19:07 -0600] [44621] [INFO] Listening at: http://127.0.0.1:6566 (44621)\n", + "[2024-10-27 14:19:07 -0600] [44621] [INFO] Using worker: uvicorn.workers.UvicornWorker\n", + "[2024-10-27 14:19:07 -0600] [44623] [INFO] Booting worker with pid: 44623\n", + "[2024-10-27 14:19:07 -0600] [44623] [INFO] Started server process [44623]\n", + "[2024-10-27 14:19:07 -0600] [44623] [INFO] Waiting for application startup.\n", + "[2024-10-27 14:19:07 -0600] [44623] [INFO] Application startup complete.\n", + "\n" + ] + } + ], + "source": [ + "# Feast online server (master and worker) processes\n", + "online_server_proc = subprocess.Popen(\n", + " \"feast --chdir ./Feature_Store serve 2>&2 & echo $! >> server_proc.txt\",\n", + " shell=True,\n", + " text=True,\n", + " stdout=subprocess.PIPE,\n", + " stderr=subprocess.PIPE,\n", + " bufsize=0\n", + ")\n", + "print_first_proc_lines(online_server_proc, 3)" + ] + }, + { + "cell_type": "markdown", + "id": "0e778173-f58a-4074-b63f-107e1f39577b", + "metadata": {}, + "source": [ + "Note that the output helpfully let's us know that the online server is \"Listening at: http://127.0.0.1:6566\" (the default host:port).\n", + "\n", + "List the running processes to verify they are up." + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "9b1a224d-884d-45c5-9711-2e2eb4351710", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + " 501 44594 1 0 2:19PM ?? 0:03.66 **/python **/feast --chdir ./Feature_Store serve_offline\n", + " 501 44621 1 0 2:19PM ?? 0:03.58 **/python **/feast --chdir ./Feature_Store serve\n", + " 501 44623 44621 0 2:19PM ?? 0:00.03 **/python **/feast --chdir ./Feature_Store serve\n", + " 501 44662 44542 0 2:19PM ?? 0:00.01 /bin/zsh -c ps -ef | grep **/feast | grep serve\n" + ] + } + ], + "source": [ + "# List running Feast processes (paths redacted)\n", + "running_procs = !ps -ef | grep feast | grep serve\n", + "\n", + "for line in running_procs:\n", + " redacted = re.sub(r'/*[^\\s]*(?P(python )|(feast ))', r'**/\\g', line)\n", + " print(redacted)" + ] + }, + { + "cell_type": "markdown", + "id": "fd52eeb4-948c-472b-9111-8549fda955a1", + "metadata": {}, + "source": [ + "Note that there are two process for the online server (master and worker)." + ] + }, + { + "cell_type": "markdown", + "id": "8258e7a8-5f6e-4737-93ee-63591518b169", + "metadata": {}, + "source": [ + "### Materialize Features to the Online Store" + ] + }, + { + "cell_type": "markdown", + "id": "21b354ab-ec22-476d-8fd9-6ffe0f3fbacb", + "metadata": {}, + "source": [ + "At this point, there is no data in the online store yet. Let's use the SDK feature store object (that we created above) to \"materialize\" data; this is Feast lingo for moving/updating data from the offline store to the online store." + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "ff6146df-03a7-4ac2-a665-ee5f440c3605", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "WARNING:root:_list_feature_views will make breaking changes. Please use _list_batch_feature_views instead. _list_feature_views will behave like _list_all_feature_views in the future.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Materializing \u001b[1m\u001b[32m2\u001b[0m feature views from \u001b[1m\u001b[32m2023-09-24 12:00:00-06:00\u001b[0m to \u001b[1m\u001b[32m2024-01-07 12:00:00-07:00\u001b[0m into the \u001b[1m\u001b[32msqlite\u001b[0m online store.\n", + "\n", + "\u001b[1m\u001b[32mdata_a\u001b[0m:\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + " 0%| | 0/1000 [00:00=7\",\"4<=X<7\"],\"statuses\":[\"PRESENT\",\"PRESENT\"],\"event_timestamps\":[\"2023-09-25T01:03:47Z\",\"2023-09-29T03:17:24Z\"]},{\"values\":[\"0<=X<200\",\"no checking\"],\"statuses\":[\"PRESENT\",\"PRESENT\"],\"event_timestamps\":[\"2023-09-25T01:03:47Z\",\"2023-09-29T03:17:24Z\"]},{\"values\":[\"existing paid\",\"critical/other existing credit\"],\"statuses\":[\"PRESENT\",\"PRESENT\"],\"event_timestamps\":[\"2023-09-25T01:03:47Z\",\"2023-09-29T03:17:24Z\"]},{\"values\":[\"female div/dep/mar\",\"male mar/wid\"],\"statuses\":[\"PRESENT\",\"PRESENT\"],\"event_timestamps\":[\"2023-09-25T01:03:47Z\",\"2023-09-29T03:17:24Z\"]},{\"values\":[12579.0,2463.0],\"statuses\":[\"PRESENT\",\"PRESENT\"],\"event_timestamps\":[\"2023-09-25T01:03:47Z\",\"2023-09-29T03:17:24Z\"]},{\"values\":[\"none\",\"none\"],\"statuses\":[\"PRESENT\",\"PRESENT\"],\"event_timestamps\":[\"2023-09-25T01:03:47Z\",\"2023-09-29T03:17:24Z\"]},{\"values\":[\"used car\",\"new car\"],\"statuses\":[\"PRESENT\",\"PRESENT\"],\"event_timestamps\":[\"2023-09-25T01:03:47Z\",\"2023-09-29T03:17:24Z\"]},{\"values\":[24.0,24.0],\"statuses\":[\"PRESENT\",\"PRESENT\"],\"event_timestamps\":[\"2023-09-25T01:03:47Z\",\"2023-09-29T03:17:24Z\"]},{\"values\":[4.0,4.0],\"statuses\":[\"PRESENT\",\"PRESENT\"],\"event_timestamps\":[\"2023-09-25T01:03:47Z\",\"2023-09-29T03:17:24Z\"]},{\"values\":[\"yes\",\"yes\"],\"statuses\":[\"PRESENT\",\"PRESENT\"],\"event_timestamps\":[\"2023-09-25T01:03:47Z\",\"2023-09-29T03:17:24Z\"]},{\"values\":[2.0,3.0],\"statuses\":[\"PRESENT\",\"PRESENT\"],\"event_timestamps\":[\"2023-09-25T01:03:47Z\",\"2023-09-29T03:17:24Z\"]},{\"values\":[1.0,1.0],\"statuses\":[\"PRESENT\",\"PRESENT\"],\"event_timestamps\":[\"2023-09-25T01:03:47Z\",\"2023-09-29T03:17:24Z\"]},{\"values\":[44.0,27.0],\"statuses\":[\"PRESENT\",\"PRESENT\"],\"event_timestamps\":[\"2023-09-25T01:03:47Z\",\"2023-09-29T03:17:24Z\"]},{\"values\":[\"none\",\"none\"],\"statuses\":[\"PRESENT\",\"PRESENT\"],\"event_timestamps\":[\"2023-09-25T01:03:47Z\",\"2023-09-29T03:17:24Z\"]},{\"values\":[\"for free\",\"own\"],\"statuses\":[\"PRESENT\",\"PRESENT\"],\"event_timestamps\":[\"2023-09-25T01:03:47Z\",\"2023-09-29T03:17:24Z\"]},{\"values\":[1.0,2.0],\"statuses\":[\"PRESENT\",\"PRESENT\"],\"event_timestamps\":[\"2023-09-25T01:03:47Z\",\"2023-09-29T03:17:24Z\"]},{\"values\":[\"high qualif/self emp/mgmt\",\"skilled\"],\"statuses\":[\"PRESENT\",\"PRESENT\"],\"event_timestamps\":[\"2023-09-25T01:03:47Z\",\"2023-09-29T03:17:24Z\"]},{\"values\":[\"no known property\",\"life insurance\"],\"statuses\":[\"PRESENT\",\"PRESENT\"],\"event_timestamps\":[\"2023-09-25T01:03:47Z\",\"2023-09-29T03:17:24Z\"]},{\"values\":[\"yes\",\"yes\"],\"statuses\":[\"PRESENT\",\"PRESENT\"],\"event_timestamps\":[\"2023-09-25T01:03:47Z\",\"2023-09-29T03:17:24Z\"]}]}']" + ] + }, + "execution_count": 16, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "response" + ] + }, + { + "cell_type": "markdown", + "id": "01d20196-1d42-486d-a0bd-97193c953785", + "metadata": {}, + "source": [ + "The `curl` command gave us a quick validation. In the [04_Credit_Risk_Model_Serving.ipynb](04_Credit_Risk_Model_Serving.ipynb) notebook, we'll use the Python `requests` library to handle the query better." + ] + }, + { + "cell_type": "markdown", + "id": "d74a5117-dd34-4dde-93a8-ea6e8c4c545a", + "metadata": {}, + "source": [ + "Now that the feature stores and their respective servers have been configured and deployed, we can proceed to train an AI model in [03_Credit_Risk_Model_Training.ipynb](03_Credit_Risk_Model_Training.ipynb)." + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.9" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/examples/credit-risk-end-to-end/03_Credit_Risk_Model_Training.ipynb b/examples/credit-risk-end-to-end/03_Credit_Risk_Model_Training.ipynb new file mode 100644 index 00000000000..ca0d0e29d95 --- /dev/null +++ b/examples/credit-risk-end-to-end/03_Credit_Risk_Model_Training.ipynb @@ -0,0 +1,1541 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "54f2ab19-68e1-4725-b6e7-efd8eedebe1a", + "metadata": {}, + "source": [ + "
" + ] + }, + { + "cell_type": "markdown", + "id": "69a40de4-65cf-4b45-b321-2b7ce571f8cb", + "metadata": {}, + "source": [ + "# Credit Risk Model Training" + ] + }, + { + "cell_type": "markdown", + "id": "fe641d83-1e28-4f7f-895c-8ca038f6cc53", + "metadata": {}, + "source": [ + "### Introduction" + ] + }, + { + "cell_type": "markdown", + "id": "8f04f635-401b-47b6-b807-df61d42ec752", + "metadata": {}, + "source": [ + "AI models have played a central role in modern credit risk assessment systems. In this example, we develop a credit risk model to predict whether a future loan will be good or bad, given some context data (presumably supplied from the loan application process). We use the modeling process to demonstrate how Feast can be used to facilitate the serving of data for training and inference use-cases.\n", + "\n", + "In this notebook, we train our AI model. We will use the popular scikit-learn library (sklearn) to train a RandomForestClassifier, as this is a relatively easy choice for a baseline model." + ] + }, + { + "cell_type": "markdown", + "id": "a96bf1aa-c450-4201-83a4-e25b08bdd12d", + "metadata": {}, + "source": [ + "### Setup" + ] + }, + { + "cell_type": "markdown", + "id": "a47b33bc-bc06-4de0-8f3a-beea8179035c", + "metadata": {}, + "source": [ + "*The following code assumes that you have read the example README.md file, and that you have setup an environment where the code can be run. Please make sure you have addressed the prerequisite needs.*" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "c66a3dab-fdbf-40be-8227-6180dc314a84", + "metadata": {}, + "outputs": [], + "source": [ + "# Imports\n", + "import warnings\n", + "import datetime\n", + "import feast\n", + "import joblib\n", + "import pandas as pd\n", + "import seaborn as sns\n", + "\n", + "from feast import FeatureStore, RepoConfig\n", + "from sklearn.model_selection import train_test_split\n", + "from sklearn.preprocessing import OrdinalEncoder\n", + "from sklearn.compose import ColumnTransformer\n", + "from sklearn.pipeline import Pipeline\n", + "from sklearn.ensemble import RandomForestClassifier\n", + "from sklearn.metrics import classification_report" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "2a841445-fa47-4826-a874-28ac0e4ea57f", + "metadata": {}, + "outputs": [], + "source": [ + "# Ignore warnings\n", + "warnings.filterwarnings(action=\"ignore\")" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "23579727-7797-4101-a70d-b0d4c24b0fdf", + "metadata": {}, + "outputs": [], + "source": [ + "# Random seed\n", + "SEED = 142" + ] + }, + { + "cell_type": "markdown", + "id": "fc5be519-7733-449b-8dc3-411e86371315", + "metadata": {}, + "source": [ + "This notebook assumes that you have already done the following:\n", + "\n", + "1. Run the [01_Credit_Risk_Data_Prep.ipynb](01_Credit_Risk_Data_Prep.ipynb) notebook to prepare the data.\n", + "2. Run the [02_Deploying_the_Feature_Store.ipynb](02_Deploying_the_Feature_Store.ipynb) notebook to configure the feature stores and launch the feature store servers.\n", + "\n", + "If you have not completed the above steps, please go back and do so before continuing. This notebook relies on the data prepared by 1, and it uses the Feast offline server stood up by 2." + ] + }, + { + "cell_type": "markdown", + "id": "1ca99047-e508-4b1f-9f4c-f11e38587d70", + "metadata": {}, + "source": [ + "### Load Label (Outcome) Data" + ] + }, + { + "cell_type": "markdown", + "id": "89b49268-b7a5-4abc-8d82-1cdbf9bb4473", + "metadata": {}, + "source": [ + "From our previous data exploration, remember that the label data represents whether the loan was classed as \"good\" (1) or \"bad\" (0). Let's pull the labels for training, as we will use them as our \"entity dataframe\" when pulling features.\n", + "\n", + "This is also a good time to remember that the label timestamps are lagged by 30-90 days from the context data records." + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "6a227a12-7b3e-462a-8f6e-38a7690df1c4", + "metadata": {}, + "outputs": [], + "source": [ + "labels = pd.read_parquet(\"Feature_Store/data/labels.parquet\")" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "31a39cad-0a85-4d98-ad95-008c81bb6fe0", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
IDclassoutcome_timestamp
00good2023-11-24 22:50:13
11bad2023-11-03 12:10:13
22good2023-11-30 22:06:03
33good2023-11-17 07:37:19
44bad2023-12-01 05:01:48
\n", + "
" + ], + "text/plain": [ + " ID class outcome_timestamp\n", + "0 0 good 2023-11-24 22:50:13\n", + "1 1 bad 2023-11-03 12:10:13\n", + "2 2 good 2023-11-30 22:06:03\n", + "3 3 good 2023-11-17 07:37:19\n", + "4 4 bad 2023-12-01 05:01:48" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "labels.head()" + ] + }, + { + "cell_type": "markdown", + "id": "857f29fd-46d3-444b-b24f-eaccd82ab7d3", + "metadata": {}, + "source": [ + "### Pull Feature Data from Feast Offline Store" + ] + }, + { + "cell_type": "markdown", + "id": "07c13b69-3d26-484c-97cd-97734cc812bd", + "metadata": {}, + "source": [ + "In order to pull feature data from the offline store, we create a FeatureStore object that connects to the offline server (continuously running in the previous notebook)." + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "9e9828f8-f210-4586-ac36-3f7e17f4f1e8", + "metadata": {}, + "outputs": [], + "source": [ + "# Create FeatureStore object\n", + "# (connects to the offline server deployed in 02_Deploying_the_Feature_Store.ipynb) \n", + "store = FeatureStore(config=RepoConfig(\n", + " project=\"loan_applications\",\n", + " provider=\"local\",\n", + " registry=\"Feature_Store/data/registry.db\",\n", + " offline_store={\n", + " \"type\": \"remote\",\n", + " \"host\": \"localhost\",\n", + " \"port\": 8815\n", + " },\n", + " entity_key_serialization_version=2\n", + "))" + ] + }, + { + "cell_type": "markdown", + "id": "c007e7ca-40c1-4850-abed-73b6171ad08d", + "metadata": {}, + "source": [ + "Now, we can retrieve feature data by supplying our entity dataframe and feature specifications to the `get_historical_features` function. Note that this function performs a fuzzy lookback (\"point-in-time\") join, matching the lagged outcome timestamp to the closest application timestamp (per ID) in the context data; it also joins the \"a\" and \"b\" features that we had previously split into two tables.\n", + "\n", + "To keep this example simple, we will limit our feature set to the numerical features plus two categorical features." + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "dd2e3cb5-c865-48f4-80b6-8a14a1ff09ab", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "WARNING:root:_list_feature_views will make breaking changes. Please use _list_batch_feature_views instead. _list_feature_views will behave like _list_all_feature_views in the future.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Using outcome_timestamp as the event timestamp. To specify a column explicitly, please name it event_timestamp.\n" + ] + } + ], + "source": [ + "# Get feature data\n", + "# (Joins a and b data, and selects records with the right timestamps)\n", + "df = store.get_historical_features(\n", + " entity_df=labels,\n", + " features=[\n", + " \"data_a:duration\",\n", + " \"data_a:credit_amount\",\n", + " \"data_a:installment_commitment\",\n", + " \"data_a:checking_status\",\n", + " \"data_b:residence_since\",\n", + " \"data_b:age\",\n", + " \"data_b:existing_credits\",\n", + " \"data_b:num_dependents\",\n", + " \"data_b:housing\"\n", + " ]\n", + ").to_df()" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "c72f6cb1-bbbf-4512-98cd-0abe5ff0c24b", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "RangeIndex: 1000 entries, 0 to 999\n", + "Data columns (total 12 columns):\n", + " # Column Non-Null Count Dtype \n", + "--- ------ -------------- ----- \n", + " 0 ID 1000 non-null int64 \n", + " 1 class 1000 non-null category \n", + " 2 outcome_timestamp 1000 non-null datetime64[ns, UTC]\n", + " 3 duration 1000 non-null int64 \n", + " 4 credit_amount 1000 non-null int64 \n", + " 5 installment_commitment 1000 non-null int64 \n", + " 6 checking_status 1000 non-null category \n", + " 7 residence_since 1000 non-null int64 \n", + " 8 age 1000 non-null int64 \n", + " 9 existing_credits 1000 non-null int64 \n", + " 10 num_dependents 1000 non-null int64 \n", + " 11 housing 1000 non-null category \n", + "dtypes: category(3), datetime64[ns, UTC](1), int64(8)\n", + "memory usage: 73.8 KB\n" + ] + } + ], + "source": [ + "# Check the data info\n", + "df.info()" + ] + }, + { + "cell_type": "markdown", + "id": "110ea48c-0a5a-4642-aaba-a9eeb4a7da48", + "metadata": {}, + "source": [ + "### Split the Data" + ] + }, + { + "cell_type": "markdown", + "id": "f6669dce-a8b0-4d80-9a15-70b7dfd2d718", + "metadata": {}, + "source": [ + "Next, we split the data into a `train` and `validate` set, which we will use to train and then validate a model. The validation set will allow us to more accurately assess the model's performance on data that it has not seen during the training phase." + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "036b0a54-48e4-4414-bb8c-0c30b6ab7469", + "metadata": {}, + "outputs": [], + "source": [ + "# Split data into train and validate datasets\n", + "train, validate = train_test_split(df, test_size=0.2, random_state=SEED)" + ] + }, + { + "cell_type": "markdown", + "id": "4b65cbf7-5981-4f51-97aa-a3ff7027f2f3", + "metadata": {}, + "source": [ + "### Exploratory Data Analysis" + ] + }, + { + "cell_type": "markdown", + "id": "e516ded8-10ad-4274-a736-f288290b5883", + "metadata": {}, + "source": [ + "Before building a model, a data scientist needs to gain understanding of the data to make sure it meets important statistical assumptions, and to identify potential opportunities and issues. As the purpose of this particular example is to show working with Feast, we will take the view of a data scientist looking to build a quick baseline model to establish some low-end metrics.\n", + "\n", + "Note that this data set is very \"clean\", as it has already been prepared. In real-life, production credit risk data can be much more complex, and have many issues that need to be understood and addressed before modeling." + ] + }, + { + "cell_type": "markdown", + "id": "553986a0-c804-4ab4-a4b9-48b16c72fd4f", + "metadata": {}, + "source": [ + "Let's look at counts for the target variable `class`, which tells us whether a (historical) loan was good or bad. We can see that there were many more good loans than bad, making the dataset imbalanced." + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "607bd29b-eaf4-41a6-aaca-a8eaaf37e2d2", + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjsAAAHHCAYAAABZbpmkAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8hTgPZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAysElEQVR4nO3deVxV5d7///dmRmWDoIKWqJWKOHZw2k1qkWRkeWvllKlHG8FKy2PcOWLedqwcQ6tTqWWm2WBq5kRZHcVSTFNT1MqwFCgVtnoUBNbvj37sb/ugpQhsvHw9H4/1yHVd11rrc+3d1jdr2Ngsy7IEAABgKC9PFwAAAFCRCDsAAMBohB0AAGA0wg4AADAaYQcAABiNsAMAAIxG2AEAAEYj7AAAAKMRdgAAgNEIO8AlyGazafz48Z4u45LD6wZcngg7QDmYN2+ebDab21KnTh116dJFn3zyiafLq3T5+fmaNWuWbrjhBtWsWVN+fn6qV6+e7rzzTr3zzjsqKirydInndODAAdlsNr3wwgueLuWCOZ1OTZgwQa1bt1aNGjUUGBioFi1aaNSoUTp06JCny5MkrVy5ksCJSufj6QIAkyQnJ6tRo0ayLEvZ2dmaN2+ebr/9di1fvlx33HGHp8urFL/++qu6deum9PR0xcXFafTo0QoNDVVWVpbWrVunfv36af/+/RozZoynSzXKDz/8oNjYWGVmZuqee+7Rgw8+KD8/P3377bd6/fXX9eGHH2rv3r2eLlMrV65USkoKgQeVirADlKNu3bqpbdu2rvUhQ4YoPDxc77zzzmUTdgYMGKBvvvlG77//vnr27OnWl5SUpC1btigjI8ND1ZmpsLBQPXv2VHZ2ttavX68bbrjBrX/SpEn65z//6aHqAM/jMhZQgUJCQhQYGCgfH/efK1544QVdd911CgsLU2BgoGJiYvTee++V2j4/P1/Dhw9X7dq1FRQUpDvvvFM///zzXx43OztbPj4+mjBhQqm+jIwM2Ww2vfTSS5KkM2fOaMKECWrcuLECAgIUFhamG264QWvXrr3g+aalpWn16tV68MEHSwWdEm3btlX//v3d2nJyclzBMCAgQK1bt9b8+fNLbXvy5Ek9+eSTql+/vvz9/dW0aVO98MILsizLbVxZX7cLcb41n+97bbPZlJiYqKVLl6pFixby9/dX8+bNtWrVqr+s5f3339f27dv1zDPPlAo6kmS32zVp0iS3tiVLligmJkaBgYGqVauW7rvvPv3yyy9uYzp37qzOnTuX2t+gQYPUsGFD1/ofL/29+uqruvrqq+Xv76927dpp8+bNbtulpKS45luyABWNMztAOcrLy9Nvv/0my7KUk5OjWbNm6cSJE7rvvvvcxs2YMUN33nmn+vfvr4KCAi1atEj33HOPVqxYofj4eNe4oUOHasGCBerXr5+uu+46ffrpp2795xIeHq5OnTrp3Xff1bhx49z6Fi9eLG9vb91zzz2SpPHjx2vy5MkaOnSo2rdvL6fTqS1btmjr1q269dZbL2j+y5cvl6RS8/0zp06dUufOnbV//34lJiaqUaNGWrJkiQYNGqTc3Fw9/vjjkiTLsnTnnXfqs88+05AhQ9SmTRutXr1aI0eO1C+//KJp06a59lnW1628a5bO/72WpH//+9/64IMP9OijjyooKEgzZ85Ur169lJmZqbCwsHPWs2zZMkm/n1U7H/PmzdPgwYPVrl07TZ48WdnZ2ZoxY4Y2bNigb775RiEhIRf+okhauHChjh8/roceekg2m01TpkxRz5499cMPP8jX11cPPfSQDh06pLVr1+qtt94q0zGAMrEAXLS5c+dakkot/v7+1rx580qN/89//uO2XlBQYLVo0cK6+eabXW3btm2zJFmPPvqo29h+/fpZkqxx48b9aU2vvPKKJcnasWOHW3t0dLTbcVq3bm3Fx8ef71T/1P/8z/9Ykqzc3Fy39lOnTlm//vqrazl27Jirb/r06ZYka8GCBa62goICy+FwWDVq1LCcTqdlWZa1dOlSS5L17LPPuu377rvvtmw2m7V//37Lsi7+dfvxxx8tSdbzzz9/zjHnW7Nlnd97bVmWJcny8/NzzcOyLGv79u2WJGvWrFl/WvO1115rBQcH/+mYPx6/Tp06VosWLaxTp0652lesWGFJssaOHetq69Spk9WpU6dS+xg4cKDVoEED13rJaxYWFmYdPXrU1f7RRx9Zkqzly5e72hISEiz+6UFl4zIWUI5SUlK0du1arV27VgsWLFCXLl00dOhQffDBB27jAgMDXX8+duyY8vLydOONN2rr1q2u9pUrV0qSHnvsMbdtn3jiifOqpWfPnvLx8dHixYtdbTt37tR3332n3r17u9pCQkK0a9cu7du377zneS5Op1OSVKNGDbf2l19+WbVr13Ytf7zUsnLlSkVERKhv376uNl9fXz322GM6ceKEPv/8c9c4b2/vUq/Hk08+KcuyXE+9Xezrdj7Ot2bp/N7rErGxsbr66qtd661atZLdbtcPP/zwp/U4nU4FBQWdV+1btmxRTk6OHn30UQUEBLja4+PjFRUVpY8//vi89nM2vXv3Vs2aNV3rN954oyT9Zf1ARSPsAOWoffv2io2NVWxsrPr376+PP/5Y0dHRSkxMVEFBgWvcihUr1LFjRwUEBCg0NFS1a9fWnDlzlJeX5xrz008/ycvLy+0fP0lq2rTpedVSq1Yt3XLLLXr33XddbYsXL5aPj4/b/TTJycnKzc1VkyZN1LJlS40cOVLffvttmeZf8g/uiRMn3Np79erlCoGtWrVy6/vpp5/UuHFjeXm5/3XUrFkzV3/Jf+vVq1fqH/WzjbuY1+18nG/N0vm91yUiIyNLtdWsWVPHjh3703rsdruOHz9+3rVLZ389oqKi3Gq/UP9df0nw+av6gYpG2AEqkJeXl7p06aLDhw+7zpx8+eWXuvPOOxUQEKDZs2dr5cqVWrt2rfr161fqRtuL1adPH+3du1fbtm2TJL377ru65ZZbVKtWLdeYm266Sd9//73eeOMNtWjRQq+99pr+9re/6bXXXrvg40VFRUn6/QzSH9WvX98VAv/4k7/pLvS99vb2Put+/ur/i6ioKOXl5engwYPlUneJc908fK7vSSpr/UBFI+wAFaywsFDS/zvb8f777ysgIECrV6/W3//+d3Xr1k2xsbGltmvQoIGKi4v1/fffu7VfyGPbPXr0kJ+fnxYvXqxt27Zp79696tOnT6lxoaGhGjx4sN555x0dPHhQrVq1KtP3oJQ8Xv/222+f9zYNGjTQvn37VFxc7Na+Z88eV3/Jfw8dOlTqDMbZxl3s61ZeNZ/ve32xunfvLklasGDBX44tqe1sr0dGRoarX/r9zExubm6pcRdz9oenr+AJhB2gAp05c0Zr1qyRn5+f6xKHt7e3bDab20/HBw4c0NKlS9227datmyRp5syZbu3Tp08/7+OHhIQoLi5O7777rhYtWiQ/Pz/16NHDbcyRI0fc1mvUqKFrrrlG+fn5rra8vDzt2bPnrJde/uj666/XrbfeqldffVUfffTRWcf890/5t99+u7KystzuLSosLNSsWbNUo0YNderUyTWuqKjI9ch8iWnTpslms7ler/J43f7K+dZ8vu/1xbr77rvVsmVLTZo0SWlpaaX6jx8/rmeeeUbS74/+16lTRy+//LLbe/zJJ59o9+7dbk+IXX311dqzZ49+/fVXV9v27du1YcOGMtdavXp1STpriAIqCo+eA+Xok08+cf10n5OTo4ULF2rfvn16+umnZbfbJf1+I+jUqVN12223qV+/fsrJyVFKSoquueYat3tl2rRpo759+2r27NnKy8vTddddp9TUVO3fv/+Caurdu7fuu+8+zZ49W3FxcaUeK46Ojlbnzp0VExOj0NBQbdmyRe+9954SExNdYz788EMNHjxYc+fO1aBBg/70eAsWLNBtt92mHj16uM5k1KxZ0/UNyl988YUrkEjSgw8+qFdeeUWDBg1Senq6GjZsqPfee08bNmzQ9OnTXffodO/eXV26dNEzzzyjAwcOqHXr1lqzZo0++ugjPfHEE657dMrrdUtNTdXp06dLtffo0eO8az7f9/pi+fr66oMPPlBsbKxuuukm3Xvvvbr++uvl6+urXbt2aeHChapZs6YmTZokX19f/fOf/9TgwYPVqVMn9e3b1/XoecOGDTV8+HDXfv/+979r6tSpiouL05AhQ5STk6OXX35ZzZs3d92MfqFiYmIk/X4DeVxcnLy9vc96thEoV558FAwwxdkePQ8ICLDatGljzZkzxyouLnYb//rrr1uNGze2/P39raioKGvu3LnWuHHjSj2Se+rUKeuxxx6zwsLCrOrVq1vdu3e3Dh48eF6PUJdwOp1WYGBgqUelSzz77LNW+/btrZCQECswMNCKioqyJk2aZBUUFJSa39y5c8/rmKdOnbKmT59uORwOy263Wz4+PlZERIR1xx13WG+//bZVWFjoNj47O9saPHiwVatWLcvPz89q2bLlWY91/Phxa/jw4Va9evUsX19fq3Hjxtbzzz9f6vW9mNet5DHqcy1vvfXWBdV8vu+1JCshIaHU9g0aNLAGDhz4pzWXOHbsmDV27FirZcuWVrVq1ayAgACrRYsWVlJSknX48GG3sYsXL7auvfZay9/f3woNDbX69+9v/fzzz6X2uWDBAuuqq66y/Pz8rDZt2lirV68+56PnZ3tc/79f88LCQmvYsGFW7dq1LZvNxmPoqBQ2y+LOMQAAYC7u2QEAAEYj7AAAAKMRdgAAgNEIOwAAwGiEHQAAYDTCDgAAMBphR79/o6vT6eT3twAAYCDCjn7/KvXg4ODz/q3BAADg0kHYAQAARiPsAAAAoxF2AACA0Qg7AADAaIQdAABgNMIOAAAwGmEHAAAYjbADAACMRtgBAABGI+wAAACjEXYAAIDRCDsAAMBohB0AAGA0wg4AADAaYQcAABiNsAMAAIxG2AEAAEYj7AAAAKMRdgAAgNF8PF0AAFSW02eKlHn0P54uA7gsRIZWU4Cvt6fLkETYAXAZyTz6H439aKenywAuC8l3tVCT8CBPlyGJy1gAAMBwhB0AAGA0wg4AADAaYQcAABiNsAMAAIxG2AEAAEYj7AAAAKMRdgAAgNEIOwAAwGiEHQAAYDTCDgAAMBphBwAAGI2wAwAAjEbYAQAARiPsAAAAoxF2AACA0Qg7AADAaIQdAABgNMIOAAAwGmEHAAAYjbADAACMRtgBAABGI+wAAACjEXYAAIDRCDsAAMBohB0AAGA0wg4AADAaYQcAABiNsAMAAIxG2AEAAEYj7AAAAKMRdgAAgNEIOwAAwGiEHQAAYDSPhp3x48fLZrO5LVFRUa7+06dPKyEhQWFhYapRo4Z69eql7Oxst31kZmYqPj5e1apVU506dTRy5EgVFhZW9lQAAEAV5ePpApo3b65169a51n18/l9Jw4cP18cff6wlS5YoODhYiYmJ6tmzpzZs2CBJKioqUnx8vCIiIrRx40YdPnxY999/v3x9ffV///d/lT4XAABQ9Xg87Pj4+CgiIqJUe15enl5//XUtXLhQN998syRp7ty5atasmTZt2qSOHTtqzZo1+u6777Ru3TqFh4erTZs2mjhxokaNGqXx48fLz8+vsqcDAACqGI/fs7Nv3z7Vq1dPV111lfr376/MzExJUnp6us6cOaPY2FjX2KioKEVGRiotLU2SlJaWppYtWyo8PNw1Ji4uTk6nU7t27TrnMfPz8+V0Ot0WAABgJo+GnQ4dOmjevHlatWqV5syZox9//FE33nijjh8/rqysLPn5+SkkJMRtm/DwcGVlZUmSsrKy3IJOSX9J37lMnjxZwcHBrqV+/frlOzEAAFBlePQyVrdu3Vx/btWqlTp06KAGDRro3XffVWBgYIUdNykpSSNGjHCtO51OAg8AAIby+GWsPwoJCVGTJk20f/9+RUREqKCgQLm5uW5jsrOzXff4RERElHo6q2T9bPcBlfD395fdbndbAACAmapU2Dlx4oS+//571a1bVzExMfL19VVqaqqrPyMjQ5mZmXI4HJIkh8OhHTt2KCcnxzVm7dq1stvtio6OrvT6AQBA1ePRy1hPPfWUunfvrgYNGujQoUMaN26cvL291bdvXwUHB2vIkCEaMWKEQkNDZbfbNWzYMDkcDnXs2FGS1LVrV0VHR2vAgAGaMmWKsrKyNHr0aCUkJMjf39+TUwMAAFWER8POzz//rL59++rIkSOqXbu2brjhBm3atEm1a9eWJE2bNk1eXl7q1auX8vPzFRcXp9mzZ7u29/b21ooVK/TII4/I4XCoevXqGjhwoJKTkz01JQAAUMXYLMuyPF2EpzmdTgUHBysvL4/7dwCD7c0+rrEf7fR0GcBlIfmuFmoSHuTpMiRVsXt2AAAAyhthBwAAGI2wAwAAjEbYAQAARiPsAAAAoxF2AACA0Qg7AADAaIQdAABgNMIOAAAwGmEHAAAYjbADAACMRtgBAABGI+wAAACjEXYAAIDRCDsAAMBohB0AAGA0wg4AADAaYQcAABiNsAMAAIxG2AEAAEYj7AAAAKMRdgAAgNEIOwAAwGiEHQAAYDTCDgAAMBphBwAAGI2wAwAAjEbYAQAARiPsAAAAoxF2AACA0Qg7AADAaIQdAABgNMIOAAAwGmEHAAAYjbADAACMRtgBAABGI+wAAACjEXYAAIDRCDsAAMBohB0AAGA0wg4AADAaYQcAABiNsAMAAIxG2AEAAEYj7AAAAKMRdgAAgNEIOwAAwGiEHQAAYDTCDgAAMBphBwAAGI2wAwAAjFZlws5zzz0nm82mJ554wtV2+vRpJSQkKCwsTDVq1FCvXr2UnZ3ttl1mZqbi4+NVrVo11alTRyNHjlRhYWElVw8AAKqqKhF2Nm/erFdeeUWtWrVyax8+fLiWL1+uJUuW6PPPP9ehQ4fUs2dPV39RUZHi4+NVUFCgjRs3av78+Zo3b57Gjh1b2VMAAABVlMfDzokTJ9S/f3/961//Us2aNV3teXl5ev311zV16lTdfPPNiomJ0dy5c7Vx40Zt2rRJkrRmzRp99913WrBggdq0aaNu3bpp4sSJSklJUUFBgaemBAAAqhCPh52EhATFx8crNjbWrT09PV1nzpxxa4+KilJkZKTS0tIkSWlpaWrZsqXCw8NdY+Li4uR0OrVr165zHjM/P19Op9NtAQAAZvLx5MEXLVqkrVu3avPmzaX6srKy5Ofnp5CQELf28PBwZWVlucb8MeiU9Jf0ncvkyZM1YcKEi6weAABcCjx2ZufgwYN6/PHH9fbbbysgIKBSj52UlKS8vDzXcvDgwUo9PgAAqDweCzvp6enKycnR3/72N/n4+MjHx0eff/65Zs6cKR8fH4WHh6ugoEC5ublu22VnZysiIkKSFBERUerprJL1kjFn4+/vL7vd7rYAAAAzeSzs3HLLLdqxY4e2bdvmWtq2bav+/fu7/uzr66vU1FTXNhkZGcrMzJTD4ZAkORwO7dixQzk5Oa4xa9euld1uV3R0dKXPCQAAVD0eu2cnKChILVq0cGurXr26wsLCXO1DhgzRiBEjFBoaKrvdrmHDhsnhcKhjx46SpK5duyo6OloDBgzQlClTlJWVpdGjRyshIUH+/v6VPicAAFD1ePQG5b8ybdo0eXl5qVevXsrPz1dcXJxmz57t6vf29taKFSv0yCOPyOFwqHr16ho4cKCSk5M9WDUAAKhKbJZlWZ4uwtOcTqeCg4OVl5fH/TuAwfZmH9fYj3Z6ugzgspB8Vws1CQ/ydBmSqsD37AAAAFQkwg4AADAaYQcAABiNsAMAAIxG2AEAAEYj7AAAAKMRdgAAgNEIOwAAwGiEHQAAYDTCDgAAMBphBwAAGI2wAwAAjEbYAQAARiPsAAAAoxF2AACA0Qg7AADAaIQdAABgNMIOAAAwGmEHAAAYjbADAACMRtgBAABGI+wAAACjEXYAAIDRCDsAAMBohB0AAGA0wg4AADAaYQcAABiNsAMAAIxG2AEAAEYj7AAAAKMRdgAAgNEIOwAAwGiEHQAAYDTCDgAAMBphBwAAGI2wAwAAjEbYAQAARiPsAAAAo5Up7Fx11VU6cuRIqfbc3FxdddVVF10UAABAeSlT2Dlw4ICKiopKtefn5+uXX3656KIAAADKi8+FDF62bJnrz6tXr1ZwcLBrvaioSKmpqWrYsGG5FQcAAHCxLijs9OjRQ5Jks9k0cOBAtz5fX181bNhQL774YrkVBwAAcLEuKOwUFxdLkho1aqTNmzerVq1aFVIUAABAebmgsFPixx9/LO86AAAAKkSZwo4kpaamKjU1VTk5Oa4zPiXeeOONiy4MAACgPJQp7EyYMEHJyclq27at6tatK5vNVt51AQAAlIsyhZ2XX35Z8+bN04ABA8q7HgAAgHJVpu/ZKSgo0HXXXVfetQAAAJS7MoWdoUOHauHCheVdCwAAQLkr02Ws06dP69VXX9W6devUqlUr+fr6uvVPnTq1XIoDAAC4WGUKO99++63atGkjSdq5c6dbHzcrAwCAqqRMl7E+++yzcy6ffvrpee9nzpw5atWqlex2u+x2uxwOhz755BNX/+nTp5WQkKCwsDDVqFFDvXr1UnZ2tts+MjMzFR8fr2rVqqlOnToaOXKkCgsLyzItAABgoDKFnfJy5ZVX6rnnnlN6erq2bNmim2++WXfddZd27dolSRo+fLiWL1+uJUuW6PPPP9ehQ4fUs2dP1/ZFRUWKj49XQUGBNm7cqPnz52vevHkaO3asp6YEAACqGJtlWdaFbtSlS5c/vVx1IWd3/ltoaKief/553X333apdu7YWLlyou+++W5K0Z88eNWvWTGlpaerYsaM++eQT3XHHHTp06JDCw8Ml/f5Y/KhRo/Trr7/Kz8/vvI7pdDoVHBysvLw82e32MtcOoGrbm31cYz/a+dcDAVy05LtaqEl4kKfLkFTGMztt2rRR69atXUt0dLQKCgq0detWtWzZskyFFBUVadGiRTp58qQcDofS09N15swZxcbGusZERUUpMjJSaWlpkqS0tDS1bNnSFXQkKS4uTk6n03V26Gzy8/PldDrdFgAAYKYy3aA8bdq0s7aPHz9eJ06cuKB97dixQw6HQ6dPn1aNGjX04YcfKjo6Wtu2bZOfn59CQkLcxoeHhysrK0uSlJWV5RZ0SvpL+s5l8uTJmjBhwgXVCQAALk3les/Offfdd8G/F6tp06batm2bvvrqKz3yyCMaOHCgvvvuu/Isq5SkpCTl5eW5loMHD1bo8QAAgOeU+ReBnk1aWpoCAgIuaBs/Pz9dc801kqSYmBht3rxZM2bMUO/evVVQUKDc3Fy3szvZ2dmKiIiQJEVEROjrr79221/J01olY87G399f/v7+F1QnAAC4NJUp7PzxiShJsixLhw8f1pYtWzRmzJiLKqi4uFj5+fmKiYmRr6+vUlNT1atXL0lSRkaGMjMz5XA4JEkOh0OTJk1STk6O6tSpI0lau3at7Ha7oqOjL6oOAABghjKFneDgYLd1Ly8vNW3aVMnJyeratet57ycpKUndunVTZGSkjh8/roULF2r9+vVavXq1goODNWTIEI0YMUKhoaGy2+0aNmyYHA6HOnbsKEnq2rWroqOjNWDAAE2ZMkVZWVkaPXq0EhISOHMDAAAklTHszJ07t1wOnpOTo/vvv1+HDx9WcHCwWrVqpdWrV+vWW2+V9PuN0F5eXurVq5fy8/MVFxen2bNnu7b39vbWihUr9Mgjj8jhcKh69eoaOHCgkpOTy6U+AABw6SvT9+yUSE9P1+7duyVJzZs317XXXltuhVUmvmcHuDzwPTtA5alK37NTpjM7OTk56tOnj9avX++6eTg3N1ddunTRokWLVLt27fKsEQAAoMzK9Oj5sGHDdPz4ce3atUtHjx7V0aNHtXPnTjmdTj322GPlXSMAAECZlenMzqpVq7Ru3To1a9bM1RYdHa2UlJQLukEZAACgopXpzE5xcbF8fX1Ltfv6+qq4uPiiiwIAACgvZQo7N998sx5//HEdOnTI1fbLL79o+PDhuuWWW8qtOAAAgItVprDz0ksvyel0qmHDhrr66qt19dVXq1GjRnI6nZo1a1Z51wgAAFBmZbpnp379+tq6davWrVunPXv2SJKaNWvm9hvKAQAAqoILOrPz6aefKjo6Wk6nUzabTbfeequGDRumYcOGqV27dmrevLm+/PLLiqoVAADggl1Q2Jk+fboeeOCBs37xXnBwsB566CFNnTq13IoDAAC4WBcUdrZv367bbrvtnP1du3ZVenr6RRcFAABQXi4o7GRnZ5/1kfMSPj4++vXXXy+6KAAAgPJyQWHniiuu0M6d5/69Mt9++63q1q170UUBAACUlwsKO7fffrvGjBmj06dPl+o7deqUxo0bpzvuuKPcigMAALhYF/To+ejRo/XBBx+oSZMmSkxMVNOmTSVJe/bsUUpKioqKivTMM89USKEAAABlcUFhJzw8XBs3btQjjzyipKQkWZYlSbLZbIqLi1NKSorCw8MrpFAAAICyuOAvFWzQoIFWrlypY8eOaf/+/bIsS40bN1bNmjUroj4AAICLUqZvUJakmjVrql27duVZCwAAQLkr0+/GAgAAuFQQdgAAgNEIOwAAwGiEHQAAYDTCDgAAMBphBwAAGI2wAwAAjEbYAQAARiPsAAAAoxF2AACA0Qg7AADAaIQdAABgNMIOAAAwGmEHAAAYjbADAACMRtgBAABGI+wAAACjEXYAAIDRCDsAAMBohB0AAGA0H08XcLk4faZImUf/4+kygMvGNbVryMvL5ukyAFQBhJ1Kknn0Pxr70U5PlwFcNuYOaq9AP29PlwGgCuAyFgAAMBphBwAAGI2wAwAAjEbYAQAARiPsAAAAoxF2AACA0Qg7AADAaIQdAABgNMIOAAAwGmEHAAAYjbADAACM5tGwM3nyZLVr105BQUGqU6eOevTooYyMDLcxp0+fVkJCgsLCwlSjRg316tVL2dnZbmMyMzMVHx+vatWqqU6dOho5cqQKCwsrcyoAAKCK8mjY+fzzz5WQkKBNmzZp7dq1OnPmjLp27aqTJ0+6xgwfPlzLly/XkiVL9Pnnn+vQoUPq2bOnq7+oqEjx8fEqKCjQxo0bNX/+fM2bN09jx471xJQAAEAV49Hfer5q1Sq39Xnz5qlOnTpKT0/XTTfdpLy8PL3++utauHChbr75ZknS3Llz1axZM23atEkdO3bUmjVr9N1332ndunUKDw9XmzZtNHHiRI0aNUrjx4+Xn5+fJ6YGAACqiCp1z05eXp4kKTQ0VJKUnp6uM2fOKDY21jUmKipKkZGRSktLkySlpaWpZcuWCg8Pd42Ji4uT0+nUrl27KrF6AABQFXn0zM4fFRcX64knntD111+vFi1aSJKysrLk5+enkJAQt7Hh4eHKyspyjflj0CnpL+k7m/z8fOXn57vWnU5neU0DAABUMVXmzE5CQoJ27typRYsWVfixJk+erODgYNdSv379Cj8mAADwjCoRdhITE7VixQp99tlnuvLKK13tERERKigoUG5urtv47OxsRUREuMb899NZJeslY/5bUlKS8vLyXMvBgwfLcTYAAKAq8WjYsSxLiYmJ+vDDD/Xpp5+qUaNGbv0xMTHy9fVVamqqqy0jI0OZmZlyOBySJIfDoR07dignJ8c1Zu3atbLb7YqOjj7rcf39/WW3290WAABgJo/es5OQkKCFCxfqo48+UlBQkOsem+DgYAUGBio4OFhDhgzRiBEjFBoaKrvdrmHDhsnhcKhjx46SpK5duyo6OloDBgzQlClTlJWVpdGjRyshIUH+/v6enB4AAKgCPBp25syZI0nq3LmzW/vcuXM1aNAgSdK0adPk5eWlXr16KT8/X3FxcZo9e7ZrrLe3t1asWKFHHnlEDodD1atX18CBA5WcnFxZ0wAAAFWYR8OOZVl/OSYgIEApKSlKSUk555gGDRpo5cqV5VkaAAAwRJW4QRkAAKCiEHYAAIDRCDsAAMBohB0AAGA0wg4AADAaYQcAABiNsAMAAIxG2AEAAEYj7AAAAKMRdgAAgNEIOwAAwGiEHQAAYDTCDgAAMBphBwAAGI2wAwAAjEbYAQAARiPsAAAAoxF2AACA0Qg7AADAaIQdAABgNMIOAAAwGmEHAAAYjbADAACMRtgBAABGI+wAAACjEXYAAIDRCDsAAMBohB0AAGA0wg4AADAaYQcAABiNsAMAAIxG2AEAAEYj7AAAAKMRdgAAgNEIOwAAwGiEHQAAYDTCDgAAMBphBwAAGI2wAwAAjEbYAQAARiPsAAAAoxF2AACA0Qg7AADAaIQdAABgNMIOAAAwGmEHAAAYjbADAACMRtgBAABGI+wAAACjEXYAAIDRCDsAAMBoHg07X3zxhbp376569erJZrNp6dKlbv2WZWns2LGqW7euAgMDFRsbq3379rmNOXr0qPr37y+73a6QkBANGTJEJ06cqMRZAACAqsyjYefkyZNq3bq1UlJSzto/ZcoUzZw5Uy+//LK++uorVa9eXXFxcTp9+rRrTP/+/bVr1y6tXbtWK1as0BdffKEHH3ywsqYAAACqOB9PHrxbt27q1q3bWfssy9L06dM1evRo3XXXXZKkN998U+Hh4Vq6dKn69Omj3bt3a9WqVdq8ebPatm0rSZo1a5Zuv/12vfDCC6pXr16lzQUAAFRNVfaenR9//FFZWVmKjY11tQUHB6tDhw5KS0uTJKWlpSkkJMQVdCQpNjZWXl5e+uqrryq9ZgAAUPV49MzOn8nKypIkhYeHu7WHh4e7+rKyslSnTh23fh8fH4WGhrrGnE1+fr7y8/Nd606ns7zKBgAAVUyVPbNTkSZPnqzg4GDXUr9+fU+XBAAAKkiVDTsRERGSpOzsbLf27OxsV19ERIRycnLc+gsLC3X06FHXmLNJSkpSXl6eazl48GA5Vw8AAKqKKht2GjVqpIiICKWmprranE6nvvrqKzkcDkmSw+FQbm6u0tPTXWM+/fRTFRcXq0OHDufct7+/v+x2u9sCAADM5NF7dk6cOKH9+/e71n/88Udt27ZNoaGhioyM1BNPPKFnn31WjRs3VqNGjTRmzBjVq1dPPXr0kCQ1a9ZMt912mx544AG9/PLLOnPmjBITE9WnTx+exAIAAJI8HHa2bNmiLl26uNZHjBghSRo4cKDmzZunf/zjHzp58qQefPBB5ebm6oYbbtCqVasUEBDg2ubtt99WYmKibrnlFnl5ealXr16aOXNmpc8FAABUTR4NO507d5ZlWefst9lsSk5OVnJy8jnHhIaGauHChRVRHgAAMECVvWcHAACgPBB2AACA0Qg7AADAaIQdAABgNMIOAAAwGmEHAAAYjbADAACMRtgBAABGI+wAAACjEXYAAIDRCDsAAMBohB0AAGA0wg4AADAaYQcAABiNsAMAAIxG2AEAAEYj7AAAAKMRdgAAgNEIOwAAwGiEHQAAYDTCDgAAMBphBwAAGI2wAwAAjEbYAQAARiPsAAAAoxF2AACA0Qg7AADAaIQdAABgNMIOAAAwGmEHAAAYjbADAACMRtgBAABGI+wAAACjEXYAAIDRCDsAAMBohB0AAGA0wg4AADAaYQcAABiNsAMAAIxG2AEAAEYj7AAAAKMRdgAAgNEIOwAAwGiEHQAAYDTCDgAAMBphBwAAGI2wAwAAjEbYAQAARiPsAAAAoxF2AACA0Qg7AADAaMaEnZSUFDVs2FABAQHq0KGDvv76a0+XBAAAqgAjws7ixYs1YsQIjRs3Tlu3blXr1q0VFxennJwcT5cGAAA8zGZZluXpIi5Whw4d1K5dO7300kuSpOLiYtWvX1/Dhg3T008//ZfbO51OBQcHKy8vT3a7vUJqPH2mSJlH/1Mh+wZQ2jW1a8jLy+bWxucQqDyRodUU4Ovt6TIkST6eLuBiFRQUKD09XUlJSa42Ly8vxcbGKi0tzYOVuQvw9VaT8CBPlwFc1vgcApenSz7s/PbbbyoqKlJ4eLhbe3h4uPbs2XPWbfLz85Wfn+9az8vLk/T7GR4AAHBpCQoKks1mO2f/JR92ymLy5MmaMGFCqfb69et7oBoAAHAx/uo2lEs+7NSqVUve3t7Kzs52a8/OzlZERMRZt0lKStKIESNc68XFxTp69KjCwsL+NBni8uN0OlW/fn0dPHiwwu7nAnBufAZxPoKC/vzy9CUfdvz8/BQTE6PU1FT16NFD0u/hJTU1VYmJiWfdxt/fX/7+/m5tISEhFVwpLmV2u52/aAEP4jOIi3HJhx1JGjFihAYOHKi2bduqffv2mj59uk6ePKnBgwd7ujQAAOBhRoSd3r1769dff9XYsWOVlZWlNm3aaNWqVaVuWgYAAJcfI8KOJCUmJp7zshVQVv7+/ho3blypy54AKgefQZQHI75UEAAA4FyM+HURAAAA50LYAQAARiPsAAAAoxF2cNnp3LmznnjiiXLd5/r162Wz2ZSbm1uu+wVwcRo2bKjp06d7ugx4GGEHAAAYjbADAACMRtjBZamwsFCJiYkKDg5WrVq1NGbMGJV8C8Nbb72ltm3bKigoSBEREerXr59ycnLctl+5cqWaNGmiwMBAdenSRQcOHPDALIBLx/Hjx9W/f39Vr15ddevW1bRp09wuKR87dkz333+/atasqWrVqqlbt27at2+f2z7ef/99NW/eXP7+/mrYsKFefPFFt/6cnBx1795dgYGBatSokd5+++3Kmh6qOMIOLkvz58+Xj4+Pvv76a82YMUNTp07Va6+9Jkk6c+aMJk6cqO3bt2vp0qU6cOCABg0a5Nr24MGD6tmzp7p3765t27Zp6NChevrppz00E+DSMGLECG3YsEHLli3T2rVr9eWXX2rr1q2u/kGDBmnLli1atmyZ0tLSZFmWbr/9dp05c0aSlJ6ernvvvVd9+vTRjh07NH78eI0ZM0bz5s1z28fBgwf12Wef6b333tPs2bNL/aCCy5QFXGY6depkNWvWzCouLna1jRo1ymrWrNlZx2/evNmSZB0/ftyyLMtKSkqyoqOj3caMGjXKkmQdO3aswuoGLlVOp9Py9fW1lixZ4mrLzc21qlWrZj3++OPW3r17LUnWhg0bXP2//fabFRgYaL377ruWZVlWv379rFtvvdVtvyNHjnR9FjMyMixJ1tdff+3q3717tyXJmjZtWgXODpcCzuzgstSxY0fZbDbXusPh0L59+1RUVKT09HR1795dkZGRCgoKUqdOnSRJmZmZkqTdu3erQ4cObvtzOByVVzxwifnhhx905swZtW/f3tUWHByspk2bSvr9M+Xj4+P2uQoLC1PTpk21e/du15jrr7/ebb/XX3+963Nbso+YmBhXf1RUlEJCQipwZrhUEHaAPzh9+rTi4uJkt9v19ttva/Pmzfrwww8lSQUFBR6uDgBQFoQdXJa++uort/VNmzapcePG2rNnj44cOaLnnntON954o6Kiokpd82/WrJm+/vrrUtsDOLurrrpKvr6+2rx5s6stLy9Pe/fulfT7Z6qwsNDtc3nkyBFlZGQoOjraNWbDhg1u+92wYYOaNGkib29vRUVFqbCwUOnp6a7+jIwMvvsKkgg7uExlZmZqxIgRysjI0DvvvKNZs2bp8ccfV2RkpPz8/DRr1iz98MMPWrZsmSZOnOi27cMPP6x9+/Zp5MiRysjI0MKFC91ukgTgLigoSAMHDtTIkSP12WefadeuXRoyZIi8vLxks9nUuHFj3XXXXXrggQf073//W9u3b9d9992nK664QnfddZck6cknn1RqaqomTpyovXv3av78+XrppZf01FNPSZKaNm2q2267TQ899JC++uorpaena+jQoQoMDPTk1FFVePqmIaCyderUyXr00Uethx9+2LLb7VbNmjWt//3f/3XdsLxw4UKrYcOGlr+/v+VwOKxly5ZZkqxvvvnGtY/ly5db11xzjeXv72/deOON1htvvMENysCfcDqdVr9+/axq1apZERER1tSpU6327dtbTz/9tGVZlnX06FFrwIABVnBwsBUYGGjFxcVZe/fuddvHe++9Z0VHR1u+vr5WZGSk9fzzz7v1Hz582IqPj7f8/f2tyMhI680337QaNGjADcqwbJb1/3+5CAAAleTkyZO64oor9OKLL2rIkCGeLgeG8/F0AQAA833zzTfas2eP2rdvr7y8PCUnJ0uS6zIVUJEIOwCASvHCCy8oIyNDfn5+iomJ0ZdffqlatWp5uixcBriMBQAAjMbTWAAAwGiEHQAAYDTCDgAAMBphBwAAGI2wA+CSdeDAAdlsNm3bts3TpQCowgg7AADAaIQdAABgNMIOgCqvuLhYU6ZM0TXXXCN/f39FRkZq0qRJpcYVFRVpyJAhatSokQIDA9W0aVPNmDHDbcz69evVvn17Va9eXSEhIbr++uv1008/SZK2b9+uLl26KCgoSHa7XTExMdqyZUulzBFAxeEblAFUeUlJSfrXv/6ladOm6YYbbtDhw4e1Z8+eUuOKi4t15ZVXasmSJQoLC9PGjRv14IMPqm7durr33ntVWFioHj166IEHHtA777yjgoICff3117LZbJKk/v3769prr9WcOXPk7e2tbdu2ydfXt7KnC6Cc8Q3KAKq048ePq3bt2nrppZc0dOhQt74DBw6oUaNG+uabb9SmTZuzbp+YmKisrCy99957Onr0qMLCwrR+/Xp16tSp1Fi73a5Zs2Zp4MCBFTEVAB7CZSwAVdru3buVn5+vW2655bzGp6SkKCYmRrVr11aNGjX06quvKjMzU5IUGhqqQYMGKS4uTt27d9eMGTN0+PBh17YjRozQ0KFDFRsbq+eee07ff/99hcwJQOUi7ACo0gIDA8977KJFi/TUU09pyJAhWrNmjbZt26bBgweroKDANWbu3LlKS0vTddddp8WLF6tJkybatGmTJGn8+PHatWuX4uPj9emnnyo6Oloffvhhuc8JQOXiMhaAKu306dMKDQ3VzJkz//Iy1rBhw/Tdd98pNTXVNSY2Nla//fbbOb+Lx+FwqF27dpo5c2apvr59++rkyZNatmxZuc4JQOXizA6AKi0gIECjRo3SP/7xD7355pv6/vvvtWnTJr3++uulxjZu3FhbtmzR6tWrtXfvXo0ZM0abN2929f/4449KSkpSWlqafvrpJ61Zs0b79u1Ts2bNdOrUKSUmJmr9+vX66aeftGHDBm3evFnNmjWrzOkCqAA8jQWgyhszZox8fHw0duxYHTp0SHXr1tXDDz9catxDDz2kb775Rr1795bNZlPfvn316KOP6pNPPpEkVatWTXv27NH8+fN15MgR1a1bVwkJCXrooYdUWFioI0eO6P7771d2drZq1aqlnj17asKECZU9XQDljMtYAADAaFzGAgAARiPsAAAAoxF2AACA0Qg7AADAaIQdAABgNMIOAAAwGmEHAAAYjbADAACMRtgBAABGI+wAAACjEXYAAIDRCDsAAMBo/x9dWkm/NZ32CAAAAABJRU5ErkJggg==", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "# plot the target variable \"class\"\n", + "p = sns.histplot(train[\"class\"], ec=\"w\", lw=4)\n", + "_ = p.set_title(\"Bad vs. Good Loan Count\")\n", + "_ = p.spines[\"top\"].set_visible(False)\n", + "_ = p.spines[\"right\"].set_visible(False)" + ] + }, + { + "cell_type": "markdown", + "id": "c6a697a5-5709-4a69-b644-62779b4f8bc5", + "metadata": {}, + "source": [ + "Now, view the first few records of the context data." + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "79424785-129d-4007-84a5-041b6d38457d", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
IDclassoutcome_timestampdurationcredit_amountinstallment_commitmentchecking_statusresidence_sinceageexisting_creditsnum_dependentshousing
18473good2023-12-16 03:29:12+00:00612384no checking43612own
764894good2023-11-15 23:19:35+00:001811694no checking32921own
504318good2023-11-23 13:03:53+00:00127014no checking23221own
454340good2023-12-26 17:59:37+00:0024574320<=X<20042421for free
453605good2023-12-18 11:27:02+00:002428284<042211own
\n", + "
" + ], + "text/plain": [ + " ID class outcome_timestamp duration credit_amount \\\n", + "18 473 good 2023-12-16 03:29:12+00:00 6 1238 \n", + "764 894 good 2023-11-15 23:19:35+00:00 18 1169 \n", + "504 318 good 2023-11-23 13:03:53+00:00 12 701 \n", + "454 340 good 2023-12-26 17:59:37+00:00 24 5743 \n", + "453 605 good 2023-12-18 11:27:02+00:00 24 2828 \n", + "\n", + " installment_commitment checking_status residence_since age \\\n", + "18 4 no checking 4 36 \n", + "764 4 no checking 3 29 \n", + "504 4 no checking 2 32 \n", + "454 2 0<=X<200 4 24 \n", + "453 4 <0 4 22 \n", + "\n", + " existing_credits num_dependents housing \n", + "18 1 2 own \n", + "764 2 1 own \n", + "504 2 1 own \n", + "454 2 1 for free \n", + "453 1 1 own " + ] + }, + "execution_count": 11, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# View first records in training data\n", + "train.head()" + ] + }, + { + "cell_type": "markdown", + "id": "fd52f5bc-aa0f-48db-b356-c52aa7ce3724", + "metadata": {}, + "source": [ + "### Feature Engineering" + ] + }, + { + "cell_type": "markdown", + "id": "3e5b5c02-ad4d-400e-bdac-bfdf2799f575", + "metadata": {}, + "source": [ + "Once data columns have been prepared so that they can be used to train an AI model, it is common to refer to them as \"features\". The process of preparing features is referred to as \"feature engineering\". \n", + "\n", + "Below, we will train a random forest model. Random forests are relatively robust to non-standardized, non-normalized data, making it easier for us to getting started. As such, the numerical columns are ready for a simple baseline training. \n", + "\n", + "We have pulled two categorical columns, wich we will need to engineer into numerical features." + ] + }, + { + "cell_type": "markdown", + "id": "45a6fb27-140c-4f5a-b464-1f5e5d81d086", + "metadata": {}, + "source": [ + "The `checking_status` column tells us roughly how much money the applicant has in their checking account, while the `housing` column shows the applicant's housing status. We presume that more money in checking correlates inversely with credit risk, while owing vs. renting, vs. living for free correlates directly with credit risk. Hence, converting these to ordinal features makes sense. Of course, in a real study we would want to quantitatively verify these presumptions." + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "9e374096-b02d-4cbb-8fca-dcc451c90c50", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "checking_status\n", + "no checking 0.39375\n", + "0<=X<200 0.27500\n", + "<0 0.26125\n", + ">=200 0.07000\n", + "Name: proportion, dtype: float64" + ] + }, + "execution_count": 12, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# Inspect the `checking_status` column distibution\n", + "train.checking_status.value_counts(normalize=True)" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "0144b525-244b-4526-8e4b-d393cb174d06", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "housing\n", + "own 0.7225\n", + "rent 0.1675\n", + "for free 0.1100\n", + "Name: proportion, dtype: float64" + ] + }, + "execution_count": 13, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# Inspect the `housing` column distribution\n", + "train.housing.value_counts(normalize=True)" + ] + }, + { + "cell_type": "markdown", + "id": "2cb340b4-7d21-4810-8be2-1633da2e4396", + "metadata": {}, + "source": [ + "We define a tranformer that can be used to convert `checking_status` and `housing` to ordinal variables. The transformer will also drop the non-feature columns (`class`, `ID`, and `application_timestamp`) from the feature data." + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "id": "27796e23-c12e-4e51-8fb4-090b26aff2ef", + "metadata": {}, + "outputs": [], + "source": [ + "# Feature lists\n", + "cat_features = [\"checking_status\", \"housing\"]\n", + "num_features = [\n", + " \"duration\", \"credit_amount\", \"installment_commitment\",\n", + " \"residence_since\", \"age\", \"existing_credits\", \"num_dependents\"\n", + "]\n", + "\n", + "# Ordinal encoder for cat_features\n", + "# (We use a ColumnTransformer to passthrough numerical feature columns)\n", + "col_transform = ColumnTransformer([\n", + " (\"cat_features\", OrdinalEncoder(), cat_features),\n", + " (\"num_features\", \"passthrough\", num_features),\n", + " ],\n", + " remainder=\"drop\",\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "id": "318429b9-e008-4cc7-8108-779934f9ac2f", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
checking_statushousingdurationcredit_amountinstallment_commitmentresidence_sinceageexisting_creditsnum_dependents
183.01.06.01238.04.04.036.01.02.0
7643.01.018.01169.04.03.029.02.01.0
5043.01.012.0701.04.02.032.02.01.0
4540.00.024.05743.02.04.024.02.01.0
4531.01.024.02828.04.04.022.01.01.0
\n", + "
" + ], + "text/plain": [ + " checking_status housing duration credit_amount \\\n", + "18 3.0 1.0 6.0 1238.0 \n", + "764 3.0 1.0 18.0 1169.0 \n", + "504 3.0 1.0 12.0 701.0 \n", + "454 0.0 0.0 24.0 5743.0 \n", + "453 1.0 1.0 24.0 2828.0 \n", + "\n", + " installment_commitment residence_since age existing_credits \\\n", + "18 4.0 4.0 36.0 1.0 \n", + "764 4.0 3.0 29.0 2.0 \n", + "504 4.0 2.0 32.0 2.0 \n", + "454 2.0 4.0 24.0 2.0 \n", + "453 4.0 4.0 22.0 1.0 \n", + "\n", + " num_dependents \n", + "18 2.0 \n", + "764 1.0 \n", + "504 1.0 \n", + "454 1.0 \n", + "453 1.0 " + ] + }, + "execution_count": 15, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# Check the tranform outputs features as expected\n", + "# (Note: transform output is an array, so we convert it\n", + "# back to dataframe for inspection)\n", + "pd.DataFrame(\n", + " index=train.index,\n", + " columns=cat_features + num_features,\n", + " data= col_transform.fit_transform(train)\n", + ").head()" + ] + }, + { + "cell_type": "markdown", + "id": "a3785c93-8830-4fa2-bb9d-31b6e8fecb01", + "metadata": {}, + "source": [ + "Finally, let's separate out the labels, and engineer them from categorical (\"good\" | \"bad\") to float (1.0 | 0.0). We do this for both the training and validation data." + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "id": "30ebff90-a193-43a2-86fb-cf09e7d03777", + "metadata": {}, + "outputs": [], + "source": [ + "# Make \"class\" target variable numeric\n", + "train_y = (train[\"class\"] == \"good\").astype(float)\n", + "validate_y = (validate[\"class\"] == \"good\").astype(float)" + ] + }, + { + "cell_type": "markdown", + "id": "b052f6b2-2a34-441d-8a5f-2aad4e4db022", + "metadata": {}, + "source": [ + "### Train the Model" + ] + }, + { + "cell_type": "markdown", + "id": "c4f14590-31f4-4680-b1a1-75755a78513e", + "metadata": {}, + "source": [ + "Now that the features are prepared, we can train (fit) our baseline model on the feature data." + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "id": "0ff48f34-dbb6-4221-aefc-3c9b3f9da3e3", + "metadata": {}, + "outputs": [], + "source": [ + "# Specify the model\n", + "rf_model = RandomForestClassifier(\n", + " n_estimators=400,\n", + " criterion=\"entropy\",\n", + " max_depth=4,\n", + " min_samples_leaf=10,\n", + " class_weight={0:5, 1:1},\n", + " random_state=SEED\n", + ")\n", + "\n", + "# Package transform and model in pipeline\n", + "model = Pipeline([(\"transform\", col_transform), (\"rf_model\", rf_model)])" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "id": "1d6ef38a-23b0-4056-a108-960495521164", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
Pipeline(steps=[('transform',\n",
+       "                 ColumnTransformer(transformers=[('cat_features',\n",
+       "                                                  OrdinalEncoder(),\n",
+       "                                                  ['checking_status',\n",
+       "                                                   'housing']),\n",
+       "                                                 ('num_features', 'passthrough',\n",
+       "                                                  ['duration', 'credit_amount',\n",
+       "                                                   'installment_commitment',\n",
+       "                                                   'residence_since', 'age',\n",
+       "                                                   'existing_credits',\n",
+       "                                                   'num_dependents'])])),\n",
+       "                ('rf_model',\n",
+       "                 RandomForestClassifier(class_weight={0: 5, 1: 1},\n",
+       "                                        criterion='entropy', max_depth=4,\n",
+       "                                        min_samples_leaf=10, n_estimators=400,\n",
+       "                                        random_state=142))])
In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook.
On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.
" + ], + "text/plain": [ + "Pipeline(steps=[('transform',\n", + " ColumnTransformer(transformers=[('cat_features',\n", + " OrdinalEncoder(),\n", + " ['checking_status',\n", + " 'housing']),\n", + " ('num_features', 'passthrough',\n", + " ['duration', 'credit_amount',\n", + " 'installment_commitment',\n", + " 'residence_since', 'age',\n", + " 'existing_credits',\n", + " 'num_dependents'])])),\n", + " ('rf_model',\n", + " RandomForestClassifier(class_weight={0: 5, 1: 1},\n", + " criterion='entropy', max_depth=4,\n", + " min_samples_leaf=10, n_estimators=400,\n", + " random_state=142))])" + ] + }, + "execution_count": 18, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# Fit the model\n", + "model.fit(train, train_y)" + ] + }, + { + "cell_type": "markdown", + "id": "73c45c39-9d8e-4f76-aca5-9f0c1568d263", + "metadata": {}, + "source": [ + "### Evaluate the Model" + ] + }, + { + "cell_type": "markdown", + "id": "ef58d432-80ba-428f-b59f-621a9e53b331", + "metadata": {}, + "source": [ + "Let's evaluate our baseline model performance. With credit risk, recall is going to be an important measure to look at. We compare the performance on the training data, with the performance on the validation data through a classification report." + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "id": "8c5472f6-2ddc-437d-8102-4d5bd2c9f39c", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + " precision recall f1-score support\n", + "\n", + " 0.0 0.42 0.92 0.58 232\n", + " 1.0 0.94 0.49 0.64 568\n", + "\n", + " accuracy 0.61 800\n", + " macro avg 0.68 0.70 0.61 800\n", + "weighted avg 0.79 0.61 0.63 800\n", + "\n" + ] + } + ], + "source": [ + "# Evaluate training set performance\n", + "train_preds = model.predict(train)\n", + "print(classification_report(train_y, train_preds))" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "id": "c296bbd3-603e-4615-abbe-2689ebcf5d8c", + "metadata": { + "scrolled": true + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + " precision recall f1-score support\n", + "\n", + " 0.0 0.46 0.87 0.61 68\n", + " 1.0 0.88 0.48 0.62 132\n", + "\n", + " accuracy 0.61 200\n", + " macro avg 0.67 0.68 0.61 200\n", + "weighted avg 0.74 0.61 0.62 200\n", + "\n" + ] + } + ], + "source": [ + "# Evaluate validation data performance\n", + "print(classification_report(validate_y, model.predict(validate)))" + ] + }, + { + "cell_type": "markdown", + "id": "d57ffbdc-f0b3-4fb6-9575-5acd983082cf", + "metadata": {}, + "source": [ + "The recall on the validation set for bad loans (0 class) is 0.87, meaning that the model correctly identified close to 90% of the bad loans. However, the precision of 0.46 tells us that the model is also classifying many loans that were actually good as bad. Precision and recall are technical metrics. In order to truly assess the models value, we would need feedback from the business side on the impact of misclassifications (for both good and bad loans).\n", + "\n", + "The difference in performance on the training vs. validation data, tells us that the model is slightly overfitting the data. Remember that this is just a quick baseline model. To improve further, we could do things like:\n", + "- gather more data\n", + "- engineer features\n", + "- experiment with hyperparameter settings\n", + "- experiment with other model types\n", + "\n", + "In fact, this is just a start. Creating AI models that meet business needs often requires a lot of guided experimentation." + ] + }, + { + "cell_type": "markdown", + "id": "0378d21a-d6db-42f9-851a-ce71f68c6802", + "metadata": {}, + "source": [ + "### Save the Model" + ] + }, + { + "cell_type": "markdown", + "id": "4450a328-f00c-4579-8e08-b2ebe5046961", + "metadata": {}, + "source": [ + "The last thing we do is save our trained model, so that we can pick it up later in the serving environment." + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "id": "da7a7906-d54f-4f2d-9803-6c82c86b28ad", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "['rf_model.pkl']" + ] + }, + "execution_count": 21, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# Save the model to a pickle file\n", + "joblib.dump(model, \"rf_model.pkl\")" + ] + }, + { + "cell_type": "markdown", + "id": "299588b8-ab67-4155-97a9-770e8e4a7476", + "metadata": {}, + "source": [ + "In the next notebook, [04_Credit_Risk_Model_Serving.ipynb](04_Credit_Risk_Model_Serving.ipynb), we will load the trained model and request predictions, with input features provided by the Feast online feature server." + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.9" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/examples/credit-risk-end-to-end/04_Credit_Risk_Model_Serving.ipynb b/examples/credit-risk-end-to-end/04_Credit_Risk_Model_Serving.ipynb new file mode 100644 index 00000000000..f263dd6cd7b --- /dev/null +++ b/examples/credit-risk-end-to-end/04_Credit_Risk_Model_Serving.ipynb @@ -0,0 +1,697 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "9c870dcb-c66d-454d-a3fa-5f9a723bf8af", + "metadata": {}, + "source": [ + "
" + ] + }, + { + "cell_type": "markdown", + "id": "339ab741-ac90-4763-9971-3b274f6a90b4", + "metadata": {}, + "source": [ + "# Credit Risk Model Serving" + ] + }, + { + "cell_type": "markdown", + "id": "31d29794-4c33-4bc1-9bb4-e238c59f882d", + "metadata": {}, + "source": [ + "### Introduction" + ] + }, + { + "cell_type": "markdown", + "id": "d6553fe7-5427-4ecc-b638-615b47acf1a8", + "metadata": {}, + "source": [ + "Model serving is an exciting part of AI/ML. All of our previous work was building to this phase where we can actually serve loan predictions. \n", + "\n", + "So what role does Feast play in model serving? We've already seen that Feast can \"materialize\" data from the training offline store to the serving online store. This comes in handy because many models need contextual features at inference time. \n", + "\n", + "With this example, we can imagine a scenario something like this:\n", + "1. A bank customer submits a loan application on a website. \n", + "2. The website backend requests features, supplying the customer's ID as input.\n", + "3. The backend retrieves feature data for the ID in question.\n", + "4. The backend submits the feature data to the model to obtain a prediction.\n", + "5. The backend uses the prediction to make a decision.\n", + "6. The response is recorded and made available to the user.\n", + "\n", + "With online requests like this, time and resource usage often matter a lot. Feast facilitates quickly retrieving the correct feature data.\n", + "\n", + "In real-life, some of the contextual feature data points could be requested from the user, while others are retrieved from data sources. While outside the scope of this example, Feast does facilitate retrieving request data, and joining it with feature data. (See [Request Source](https://rtd.feast.dev/en/master/#request-source)).\n", + "\n", + "In this notebook, we request feature data from the online store for a small batch of users. We then get outcome predictions from our trained model. This notebook is a continuation of the work done in the previous notebooks; it comes as the step after [03_Credit_Risk_Model_Training.ipynb](03_Credit_Risk_Model_Training.ipynb)." + ] + }, + { + "cell_type": "markdown", + "id": "53818109-c357-435f-8a8b-2a62982fa9a8", + "metadata": {}, + "source": [ + "### Setup" + ] + }, + { + "cell_type": "markdown", + "id": "92b5ab1b-186d-4b76-aac7-9b5110f8673e", + "metadata": {}, + "source": [ + "*The following code assumes that you have read the example README.md file, and that you have setup an environment where the code can be run. Please make sure you have addressed the prerequisite needs.*" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "378189ed-e967-4b2b-b591-aab980a685b3", + "metadata": {}, + "outputs": [], + "source": [ + "# Imports\n", + "import os\n", + "import joblib\n", + "import json\n", + "import requests\n", + "import warnings\n", + "import pandas as pd\n", + "\n", + "from feast import FeatureStore" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "ea90edb2-16f0-4d40-a280-4e6ea79ea5be", + "metadata": {}, + "outputs": [], + "source": [ + "# ingnore warnings\n", + "warnings.filterwarnings(action=\"ignore\")" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "55f8ed91-7c13-44f7-a294-b6cacd43f8db", + "metadata": {}, + "outputs": [], + "source": [ + "# Load the model\n", + "model = joblib.load(\"rf_model.pkl\")" + ] + }, + { + "cell_type": "markdown", + "id": "3093e1b6-66d9-4936-b197-d853631914db", + "metadata": {}, + "source": [ + "### Query Feast Online Server for Feature Data" + ] + }, + { + "cell_type": "markdown", + "id": "2b5bbc4a-e2d3-4b7b-8309-434ff3b3e2cf", + "metadata": {}, + "source": [ + "Here, we show two different ways to retrieve data from the online feature server. The first is using the Python `requests` library, and the second is using the Feast Python SDK.\n", + "\n", + "We can use the Python requests library to request feature data from the online feature server (that we deployed in notebook [02_Deploying_the_Feature_Store.ipynb](02_Deploying_the_Feature_Store.ipynb)). The request takes the form of an HTTP POST command sent to the server endpoint (`url`). We request the data we need by supplying the entity and feature information in the data payload. We also need to specify an `application/json` content type in the request header." + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "c6fd4f1a-917b-4a98-9bf6-101b4a074b64", + "metadata": {}, + "outputs": [], + "source": [ + "# ID examples\n", + "ids = [18, 764, 504, 454, 453, 0, 1, 2, 3, 4, 5, 6, 7, 8]\n", + "\n", + "# Submit get_online_features request to Feast online store server\n", + "response = requests.post(\n", + " url=\"http://localhost:6566/get-online-features\",\n", + " headers = {'Content-Type': 'application/json'},\n", + " data=json.dumps({\n", + " \"entities\": {\"ID\": ids},\n", + " \"features\": [\n", + " \"data_a:duration\",\n", + " \"data_a:credit_amount\",\n", + " \"data_a:installment_commitment\",\n", + " \"data_a:checking_status\",\n", + " \"data_b:residence_since\",\n", + " \"data_b:age\",\n", + " \"data_b:existing_credits\",\n", + " \"data_b:num_dependents\",\n", + " \"data_b:housing\"\n", + " ]\n", + " })\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "8e616a52-c18c-44a9-9e63-3aba071d7e79", + "metadata": {}, + "source": [ + "The response is returned as JSON, with feature values for each of the IDs." + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "cf8948b7-4ed7-4c45-8acf-462331d9e4d2", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "'{\"metadata\":{\"feature_names\":[\"ID\",\"checking_status\",\"duration\",\"installment_commitment\",\"credit_amount\",\"residence_since\",\"num_dependents\",\"age\",\"housing\",\"existing_credits\"]},\"results\":[{\"values\":[18,764,504,454,453,0,1,2,3,4,5,6,7,8],\"statuses\":[\"PRESENT\",\"PRESENT\",\"PRESENT\",\"PRESENT\",\"PRESENT\",\"PRESENT\",\"PRESENT\",\"PRESENT\",\"PRESENT\",\"PRESENT\",\"PRESENT\",\"PRESENT\",\"PRESENT\",\"PRESENT\"],\"event_timestamps\":[\"1970-01-01T00:00:00Z\",\"1970-01-01T00:00:00Z\",\"1970-01-01T00:00:00Z\",\"1970-01-01T00:00:00Z\",\"1970-01-01T00:00:00Z\",\"1970-01-01T00:00:00Z\",\"1970-01-01T00:00:00Z\",\"1970-01-01T00:00:00Z\",\"1970-01-01T00:00:00Z\",\"1970-01-01T00:00:00Z\",\"1970-01-01T00:00:00Z\",\"1970-01-01T00:00:00Z\",\"1970-01-01T00:00:00Z\",\"1970-01-01T00:00:00Z\"]},{\"values\":[\"0<=X<200\",\"no checking\",\"<0\",\"<0\",\"no checking\",\"<0\",\"0<=X<200\",\"no checking\",\"<0\",\"<0\",\"no checking\",\"no checking\",\"0<=X<200\",\"no checking\"],\"statuses\":[\"PRESENT\",\"PRESENT\",\"PRESENT\",\"PRESENT\",\"PRESENT\",\"PRESENT\",\"PRESENT\",\"PRESENT\",\"PRESENT\",\"PRESENT\",'" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# Show first 1000 characters of response\n", + "response.text[:1000]" + ] + }, + { + "cell_type": "markdown", + "id": "c719f702-578a-4f35-b8ff-e41707cda23e", + "metadata": {}, + "source": [ + "As the response data comes in JSON format, there is a little formatting required to organize the data into a dataframe with one record per row (and features as columns)." + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "b992063d-8d83-4bf7-8153-f690b0410359", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
IDchecking_statusdurationinstallment_commitmentcredit_amountresidence_sincenum_dependentsagehousingexisting_credits
0180<=X<20024.04.012579.02.01.044.0for free1.0
1764no checking24.04.02463.03.01.027.0own2.0
2504<024.04.01207.04.01.024.0rent1.0
\n", + "
" + ], + "text/plain": [ + " ID checking_status duration installment_commitment credit_amount \\\n", + "0 18 0<=X<200 24.0 4.0 12579.0 \n", + "1 764 no checking 24.0 4.0 2463.0 \n", + "2 504 <0 24.0 4.0 1207.0 \n", + "\n", + " residence_since num_dependents age housing existing_credits \n", + "0 2.0 1.0 44.0 for free 1.0 \n", + "1 3.0 1.0 27.0 own 2.0 \n", + "2 4.0 1.0 24.0 rent 1.0 " + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# Inspect the response\n", + "resp_data = json.loads(response.text)\n", + "\n", + "# Transform JSON into dataframe\n", + "records = pd.DataFrame(\n", + " columns=resp_data[\"metadata\"][\"feature_names\"], \n", + " data=[[r[\"values\"][i] for r in resp_data[\"results\"]] for i in range(len(ids))]\n", + ")\n", + "records.head(3)" + ] + }, + { + "cell_type": "markdown", + "id": "6db9b8ac-146e-40d3-b35a-cf4f4b6bbc8a", + "metadata": {}, + "source": [ + "Now, let's see how we can do the same with the Feast Python SDK. Note that we instantiate our `FeatureStore` object with the configuration that we set up in [02_Deploying_the_Feature_Store.ipynb](02_Deploying_the_Feature_Store.ipynb), by pointing to the `./Feature_Store` directory." + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "765dc62b-e1e7-45fe-88b4-cc0235519ff8", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "WARNING:root:_list_feature_views will make breaking changes. Please use _list_batch_feature_views instead. _list_feature_views will behave like _list_all_feature_views in the future.\n", + "WARNING:root:Cannot use sqlite_vec for vector search\n" + ] + } + ], + "source": [ + "# Instantiate FeatureStore object\n", + "store = FeatureStore(repo_path=\"./Feature_Store\")\n", + "\n", + "# Retrieve features\n", + "records = store.get_online_features(\n", + " entity_rows=[{\"ID\":v} for v in ids],\n", + " features=[\n", + " \"data_a:duration\",\n", + " \"data_a:credit_amount\",\n", + " \"data_a:installment_commitment\",\n", + " \"data_a:checking_status\",\n", + " \"data_b:residence_since\",\n", + " \"data_b:age\",\n", + " \"data_b:existing_credits\",\n", + " \"data_b:num_dependents\",\n", + " \"data_b:housing\" \n", + " ]\n", + ").to_df()" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "1d214e55-df0b-460d-936c-8951f7365a93", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
IDcredit_amountinstallment_commitmentchecking_statusdurationnum_dependentshousingageresidence_sinceexisting_credits
01812579.04.00<=X<20024.01.0for free44.02.01.0
17642463.04.0no checking24.01.0own27.03.02.0
25041207.04.0<024.01.0rent24.04.01.0
\n", + "
" + ], + "text/plain": [ + " ID credit_amount installment_commitment checking_status duration \\\n", + "0 18 12579.0 4.0 0<=X<200 24.0 \n", + "1 764 2463.0 4.0 no checking 24.0 \n", + "2 504 1207.0 4.0 <0 24.0 \n", + "\n", + " num_dependents housing age residence_since existing_credits \n", + "0 1.0 for free 44.0 2.0 1.0 \n", + "1 1.0 own 27.0 3.0 2.0 \n", + "2 1.0 rent 24.0 4.0 1.0 " + ] + }, + "execution_count": 8, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "records.head(3)" + ] + }, + { + "cell_type": "markdown", + "id": "fd828758-6c57-4f9e-bbda-3983b6579da2", + "metadata": {}, + "source": [ + "### Get Predictions from the Model" + ] + }, + { + "cell_type": "markdown", + "id": "f446d7ec-0dae-409a-82a2-c0d7016c2001", + "metadata": {}, + "source": [ + "Now we can request predictions from our trained model. \n", + "\n", + "For convenience, we output the predictions along with the implied loan designations. Remember that these are predictions on loan outcomes, given context data from the loan application process. Since we have access to the actual `class` outcomes, we display those as well to see how the model did.|" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "70203f7b-f1e5-46ba-8623-f10bf3a5abf8", + "metadata": {}, + "outputs": [], + "source": [ + "# Get predictions from the model\n", + "preds = model.predict(records)" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "27001dde-8bdb-4de1-8c33-a76f030748e0", + "metadata": {}, + "outputs": [], + "source": [ + "# Load labels\n", + "labels = pd.read_parquet(\"Feature_Store/data/labels.parquet\")" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "ddc958e8-8ff8-49b1-ac10-fc965f3bf21c", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
IDPredictionLoan_DesignationTrue_Value
18180.0badbad
7647641.0goodgood
5045040.0badbad
4544540.0badbad
4534531.0goodgood
001.0goodgood
110.0badbad
221.0goodgood
330.0badgood
440.0badbad
551.0goodgood
661.0goodgood
770.0badgood
881.0goodgood
\n", + "
" + ], + "text/plain": [ + " ID Prediction Loan_Designation True_Value\n", + "18 18 0.0 bad bad\n", + "764 764 1.0 good good\n", + "504 504 0.0 bad bad\n", + "454 454 0.0 bad bad\n", + "453 453 1.0 good good\n", + "0 0 1.0 good good\n", + "1 1 0.0 bad bad\n", + "2 2 1.0 good good\n", + "3 3 0.0 bad good\n", + "4 4 0.0 bad bad\n", + "5 5 1.0 good good\n", + "6 6 1.0 good good\n", + "7 7 0.0 bad good\n", + "8 8 1.0 good good" + ] + }, + "execution_count": 11, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# Show preds\n", + "pd.DataFrame({\n", + " \"ID\": ids,\n", + " \"Prediction\": preds,\n", + " \"Loan_Designation\": [\"bad\" if i==0.0 else \"good\" for i in preds],\n", + " \"True_Value\": labels.loc[ids, \"class\"]\n", + "})" + ] + }, + { + "cell_type": "markdown", + "id": "87cd592a-61fc-4553-b84a-941d1785910d", + "metadata": {}, + "source": [ + "It's important to remember that the model's predictions are like educated guesses based on learned patterns. The model will get some predictions right, and other wrong. With the example records above, it looks like the model did pretty good! An AI/ML team's task is generally to make the model's predictions as useful as possible in helping the organization make decisions (for example, on loan approvals).\n", + "\n", + "In this case, we have a baseline model. While not ready for production, this model has set a low bar by which other models can be measured. Teams can also use a model like this to help with early testing, and with proving out things like pipelines and infrastructure before more sophisticated models are available.\n", + "\n", + "We have used Feast to query the feature data in support of model serving. The next notebook, [05_Credit_Risk_Cleanup.ipynb](05_Credit_Risk_Cleanup.ipynb), cleans up resources created in this and previous notebooks." + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.9" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/examples/credit-risk-end-to-end/05_Credit_Risk_Cleanup.ipynb b/examples/credit-risk-end-to-end/05_Credit_Risk_Cleanup.ipynb new file mode 100644 index 00000000000..846748dc425 --- /dev/null +++ b/examples/credit-risk-end-to-end/05_Credit_Risk_Cleanup.ipynb @@ -0,0 +1,296 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "cf46ec61-7914-4677-b12b-a9e478e88d3f", + "metadata": {}, + "source": [ + "# Credit Risk Cleanup" + ] + }, + { + "cell_type": "markdown", + "id": "6ae8aaec-e01d-48d3-b768-98661ad1ec85", + "metadata": {}, + "source": [ + "Run this notebook if you are done experimenting with this demo, or if you wish to start again with a clean slate.\n", + "\n", + "**RUNNING THE FOLLOWING CODE WILL REMOVE FILES AND PROCESSES CREATED BY THE PREVIOUS EXAMPLE NOTEBOOKS.**\n", + "\n", + "The notebook progresses in reverse order of how the files and processes were added. (The reverse order makes it possible to partially revert changes by running cells up to a certain point.)" + ] + }, + { + "cell_type": "markdown", + "id": "6feaa771-4226-459f-b6dd-214024cb5c7c", + "metadata": {}, + "source": [ + "#### Setup" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "20a39e94-920d-4108-aa6b-1e29d2224f71", + "metadata": {}, + "outputs": [], + "source": [ + "# Imports\n", + "import os\n", + "import time\n", + "import psutil" + ] + }, + { + "cell_type": "markdown", + "id": "3f124260-a8b2-475d-9103-8d336c543fce", + "metadata": {}, + "source": [ + "#### Remove Trained Model File" + ] + }, + { + "cell_type": "markdown", + "id": "f7a05a2b-9a26-4722-a526-84da99fc0b29", + "metadata": {}, + "source": [ + "This removes the model that was created and saved in [03_Credit_Risk_Model_Training.ipynb](03_Credit_Risk_Model_Training.ipynb)." + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "a6b21063-ea43-4329-be0c-c1644c705db2", + "metadata": {}, + "outputs": [], + "source": [ + "# Remove the model file that was saved in model training.\n", + "model_path = \"./rf_model.pkl\"\n", + "os.remove(model_path)" + ] + }, + { + "cell_type": "markdown", + "id": "ed97c24a-8f25-4e77-9037-f9cf4ad68dfa", + "metadata": {}, + "source": [ + "#### Shutdown Servers" + ] + }, + { + "cell_type": "markdown", + "id": "2f825d10-c13d-4701-b102-e15ad1c0bd3b", + "metadata": {}, + "source": [ + "Shut down the servers that were launched in [02_Deploying_the_Feature_Store.ipynb](02_Deploying_the_Feature_Store.ipynb); also remove the `server_proc.txt` that held the process PIDs." + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "66db4d46-a895-4041-ad87-ab0a77f13211", + "metadata": {}, + "outputs": [], + "source": [ + "# Load server process objects\n", + "server_pids = open(\"server_proc.txt\").readlines()\n", + "offline_server_proc = psutil.Process(int(server_pids[0].strip()))\n", + "online_server_proc = psutil.Process(int(server_pids[1].strip()))" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "001fd472-2e28-499e-9eac-0a16ad8187a0", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Online server : psutil.Process(pid=44621, name='python3.11', status='running', started='14:19:05')\n", + "Online server is running: True\n", + "\n", + "Offline server PID: psutil.Process(pid=44594, name='python3.11', status='running', started='14:19:03')\n", + "Offline server is running: True\n" + ] + } + ], + "source": [ + "# Verify if servers are running\n", + "def verify_servers():\n", + " # online server\n", + " print(f\"Online server : {online_server_proc}\")\n", + " print(f\"Online server is running: {online_server_proc.is_running()}\", end='\\n\\n')\n", + " # offline server\n", + " print(f\"Offline server PID: {offline_server_proc}\")\n", + " print(f\"Offline server is running: {offline_server_proc.is_running()}\")\n", + " \n", + "verify_servers()" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "68376350-790a-4e7e-9325-c7de4d22e54b", + "metadata": {}, + "outputs": [], + "source": [ + "# Terminate offline server\n", + "offline_server_proc.terminate()" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "446b6bf9-aef2-4873-b477-8bf595a8eabf", + "metadata": {}, + "outputs": [], + "source": [ + "# Terminate online server (master and worker)\n", + "for child in online_server_proc.children(recursive=True):\n", + " child.terminate()\n", + "online_server_proc.terminate()\n", + "time.sleep(2)" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "774827f6-4dcd-495b-b5c5-186b97148619", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Online server : psutil.Process(pid=44621, name='python3.11', status='terminated', started='14:19:05')\n", + "Online server is running: False\n", + "\n", + "Offline server PID: psutil.Process(pid=44594, name='python3.11', status='terminated', started='14:19:03')\n", + "Offline server is running: False\n" + ] + } + ], + "source": [ + "# Verify termination\n", + "verify_servers()" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "f8a155e4-23b3-4fb3-b868-02ba2e0a4a31", + "metadata": {}, + "outputs": [], + "source": [ + "# Remove server_proc.txt (file for keeping track of pids)\n", + "os.remove(\"server_proc.txt\")" + ] + }, + { + "cell_type": "markdown", + "id": "ed7d6f25-d255-4986-9cf2-9876f6c558cc", + "metadata": {}, + "source": [ + "#### Remove Feast Applied Configuration Files" + ] + }, + { + "cell_type": "markdown", + "id": "d73efe15-a1d9-459b-8142-835dc2bf1c9f", + "metadata": {}, + "source": [ + "Remove the registry and online store (SQLite) files created on`feast apply` created in [02_Deploying_the_Feature_Store.ipynb](02_Deploying_the_Feature_Store.ipynb)." + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "0f13a4ac-d2ad-462b-b65e-4266b7cb4922", + "metadata": {}, + "outputs": [], + "source": [ + "os.remove(\"Feature_Store/data/online_store.db\")\n", + "os.remove(\"Feature_Store/data/registry.db\")" + ] + }, + { + "cell_type": "markdown", + "id": "eb0494cd-0143-4f5f-b7d6-9675e1403d9f", + "metadata": {}, + "source": [ + "#### Remove Feast Configuration Files" + ] + }, + { + "cell_type": "markdown", + "id": "86c33ac7-9e1f-4798-9f14-77773a1c13bd", + "metadata": {}, + "source": [ + "Remove the configution and feature definition files created in [02_Deploying_the_Feature_Store.ipynb](02_Deploying_the_Feature_Store.ipynb)." + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "a747043f-05fe-4b44-979d-9b30565074ee", + "metadata": {}, + "outputs": [], + "source": [ + "os.remove(\"Feature_Store/feature_store.yaml\")\n", + "os.remove(\"Feature_Store/feature_definitions.py\")" + ] + }, + { + "cell_type": "markdown", + "id": "81975a0f-7fd6-4ed3-91cf-812946df4713", + "metadata": {}, + "source": [ + "#### Remove Data Files" + ] + }, + { + "cell_type": "markdown", + "id": "8182dc1e-d5c1-4739-b7c7-0620e93c5b64", + "metadata": {}, + "source": [ + "Remove the data files created in [01_Credit_Risk_Data_Prep.ipynb](01_Credit_Risk_Data_Prep.ipynb)." + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "4ddb4fb2-fea1-4b70-8978-732af9a1cd3f", + "metadata": {}, + "outputs": [], + "source": [ + "for f in [\"data_a.parquet\", \"data_b.parquet\", \"labels.parquet\"]:\n", + " os.remove(f\"Feature_Store/data/{f}\")\n", + "os.rmdir(\"Feature_Store/data\")\n", + "os.rmdir(\"Feature_Store\")" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.9" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/examples/credit-risk-end-to-end/README.md b/examples/credit-risk-end-to-end/README.md new file mode 100644 index 00000000000..5f59c750784 --- /dev/null +++ b/examples/credit-risk-end-to-end/README.md @@ -0,0 +1,39 @@ + +![Feast_Logo](https://raw.githubusercontent.com/feast-dev/feast/master/docs/assets/feast_logo.png) + +# Feast Credit Risk Classification End-to-End Example + +This example starts with an [OpenML](https://openml.org) credit risk dataset, and walks through the steps of preparing the data, setting up feature store resources, and serving features; this is all done inside the paradigm of an ML workflow, with the goal of helping users understand how Feast fits in the progression from data preparation, to model training and model serving. + +The example is organized in five notebooks: +1. [01_Credit_Risk_Data_Prep.ipynb](01_Credit_Risk_Data_Prep.ipynb) +2. [02_Deploying_the_Feature_Store.ipynb](02_Deploying_the_Feature_Store.ipynb) +3. [03_Credit_Risk_Model_Training.ipynb](03_Credit_Risk_Model_Training.ipynb) +4. [04_Credit_Risk_Model_Serving.ipynb](04_Credit_Risk_Model_Serving.ipynb) +5. [05_Credit_Risk_Cleanup.ipynb](05_Credit_Risk_Cleanup.ipynb) + +Run the notebooks in order to progress through the example. See below for prerequisite setup steps. + +### Preparing your Environment +To run the example, install the Python dependencies. You may wish to do so inside a virtual environment. Open a command terminal, and run the following: + +``` +# create venv-example virtual environment +python -m venv venv-example +# activate environment +source venv-example/bin/activate +``` + +Install the Python dependencies: +``` +pip install -r requirements.txt +``` + +Note that this example was tested with Python 3.11, but it should also work with other similar versions. + +### Running the Notebooks +Once you have installed the Python dependencies, you can run the example notebooks. To run the notebooks locally, execute the following command in a terminal window: + +```jupyter notebook``` + +You should see a browser window open a page where you can navigate to the example notebook (.ipynb) files and open them. diff --git a/examples/credit-risk-end-to-end/requirements.txt b/examples/credit-risk-end-to-end/requirements.txt new file mode 100644 index 00000000000..8b9b1313e78 --- /dev/null +++ b/examples/credit-risk-end-to-end/requirements.txt @@ -0,0 +1,6 @@ +feast +jupyter==1.1.1 +scikit-learn==1.5.2 +pandas==2.2.3 +matplotlib==3.9.2 +seaborn==0.13.2 \ No newline at end of file diff --git a/examples/python-helm-demo/README.md b/examples/python-helm-demo/README.md index 90469e746d4..078550ae392 100644 --- a/examples/python-helm-demo/README.md +++ b/examples/python-helm-demo/README.md @@ -3,87 +3,168 @@ For this tutorial, we set up Feast with Redis. -We use the Feast CLI to register and materialize features, and then retrieving via a Feast Python feature server deployed in Kubernetes +We use the Feast CLI to register and materialize features from the current machine, and then retrieving via a +Feast Python feature server deployed in Kubernetes ## First, let's set up a Redis cluster 1. Start minikube (`minikube start`) -2. Use helm to install a default Redis cluster +1. Use helm to install a default Redis cluster ```bash helm repo add bitnami https://charts.bitnami.com/bitnami helm repo update helm install my-redis bitnami/redis ``` ![](redis-screenshot.png) -3. Port forward Redis so we can materialize features to it +1. Port forward Redis so we can materialize features to it ```bash kubectl port-forward --namespace default svc/my-redis-master 6379:6379 ``` -4. Get your Redis password using the command (pasted below for convenience). We'll need this to tell Feast how to communicate with the cluster. +1. Get your Redis password using the command (pasted below for convenience). We'll need this to tell Feast how to communicate with the cluster. ```bash export REDIS_PASSWORD=$(kubectl get secret --namespace default my-redis -o jsonpath="{.data.redis-password}" | base64 --decode) echo $REDIS_PASSWORD ``` +## Then, let's set up a MinIO S3 store +Manifests have been taken from [Deploy Minio in your project](https://ai-on-openshift.io/tools-and-applications/minio/minio/#deploy-minio-in-your-project). + +1. Deploy MinIO instance: + ``` + kubectl apply -f minio-dev.yaml + ``` + +1. Forward the UI port: + ```console + kubectl port-forward svc/minio-service 9090:9090 + ``` +1. Login to (localhost:9090)[http://localhost:9090] as `minio`/`minio123` and create bucket called `feast-demo`. +1. Stop previous port forwarding and forward the API port instead: + ```console + kubectl port-forward svc/minio-service 9000:9000 + ``` + ## Next, we setup a local Feast repo -1. Install Feast with Redis dependencies `pip install "feast[redis]"` -2. Make a bucket in GCS (or S3) -3. The feature repo is already setup here, so you just need to swap in your GCS bucket and Redis credentials. - We need to modify the `feature_store.yaml`, which has two fields for you to replace: +1. Install Feast with Redis dependencies `pip install "feast[redis,aws]"` +1. The feature repo is already setup here, so you just need to swap in your Redis credentials. + We need to modify the `feature_store.yaml`, which has one field for you to replace: + ```console + sed "s/_REDIS_PASSWORD_/${REDIS_PASSWORD}/" feature_repo/feature_store.yaml.template > feature_repo/feature_store.yaml + cat feature_repo/feature_store.yaml + ``` + + Example repo: ```yaml - registry: gs://[YOUR GCS BUCKET]/demo-repo/registry.db + registry: s3://localhost:9000/feast-demo/registry.db project: feast_python_demo - provider: gcp + provider: local online_store: type: redis - # Note: this would normally be using instance URL's to access Redis - connection_string: localhost:6379,password=[YOUR PASSWORD] + connection_string: localhost:6379,password=**** offline_store: type: file entity_key_serialization_version: 2 ``` -4. Run `feast apply` from within the `feature_repo` directory to apply your local features to the remote registry - - Note: you may need to authenticate to gcloud first with `gcloud auth login` -5. Materialize features to the online store: +1. To run `feast apply` from the current machine we need to define the AWS credentials to connect the MinIO S3 store, which +are defined in [minio.env](./minio.env): + ```console + source minio.env + cd feature_repo + feast apply + ``` +1. Let's validate the setup by running some queries + ```console + feast entities list + feast feature-views list + ``` +1. Materialize features to the online store: ```bash + cd feature_repo CURRENT_TIME=$(date -u +"%Y-%m-%dT%H:%M:%S") feast materialize-incremental $CURRENT_TIME ``` ## Now let's setup the Feast Server -1. Add the gcp-auth addon to mount GCP credentials: - ```bash - minikube addons enable gcp-auth - ``` -2. Add Feast's Python/Go feature server chart repo +1. Add Feast's Python feature server chart repo ```bash helm repo add feast-charts https://feast-helm-charts.storage.googleapis.com helm repo update ``` -3. For this tutorial, because we don't have a direct hosted endpoint into Redis, we need to change `feature_store.yaml` to talk to the Kubernetes Redis service - ```bash - sed -i '' 's/localhost:6379/my-redis-master:6379/g' feature_store.yaml - ``` -4. Install the Feast helm chart: `helm install feast-release feast-charts/feast-feature-server --set feature_store_yaml_base64=$(base64 feature_store.yaml)` - > **Dev instructions**: if you're changing the java logic or chart, you can do - 1. `eval $(minikube docker-env)` - 2. `make build-feature-server-dev` - 3. `helm install feast-release ../../../infra/charts/feast-feature-server --set image.tag=dev --set feature_store_yaml_base64=$(base64 feature_store.yaml)` -5. (Optional): check logs of the server to make sure it’s working +1. For this tutorial, we'll use a predefined configuration where we just needs to inject the Redis service password: + ```console + sed "s/_REDIS_PASSWORD_/$REDIS_PASSWORD/" online_feature_store.yaml.template > online_feature_store.yaml + cat online_feature_store.yaml + ``` + As you see, the connection points to `my-redis-master:6379` instead of `localhost:6379`. + +1. Install the Feast helm chart: + ```console + helm upgrade --install feast-online feast-charts/feast-feature-server \ + --set fullnameOverride=online-server --set feast_mode=online \ + --set feature_store_yaml_base64=$(base64 -i 'online_feature_store.yaml') + ``` +1. Patch the deployment to include MinIO settings: + ```console + kubectl patch deployment online-server --type='json' -p='[ + { + "op": "add", + "path": "/spec/template/spec/containers/0/env/-", + "value": { + "name": "AWS_ACCESS_KEY_ID", + "value": "minio" + } + }, + { + "op": "add", + "path": "/spec/template/spec/containers/0/env/-", + "value": { + "name": "AWS_SECRET_ACCESS_KEY", + "value": "minio123" + } + }, + { + "op": "add", + "path": "/spec/template/spec/containers/0/env/-", + "value": { + "name": "AWS_DEFAULT_REGION", + "value": "default" + } + }, + { + "op": "add", + "path": "/spec/template/spec/containers/0/env/-", + "value": { + "name": "FEAST_S3_ENDPOINT_URL", + "value": "http://minio-service:9000" + } + } + ]' + kubectl wait --for=condition=available deployment/online-server --timeout=2m + ``` +1. (Optional): check logs of the server to make sure it’s working ```bash - kubectl logs svc/feast-release-feast-feature-server + kubectl logs svc/online-server ``` -6. Port forward to expose the grpc endpoint: +1. Port forward to expose the grpc endpoint: ```bash - kubectl port-forward svc/feast-release-feast-feature-server 6566:80 + kubectl port-forward svc/online-server 6566:80 ``` -7. Run test fetches for online features:8. - - First: change back the Redis connection string to allow localhost connections to Redis +1. Run test fetches for online features:8. ```bash - sed -i '' 's/my-redis-master:6379/localhost:6379/g' feature_store.yaml + source minio.env + cd test + python test_python_fetch.py ``` - - Then run the included fetch script, which fetches both via the HTTP endpoint and for comparison, via the Python SDK - ```bash - python test_python_fetch.py + + Output example: + ```console + --- Online features with SDK --- + WARNING:root:_list_feature_views will make breaking changes. Please use _list_batch_feature_views instead. _list_feature_views will behave like _list_all_feature_views in the future. + conv_rate : [0.6799587607383728, 0.9761165976524353] + driver_id : [1001, 1002] + + --- Online features with HTTP endpoint --- + conv_rate : [0.67995876 0.9761166 ] + driver_id : [1001 1002] ``` \ No newline at end of file diff --git a/examples/python-helm-demo/feature_repo/data/driver_stats_with_string.parquet b/examples/python-helm-demo/feature_repo/data/driver_stats_with_string.parquet index 83b8c31aa51..ae8f17e45d3 100644 Binary files a/examples/python-helm-demo/feature_repo/data/driver_stats_with_string.parquet and b/examples/python-helm-demo/feature_repo/data/driver_stats_with_string.parquet differ diff --git a/examples/python-helm-demo/feature_repo/feature_store.yaml b/examples/python-helm-demo/feature_repo/feature_store.yaml deleted file mode 100644 index d49c0cbd0eb..00000000000 --- a/examples/python-helm-demo/feature_repo/feature_store.yaml +++ /dev/null @@ -1,10 +0,0 @@ -registry: gs://[YOUR GCS BUCKET]/demo-repo/registry.db -project: feast_python_demo -provider: gcp -online_store: - type: redis - # Note: this would normally be using instance URL's to access Redis - connection_string: localhost:6379,password=[YOUR PASSWORD] -offline_store: - type: file -entity_key_serialization_version: 2 \ No newline at end of file diff --git a/examples/python-helm-demo/feature_repo/feature_store.yaml.template b/examples/python-helm-demo/feature_repo/feature_store.yaml.template new file mode 100644 index 00000000000..585ba23e63b --- /dev/null +++ b/examples/python-helm-demo/feature_repo/feature_store.yaml.template @@ -0,0 +1,9 @@ +registry: s3://feast-demo/registry.db +project: feast_python_demo +provider: local +online_store: + type: redis + connection_string: localhost:6379,password=_REDIS_PASSWORD_ +offline_store: + type: file +entity_key_serialization_version: 2 \ No newline at end of file diff --git a/examples/python-helm-demo/minio-dev.yaml b/examples/python-helm-demo/minio-dev.yaml new file mode 100644 index 00000000000..9285cbca983 --- /dev/null +++ b/examples/python-helm-demo/minio-dev.yaml @@ -0,0 +1,128 @@ +--- +kind: PersistentVolumeClaim +apiVersion: v1 +metadata: + name: minio-pvc +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 20Gi + volumeMode: Filesystem +--- +kind: Secret +apiVersion: v1 +metadata: + name: minio-secret +stringData: + # change the username and password to your own values. + # ensure that the user is at least 3 characters long and the password at least 8 + minio_root_user: minio + minio_root_password: minio123 +--- +kind: Deployment +apiVersion: apps/v1 +metadata: + name: minio +spec: + replicas: 1 + selector: + matchLabels: + app: minio + template: + metadata: + labels: + app: minio + spec: + volumes: + - name: data + persistentVolumeClaim: + claimName: minio-pvc + containers: + - resources: + limits: + cpu: 250m + memory: 1Gi + requests: + cpu: 20m + memory: 100Mi + readinessProbe: + tcpSocket: + port: 9000 + initialDelaySeconds: 5 + timeoutSeconds: 1 + periodSeconds: 5 + successThreshold: 1 + failureThreshold: 3 + terminationMessagePath: /dev/termination-log + name: minio + livenessProbe: + tcpSocket: + port: 9000 + initialDelaySeconds: 30 + timeoutSeconds: 1 + periodSeconds: 5 + successThreshold: 1 + failureThreshold: 3 + env: + - name: MINIO_ROOT_USER + valueFrom: + secretKeyRef: + name: minio-secret + key: minio_root_user + - name: MINIO_ROOT_PASSWORD + valueFrom: + secretKeyRef: + name: minio-secret + key: minio_root_password + ports: + - containerPort: 9000 + protocol: TCP + - containerPort: 9090 + protocol: TCP + imagePullPolicy: IfNotPresent + volumeMounts: + - name: data + mountPath: /data + subPath: minio + terminationMessagePolicy: File + image: >- + quay.io/minio/minio:RELEASE.2023-06-19T19-52-50Z + args: + - server + - /data + - --console-address + - :9090 + restartPolicy: Always + terminationGracePeriodSeconds: 30 + dnsPolicy: ClusterFirst + securityContext: {} + schedulerName: default-scheduler + strategy: + type: Recreate + revisionHistoryLimit: 10 + progressDeadlineSeconds: 600 +--- +kind: Service +apiVersion: v1 +metadata: + name: minio-service +spec: + ipFamilies: + - IPv4 + ports: + - name: api + protocol: TCP + port: 9000 + targetPort: 9000 + - name: ui + protocol: TCP + port: 9090 + targetPort: 9090 + internalTrafficPolicy: Cluster + type: ClusterIP + ipFamilyPolicy: SingleStack + sessionAffinity: None + selector: + app: minio \ No newline at end of file diff --git a/examples/python-helm-demo/minio.env b/examples/python-helm-demo/minio.env new file mode 100644 index 00000000000..b19ec5083f5 --- /dev/null +++ b/examples/python-helm-demo/minio.env @@ -0,0 +1,7 @@ +export AWS_ACCESS_KEY_ID=minio +export AWS_DEFAULT_REGION=default +#export AWS_S3_BUCKET=feast-demo +#export AWS_S3_ENDPOINT=http://localhost:9000 +export FEAST_S3_ENDPOINT_URL=http://localhost:9000 +export AWS_SECRET_ACCESS_KEY=minio123 + diff --git a/examples/python-helm-demo/online_feature_store.yaml.template b/examples/python-helm-demo/online_feature_store.yaml.template new file mode 100644 index 00000000000..7acb9582c51 --- /dev/null +++ b/examples/python-helm-demo/online_feature_store.yaml.template @@ -0,0 +1,7 @@ +project: feast_python_demo +provider: local +registry: s3://feast-demo/registry.db +online_store: + type: redis + connection_string: my-redis-master:6379,password=_REDIS_PASSWORD_ +entity_key_serialization_version: 2 \ No newline at end of file diff --git a/examples/python-helm-demo/test/feature_store.yaml b/examples/python-helm-demo/test/feature_store.yaml new file mode 100644 index 00000000000..13e99873ee7 --- /dev/null +++ b/examples/python-helm-demo/test/feature_store.yaml @@ -0,0 +1,7 @@ +registry: s3://feast-demo/registry.db +project: feast_python_demo +provider: local +online_store: + path: http://localhost:6566 + type: remote +entity_key_serialization_version: 2 \ No newline at end of file diff --git a/examples/python-helm-demo/feature_repo/test_python_fetch.py b/examples/python-helm-demo/test/test_python_fetch.py similarity index 73% rename from examples/python-helm-demo/feature_repo/test_python_fetch.py rename to examples/python-helm-demo/test/test_python_fetch.py index f9c7c62f4fd..715912422f3 100644 --- a/examples/python-helm-demo/feature_repo/test_python_fetch.py +++ b/examples/python-helm-demo/test/test_python_fetch.py @@ -1,6 +1,7 @@ from feast import FeatureStore import requests import json +import pandas as pd def run_demo_http(): @@ -14,7 +15,14 @@ def run_demo_http(): r = requests.post( "http://localhost:6566/get-online-features", data=json.dumps(online_request) ) - print(json.dumps(r.json(), indent=4, sort_keys=True)) + + resp_data = json.loads(r.text) + records = pd.DataFrame.from_records( + columns=resp_data["metadata"]["feature_names"], + data=[[r["values"][i] for r in resp_data["results"]] for i in range(len(resp_data["results"]))] + ) + for col in sorted(records.columns): + print(col, " : ", records[col].values) def run_demo_sdk(): diff --git a/go/internal/feast/errors.go b/go/internal/feast/errors.go new file mode 100644 index 00000000000..f42b4aad82d --- /dev/null +++ b/go/internal/feast/errors.go @@ -0,0 +1,22 @@ +package feast + +import ( + "google.golang.org/genproto/googleapis/rpc/errdetails" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +type FeastTransformationServiceNotConfigured struct{} + +func (FeastTransformationServiceNotConfigured) GRPCStatus() *status.Status { + errorStatus := status.New(codes.Internal, "No transformation service configured") + ds, err := errorStatus.WithDetails(&errdetails.LocalizedMessage{Message: "No transformation service configured, required for on-demand feature transformations"}) + if err != nil { + return errorStatus + } + return ds +} + +func (e FeastTransformationServiceNotConfigured) Error() string { + return e.GRPCStatus().Err().Error() +} diff --git a/go/internal/feast/featurestore.go b/go/internal/feast/featurestore.go index df4df7e1995..abe1d195def 100644 --- a/go/internal/feast/featurestore.go +++ b/go/internal/feast/featurestore.go @@ -3,8 +3,9 @@ package feast import ( "context" "errors" - "fmt" + "github.com/apache/arrow/go/v17/arrow/memory" + //"gopkg.in/DataDog/dd-trace-go.v1/ddtrace/tracer" "github.com/feast-dev/feast/go/internal/feast/model" @@ -60,17 +61,14 @@ func NewFeatureStore(config *registry.RepoConfig, callback transformation.Transf return nil, err } - // Use a scalable transformation service like Python Transformation Service. - // Assume the user will define the "transformation_service_endpoint" in the feature_store.yaml file - // under the "feature_server" section. - transformationServerEndpoint, ok := config.FeatureServer["transformation_service_endpoint"] - if !ok { - fmt.Println("Errors while reading transformation_service_endpoint info") - panic("No transformation service endpoint provided in the feature_store.yaml file.") + var transformationService *transformation.GrpcTransformationService + if transformationServerEndpoint, ok := config.FeatureServer["transformation_service_endpoint"]; ok { + // Use a scalable transformation service like Python Transformation Service. + // Assume the user will define the "transformation_service_endpoint" in the feature_store.yaml file + // under the "feature_server" section. + transformationService, _ = transformation.NewGrpcTransformationService(config, transformationServerEndpoint.(string)) } - transformationService, _ := transformation.NewGrpcTransformationService(config, transformationServerEndpoint.(string)) - return &FeatureStore{ config: config, registry: registry, @@ -112,6 +110,10 @@ func (fs *FeatureStore) GetOnlineFeatures( return nil, err } + if len(requestedOnDemandFeatureViews) > 0 && fs.transformationService == nil { + return nil, FeastTransformationServiceNotConfigured{} + } + entityNameToJoinKeyMap, expectedJoinKeysSet, err := onlineserving.GetEntityMaps(requestedFeatureViews, entities) if err != nil { return nil, err diff --git a/go/internal/feast/featurestore_test.go b/go/internal/feast/featurestore_test.go index f066b39df2d..e1f908b9062 100644 --- a/go/internal/feast/featurestore_test.go +++ b/go/internal/feast/featurestore_test.go @@ -2,124 +2,241 @@ package feast import ( "context" + "log" + "os" "path/filepath" "runtime" "testing" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" "github.com/feast-dev/feast/go/internal/feast/onlinestore" "github.com/feast-dev/feast/go/internal/feast/registry" + "github.com/feast-dev/feast/go/internal/test" + "github.com/feast-dev/feast/go/protos/feast/serving" "github.com/feast-dev/feast/go/protos/feast/types" ) -// Return absolute path to the test_repo registry regardless of the working directory -func getRegistryPath() map[string]interface{} { +var featureRepoBasePath string +var featureRepoRegistryFile string + +func TestMain(m *testing.M) { // Get the file path of this source file, regardless of the working directory _, filename, _, ok := runtime.Caller(0) if !ok { - panic("couldn't find file path of the test file") + log.Print("couldn't find file path of the test file") + os.Exit(1) } - registry := map[string]interface{}{ - "path": filepath.Join(filename, "..", "..", "..", "feature_repo/data/registry.db"), + featureRepoBasePath = filepath.Join(filename, "..", "..", "test") + featureRepoRegistryFile = filepath.Join(featureRepoBasePath, "feature_repo", "data", "registry.db") + if err := test.SetupInitializedRepo(featureRepoBasePath); err != nil { + log.Print("Could not initialize test repo: ", err) + os.Exit(1) } - return registry + os.Exit(m.Run()) } func TestNewFeatureStore(t *testing.T) { - t.Skip("@todo(achals): feature_repo isn't checked in yet") - config := registry.RepoConfig{ - Project: "feature_repo", - Registry: getRegistryPath(), - Provider: "local", - OnlineStore: map[string]interface{}{ - "type": "redis", - }, - } - fs, err := NewFeatureStore(&config, nil) - assert.Nil(t, err) - assert.IsType(t, &onlinestore.RedisOnlineStore{}, fs.onlineStore) - - t.Run("valid config", func(t *testing.T) { - config := ®istry.RepoConfig{ - Project: "feature_repo", - Registry: getRegistryPath(), - Provider: "local", - OnlineStore: map[string]interface{}{ - "type": "redis", + tests := []struct { + name string + config *registry.RepoConfig + expectOnlineStoreType interface{} + errMessage string + }{ + { + name: "valid config", + config: ®istry.RepoConfig{ + Project: "feature_repo", + Registry: map[string]interface{}{ + "path": featureRepoRegistryFile, + }, + Provider: "local", + OnlineStore: map[string]interface{}{ + "type": "redis", + }, }, - FeatureServer: map[string]interface{}{ - "transformation_service_endpoint": "localhost:50051", + expectOnlineStoreType: &onlinestore.RedisOnlineStore{}, + }, + { + name: "valid config with transformation service endpoint", + config: ®istry.RepoConfig{ + Project: "feature_repo", + Registry: map[string]interface{}{ + "path": featureRepoRegistryFile, + }, + Provider: "local", + OnlineStore: map[string]interface{}{ + "type": "redis", + }, + FeatureServer: map[string]interface{}{ + "transformation_service_endpoint": "localhost:50051", + }, }, - } - fs, err := NewFeatureStore(config, nil) - assert.Nil(t, err) - assert.NotNil(t, fs) - assert.IsType(t, &onlinestore.RedisOnlineStore{}, fs.onlineStore) - assert.NotNil(t, fs.transformationService) - }) - - t.Run("missing transformation service endpoint", func(t *testing.T) { - config := ®istry.RepoConfig{ - Project: "feature_repo", - Registry: getRegistryPath(), - Provider: "local", - OnlineStore: map[string]interface{}{ - "type": "redis", + expectOnlineStoreType: &onlinestore.RedisOnlineStore{}, + }, + { + name: "invalid online store config", + config: ®istry.RepoConfig{ + Project: "feature_repo", + Registry: map[string]interface{}{ + "path": featureRepoRegistryFile, + }, + Provider: "local", + OnlineStore: map[string]interface{}{ + "type": "invalid_store", + }, }, - } - defer func() { - if r := recover(); r == nil { - t.Errorf("The code did not panic") + errMessage: "invalid_store online store type is currently not supported", + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + got, err := NewFeatureStore(test.config, nil) + if test.errMessage != "" { + assert.Nil(t, got) + require.Error(t, err) + assert.ErrorContains(t, err, test.errMessage) + + } else { + require.NoError(t, err) + assert.NotNil(t, got) + assert.IsType(t, test.expectOnlineStoreType, got.onlineStore) } - }() - NewFeatureStore(config, nil) - }) - - t.Run("invalid online store config", func(t *testing.T) { - config := ®istry.RepoConfig{ - Project: "feature_repo", - Registry: getRegistryPath(), - Provider: "local", - OnlineStore: map[string]interface{}{ - "type": "invalid_store", + }) + } + +} + +type MockRedis struct { + mock.Mock +} + +func (m *MockRedis) Destruct() {} +func (m *MockRedis) OnlineRead(ctx context.Context, entityKeys []*types.EntityKey, featureViewNames []string, featureNames []string) ([][]onlinestore.FeatureData, error) { + args := m.Called(ctx, entityKeys, featureViewNames, featureNames) + var fd [][]onlinestore.FeatureData + if args.Get(0) != nil { + fd = args.Get(0).([][]onlinestore.FeatureData) + } + return fd, args.Error(1) +} + +func TestGetOnlineFeatures(t *testing.T) { + tests := []struct { + name string + config *registry.RepoConfig + fn func(*testing.T, *FeatureStore) + }{ + { + name: "redis with simple features", + config: ®istry.RepoConfig{ + Project: "feature_repo", + Registry: map[string]interface{}{ + "path": featureRepoRegistryFile, + }, + Provider: "local", + OnlineStore: map[string]interface{}{ + "type": "redis", + "connection_string": "localhost:6379", + }, }, - FeatureServer: map[string]interface{}{ - "transformation_service_endpoint": "localhost:50051", + fn: testRedisSimpleFeatures, + }, + { + name: "redis with On-demand feature views, no transformation service endpoint", + config: ®istry.RepoConfig{ + Project: "feature_repo", + Registry: map[string]interface{}{ + "path": featureRepoRegistryFile, + }, + Provider: "local", + OnlineStore: map[string]interface{}{ + "type": "redis", + "connection_string": "localhost:6379", + }, }, - } - fs, err := NewFeatureStore(config, nil) - assert.NotNil(t, err) - assert.Nil(t, fs) - }) + fn: testRedisODFVNoTransformationService, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + + fs, err := NewFeatureStore(test.config, nil) + require.Nil(t, err) + fs.onlineStore = new(MockRedis) + test.fn(t, fs) + }) + + } } -func TestGetOnlineFeaturesRedis(t *testing.T) { - t.Skip("@todo(achals): feature_repo isn't checked in yet") - config := registry.RepoConfig{ - Project: "feature_repo", - Registry: getRegistryPath(), - Provider: "local", - OnlineStore: map[string]interface{}{ - "type": "redis", - "connection_string": "localhost:6379", +func testRedisSimpleFeatures(t *testing.T, fs *FeatureStore) { + + featureNames := []string{"driver_hourly_stats:conv_rate", + "driver_hourly_stats:acc_rate", + "driver_hourly_stats:avg_daily_trips", + } + entities := map[string]*types.RepeatedValue{"driver_id": {Val: []*types.Value{{Val: &types.Value_Int64Val{Int64Val: 1001}}, + {Val: &types.Value_Int64Val{Int64Val: 1002}}, + }}} + + results := [][]onlinestore.FeatureData{ + { + { + Reference: serving.FeatureReferenceV2{FeatureViewName: "driver_hourly_stats", FeatureName: "conv_rate"}, + Value: types.Value{Val: &types.Value_FloatVal{FloatVal: 12.0}}, + }, + { + Reference: serving.FeatureReferenceV2{FeatureViewName: "driver_hourly_stats", FeatureName: "acc_rate"}, + Value: types.Value{Val: &types.Value_FloatVal{FloatVal: 1.0}}, + }, + { + Reference: serving.FeatureReferenceV2{FeatureViewName: "driver_hourly_stats", FeatureName: "avg_daily_trips"}, + Value: types.Value{Val: &types.Value_Int64Val{Int64Val: 100}}, + }, + }, + { + + { + Reference: serving.FeatureReferenceV2{FeatureViewName: "driver_hourly_stats", FeatureName: "conv_rate"}, + Value: types.Value{Val: &types.Value_FloatVal{FloatVal: 24.0}}, + }, + { + Reference: serving.FeatureReferenceV2{FeatureViewName: "driver_hourly_stats", FeatureName: "acc_rate"}, + Value: types.Value{Val: &types.Value_FloatVal{FloatVal: 2.0}}, + }, + { + Reference: serving.FeatureReferenceV2{FeatureViewName: "driver_hourly_stats", FeatureName: "avg_daily_trips"}, + Value: types.Value{Val: &types.Value_Int64Val{Int64Val: 130}}, + }, }, } + ctx := context.Background() + mr := fs.onlineStore.(*MockRedis) + mr.On("OnlineRead", ctx, mock.Anything, mock.Anything, mock.Anything).Return(results, nil) + response, err := fs.GetOnlineFeatures(ctx, featureNames, nil, entities, map[string]*types.RepeatedValue{}, true) + require.Nil(t, err) + assert.Len(t, response, 4) // 3 Features + 1 entity = 4 columns (feature vectors) in response +} +func testRedisODFVNoTransformationService(t *testing.T, fs *FeatureStore) { featureNames := []string{"driver_hourly_stats:conv_rate", "driver_hourly_stats:acc_rate", "driver_hourly_stats:avg_daily_trips", + "transformed_conv_rate:conv_rate_plus_val1", } entities := map[string]*types.RepeatedValue{"driver_id": {Val: []*types.Value{{Val: &types.Value_Int64Val{Int64Val: 1001}}, {Val: &types.Value_Int64Val{Int64Val: 1002}}, {Val: &types.Value_Int64Val{Int64Val: 1003}}}}, } - fs, err := NewFeatureStore(&config, nil) - assert.Nil(t, err) ctx := context.Background() - response, err := fs.GetOnlineFeatures( - ctx, featureNames, nil, entities, map[string]*types.RepeatedValue{}, true) - assert.Nil(t, err) - assert.Len(t, response, 4) // 3 Features + 1 entity = 4 columns (feature vectors) in response + mr := fs.onlineStore.(*MockRedis) + mr.On("OnlineRead", ctx, mock.Anything, mock.Anything, mock.Anything).Return(nil, nil) + response, err := fs.GetOnlineFeatures(ctx, featureNames, nil, entities, map[string]*types.RepeatedValue{}, true) + assert.Nil(t, response) + assert.ErrorAs(t, err, &FeastTransformationServiceNotConfigured{}) + } diff --git a/go/internal/test/feature_repo/example.py b/go/internal/test/feature_repo/example.py index 70843610075..a814b58913b 100644 --- a/go/internal/test/feature_repo/example.py +++ b/go/internal/test/feature_repo/example.py @@ -2,10 +2,12 @@ from datetime import timedelta -from feast import Entity, Feature, FeatureView, Field, FileSource, FeatureService +from feast import Entity, Feature, FeatureView, Field, FileSource, FeatureService, RequestSource from feast.feature_logging import LoggingConfig from feast.infra.offline_stores.file_source import FileLoggingDestination -from feast.types import Float32, Int64 +from feast.types import Float32, Float64, Int64, PrimitiveFeastType +from feast.on_demand_feature_view import on_demand_feature_view +import pandas as pd # Read data from parquet files. Parquet is convenient for local development mode. For # production, you can use your favorite DWH, such as BigQuery. See Feast documentation @@ -41,4 +43,32 @@ name="test_service", features=[driver_hourly_stats_view], logging_config=LoggingConfig(destination=FileLoggingDestination(path="")) -) \ No newline at end of file +) + + +# Define a request data source which encodes features / information only +# available at request time (e.g. part of the user initiated HTTP request) +input_request = RequestSource( + name="vals_to_add", + schema=[ + Field(name="val_to_add", dtype=PrimitiveFeastType.INT64), + Field(name="val_to_add_2", dtype=PrimitiveFeastType.INT64), + ] +) + +# Use the input data and feature view features to create new features +@on_demand_feature_view( + sources=[ + driver_hourly_stats_view, + input_request + ], + schema=[ + Field(name='conv_rate_plus_val1', dtype=Float64), + Field(name='conv_rate_plus_val2', dtype=Float64) + ] +) +def transformed_conv_rate(features_df: pd.DataFrame) -> pd.DataFrame: + df = pd.DataFrame() + df['conv_rate_plus_val1'] = (features_df['conv_rate'] + features_df['val_to_add']) + df['conv_rate_plus_val2'] = (features_df['conv_rate'] + features_df['val_to_add_2']) + return df diff --git a/infra/charts/feast-feature-server/Chart.yaml b/infra/charts/feast-feature-server/Chart.yaml index 15cb6141236..7b8e51b3b52 100644 --- a/infra/charts/feast-feature-server/Chart.yaml +++ b/infra/charts/feast-feature-server/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: feast-feature-server description: Feast Feature Server in Go or Python type: application -version: 0.42.0 +version: 0.43.0 keywords: - machine learning - big data diff --git a/infra/charts/feast-feature-server/README.md b/infra/charts/feast-feature-server/README.md index a1578196b91..f8554cb4dff 100644 --- a/infra/charts/feast-feature-server/README.md +++ b/infra/charts/feast-feature-server/README.md @@ -1,6 +1,6 @@ # Feast Python / Go Feature Server Helm Charts -Current chart version is `0.42.0` +Current chart version is `0.43.0` ## Installation @@ -40,7 +40,7 @@ See [here](https://github.com/feast-dev/feast/tree/master/examples/python-helm-d | fullnameOverride | string | `""` | | | image.pullPolicy | string | `"IfNotPresent"` | | | image.repository | string | `"feastdev/feature-server"` | Docker image for Feature Server repository | -| image.tag | string | `"0.42.0"` | The Docker image tag (can be overwritten if custom feature server deps are needed for on demand transforms) | +| image.tag | string | `"0.43.0"` | The Docker image tag (can be overwritten if custom feature server deps are needed for on demand transforms) | | imagePullSecrets | list | `[]` | | | livenessProbe.initialDelaySeconds | int | `30` | | | livenessProbe.periodSeconds | int | `30` | | @@ -56,6 +56,7 @@ See [here](https://github.com/feast-dev/feast/tree/master/examples/python-helm-d | readinessProbe.periodSeconds | int | `10` | | | replicaCount | int | `1` | | | resources | object | `{}` | | +| route.enabled | bool | `false` | | | securityContext | object | `{}` | | | service.port | int | `80` | | | service.type | string | `"ClusterIP"` | | diff --git a/infra/charts/feast-feature-server/templates/route.yaml b/infra/charts/feast-feature-server/templates/route.yaml new file mode 100644 index 00000000000..2f4d36d9e5a --- /dev/null +++ b/infra/charts/feast-feature-server/templates/route.yaml @@ -0,0 +1,18 @@ +{{- if and (.Values.route.enabled) (eq .Values.feast_mode "ui") }} +--- +kind: Route +apiVersion: route.openshift.io/v1 +metadata: + name: {{ include "feast-feature-server.fullname" . }} + labels: + {{- include "feast-feature-server.labels" . | nindent 4 }} +spec: + to: + kind: Service + name: {{ include "feast-feature-server.fullname" . }} + port: + targetPort: http + tls: + termination: edge + insecureEdgeTerminationPolicy: Redirect +{{- end}} \ No newline at end of file diff --git a/infra/charts/feast-feature-server/values.yaml b/infra/charts/feast-feature-server/values.yaml index ed54d328d10..3bbf9ec2b20 100644 --- a/infra/charts/feast-feature-server/values.yaml +++ b/infra/charts/feast-feature-server/values.yaml @@ -9,7 +9,7 @@ image: repository: feastdev/feature-server pullPolicy: IfNotPresent # image.tag -- The Docker image tag (can be overwritten if custom feature server deps are needed for on demand transforms) - tag: 0.42.0 + tag: 0.43.0 logLevel: "WARNING" # Set log level DEBUG, INFO, WARNING, ERROR, and CRITICAL (case-insensitive) @@ -74,3 +74,7 @@ livenessProbe: readinessProbe: initialDelaySeconds: 20 periodSeconds: 10 + +# to create OpenShift Route object for UI +route: + enabled: false \ No newline at end of file diff --git a/infra/charts/feast/Chart.yaml b/infra/charts/feast/Chart.yaml index c3dddef6f95..58c4ce34fbb 100644 --- a/infra/charts/feast/Chart.yaml +++ b/infra/charts/feast/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v1 description: Feature store for machine learning name: feast -version: 0.42.0 +version: 0.43.0 keywords: - machine learning - big data diff --git a/infra/charts/feast/README.md b/infra/charts/feast/README.md index e2b92ec44c0..4170541543d 100644 --- a/infra/charts/feast/README.md +++ b/infra/charts/feast/README.md @@ -8,7 +8,7 @@ This repo contains Helm charts for Feast Java components that are being installe ## Chart: Feast -Feature store for machine learning Current chart version is `0.42.0` +Feature store for machine learning Current chart version is `0.43.0` ## Installation @@ -65,8 +65,8 @@ See [here](https://github.com/feast-dev/feast/tree/master/examples/java-demo) fo | Repository | Name | Version | |------------|------|---------| | https://charts.helm.sh/stable | redis | 10.5.6 | -| https://feast-helm-charts.storage.googleapis.com | feature-server(feature-server) | 0.42.0 | -| https://feast-helm-charts.storage.googleapis.com | transformation-service(transformation-service) | 0.42.0 | +| https://feast-helm-charts.storage.googleapis.com | feature-server(feature-server) | 0.43.0 | +| https://feast-helm-charts.storage.googleapis.com | transformation-service(transformation-service) | 0.43.0 | ## Values diff --git a/infra/charts/feast/charts/feature-server/Chart.yaml b/infra/charts/feast/charts/feature-server/Chart.yaml index ef4282edaee..f1468973eb2 100644 --- a/infra/charts/feast/charts/feature-server/Chart.yaml +++ b/infra/charts/feast/charts/feature-server/Chart.yaml @@ -1,8 +1,8 @@ apiVersion: v1 description: "Feast Feature Server: Online feature serving service for Feast" name: feature-server -version: 0.42.0 -appVersion: v0.42.0 +version: 0.43.0 +appVersion: v0.43.0 keywords: - machine learning - big data diff --git a/infra/charts/feast/charts/feature-server/README.md b/infra/charts/feast/charts/feature-server/README.md index 3c447b4aa63..990a1154884 100644 --- a/infra/charts/feast/charts/feature-server/README.md +++ b/infra/charts/feast/charts/feature-server/README.md @@ -1,6 +1,6 @@ # feature-server -![Version: 0.42.0](https://img.shields.io/badge/Version-0.42.0-informational?style=flat-square) ![AppVersion: v0.42.0](https://img.shields.io/badge/AppVersion-v0.42.0-informational?style=flat-square) +![Version: 0.43.0](https://img.shields.io/badge/Version-0.43.0-informational?style=flat-square) ![AppVersion: v0.43.0](https://img.shields.io/badge/AppVersion-v0.43.0-informational?style=flat-square) Feast Feature Server: Online feature serving service for Feast @@ -17,7 +17,7 @@ Feast Feature Server: Online feature serving service for Feast | envOverrides | object | `{}` | Extra environment variables to set | | image.pullPolicy | string | `"IfNotPresent"` | Image pull policy | | image.repository | string | `"feastdev/feature-server-java"` | Docker image for Feature Server repository | -| image.tag | string | `"0.42.0"` | Image tag | +| image.tag | string | `"0.43.0"` | Image tag | | ingress.grpc.annotations | object | `{}` | Extra annotations for the ingress | | ingress.grpc.auth.enabled | bool | `false` | Flag to enable auth | | ingress.grpc.class | string | `"nginx"` | Which ingress controller to use | diff --git a/infra/charts/feast/charts/feature-server/values.yaml b/infra/charts/feast/charts/feature-server/values.yaml index e53d0293bcb..b7098ef5ac4 100644 --- a/infra/charts/feast/charts/feature-server/values.yaml +++ b/infra/charts/feast/charts/feature-server/values.yaml @@ -5,7 +5,7 @@ image: # image.repository -- Docker image for Feature Server repository repository: feastdev/feature-server-java # image.tag -- Image tag - tag: 0.42.0 + tag: 0.43.0 # image.pullPolicy -- Image pull policy pullPolicy: IfNotPresent diff --git a/infra/charts/feast/charts/transformation-service/Chart.yaml b/infra/charts/feast/charts/transformation-service/Chart.yaml index 245ad022180..1c19dd8b7d6 100644 --- a/infra/charts/feast/charts/transformation-service/Chart.yaml +++ b/infra/charts/feast/charts/transformation-service/Chart.yaml @@ -1,8 +1,8 @@ apiVersion: v1 description: "Transformation service: to compute on-demand features" name: transformation-service -version: 0.42.0 -appVersion: v0.42.0 +version: 0.43.0 +appVersion: v0.43.0 keywords: - machine learning - big data diff --git a/infra/charts/feast/charts/transformation-service/README.md b/infra/charts/feast/charts/transformation-service/README.md index a69d2d5911f..27bd8afc609 100644 --- a/infra/charts/feast/charts/transformation-service/README.md +++ b/infra/charts/feast/charts/transformation-service/README.md @@ -1,6 +1,6 @@ # transformation-service -![Version: 0.42.0](https://img.shields.io/badge/Version-0.42.0-informational?style=flat-square) ![AppVersion: v0.42.0](https://img.shields.io/badge/AppVersion-v0.42.0-informational?style=flat-square) +![Version: 0.43.0](https://img.shields.io/badge/Version-0.43.0-informational?style=flat-square) ![AppVersion: v0.43.0](https://img.shields.io/badge/AppVersion-v0.43.0-informational?style=flat-square) Transformation service: to compute on-demand features @@ -13,7 +13,7 @@ Transformation service: to compute on-demand features | envOverrides | object | `{}` | Extra environment variables to set | | image.pullPolicy | string | `"IfNotPresent"` | Image pull policy | | image.repository | string | `"feastdev/feature-transformation-server"` | Docker image for Transformation Server repository | -| image.tag | string | `"0.42.0"` | Image tag | +| image.tag | string | `"0.43.0"` | Image tag | | nodeSelector | object | `{}` | Node labels for pod assignment | | podLabels | object | `{}` | Labels to be added to Feast Serving pods | | replicaCount | int | `1` | Number of pods that will be created | diff --git a/infra/charts/feast/charts/transformation-service/values.yaml b/infra/charts/feast/charts/transformation-service/values.yaml index d765c3d9130..bb2cd8e4ea5 100644 --- a/infra/charts/feast/charts/transformation-service/values.yaml +++ b/infra/charts/feast/charts/transformation-service/values.yaml @@ -5,7 +5,7 @@ image: # image.repository -- Docker image for Transformation Server repository repository: feastdev/feature-transformation-server # image.tag -- Image tag - tag: 0.42.0 + tag: 0.43.0 # image.pullPolicy -- Image pull policy pullPolicy: IfNotPresent diff --git a/infra/charts/feast/requirements.yaml b/infra/charts/feast/requirements.yaml index 41d6dc25c2b..82c6351eaf5 100644 --- a/infra/charts/feast/requirements.yaml +++ b/infra/charts/feast/requirements.yaml @@ -1,12 +1,12 @@ dependencies: - name: feature-server alias: feature-server - version: 0.42.0 + version: 0.43.0 condition: feature-server.enabled repository: https://feast-helm-charts.storage.googleapis.com - name: transformation-service alias: transformation-service - version: 0.42.0 + version: 0.43.0 condition: transformation-service.enabled repository: https://feast-helm-charts.storage.googleapis.com - name: redis diff --git a/infra/feast-helm-operator/Makefile b/infra/feast-helm-operator/Makefile index 4b1a9ec56a4..e3edf0622f5 100644 --- a/infra/feast-helm-operator/Makefile +++ b/infra/feast-helm-operator/Makefile @@ -3,7 +3,7 @@ # To re-generate a bundle for another specific version without changing the standard setup, you can: # - use the VERSION as arg of the bundle target (e.g make bundle VERSION=0.0.2) # - use environment variables to overwrite this value (e.g export VERSION=0.0.2) -VERSION ?= 0.42.0 +VERSION ?= 0.43.0 # CHANNELS define the bundle channels used in the bundle. # Add a new line here if you would like to change its default config. (E.g CHANNELS = "candidate,fast,stable") diff --git a/infra/feast-helm-operator/config/manager/kustomization.yaml b/infra/feast-helm-operator/config/manager/kustomization.yaml index ec9247f695d..e4c3f23a824 100644 --- a/infra/feast-helm-operator/config/manager/kustomization.yaml +++ b/infra/feast-helm-operator/config/manager/kustomization.yaml @@ -5,4 +5,4 @@ kind: Kustomization images: - name: controller newName: feastdev/feast-helm-operator - newTag: 0.42.0 + newTag: 0.43.0 diff --git a/infra/feast-operator/.golangci.yml b/infra/feast-operator/.golangci.yml index ca69a11f6fd..bf54dc46026 100644 --- a/infra/feast-operator/.golangci.yml +++ b/infra/feast-operator/.golangci.yml @@ -21,7 +21,6 @@ linters: enable: - dupl - errcheck - - exportloopref - goconst - gocyclo - gofmt @@ -32,9 +31,16 @@ linters: - lll - misspell - nakedret + - ginkgolinter - prealloc + - revive - staticcheck - typecheck - unconvert - unparam - unused + +linters-settings: + revive: + rules: + - name: comment-spacings diff --git a/infra/feast-operator/Dockerfile b/infra/feast-operator/Dockerfile index aca26f92295..9fd79d7285a 100644 --- a/infra/feast-operator/Dockerfile +++ b/infra/feast-operator/Dockerfile @@ -1,9 +1,8 @@ # Build the manager binary -FROM golang:1.21 AS builder +FROM registry.access.redhat.com/ubi9/go-toolset:1.22.9 AS builder ARG TARGETOS ARG TARGETARCH -WORKDIR /workspace # Copy the Go Modules manifests COPY go.mod go.mod COPY go.sum go.sum @@ -23,11 +22,9 @@ COPY internal/controller/ internal/controller/ # by leaving it empty we can ensure that the container and binary shipped on it will have the same platform. RUN CGO_ENABLED=0 GOOS=${TARGETOS:-linux} GOARCH=${TARGETARCH} go build -a -o manager cmd/main.go -# Use distroless as minimal base image to package the manager binary -# Refer to https://github.com/GoogleContainerTools/distroless for more details -FROM gcr.io/distroless/static:nonroot +FROM registry.access.redhat.com/ubi9/ubi-micro:9.5 WORKDIR / -COPY --from=builder /workspace/manager . +COPY --from=builder /opt/app-root/src/manager . USER 65532:65532 ENTRYPOINT ["/manager"] diff --git a/infra/feast-operator/Makefile b/infra/feast-operator/Makefile index 310d64afaaa..56f388fde65 100644 --- a/infra/feast-operator/Makefile +++ b/infra/feast-operator/Makefile @@ -3,7 +3,7 @@ # To re-generate a bundle for another specific version without changing the standard setup, you can: # - use the VERSION as arg of the bundle target (e.g make bundle VERSION=0.0.2) # - use environment variables to overwrite this value (e.g export VERSION=0.0.2) -VERSION ?= 0.42.0 +VERSION ?= 0.43.0 # CHANNELS define the bundle channels used in the bundle. # Add a new line here if you would like to change its default config. (E.g CHANNELS = "candidate,fast,stable") @@ -48,11 +48,11 @@ endif # Set the Operator SDK version to use. By default, what is installed on the system is used. # This is useful for CI or a project to utilize a specific version of the operator-sdk toolkit. -OPERATOR_SDK_VERSION ?= v1.37.0 +OPERATOR_SDK_VERSION ?= v1.38.0 # Image URL to use all building/pushing image targets IMG ?= $(IMAGE_TAG_BASE):$(VERSION) # ENVTEST_K8S_VERSION refers to the version of kubebuilder assets to be downloaded by envtest binary. -ENVTEST_K8S_VERSION = 1.29.0 +ENVTEST_K8S_VERSION = 1.30.0 # Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set) ifeq (,$(shell go env GOBIN)) @@ -117,7 +117,7 @@ test: build-installer fmt vet lint envtest ## Run tests. # Utilize Kind or modify the e2e tests to load the image locally, enabling compatibility with other vendors. .PHONY: test-e2e # Run the e2e tests against a Kind k8s instance that is spun up. test-e2e: - go test ./test/e2e/ -v -ginkgo.v + go test -timeout 30m ./test/e2e/ -v -ginkgo.v .PHONY: lint lint: golangci-lint ## Run golangci-lint linter & yamllint @@ -142,7 +142,7 @@ run: manifests generate fmt vet ## Run a controller from your host. # More info: https://docs.docker.com/develop/develop-images/build_enhancements/ .PHONY: docker-build docker-build: ## Build docker image with the manager. - $(CONTAINER_TOOL) build -t ${IMG} . + $(CONTAINER_TOOL) build -t ${IMG} --load . ## Build feast docker image. .PHONY: feast-ci-dev-docker-img @@ -167,7 +167,7 @@ docker-buildx: ## Build and push docker image for the manager for cross-platform sed -e '1 s/\(^FROM\)/FROM --platform=\$$\{BUILDPLATFORM\}/; t' -e ' 1,// s//FROM --platform=\$$\{BUILDPLATFORM\}/' Dockerfile > Dockerfile.cross - $(CONTAINER_TOOL) buildx create --name project-v3-builder $(CONTAINER_TOOL) buildx use project-v3-builder - - $(CONTAINER_TOOL) buildx build --push --platform=$(PLATFORMS) --tag ${IMG} -f Dockerfile.cross . + - $(CONTAINER_TOOL) buildx build --push --platform=$(PLATFORMS) --tag ${IMG} -f Dockerfile.cross --load . - $(CONTAINER_TOOL) buildx rm project-v3-builder rm Dockerfile.cross @@ -209,16 +209,16 @@ $(LOCALBIN): ## Tool Binaries KUBECTL ?= kubectl -KUSTOMIZE ?= $(LOCALBIN)/kustomize-$(KUSTOMIZE_VERSION) -CONTROLLER_GEN ?= $(LOCALBIN)/controller-gen-$(CONTROLLER_TOOLS_VERSION) -ENVTEST ?= $(LOCALBIN)/setup-envtest-$(ENVTEST_VERSION) -GOLANGCI_LINT = $(LOCALBIN)/golangci-lint-$(GOLANGCI_LINT_VERSION) +KUSTOMIZE ?= $(LOCALBIN)/kustomize +CONTROLLER_GEN ?= $(LOCALBIN)/controller-gen +ENVTEST ?= $(LOCALBIN)/setup-envtest +GOLANGCI_LINT = $(LOCALBIN)/golangci-lint ## Tool Versions -KUSTOMIZE_VERSION ?= v5.3.0 -CONTROLLER_TOOLS_VERSION ?= v0.14.0 -ENVTEST_VERSION ?= release-0.17 -GOLANGCI_LINT_VERSION ?= v1.57.2 +KUSTOMIZE_VERSION ?= v5.4.2 +CONTROLLER_TOOLS_VERSION ?= v0.15.0 +ENVTEST_VERSION ?= release-0.18 +GOLANGCI_LINT_VERSION ?= v1.59.1 .PHONY: kustomize kustomize: $(KUSTOMIZE) ## Download kustomize locally if necessary. @@ -238,20 +238,23 @@ $(ENVTEST): $(LOCALBIN) .PHONY: golangci-lint golangci-lint: $(GOLANGCI_LINT) ## Download golangci-lint locally if necessary. $(GOLANGCI_LINT): $(LOCALBIN) - $(call go-install-tool,$(GOLANGCI_LINT),github.com/golangci/golangci-lint/cmd/golangci-lint,${GOLANGCI_LINT_VERSION}) + $(call go-install-tool,$(GOLANGCI_LINT),github.com/golangci/golangci-lint/cmd/golangci-lint,$(GOLANGCI_LINT_VERSION)) # go-install-tool will 'go install' any package with custom target and name of binary, if it doesn't exist # $1 - target path with name of binary (ideally with version) # $2 - package url which can be installed # $3 - specific version of package define go-install-tool -@[ -f $(1) ] || { \ +@[ -f "$(1)-$(3)" ] || { \ + echo "Downloading $${package}" ;\ +rm -f $(1) || true ;\ set -e; \ package=$(2)@$(3) ;\ echo "Downloading $${package}" ;\ GOBIN=$(LOCALBIN) go install $${package} ;\ -mv "$$(echo "$(1)" | sed "s/-$(3)$$//")" $(1) ;\ -} +mv $(1) $(1)-$(3) ;\ +} ;\ +ln -sf $(1)-$(3) $(1) endef .PHONY: operator-sdk diff --git a/infra/feast-operator/README.md b/infra/feast-operator/README.md index 3012eb63d4b..9d4b51f37f4 100644 --- a/infra/feast-operator/README.md +++ b/infra/feast-operator/README.md @@ -4,7 +4,7 @@ This is a K8s Operator that can be used to deploy and manage **Feast**, an open ## Getting Started ### Prerequisites -- go version v1.21.0+ +- go version v1.22 - docker version 17.03+. - kubectl version v1.11.3+. - Access to a Kubernetes v1.11.3+ cluster. @@ -108,8 +108,8 @@ make deploy IMG=/feast-operator: ``` ### Prerequisites -- go version v1.21 -- operator-sdk version v1.37.0 +- go version v1.22 +- operator-sdk version v1.38.0 **NOTE:** Run `make help` for more information on all potential `make` targets diff --git a/infra/feast-operator/api/feastversion/version.go b/infra/feast-operator/api/feastversion/version.go index 4a20c4ca592..50fddbada8e 100644 --- a/infra/feast-operator/api/feastversion/version.go +++ b/infra/feast-operator/api/feastversion/version.go @@ -17,4 +17,4 @@ limitations under the License. package feastversion // Feast release version -const FeastVersion = "0.42.0" +const FeastVersion = "0.43.0" diff --git a/infra/feast-operator/api/v1alpha1/featurestore_types.go b/infra/feast-operator/api/v1alpha1/featurestore_types.go index 17a029c02ea..b44f776dab4 100644 --- a/infra/feast-operator/api/v1alpha1/featurestore_types.go +++ b/infra/feast-operator/api/v1alpha1/featurestore_types.go @@ -17,6 +17,7 @@ limitations under the License. package v1alpha1 import ( + appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -65,25 +66,25 @@ type FeatureStoreSpec struct { AuthzConfig *AuthzConfig `json:"authz,omitempty"` } -// FeatureStoreServices defines the desired feast service deployments. ephemeral registry is deployed by default. +// FeatureStoreServices defines the desired feast services. An ephemeral registry is deployed by default. type FeatureStoreServices struct { - OfflineStore *OfflineStore `json:"offlineStore,omitempty"` - OnlineStore *OnlineStore `json:"onlineStore,omitempty"` - Registry *Registry `json:"registry,omitempty"` + OfflineStore *OfflineStore `json:"offlineStore,omitempty"` + OnlineStore *OnlineStore `json:"onlineStore,omitempty"` + Registry *Registry `json:"registry,omitempty"` + DeploymentStrategy *appsv1.DeploymentStrategy `json:"deploymentStrategy,omitempty"` + // Disable the 'feast repo initialization' initContainer + DisableInitContainers bool `json:"disableInitContainers,omitempty"` } // OfflineStore configures the deployed offline store service type OfflineStore struct { ServiceConfigs `json:",inline"` Persistence *OfflineStorePersistence `json:"persistence,omitempty"` - TLS *OfflineTlsConfigs `json:"tls,omitempty"` -} - -// OfflineTlsConfigs configures server TLS for the offline feast service. in an openshift cluster, this is configured by default using service serving certificates. -type OfflineTlsConfigs struct { - TlsConfigs `json:",inline"` - // verify the client TLS certificate. - VerifyClient *bool `json:"verifyClient,omitempty"` + TLS *TlsConfigs `json:"tls,omitempty"` + // LogLevel sets the logging level for the offline store service + // Allowed values: "debug", "info", "warning", "error", "critical". + // +kubebuilder:validation:Enum=debug;info;warning;error;critical + LogLevel string `json:"logLevel,omitempty"` } // OfflineStorePersistence configures the persistence settings for the offline store service @@ -95,7 +96,7 @@ type OfflineStorePersistence struct { // OfflineStoreFilePersistence configures the file-based persistence for the offline store service type OfflineStoreFilePersistence struct { - // +kubebuilder:validation:Enum=dask;duckdb + // +kubebuilder:validation:Enum=file;dask;duckdb Type string `json:"type,omitempty"` PvcConfig *PvcConfig `json:"pvc,omitempty"` } @@ -103,11 +104,12 @@ type OfflineStoreFilePersistence struct { var ValidOfflineStoreFilePersistenceTypes = []string{ "dask", "duckdb", + "file", } // OfflineStoreDBStorePersistence configures the DB store persistence for the offline store service type OfflineStoreDBStorePersistence struct { - // +kubebuilder:validation:Enum=snowflake.offline;bigquery;redshift;spark;postgres;feast_trino.trino.TrinoOfflineStore;redis + // +kubebuilder:validation:Enum=snowflake.offline;bigquery;redshift;spark;postgres;trino;redis;athena;mssql Type string `json:"type"` // Data store parameters should be placed as-is from the "feature_store.yaml" under the secret key. "registry_type" & "type" fields should be removed. SecretRef corev1.LocalObjectReference `json:"secretRef"` @@ -121,8 +123,10 @@ var ValidOfflineStoreDBStorePersistenceTypes = []string{ "redshift", "spark", "postgres", - "feast_trino.trino.TrinoOfflineStore", + "trino", "redis", + "athena", + "mssql", } // OnlineStore configures the deployed online store service @@ -130,6 +134,10 @@ type OnlineStore struct { ServiceConfigs `json:",inline"` Persistence *OnlineStorePersistence `json:"persistence,omitempty"` TLS *TlsConfigs `json:"tls,omitempty"` + // LogLevel sets the logging level for the online store service + // Allowed values: "debug", "info", "warning", "error", "critical". + // +kubebuilder:validation:Enum=debug;info;warning;error;critical + LogLevel string `json:"logLevel,omitempty"` } // OnlineStorePersistence configures the persistence settings for the online store service @@ -142,7 +150,7 @@ type OnlineStorePersistence struct { // OnlineStoreFilePersistence configures the file-based persistence for the offline store service // +kubebuilder:validation:XValidation:rule="(!has(self.pvc) && has(self.path)) ? self.path.startsWith('/') : true",message="Ephemeral stores must have absolute paths." // +kubebuilder:validation:XValidation:rule="(has(self.pvc) && has(self.path)) ? !self.path.startsWith('/') : true",message="PVC path must be a file name only, with no slashes." -// +kubebuilder:validation:XValidation:rule="has(self.path) && !self.path.startsWith('s3://') && !self.path.startsWith('gs://')",message="Online store does not support S3 or GS buckets." +// +kubebuilder:validation:XValidation:rule="has(self.path) ? !(self.path.startsWith('s3://') || self.path.startsWith('gs://')) : true",message="Online store does not support S3 or GS buckets." type OnlineStoreFilePersistence struct { Path string `json:"path,omitempty"` PvcConfig *PvcConfig `json:"pvc,omitempty"` @@ -150,7 +158,7 @@ type OnlineStoreFilePersistence struct { // OnlineStoreDBStorePersistence configures the DB store persistence for the offline store service type OnlineStoreDBStorePersistence struct { - // +kubebuilder:validation:Enum=snowflake.online;redis;ikv;datastore;dynamodb;bigtable;postgres;cassandra;mysql;hazelcast;singlestore + // +kubebuilder:validation:Enum=snowflake.online;redis;ikv;datastore;dynamodb;bigtable;postgres;cassandra;mysql;hazelcast;singlestore;hbase;elasticsearch;qdrant;couchbase;milvus Type string `json:"type"` // Data store parameters should be placed as-is from the "feature_store.yaml" under the secret key. "registry_type" & "type" fields should be removed. SecretRef corev1.LocalObjectReference `json:"secretRef"` @@ -170,6 +178,11 @@ var ValidOnlineStoreDBStorePersistenceTypes = []string{ "mysql", "hazelcast", "singlestore", + "hbase", + "elasticsearch", + "qdrant", + "couchbase", + "milvus", } // LocalRegistryConfig configures the deployed registry service @@ -177,6 +190,10 @@ type LocalRegistryConfig struct { ServiceConfigs `json:",inline"` Persistence *RegistryPersistence `json:"persistence,omitempty"` TLS *TlsConfigs `json:"tls,omitempty"` + // LogLevel sets the logging level for the registry service + // Allowed values: "debug", "info", "warning", "error", "critical". + // +kubebuilder:validation:Enum=debug;info;warning;error;critical + LogLevel string `json:"logLevel,omitempty"` } // RegistryPersistence configures the persistence settings for the registry service @@ -223,13 +240,15 @@ type PvcConfig struct { Create *PvcCreate `json:"create,omitempty"` // MountPath within the container at which the volume should be mounted. // Must start by "/" and cannot contain ':'. - MountPath string `json:"mountPath,omitempty"` + MountPath string `json:"mountPath"` } // PvcCreate defines the immutable settings to create a new PVC mounted at the given path. -// The PVC name is the same as the associated deployment name. +// The PVC name is the same as the associated deployment & feast service name. // +kubebuilder:validation:XValidation:rule="self == oldSelf",message="PvcCreate is immutable" type PvcCreate struct { + // AccessModes k8s persistent volume access modes. Defaults to ["ReadWriteOnce"]. + AccessModes []corev1.PersistentVolumeAccessMode `json:"accessModes,omitempty"` // StorageClassName is the name of an existing StorageClass to which this persistent volume belongs. Empty value // means that this volume does not belong to any StorageClass and the cluster default will be used. StorageClassName *string `json:"storageClassName,omitempty"` @@ -281,6 +300,7 @@ type DefaultConfigs struct { // OptionalConfigs k8s container settings that are optional type OptionalConfigs struct { Env *[]corev1.EnvVar `json:"env,omitempty"` + EnvFrom *[]corev1.EnvFromSource `json:"envFrom,omitempty"` ImagePullPolicy *corev1.PullPolicy `json:"imagePullPolicy,omitempty"` Resources *corev1.ResourceRequirements `json:"resources,omitempty"` } @@ -354,12 +374,11 @@ type FeatureStoreStatus struct { // Shows the currently applied feast configuration, including any pertinent defaults Applied FeatureStoreSpec `json:"applied,omitempty"` // ConfigMap in this namespace containing a client `feature_store.yaml` for this feast deployment - ClientConfigMap string `json:"clientConfigMap,omitempty"` - Conditions []metav1.Condition `json:"conditions,omitempty"` - // Version of feast that's currently deployed - FeastVersion string `json:"feastVersion,omitempty"` - Phase string `json:"phase,omitempty"` - ServiceHostnames ServiceHostnames `json:"serviceHostnames,omitempty"` + ClientConfigMap string `json:"clientConfigMap,omitempty"` + Conditions []metav1.Condition `json:"conditions,omitempty"` + FeastVersion string `json:"feastVersion,omitempty"` + Phase string `json:"phase,omitempty"` + ServiceHostnames ServiceHostnames `json:"serviceHostnames,omitempty"` } // ServiceHostnames defines the service hostnames in the format of :, e.g. example.svc.cluster.local:80 @@ -369,11 +388,11 @@ type ServiceHostnames struct { Registry string `json:"registry,omitempty"` } -//+kubebuilder:object:root=true -//+kubebuilder:subresource:status -//+kubebuilder:resource:shortName=feast -//+kubebuilder:printcolumn:name="Status",type=string,JSONPath=`.status.phase` -//+kubebuilder:printcolumn:name="Age",type=date,JSONPath=`.metadata.creationTimestamp` +// +kubebuilder:object:root=true +// +kubebuilder:subresource:status +// +kubebuilder:resource:shortName=feast +// +kubebuilder:printcolumn:name="Status",type=string,JSONPath=`.status.phase` +// +kubebuilder:printcolumn:name="Age",type=date,JSONPath=`.metadata.creationTimestamp` // FeatureStore is the Schema for the featurestores API type FeatureStore struct { @@ -384,7 +403,7 @@ type FeatureStore struct { Status FeatureStoreStatus `json:"status,omitempty"` } -//+kubebuilder:object:root=true +// +kubebuilder:object:root=true // FeatureStoreList contains a list of FeatureStore type FeatureStoreList struct { diff --git a/infra/feast-operator/api/v1alpha1/zz_generated.deepcopy.go b/infra/feast-operator/api/v1alpha1/zz_generated.deepcopy.go index 3f317c650e9..6fbd44deb70 100644 --- a/infra/feast-operator/api/v1alpha1/zz_generated.deepcopy.go +++ b/infra/feast-operator/api/v1alpha1/zz_generated.deepcopy.go @@ -21,7 +21,8 @@ limitations under the License. package v1alpha1 import ( - "k8s.io/api/core/v1" + "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" runtime "k8s.io/apimachinery/pkg/runtime" ) @@ -163,6 +164,11 @@ func (in *FeatureStoreServices) DeepCopyInto(out *FeatureStoreServices) { *out = new(Registry) (*in).DeepCopyInto(*out) } + if in.DeploymentStrategy != nil { + in, out := &in.DeploymentStrategy, &out.DeploymentStrategy + *out = new(v1.DeploymentStrategy) + (*in).DeepCopyInto(*out) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new FeatureStoreServices. @@ -281,7 +287,7 @@ func (in *OfflineStore) DeepCopyInto(out *OfflineStore) { } if in.TLS != nil { in, out := &in.TLS, &out.TLS - *out = new(OfflineTlsConfigs) + *out = new(TlsConfigs) (*in).DeepCopyInto(*out) } } @@ -357,27 +363,6 @@ func (in *OfflineStorePersistence) DeepCopy() *OfflineStorePersistence { return out } -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *OfflineTlsConfigs) DeepCopyInto(out *OfflineTlsConfigs) { - *out = *in - in.TlsConfigs.DeepCopyInto(&out.TlsConfigs) - if in.VerifyClient != nil { - in, out := &in.VerifyClient, &out.VerifyClient - *out = new(bool) - **out = **in - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new OfflineTlsConfigs. -func (in *OfflineTlsConfigs) DeepCopy() *OfflineTlsConfigs { - if in == nil { - return nil - } - out := new(OfflineTlsConfigs) - in.DeepCopyInto(out) - return out -} - // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *OidcAuthz) DeepCopyInto(out *OidcAuthz) { *out = *in @@ -486,10 +471,21 @@ func (in *OptionalConfigs) DeepCopyInto(out *OptionalConfigs) { *out = *in if in.Env != nil { in, out := &in.Env, &out.Env - *out = new([]v1.EnvVar) + *out = new([]corev1.EnvVar) if **in != nil { in, out := *in, *out - *out = make([]v1.EnvVar, len(*in)) + *out = make([]corev1.EnvVar, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + } + if in.EnvFrom != nil { + in, out := &in.EnvFrom, &out.EnvFrom + *out = new([]corev1.EnvFromSource) + if **in != nil { + in, out := *in, *out + *out = make([]corev1.EnvFromSource, len(*in)) for i := range *in { (*in)[i].DeepCopyInto(&(*out)[i]) } @@ -497,12 +493,12 @@ func (in *OptionalConfigs) DeepCopyInto(out *OptionalConfigs) { } if in.ImagePullPolicy != nil { in, out := &in.ImagePullPolicy, &out.ImagePullPolicy - *out = new(v1.PullPolicy) + *out = new(corev1.PullPolicy) **out = **in } if in.Resources != nil { in, out := &in.Resources, &out.Resources - *out = new(v1.ResourceRequirements) + *out = new(corev1.ResourceRequirements) (*in).DeepCopyInto(*out) } } @@ -522,7 +518,7 @@ func (in *PvcConfig) DeepCopyInto(out *PvcConfig) { *out = *in if in.Ref != nil { in, out := &in.Ref, &out.Ref - *out = new(v1.LocalObjectReference) + *out = new(corev1.LocalObjectReference) **out = **in } if in.Create != nil { @@ -545,6 +541,11 @@ func (in *PvcConfig) DeepCopy() *PvcConfig { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *PvcCreate) DeepCopyInto(out *PvcCreate) { *out = *in + if in.AccessModes != nil { + in, out := &in.AccessModes, &out.AccessModes + *out = make([]corev1.PersistentVolumeAccessMode, len(*in)) + copy(*out, *in) + } if in.StorageClassName != nil { in, out := &in.StorageClassName, &out.StorageClassName *out = new(string) @@ -742,7 +743,7 @@ func (in *TlsConfigs) DeepCopyInto(out *TlsConfigs) { *out = *in if in.SecretRef != nil { in, out := &in.SecretRef, &out.SecretRef - *out = new(v1.LocalObjectReference) + *out = new(corev1.LocalObjectReference) **out = **in } out.SecretKeyNames = in.SecretKeyNames diff --git a/infra/feast-operator/bundle.Dockerfile b/infra/feast-operator/bundle.Dockerfile index ab3f14a9da4..685b137b92a 100644 --- a/infra/feast-operator/bundle.Dockerfile +++ b/infra/feast-operator/bundle.Dockerfile @@ -6,7 +6,7 @@ LABEL operators.operatorframework.io.bundle.manifests.v1=manifests/ LABEL operators.operatorframework.io.bundle.metadata.v1=metadata/ LABEL operators.operatorframework.io.bundle.package.v1=feast-operator LABEL operators.operatorframework.io.bundle.channels.v1=alpha -LABEL operators.operatorframework.io.metrics.builder=operator-sdk-v1.37.0 +LABEL operators.operatorframework.io.metrics.builder=operator-sdk-v1.38.0 LABEL operators.operatorframework.io.metrics.mediatype.v1=metrics+v1 LABEL operators.operatorframework.io.metrics.project_layout=go.kubebuilder.io/v4 diff --git a/infra/feast-operator/bundle/manifests/feast-operator-controller-manager-metrics-service_v1_service.yaml b/infra/feast-operator/bundle/manifests/feast-operator-controller-manager-metrics-service_v1_service.yaml index e0cd9dc2545..913517e198a 100644 --- a/infra/feast-operator/bundle/manifests/feast-operator-controller-manager-metrics-service_v1_service.yaml +++ b/infra/feast-operator/bundle/manifests/feast-operator-controller-manager-metrics-service_v1_service.yaml @@ -12,7 +12,7 @@ spec: - name: https port: 8443 protocol: TCP - targetPort: https + targetPort: 8443 selector: control-plane: controller-manager status: diff --git a/infra/feast-operator/bundle/manifests/feast-operator.clusterserviceversion.yaml b/infra/feast-operator/bundle/manifests/feast-operator.clusterserviceversion.yaml index 245db443581..bfd32b5a830 100644 --- a/infra/feast-operator/bundle/manifests/feast-operator.clusterserviceversion.yaml +++ b/infra/feast-operator/bundle/manifests/feast-operator.clusterserviceversion.yaml @@ -16,10 +16,10 @@ metadata: } ] capabilities: Basic Install - createdAt: "2024-11-01T13:05:11Z" - operators.operatorframework.io/builder: operator-sdk-v1.37.0 + createdAt: "2025-01-16T22:15:56Z" + operators.operatorframework.io/builder: operator-sdk-v1.38.0 operators.operatorframework.io/project_layout: go.kubebuilder.io/v4 - name: feast-operator.v0.41.0 + name: feast-operator.v0.42.0 namespace: placeholder spec: apiservicedefinitions: {} @@ -54,6 +54,8 @@ spec: - "" resources: - configmaps + - persistentvolumeclaims + - serviceaccounts - services verbs: - create @@ -62,6 +64,13 @@ spec: - list - update - watch + - apiGroups: + - "" + resources: + - secrets + verbs: + - get + - list - apiGroups: - feast.dev resources: @@ -88,6 +97,18 @@ spec: - get - patch - update + - apiGroups: + - rbac.authorization.k8s.io + resources: + - rolebindings + - roles + verbs: + - create + - delete + - get + - list + - update + - watch - apiGroups: - authentication.k8s.io resources: @@ -122,35 +143,17 @@ spec: spec: containers: - args: - - --secure-listen-address=0.0.0.0:8443 - - --upstream=http://127.0.0.1:8080/ - - --logtostderr=true - - --v=0 - image: gcr.io/kubebuilder/kube-rbac-proxy:v0.16.0 - name: kube-rbac-proxy - ports: - - containerPort: 8443 - name: https - protocol: TCP - resources: - limits: - cpu: 500m - memory: 128Mi - requests: - cpu: 5m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - - args: - - --health-probe-bind-address=:8081 - - --metrics-bind-address=127.0.0.1:8080 + - --metrics-bind-address=:8443 - --leader-elect + - --health-probe-bind-address=:8081 command: - /manager - image: feastdev/feast-operator:0.41.0 + env: + - name: RELATED_IMAGE_FEATURE_SERVER + value: docker.io/feastdev/feature-server:0.42.0 + - name: RELATED_IMAGE_GRPC_CURL + value: docker.io/fullstorydev/grpcurl:v1.9.1-alpine + image: feastdev/feast-operator:0.42.0 livenessProbe: httpGet: path: /healthz @@ -239,4 +242,9 @@ spec: provider: name: Feast Community url: https://lf-aidata.atlassian.net/wiki/spaces/FEAST/ - version: 0.41.0 + relatedImages: + - image: docker.io/feastdev/feature-server:0.42.0 + name: feature-server + - image: docker.io/fullstorydev/grpcurl:v1.9.1-alpine + name: grpc-curl + version: 0.42.0 diff --git a/infra/feast-operator/bundle/manifests/feast.dev_featurestores.yaml b/infra/feast-operator/bundle/manifests/feast.dev_featurestores.yaml index 2142e093eb1..ff1a77936f2 100644 --- a/infra/feast-operator/bundle/manifests/feast.dev_featurestores.yaml +++ b/infra/feast-operator/bundle/manifests/feast.dev_featurestores.yaml @@ -2,7 +2,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.14.0 + controller-gen.kubebuilder.io/version: v0.15.0 creationTimestamp: null name: featurestores.feast.dev spec: @@ -48,6 +48,57 @@ spec: spec: description: FeatureStoreSpec defines the desired state of FeatureStore properties: + authz: + description: AuthzConfig defines the authorization settings for the + deployed Feast services. + properties: + kubernetes: + description: |- + KubernetesAuthz provides a way to define the authorization settings using Kubernetes RBAC resources. + https://kubernetes.io/docs/reference/access-authn-authz/rbac/ + properties: + roles: + description: |- + The Kubernetes RBAC roles to be deployed in the same namespace of the FeatureStore. + Roles are managed by the operator and created with an empty list of rules. + See the Feast permission model at https://docs.feast.dev/getting-started/concepts/permission + The feature store admin is not obligated to manage roles using the Feast operator, roles can be managed independently. + This configuration option is only providing a way to automate this procedure. + Important note: the operator cannot ensure that these roles will match the ones used in the configured Feast permissions. + items: + type: string + type: array + type: object + oidc: + description: |- + OidcAuthz defines the authorization settings for deployments using an Open ID Connect identity provider. + https://auth0.com/docs/authenticate/protocols/openid-connect-protocol + properties: + secretRef: + description: |- + LocalObjectReference contains enough information to let you locate the + referenced object inside the same namespace. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. + type: string + type: object + x-kubernetes-map-type: atomic + required: + - secretRef + type: object + type: object + x-kubernetes-validations: + - message: One selection required between kubernetes or oidc. + rule: '[has(self.kubernetes), has(self.oidc)].exists_one(c, c)' feastProject: description: FeastProject is the Feast project id. This can be any alphanumeric string with underscores, but it cannot start with an @@ -55,9 +106,63 @@ spec: pattern: ^[A-Za-z0-9][A-Za-z0-9_]*$ type: string services: - description: FeatureStoreServices defines the desired feast service - deployments. ephemeral registry is deployed by default. + description: FeatureStoreServices defines the desired feast services. + An ephemeral registry is deployed by default. properties: + deploymentStrategy: + description: DeploymentStrategy describes how to replace existing + pods with new ones. + properties: + rollingUpdate: + description: |- + Rolling update config params. Present only if DeploymentStrategyType = + RollingUpdate. + --- + TODO: Update this to follow our convention for oneOf, whatever we decide it + to be. + properties: + maxSurge: + anyOf: + - type: integer + - type: string + description: |- + The maximum number of pods that can be scheduled above the desired number of + pods. + Value can be an absolute number (ex: 5) or a percentage of desired pods (ex: 10%). + This can not be 0 if MaxUnavailable is 0. + Absolute number is calculated from percentage by rounding up. + Defaults to 25%. + Example: when this is set to 30%, the new ReplicaSet can be scaled up immediately when + the rolling update starts, such that the total number of old and new pods do not exceed + 130% of desired pods. Once old pods have been killed, + new ReplicaSet can be scaled up further, ensuring that total number of pods running + at any time during the update is at most 130% of desired pods. + x-kubernetes-int-or-string: true + maxUnavailable: + anyOf: + - type: integer + - type: string + description: |- + The maximum number of pods that can be unavailable during the update. + Value can be an absolute number (ex: 5) or a percentage of desired pods (ex: 10%). + Absolute number is calculated from percentage by rounding down. + This can not be 0 if MaxSurge is 0. + Defaults to 25%. + Example: when this is set to 30%, the old ReplicaSet can be scaled down to 70% of desired pods + immediately when the rolling update starts. Once new pods are ready, old ReplicaSet + can be scaled down further, followed by scaling up the new ReplicaSet, ensuring + that the total number of pods available at all times during the update is at + least 70% of desired pods. + x-kubernetes-int-or-string: true + type: object + type: + description: Type of deployment. Can be "Recreate" or "RollingUpdate". + Default is RollingUpdate. + type: string + type: object + disableInitContainers: + description: Disable the 'feast repo initialization' initContainer + type: boolean offlineStore: description: OfflineStore configures the deployed offline store service @@ -94,10 +199,15 @@ spec: description: The key to select. type: string name: + default: "" description: |- Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string optional: description: Specify whether the ConfigMap or @@ -157,10 +267,15 @@ spec: from. Must be a valid secret key. type: string name: + default: "" description: |- Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string optional: description: Specify whether the Secret or its @@ -175,12 +290,223 @@ spec: - name type: object type: array + envFrom: + items: + description: EnvFromSource represents the source of a set + of ConfigMaps + properties: + configMapRef: + description: The ConfigMap to select from + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. + type: string + optional: + description: Specify whether the ConfigMap must + be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + prefix: + description: An optional identifier to prepend to each + key in the ConfigMap. Must be a C_IDENTIFIER. + type: string + secretRef: + description: The Secret to select from + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. + type: string + optional: + description: Specify whether the Secret must be + defined + type: boolean + type: object + x-kubernetes-map-type: atomic + type: object + type: array image: type: string imagePullPolicy: description: PullPolicy describes a policy for if/when to pull a container image type: string + logLevel: + description: |- + LogLevel sets the logging level for the offline store service + Allowed values: "debug", "info", "warning", "error", "critical". + enum: + - debug + - info + - warning + - error + - critical + type: string + persistence: + description: OfflineStorePersistence configures the persistence + settings for the offline store service + properties: + file: + description: OfflineStoreFilePersistence configures the + file-based persistence for the offline store service + properties: + pvc: + description: |- + PvcConfig defines the settings for a persistent file store based on PVCs. + We can refer to an existing PVC using the `Ref` field, or create a new one using the `Create` field. + properties: + create: + description: Settings for creating a new PVC + properties: + accessModes: + description: AccessModes k8s persistent volume + access modes. Defaults to ["ReadWriteOnce"]. + items: + type: string + type: array + resources: + description: |- + Resources describes the storage resource requirements for a volume. + Default requested storage size depends on the associated service: + - 10Gi for offline store + - 5Gi for online store + - 5Gi for registry + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + storageClassName: + description: |- + StorageClassName is the name of an existing StorageClass to which this persistent volume belongs. Empty value + means that this volume does not belong to any StorageClass and the cluster default will be used. + type: string + type: object + x-kubernetes-validations: + - message: PvcCreate is immutable + rule: self == oldSelf + mountPath: + description: |- + MountPath within the container at which the volume should be mounted. + Must start by "/" and cannot contain ':'. + type: string + ref: + description: Reference to an existing field + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. + type: string + type: object + x-kubernetes-map-type: atomic + required: + - mountPath + type: object + x-kubernetes-validations: + - message: One selection is required between ref and + create. + rule: '[has(self.ref), has(self.create)].exists_one(c, + c)' + - message: Mount path must start with '/' and must + not contain ':' + rule: self.mountPath.matches('^/[^:]*$') + type: + enum: + - file + - dask + - duckdb + type: string + type: object + store: + description: OfflineStoreDBStorePersistence configures + the DB store persistence for the offline store service + properties: + secretKeyName: + description: By default, the selected store "type" + is used as the SecretKeyName + type: string + secretRef: + description: Data store parameters should be placed + as-is from the "feature_store.yaml" under the secret + key. "registry_type" & "type" fields should be removed. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. + type: string + type: object + x-kubernetes-map-type: atomic + type: + enum: + - snowflake.offline + - bigquery + - redshift + - spark + - postgres + - trino + - redis + - athena + - mssql + type: string + required: + - secretRef + - type + type: object + type: object + x-kubernetes-validations: + - message: One selection required between file or store. + rule: '[has(self.file), has(self.store)].exists_one(c, c)' resources: description: ResourceRequirements describes the compute resource requirements. @@ -237,6 +563,49 @@ spec: More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ type: object type: object + tls: + description: TlsConfigs configures server TLS for a feast + service. in an openshift cluster, this is configured by + default using service serving certificates. + properties: + disable: + description: will disable TLS for the feast service. useful + in an openshift cluster, for example, where TLS is configured + by default + type: boolean + secretKeyNames: + description: SecretKeyNames defines the secret key names + for the TLS key and cert. + properties: + tlsCrt: + description: defaults to "tls.crt" + type: string + tlsKey: + description: defaults to "tls.key" + type: string + type: object + secretRef: + description: references the local k8s secret where the + TLS key and cert reside + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. + type: string + type: object + x-kubernetes-map-type: atomic + type: object + x-kubernetes-validations: + - message: '`secretRef` required if `disable` is false.' + rule: '(!has(self.disable) || !self.disable) ? has(self.secretRef) + : true' type: object onlineStore: description: OnlineStore configures the deployed online store @@ -274,10 +643,15 @@ spec: description: The key to select. type: string name: + default: "" description: |- Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string optional: description: Specify whether the ConfigMap or @@ -337,10 +711,15 @@ spec: from. Must be a valid secret key. type: string name: + default: "" description: |- Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string optional: description: Specify whether the Secret or its @@ -355,12 +734,237 @@ spec: - name type: object type: array + envFrom: + items: + description: EnvFromSource represents the source of a set + of ConfigMaps + properties: + configMapRef: + description: The ConfigMap to select from + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. + type: string + optional: + description: Specify whether the ConfigMap must + be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + prefix: + description: An optional identifier to prepend to each + key in the ConfigMap. Must be a C_IDENTIFIER. + type: string + secretRef: + description: The Secret to select from + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. + type: string + optional: + description: Specify whether the Secret must be + defined + type: boolean + type: object + x-kubernetes-map-type: atomic + type: object + type: array image: type: string imagePullPolicy: description: PullPolicy describes a policy for if/when to pull a container image type: string + logLevel: + description: |- + LogLevel sets the logging level for the online store service + Allowed values: "debug", "info", "warning", "error", "critical". + enum: + - debug + - info + - warning + - error + - critical + type: string + persistence: + description: OnlineStorePersistence configures the persistence + settings for the online store service + properties: + file: + description: OnlineStoreFilePersistence configures the + file-based persistence for the offline store service + properties: + path: + type: string + pvc: + description: |- + PvcConfig defines the settings for a persistent file store based on PVCs. + We can refer to an existing PVC using the `Ref` field, or create a new one using the `Create` field. + properties: + create: + description: Settings for creating a new PVC + properties: + accessModes: + description: AccessModes k8s persistent volume + access modes. Defaults to ["ReadWriteOnce"]. + items: + type: string + type: array + resources: + description: |- + Resources describes the storage resource requirements for a volume. + Default requested storage size depends on the associated service: + - 10Gi for offline store + - 5Gi for online store + - 5Gi for registry + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + storageClassName: + description: |- + StorageClassName is the name of an existing StorageClass to which this persistent volume belongs. Empty value + means that this volume does not belong to any StorageClass and the cluster default will be used. + type: string + type: object + x-kubernetes-validations: + - message: PvcCreate is immutable + rule: self == oldSelf + mountPath: + description: |- + MountPath within the container at which the volume should be mounted. + Must start by "/" and cannot contain ':'. + type: string + ref: + description: Reference to an existing field + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. + type: string + type: object + x-kubernetes-map-type: atomic + required: + - mountPath + type: object + x-kubernetes-validations: + - message: One selection is required between ref and + create. + rule: '[has(self.ref), has(self.create)].exists_one(c, + c)' + - message: Mount path must start with '/' and must + not contain ':' + rule: self.mountPath.matches('^/[^:]*$') + type: object + x-kubernetes-validations: + - message: Ephemeral stores must have absolute paths. + rule: '(!has(self.pvc) && has(self.path)) ? self.path.startsWith(''/'') + : true' + - message: PVC path must be a file name only, with no + slashes. + rule: '(has(self.pvc) && has(self.path)) ? !self.path.startsWith(''/'') + : true' + - message: Online store does not support S3 or GS buckets. + rule: 'has(self.path) ? !(self.path.startsWith(''s3://'') + || self.path.startsWith(''gs://'')) : true' + store: + description: OnlineStoreDBStorePersistence configures + the DB store persistence for the offline store service + properties: + secretKeyName: + description: By default, the selected store "type" + is used as the SecretKeyName + type: string + secretRef: + description: Data store parameters should be placed + as-is from the "feature_store.yaml" under the secret + key. "registry_type" & "type" fields should be removed. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. + type: string + type: object + x-kubernetes-map-type: atomic + type: + enum: + - snowflake.online + - redis + - ikv + - datastore + - dynamodb + - bigtable + - postgres + - cassandra + - mysql + - hazelcast + - singlestore + - hbase + - elasticsearch + - qdrant + - couchbase + - milvus + type: string + required: + - secretRef + - type + type: object + type: object + x-kubernetes-validations: + - message: One selection required between file or store. + rule: '[has(self.file), has(self.store)].exists_one(c, c)' resources: description: ResourceRequirements describes the compute resource requirements. @@ -417,6 +1021,49 @@ spec: More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ type: object type: object + tls: + description: TlsConfigs configures server TLS for a feast + service. in an openshift cluster, this is configured by + default using service serving certificates. + properties: + disable: + description: will disable TLS for the feast service. useful + in an openshift cluster, for example, where TLS is configured + by default + type: boolean + secretKeyNames: + description: SecretKeyNames defines the secret key names + for the TLS key and cert. + properties: + tlsCrt: + description: defaults to "tls.crt" + type: string + tlsKey: + description: defaults to "tls.key" + type: string + type: object + secretRef: + description: references the local k8s secret where the + TLS key and cert reside + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. + type: string + type: object + x-kubernetes-map-type: atomic + type: object + x-kubernetes-validations: + - message: '`secretRef` required if `disable` is false.' + rule: '(!has(self.disable) || !self.disable) ? has(self.secretRef) + : true' type: object registry: description: Registry configures the registry service. One selection @@ -458,10 +1105,15 @@ spec: description: The key to select. type: string name: + default: "" description: |- Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string optional: description: Specify whether the ConfigMap @@ -522,10 +1174,15 @@ spec: from. Must be a valid secret key. type: string name: + default: "" description: |- Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string optional: description: Specify whether the Secret @@ -540,12 +1197,237 @@ spec: - name type: object type: array + envFrom: + items: + description: EnvFromSource represents the source of + a set of ConfigMaps + properties: + configMapRef: + description: The ConfigMap to select from + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. + type: string + optional: + description: Specify whether the ConfigMap must + be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + prefix: + description: An optional identifier to prepend to + each key in the ConfigMap. Must be a C_IDENTIFIER. + type: string + secretRef: + description: The Secret to select from + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. + type: string + optional: + description: Specify whether the Secret must + be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + type: object + type: array image: type: string imagePullPolicy: description: PullPolicy describes a policy for if/when to pull a container image type: string + logLevel: + description: |- + LogLevel sets the logging level for the registry service + Allowed values: "debug", "info", "warning", "error", "critical". + enum: + - debug + - info + - warning + - error + - critical + type: string + persistence: + description: RegistryPersistence configures the persistence + settings for the registry service + properties: + file: + description: RegistryFilePersistence configures the + file-based persistence for the registry service + properties: + path: + type: string + pvc: + description: |- + PvcConfig defines the settings for a persistent file store based on PVCs. + We can refer to an existing PVC using the `Ref` field, or create a new one using the `Create` field. + properties: + create: + description: Settings for creating a new PVC + properties: + accessModes: + description: AccessModes k8s persistent + volume access modes. Defaults to ["ReadWriteOnce"]. + items: + type: string + type: array + resources: + description: |- + Resources describes the storage resource requirements for a volume. + Default requested storage size depends on the associated service: + - 10Gi for offline store + - 5Gi for online store + - 5Gi for registry + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + storageClassName: + description: |- + StorageClassName is the name of an existing StorageClass to which this persistent volume belongs. Empty value + means that this volume does not belong to any StorageClass and the cluster default will be used. + type: string + type: object + x-kubernetes-validations: + - message: PvcCreate is immutable + rule: self == oldSelf + mountPath: + description: |- + MountPath within the container at which the volume should be mounted. + Must start by "/" and cannot contain ':'. + type: string + ref: + description: Reference to an existing field + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. + type: string + type: object + x-kubernetes-map-type: atomic + required: + - mountPath + type: object + x-kubernetes-validations: + - message: One selection is required between ref + and create. + rule: '[has(self.ref), has(self.create)].exists_one(c, + c)' + - message: Mount path must start with '/' and + must not contain ':' + rule: self.mountPath.matches('^/[^:]*$') + s3_additional_kwargs: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-validations: + - message: Registry files must use absolute paths + or be S3 ('s3://') or GS ('gs://') object store + URIs. + rule: '(!has(self.pvc) && has(self.path)) ? (self.path.startsWith(''/'') + || self.path.startsWith(''s3://'') || self.path.startsWith(''gs://'')) + : true' + - message: PVC path must be a file name only, with + no slashes. + rule: '(has(self.pvc) && has(self.path)) ? !self.path.startsWith(''/'') + : true' + - message: PVC persistence does not support S3 or + GS object store URIs. + rule: '(has(self.pvc) && has(self.path)) ? !(self.path.startsWith(''s3://'') + || self.path.startsWith(''gs://'')) : true' + - message: Additional S3 settings are available only + for S3 object store URIs. + rule: '(has(self.s3_additional_kwargs) && has(self.path)) + ? self.path.startsWith(''s3://'') : true' + store: + description: RegistryDBStorePersistence configures + the DB store persistence for the registry service + properties: + secretKeyName: + description: By default, the selected store "type" + is used as the SecretKeyName + type: string + secretRef: + description: Data store parameters should be placed + as-is from the "feature_store.yaml" under the + secret key. "registry_type" & "type" fields + should be removed. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. + type: string + type: object + x-kubernetes-map-type: atomic + type: + enum: + - sql + - snowflake.registry + type: string + required: + - secretRef + - type + type: object + type: object + x-kubernetes-validations: + - message: One selection required between file or store. + rule: '[has(self.file), has(self.store)].exists_one(c, + c)' resources: description: ResourceRequirements describes the compute resource requirements. @@ -603,6 +1485,49 @@ spec: More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ type: object type: object + tls: + description: TlsConfigs configures server TLS for a feast + service. in an openshift cluster, this is configured + by default using service serving certificates. + properties: + disable: + description: will disable TLS for the feast service. + useful in an openshift cluster, for example, where + TLS is configured by default + type: boolean + secretKeyNames: + description: SecretKeyNames defines the secret key + names for the TLS key and cert. + properties: + tlsCrt: + description: defaults to "tls.crt" + type: string + tlsKey: + description: defaults to "tls.key" + type: string + type: object + secretRef: + description: references the local k8s secret where + the TLS key and cert reside + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. + type: string + type: object + x-kubernetes-map-type: atomic + type: object + x-kubernetes-validations: + - message: '`secretRef` required if `disable` is false.' + rule: '(!has(self.disable) || !self.disable) ? has(self.secretRef) + : true' type: object remote: description: |- @@ -626,6 +1551,37 @@ spec: description: Host address of the remote registry service - :, e.g. `registry..svc.cluster.local:80` type: string + tls: + description: TlsRemoteRegistryConfigs configures client + TLS for a remote feast registry. in an openshift cluster, + this is configured by default when the remote feast + registry is using service serving certificates. + properties: + certName: + description: defines the configmap key name for the + client TLS cert. + type: string + configMapRef: + description: references the local k8s configmap where + the TLS cert resides + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. + type: string + type: object + x-kubernetes-map-type: atomic + required: + - certName + - configMapRef + type: object type: object x-kubernetes-validations: - message: One selection required. @@ -646,6 +1602,58 @@ spec: description: Shows the currently applied feast configuration, including any pertinent defaults properties: + authz: + description: AuthzConfig defines the authorization settings for + the deployed Feast services. + properties: + kubernetes: + description: |- + KubernetesAuthz provides a way to define the authorization settings using Kubernetes RBAC resources. + https://kubernetes.io/docs/reference/access-authn-authz/rbac/ + properties: + roles: + description: |- + The Kubernetes RBAC roles to be deployed in the same namespace of the FeatureStore. + Roles are managed by the operator and created with an empty list of rules. + See the Feast permission model at https://docs.feast.dev/getting-started/concepts/permission + The feature store admin is not obligated to manage roles using the Feast operator, roles can be managed independently. + This configuration option is only providing a way to automate this procedure. + Important note: the operator cannot ensure that these roles will match the ones used in the configured Feast permissions. + items: + type: string + type: array + type: object + oidc: + description: |- + OidcAuthz defines the authorization settings for deployments using an Open ID Connect identity provider. + https://auth0.com/docs/authenticate/protocols/openid-connect-protocol + properties: + secretRef: + description: |- + LocalObjectReference contains enough information to let you locate the + referenced object inside the same namespace. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. + type: string + type: object + x-kubernetes-map-type: atomic + required: + - secretRef + type: object + type: object + x-kubernetes-validations: + - message: One selection required between kubernetes or oidc. + rule: '[has(self.kubernetes), has(self.oidc)].exists_one(c, + c)' feastProject: description: FeastProject is the Feast project id. This can be any alphanumeric string with underscores, but it cannot start @@ -653,9 +1661,63 @@ spec: pattern: ^[A-Za-z0-9][A-Za-z0-9_]*$ type: string services: - description: FeatureStoreServices defines the desired feast service - deployments. ephemeral registry is deployed by default. + description: FeatureStoreServices defines the desired feast services. + An ephemeral registry is deployed by default. properties: + deploymentStrategy: + description: DeploymentStrategy describes how to replace existing + pods with new ones. + properties: + rollingUpdate: + description: |- + Rolling update config params. Present only if DeploymentStrategyType = + RollingUpdate. + --- + TODO: Update this to follow our convention for oneOf, whatever we decide it + to be. + properties: + maxSurge: + anyOf: + - type: integer + - type: string + description: |- + The maximum number of pods that can be scheduled above the desired number of + pods. + Value can be an absolute number (ex: 5) or a percentage of desired pods (ex: 10%). + This can not be 0 if MaxUnavailable is 0. + Absolute number is calculated from percentage by rounding up. + Defaults to 25%. + Example: when this is set to 30%, the new ReplicaSet can be scaled up immediately when + the rolling update starts, such that the total number of old and new pods do not exceed + 130% of desired pods. Once old pods have been killed, + new ReplicaSet can be scaled up further, ensuring that total number of pods running + at any time during the update is at most 130% of desired pods. + x-kubernetes-int-or-string: true + maxUnavailable: + anyOf: + - type: integer + - type: string + description: |- + The maximum number of pods that can be unavailable during the update. + Value can be an absolute number (ex: 5) or a percentage of desired pods (ex: 10%). + Absolute number is calculated from percentage by rounding down. + This can not be 0 if MaxSurge is 0. + Defaults to 25%. + Example: when this is set to 30%, the old ReplicaSet can be scaled down to 70% of desired pods + immediately when the rolling update starts. Once new pods are ready, old ReplicaSet + can be scaled down further, followed by scaling up the new ReplicaSet, ensuring + that the total number of pods available at all times during the update is at + least 70% of desired pods. + x-kubernetes-int-or-string: true + type: object + type: + description: Type of deployment. Can be "Recreate" or + "RollingUpdate". Default is RollingUpdate. + type: string + type: object + disableInitContainers: + description: Disable the 'feast repo initialization' initContainer + type: boolean offlineStore: description: OfflineStore configures the deployed offline store service @@ -692,10 +1754,15 @@ spec: description: The key to select. type: string name: + default: "" description: |- Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string optional: description: Specify whether the ConfigMap @@ -756,10 +1823,15 @@ spec: from. Must be a valid secret key. type: string name: + default: "" description: |- Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string optional: description: Specify whether the Secret @@ -774,12 +1846,226 @@ spec: - name type: object type: array + envFrom: + items: + description: EnvFromSource represents the source of + a set of ConfigMaps + properties: + configMapRef: + description: The ConfigMap to select from + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. + type: string + optional: + description: Specify whether the ConfigMap must + be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + prefix: + description: An optional identifier to prepend to + each key in the ConfigMap. Must be a C_IDENTIFIER. + type: string + secretRef: + description: The Secret to select from + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. + type: string + optional: + description: Specify whether the Secret must + be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + type: object + type: array image: type: string imagePullPolicy: description: PullPolicy describes a policy for if/when to pull a container image type: string + logLevel: + description: |- + LogLevel sets the logging level for the offline store service + Allowed values: "debug", "info", "warning", "error", "critical". + enum: + - debug + - info + - warning + - error + - critical + type: string + persistence: + description: OfflineStorePersistence configures the persistence + settings for the offline store service + properties: + file: + description: OfflineStoreFilePersistence configures + the file-based persistence for the offline store + service + properties: + pvc: + description: |- + PvcConfig defines the settings for a persistent file store based on PVCs. + We can refer to an existing PVC using the `Ref` field, or create a new one using the `Create` field. + properties: + create: + description: Settings for creating a new PVC + properties: + accessModes: + description: AccessModes k8s persistent + volume access modes. Defaults to ["ReadWriteOnce"]. + items: + type: string + type: array + resources: + description: |- + Resources describes the storage resource requirements for a volume. + Default requested storage size depends on the associated service: + - 10Gi for offline store + - 5Gi for online store + - 5Gi for registry + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + storageClassName: + description: |- + StorageClassName is the name of an existing StorageClass to which this persistent volume belongs. Empty value + means that this volume does not belong to any StorageClass and the cluster default will be used. + type: string + type: object + x-kubernetes-validations: + - message: PvcCreate is immutable + rule: self == oldSelf + mountPath: + description: |- + MountPath within the container at which the volume should be mounted. + Must start by "/" and cannot contain ':'. + type: string + ref: + description: Reference to an existing field + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. + type: string + type: object + x-kubernetes-map-type: atomic + required: + - mountPath + type: object + x-kubernetes-validations: + - message: One selection is required between ref + and create. + rule: '[has(self.ref), has(self.create)].exists_one(c, + c)' + - message: Mount path must start with '/' and + must not contain ':' + rule: self.mountPath.matches('^/[^:]*$') + type: + enum: + - file + - dask + - duckdb + type: string + type: object + store: + description: OfflineStoreDBStorePersistence configures + the DB store persistence for the offline store service + properties: + secretKeyName: + description: By default, the selected store "type" + is used as the SecretKeyName + type: string + secretRef: + description: Data store parameters should be placed + as-is from the "feature_store.yaml" under the + secret key. "registry_type" & "type" fields + should be removed. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. + type: string + type: object + x-kubernetes-map-type: atomic + type: + enum: + - snowflake.offline + - bigquery + - redshift + - spark + - postgres + - trino + - redis + - athena + - mssql + type: string + required: + - secretRef + - type + type: object + type: object + x-kubernetes-validations: + - message: One selection required between file or store. + rule: '[has(self.file), has(self.store)].exists_one(c, + c)' resources: description: ResourceRequirements describes the compute resource requirements. @@ -837,6 +2123,49 @@ spec: More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ type: object type: object + tls: + description: TlsConfigs configures server TLS for a feast + service. in an openshift cluster, this is configured + by default using service serving certificates. + properties: + disable: + description: will disable TLS for the feast service. + useful in an openshift cluster, for example, where + TLS is configured by default + type: boolean + secretKeyNames: + description: SecretKeyNames defines the secret key + names for the TLS key and cert. + properties: + tlsCrt: + description: defaults to "tls.crt" + type: string + tlsKey: + description: defaults to "tls.key" + type: string + type: object + secretRef: + description: references the local k8s secret where + the TLS key and cert reside + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. + type: string + type: object + x-kubernetes-map-type: atomic + type: object + x-kubernetes-validations: + - message: '`secretRef` required if `disable` is false.' + rule: '(!has(self.disable) || !self.disable) ? has(self.secretRef) + : true' type: object onlineStore: description: OnlineStore configures the deployed online store @@ -874,10 +2203,15 @@ spec: description: The key to select. type: string name: + default: "" description: |- Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string optional: description: Specify whether the ConfigMap @@ -938,10 +2272,15 @@ spec: from. Must be a valid secret key. type: string name: + default: "" description: |- Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string optional: description: Specify whether the Secret @@ -956,12 +2295,241 @@ spec: - name type: object type: array + envFrom: + items: + description: EnvFromSource represents the source of + a set of ConfigMaps + properties: + configMapRef: + description: The ConfigMap to select from + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. + type: string + optional: + description: Specify whether the ConfigMap must + be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + prefix: + description: An optional identifier to prepend to + each key in the ConfigMap. Must be a C_IDENTIFIER. + type: string + secretRef: + description: The Secret to select from + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. + type: string + optional: + description: Specify whether the Secret must + be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + type: object + type: array image: type: string imagePullPolicy: description: PullPolicy describes a policy for if/when to pull a container image type: string + logLevel: + description: |- + LogLevel sets the logging level for the online store service + Allowed values: "debug", "info", "warning", "error", "critical". + enum: + - debug + - info + - warning + - error + - critical + type: string + persistence: + description: OnlineStorePersistence configures the persistence + settings for the online store service + properties: + file: + description: OnlineStoreFilePersistence configures + the file-based persistence for the offline store + service + properties: + path: + type: string + pvc: + description: |- + PvcConfig defines the settings for a persistent file store based on PVCs. + We can refer to an existing PVC using the `Ref` field, or create a new one using the `Create` field. + properties: + create: + description: Settings for creating a new PVC + properties: + accessModes: + description: AccessModes k8s persistent + volume access modes. Defaults to ["ReadWriteOnce"]. + items: + type: string + type: array + resources: + description: |- + Resources describes the storage resource requirements for a volume. + Default requested storage size depends on the associated service: + - 10Gi for offline store + - 5Gi for online store + - 5Gi for registry + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + storageClassName: + description: |- + StorageClassName is the name of an existing StorageClass to which this persistent volume belongs. Empty value + means that this volume does not belong to any StorageClass and the cluster default will be used. + type: string + type: object + x-kubernetes-validations: + - message: PvcCreate is immutable + rule: self == oldSelf + mountPath: + description: |- + MountPath within the container at which the volume should be mounted. + Must start by "/" and cannot contain ':'. + type: string + ref: + description: Reference to an existing field + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. + type: string + type: object + x-kubernetes-map-type: atomic + required: + - mountPath + type: object + x-kubernetes-validations: + - message: One selection is required between ref + and create. + rule: '[has(self.ref), has(self.create)].exists_one(c, + c)' + - message: Mount path must start with '/' and + must not contain ':' + rule: self.mountPath.matches('^/[^:]*$') + type: object + x-kubernetes-validations: + - message: Ephemeral stores must have absolute paths. + rule: '(!has(self.pvc) && has(self.path)) ? self.path.startsWith(''/'') + : true' + - message: PVC path must be a file name only, with + no slashes. + rule: '(has(self.pvc) && has(self.path)) ? !self.path.startsWith(''/'') + : true' + - message: Online store does not support S3 or GS + buckets. + rule: 'has(self.path) ? !(self.path.startsWith(''s3://'') + || self.path.startsWith(''gs://'')) : true' + store: + description: OnlineStoreDBStorePersistence configures + the DB store persistence for the offline store service + properties: + secretKeyName: + description: By default, the selected store "type" + is used as the SecretKeyName + type: string + secretRef: + description: Data store parameters should be placed + as-is from the "feature_store.yaml" under the + secret key. "registry_type" & "type" fields + should be removed. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. + type: string + type: object + x-kubernetes-map-type: atomic + type: + enum: + - snowflake.online + - redis + - ikv + - datastore + - dynamodb + - bigtable + - postgres + - cassandra + - mysql + - hazelcast + - singlestore + - hbase + - elasticsearch + - qdrant + - couchbase + - milvus + type: string + required: + - secretRef + - type + type: object + type: object + x-kubernetes-validations: + - message: One selection required between file or store. + rule: '[has(self.file), has(self.store)].exists_one(c, + c)' resources: description: ResourceRequirements describes the compute resource requirements. @@ -1019,6 +2587,49 @@ spec: More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ type: object type: object + tls: + description: TlsConfigs configures server TLS for a feast + service. in an openshift cluster, this is configured + by default using service serving certificates. + properties: + disable: + description: will disable TLS for the feast service. + useful in an openshift cluster, for example, where + TLS is configured by default + type: boolean + secretKeyNames: + description: SecretKeyNames defines the secret key + names for the TLS key and cert. + properties: + tlsCrt: + description: defaults to "tls.crt" + type: string + tlsKey: + description: defaults to "tls.key" + type: string + type: object + secretRef: + description: references the local k8s secret where + the TLS key and cert reside + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. + type: string + type: object + x-kubernetes-map-type: atomic + type: object + x-kubernetes-validations: + - message: '`secretRef` required if `disable` is false.' + rule: '(!has(self.disable) || !self.disable) ? has(self.secretRef) + : true' type: object registry: description: Registry configures the registry service. One @@ -1060,10 +2671,15 @@ spec: description: The key to select. type: string name: + default: "" description: |- Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string optional: description: Specify whether the ConfigMap @@ -1127,10 +2743,15 @@ spec: key. type: string name: + default: "" description: |- Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string optional: description: Specify whether the Secret @@ -1145,12 +2766,242 @@ spec: - name type: object type: array + envFrom: + items: + description: EnvFromSource represents the source + of a set of ConfigMaps + properties: + configMapRef: + description: The ConfigMap to select from + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. + type: string + optional: + description: Specify whether the ConfigMap + must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + prefix: + description: An optional identifier to prepend + to each key in the ConfigMap. Must be a C_IDENTIFIER. + type: string + secretRef: + description: The Secret to select from + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. + type: string + optional: + description: Specify whether the Secret + must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + type: object + type: array image: type: string imagePullPolicy: description: PullPolicy describes a policy for if/when to pull a container image type: string + logLevel: + description: |- + LogLevel sets the logging level for the registry service + Allowed values: "debug", "info", "warning", "error", "critical". + enum: + - debug + - info + - warning + - error + - critical + type: string + persistence: + description: RegistryPersistence configures the persistence + settings for the registry service + properties: + file: + description: RegistryFilePersistence configures + the file-based persistence for the registry + service + properties: + path: + type: string + pvc: + description: |- + PvcConfig defines the settings for a persistent file store based on PVCs. + We can refer to an existing PVC using the `Ref` field, or create a new one using the `Create` field. + properties: + create: + description: Settings for creating a new + PVC + properties: + accessModes: + description: AccessModes k8s persistent + volume access modes. Defaults to + ["ReadWriteOnce"]. + items: + type: string + type: array + resources: + description: |- + Resources describes the storage resource requirements for a volume. + Default requested storage size depends on the associated service: + - 10Gi for offline store + - 5Gi for online store + - 5Gi for registry + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + storageClassName: + description: |- + StorageClassName is the name of an existing StorageClass to which this persistent volume belongs. Empty value + means that this volume does not belong to any StorageClass and the cluster default will be used. + type: string + type: object + x-kubernetes-validations: + - message: PvcCreate is immutable + rule: self == oldSelf + mountPath: + description: |- + MountPath within the container at which the volume should be mounted. + Must start by "/" and cannot contain ':'. + type: string + ref: + description: Reference to an existing + field + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. + type: string + type: object + x-kubernetes-map-type: atomic + required: + - mountPath + type: object + x-kubernetes-validations: + - message: One selection is required between + ref and create. + rule: '[has(self.ref), has(self.create)].exists_one(c, + c)' + - message: Mount path must start with '/' + and must not contain ':' + rule: self.mountPath.matches('^/[^:]*$') + s3_additional_kwargs: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-validations: + - message: Registry files must use absolute paths + or be S3 ('s3://') or GS ('gs://') object + store URIs. + rule: '(!has(self.pvc) && has(self.path)) ? + (self.path.startsWith(''/'') || self.path.startsWith(''s3://'') + || self.path.startsWith(''gs://'')) : true' + - message: PVC path must be a file name only, + with no slashes. + rule: '(has(self.pvc) && has(self.path)) ? !self.path.startsWith(''/'') + : true' + - message: PVC persistence does not support S3 + or GS object store URIs. + rule: '(has(self.pvc) && has(self.path)) ? !(self.path.startsWith(''s3://'') + || self.path.startsWith(''gs://'')) : true' + - message: Additional S3 settings are available + only for S3 object store URIs. + rule: '(has(self.s3_additional_kwargs) && has(self.path)) + ? self.path.startsWith(''s3://'') : true' + store: + description: RegistryDBStorePersistence configures + the DB store persistence for the registry service + properties: + secretKeyName: + description: By default, the selected store + "type" is used as the SecretKeyName + type: string + secretRef: + description: Data store parameters should + be placed as-is from the "feature_store.yaml" + under the secret key. "registry_type" & + "type" fields should be removed. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. + type: string + type: object + x-kubernetes-map-type: atomic + type: + enum: + - sql + - snowflake.registry + type: string + required: + - secretRef + - type + type: object + type: object + x-kubernetes-validations: + - message: One selection required between file or + store. + rule: '[has(self.file), has(self.store)].exists_one(c, + c)' resources: description: ResourceRequirements describes the compute resource requirements. @@ -1208,6 +3059,49 @@ spec: More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ type: object type: object + tls: + description: TlsConfigs configures server TLS for + a feast service. in an openshift cluster, this is + configured by default using service serving certificates. + properties: + disable: + description: will disable TLS for the feast service. + useful in an openshift cluster, for example, + where TLS is configured by default + type: boolean + secretKeyNames: + description: SecretKeyNames defines the secret + key names for the TLS key and cert. + properties: + tlsCrt: + description: defaults to "tls.crt" + type: string + tlsKey: + description: defaults to "tls.key" + type: string + type: object + secretRef: + description: references the local k8s secret where + the TLS key and cert reside + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. + type: string + type: object + x-kubernetes-map-type: atomic + type: object + x-kubernetes-validations: + - message: '`secretRef` required if `disable` is false.' + rule: '(!has(self.disable) || !self.disable) ? has(self.secretRef) + : true' type: object remote: description: |- @@ -1231,6 +3125,37 @@ spec: description: Host address of the remote registry service - :, e.g. `registry..svc.cluster.local:80` type: string + tls: + description: TlsRemoteRegistryConfigs configures client + TLS for a remote feast registry. in an openshift + cluster, this is configured by default when the + remote feast registry is using service serving certificates. + properties: + certName: + description: defines the configmap key name for + the client TLS cert. + type: string + configMapRef: + description: references the local k8s configmap + where the TLS cert resides + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. + type: string + type: object + x-kubernetes-map-type: atomic + required: + - certName + - configMapRef + type: object type: object x-kubernetes-validations: - message: One selection required. @@ -1319,7 +3244,6 @@ spec: type: object type: array feastVersion: - description: Version of feast that's currently deployed type: string phase: type: string diff --git a/infra/feast-operator/bundle/metadata/annotations.yaml b/infra/feast-operator/bundle/metadata/annotations.yaml index bf929b9755b..5e280a43e24 100644 --- a/infra/feast-operator/bundle/metadata/annotations.yaml +++ b/infra/feast-operator/bundle/metadata/annotations.yaml @@ -5,7 +5,7 @@ annotations: operators.operatorframework.io.bundle.metadata.v1: metadata/ operators.operatorframework.io.bundle.package.v1: feast-operator operators.operatorframework.io.bundle.channels.v1: alpha - operators.operatorframework.io.metrics.builder: operator-sdk-v1.37.0 + operators.operatorframework.io.metrics.builder: operator-sdk-v1.38.0 operators.operatorframework.io.metrics.mediatype.v1: metrics+v1 operators.operatorframework.io.metrics.project_layout: go.kubebuilder.io/v4 diff --git a/infra/feast-operator/cmd/main.go b/infra/feast-operator/cmd/main.go index 23a0309041b..cf7b9a76f21 100644 --- a/infra/feast-operator/cmd/main.go +++ b/infra/feast-operator/cmd/main.go @@ -33,13 +33,14 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/healthz" "sigs.k8s.io/controller-runtime/pkg/log/zap" + "sigs.k8s.io/controller-runtime/pkg/metrics/filters" metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" "sigs.k8s.io/controller-runtime/pkg/webhook" feastdevv1alpha1 "github.com/feast-dev/feast/infra/feast-operator/api/v1alpha1" "github.com/feast-dev/feast/infra/feast-operator/internal/controller" "github.com/feast-dev/feast/infra/feast-operator/internal/controller/services" - //+kubebuilder:scaffold:imports + // +kubebuilder:scaffold:imports ) var ( @@ -51,7 +52,7 @@ func init() { utilruntime.Must(clientgoscheme.AddToScheme(scheme)) utilruntime.Must(feastdevv1alpha1.AddToScheme(scheme)) - //+kubebuilder:scaffold:scheme + // +kubebuilder:scaffold:scheme } func main() { @@ -60,13 +61,15 @@ func main() { var probeAddr string var secureMetrics bool var enableHTTP2 bool - flag.StringVar(&metricsAddr, "metrics-bind-address", ":8080", "The address the metric endpoint binds to.") + var tlsOpts []func(*tls.Config) + flag.StringVar(&metricsAddr, "metrics-bind-address", "0", "The address the metrics endpoint binds to. "+ + "Use :8443 for HTTPS or :8080 for HTTP, or leave as 0 to disable the metrics service.") flag.StringVar(&probeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.") flag.BoolVar(&enableLeaderElection, "leader-elect", false, "Enable leader election for controller manager. "+ "Enabling this will ensure there is only one active controller manager.") - flag.BoolVar(&secureMetrics, "metrics-secure", false, - "If set the metrics endpoint is served securely") + flag.BoolVar(&secureMetrics, "metrics-secure", true, + "If set, the metrics endpoint is served securely via HTTPS. Use --metrics-secure=false to use HTTP instead.") flag.BoolVar(&enableHTTP2, "enable-http2", false, "If set, HTTP/2 will be enabled for the metrics and webhook servers") opts := zap.Options{ @@ -88,7 +91,6 @@ func main() { c.NextProtos = []string{"http/1.1"} } - tlsOpts := []func(*tls.Config){} if !enableHTTP2 { tlsOpts = append(tlsOpts, disableHTTP2) } @@ -97,13 +99,33 @@ func main() { TLSOpts: tlsOpts, }) + // Metrics endpoint is enabled in 'config/default/kustomization.yaml'. The Metrics options configure the server. + // More info: + // - https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.18.4/pkg/metrics/server + // - https://book.kubebuilder.io/reference/metrics.html + metricsServerOptions := metricsserver.Options{ + BindAddress: metricsAddr, + SecureServing: secureMetrics, + // TODO(user): TLSOpts is used to allow configuring the TLS config used for the server. If certificates are + // not provided, self-signed certificates will be generated by default. This option is not recommended for + // production environments as self-signed certificates do not offer the same level of trust and security + // as certificates issued by a trusted Certificate Authority (CA). The primary risk is potentially allowing + // unauthorized access to sensitive metrics data. Consider replacing with CertDir, CertName, and KeyName + // to provide certificates, ensuring the server communicates using trusted and secure certificates. + TLSOpts: tlsOpts, + } + + if secureMetrics { + // FilterProvider is used to protect the metrics endpoint with authn/authz. + // These configurations ensure that only authorized users and service accounts + // can access the metrics endpoint. The RBAC are configured in 'config/rbac/kustomization.yaml'. More info: + // https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.18.4/pkg/metrics/filters#WithAuthenticationAndAuthorization + metricsServerOptions.FilterProvider = filters.WithAuthenticationAndAuthorization + } + mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{ - Scheme: scheme, - Metrics: metricsserver.Options{ - BindAddress: metricsAddr, - SecureServing: secureMetrics, - TLSOpts: tlsOpts, - }, + Scheme: scheme, + Metrics: metricsServerOptions, WebhookServer: webhookServer, HealthProbeBindAddress: probeAddr, LeaderElection: enableLeaderElection, @@ -142,7 +164,7 @@ func main() { setupLog.Error(err, "unable to create controller", "controller", "FeatureStore") os.Exit(1) } - //+kubebuilder:scaffold:builder + // +kubebuilder:scaffold:builder if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil { setupLog.Error(err, "unable to set up health check") diff --git a/infra/feast-operator/config/crd/bases/feast.dev_featurestores.yaml b/infra/feast-operator/config/crd/bases/feast.dev_featurestores.yaml index 1402a64056c..a509caf329f 100644 --- a/infra/feast-operator/config/crd/bases/feast.dev_featurestores.yaml +++ b/infra/feast-operator/config/crd/bases/feast.dev_featurestores.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.14.0 + controller-gen.kubebuilder.io/version: v0.15.0 name: featurestores.feast.dev spec: group: feast.dev @@ -80,10 +80,15 @@ spec: referenced object inside the same namespace. properties: name: + default: "" description: |- Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string type: object x-kubernetes-map-type: atomic @@ -101,9 +106,63 @@ spec: pattern: ^[A-Za-z0-9][A-Za-z0-9_]*$ type: string services: - description: FeatureStoreServices defines the desired feast service - deployments. ephemeral registry is deployed by default. + description: FeatureStoreServices defines the desired feast services. + An ephemeral registry is deployed by default. properties: + deploymentStrategy: + description: DeploymentStrategy describes how to replace existing + pods with new ones. + properties: + rollingUpdate: + description: |- + Rolling update config params. Present only if DeploymentStrategyType = + RollingUpdate. + --- + TODO: Update this to follow our convention for oneOf, whatever we decide it + to be. + properties: + maxSurge: + anyOf: + - type: integer + - type: string + description: |- + The maximum number of pods that can be scheduled above the desired number of + pods. + Value can be an absolute number (ex: 5) or a percentage of desired pods (ex: 10%). + This can not be 0 if MaxUnavailable is 0. + Absolute number is calculated from percentage by rounding up. + Defaults to 25%. + Example: when this is set to 30%, the new ReplicaSet can be scaled up immediately when + the rolling update starts, such that the total number of old and new pods do not exceed + 130% of desired pods. Once old pods have been killed, + new ReplicaSet can be scaled up further, ensuring that total number of pods running + at any time during the update is at most 130% of desired pods. + x-kubernetes-int-or-string: true + maxUnavailable: + anyOf: + - type: integer + - type: string + description: |- + The maximum number of pods that can be unavailable during the update. + Value can be an absolute number (ex: 5) or a percentage of desired pods (ex: 10%). + Absolute number is calculated from percentage by rounding down. + This can not be 0 if MaxSurge is 0. + Defaults to 25%. + Example: when this is set to 30%, the old ReplicaSet can be scaled down to 70% of desired pods + immediately when the rolling update starts. Once new pods are ready, old ReplicaSet + can be scaled down further, followed by scaling up the new ReplicaSet, ensuring + that the total number of pods available at all times during the update is at + least 70% of desired pods. + x-kubernetes-int-or-string: true + type: object + type: + description: Type of deployment. Can be "Recreate" or "RollingUpdate". + Default is RollingUpdate. + type: string + type: object + disableInitContainers: + description: Disable the 'feast repo initialization' initContainer + type: boolean offlineStore: description: OfflineStore configures the deployed offline store service @@ -140,10 +199,15 @@ spec: description: The key to select. type: string name: + default: "" description: |- Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string optional: description: Specify whether the ConfigMap or @@ -203,10 +267,15 @@ spec: from. Must be a valid secret key. type: string name: + default: "" description: |- Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string optional: description: Specify whether the Secret or its @@ -221,12 +290,74 @@ spec: - name type: object type: array + envFrom: + items: + description: EnvFromSource represents the source of a set + of ConfigMaps + properties: + configMapRef: + description: The ConfigMap to select from + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. + type: string + optional: + description: Specify whether the ConfigMap must + be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + prefix: + description: An optional identifier to prepend to each + key in the ConfigMap. Must be a C_IDENTIFIER. + type: string + secretRef: + description: The Secret to select from + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. + type: string + optional: + description: Specify whether the Secret must be + defined + type: boolean + type: object + x-kubernetes-map-type: atomic + type: object + type: array image: type: string imagePullPolicy: description: PullPolicy describes a policy for if/when to pull a container image type: string + logLevel: + description: |- + LogLevel sets the logging level for the offline store service + Allowed values: "debug", "info", "warning", "error", "critical". + enum: + - debug + - info + - warning + - error + - critical + type: string persistence: description: OfflineStorePersistence configures the persistence settings for the offline store service @@ -243,6 +374,12 @@ spec: create: description: Settings for creating a new PVC properties: + accessModes: + description: AccessModes k8s persistent volume + access modes. Defaults to ["ReadWriteOnce"]. + items: + type: string + type: array resources: description: |- Resources describes the storage resource requirements for a volume. @@ -294,13 +431,20 @@ spec: description: Reference to an existing field properties: name: + default: "" description: |- Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string type: object x-kubernetes-map-type: atomic + required: + - mountPath type: object x-kubernetes-validations: - message: One selection is required between ref and @@ -312,6 +456,7 @@ spec: rule: self.mountPath.matches('^/[^:]*$') type: enum: + - file - dask - duckdb type: string @@ -330,10 +475,15 @@ spec: key. "registry_type" & "type" fields should be removed. properties: name: + default: "" description: |- Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string type: object x-kubernetes-map-type: atomic @@ -344,8 +494,10 @@ spec: - redshift - spark - postgres - - feast_trino.trino.TrinoOfflineStore + - trino - redis + - athena + - mssql type: string required: - secretRef @@ -412,9 +564,9 @@ spec: type: object type: object tls: - description: OfflineTlsConfigs configures server TLS for the - offline feast service. in an openshift cluster, this is - configured by default using service serving certificates. + description: TlsConfigs configures server TLS for a feast + service. in an openshift cluster, this is configured by + default using service serving certificates. properties: disable: description: will disable TLS for the feast service. useful @@ -437,16 +589,18 @@ spec: TLS key and cert reside properties: name: + default: "" description: |- Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string type: object x-kubernetes-map-type: atomic - verifyClient: - description: verify the client TLS certificate. - type: boolean type: object x-kubernetes-validations: - message: '`secretRef` required if `disable` is false.' @@ -489,10 +643,15 @@ spec: description: The key to select. type: string name: + default: "" description: |- Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string optional: description: Specify whether the ConfigMap or @@ -552,10 +711,15 @@ spec: from. Must be a valid secret key. type: string name: + default: "" description: |- Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string optional: description: Specify whether the Secret or its @@ -570,12 +734,74 @@ spec: - name type: object type: array + envFrom: + items: + description: EnvFromSource represents the source of a set + of ConfigMaps + properties: + configMapRef: + description: The ConfigMap to select from + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. + type: string + optional: + description: Specify whether the ConfigMap must + be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + prefix: + description: An optional identifier to prepend to each + key in the ConfigMap. Must be a C_IDENTIFIER. + type: string + secretRef: + description: The Secret to select from + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. + type: string + optional: + description: Specify whether the Secret must be + defined + type: boolean + type: object + x-kubernetes-map-type: atomic + type: object + type: array image: type: string imagePullPolicy: description: PullPolicy describes a policy for if/when to pull a container image type: string + logLevel: + description: |- + LogLevel sets the logging level for the online store service + Allowed values: "debug", "info", "warning", "error", "critical". + enum: + - debug + - info + - warning + - error + - critical + type: string persistence: description: OnlineStorePersistence configures the persistence settings for the online store service @@ -594,6 +820,12 @@ spec: create: description: Settings for creating a new PVC properties: + accessModes: + description: AccessModes k8s persistent volume + access modes. Defaults to ["ReadWriteOnce"]. + items: + type: string + type: array resources: description: |- Resources describes the storage resource requirements for a volume. @@ -645,13 +877,20 @@ spec: description: Reference to an existing field properties: name: + default: "" description: |- Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string type: object x-kubernetes-map-type: atomic + required: + - mountPath type: object x-kubernetes-validations: - message: One selection is required between ref and @@ -671,8 +910,8 @@ spec: rule: '(has(self.pvc) && has(self.path)) ? !self.path.startsWith(''/'') : true' - message: Online store does not support S3 or GS buckets. - rule: has(self.path) && !self.path.startsWith('s3://') - && !self.path.startsWith('gs://') + rule: 'has(self.path) ? !(self.path.startsWith(''s3://'') + || self.path.startsWith(''gs://'')) : true' store: description: OnlineStoreDBStorePersistence configures the DB store persistence for the offline store service @@ -687,10 +926,15 @@ spec: key. "registry_type" & "type" fields should be removed. properties: name: + default: "" description: |- Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string type: object x-kubernetes-map-type: atomic @@ -707,6 +951,11 @@ spec: - mysql - hazelcast - singlestore + - hbase + - elasticsearch + - qdrant + - couchbase + - milvus type: string required: - secretRef @@ -798,10 +1047,15 @@ spec: TLS key and cert reside properties: name: + default: "" description: |- Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string type: object x-kubernetes-map-type: atomic @@ -851,10 +1105,15 @@ spec: description: The key to select. type: string name: + default: "" description: |- Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string optional: description: Specify whether the ConfigMap @@ -915,10 +1174,15 @@ spec: from. Must be a valid secret key. type: string name: + default: "" description: |- Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string optional: description: Specify whether the Secret @@ -933,12 +1197,74 @@ spec: - name type: object type: array + envFrom: + items: + description: EnvFromSource represents the source of + a set of ConfigMaps + properties: + configMapRef: + description: The ConfigMap to select from + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. + type: string + optional: + description: Specify whether the ConfigMap must + be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + prefix: + description: An optional identifier to prepend to + each key in the ConfigMap. Must be a C_IDENTIFIER. + type: string + secretRef: + description: The Secret to select from + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. + type: string + optional: + description: Specify whether the Secret must + be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + type: object + type: array image: type: string imagePullPolicy: description: PullPolicy describes a policy for if/when to pull a container image type: string + logLevel: + description: |- + LogLevel sets the logging level for the registry service + Allowed values: "debug", "info", "warning", "error", "critical". + enum: + - debug + - info + - warning + - error + - critical + type: string persistence: description: RegistryPersistence configures the persistence settings for the registry service @@ -957,6 +1283,12 @@ spec: create: description: Settings for creating a new PVC properties: + accessModes: + description: AccessModes k8s persistent + volume access modes. Defaults to ["ReadWriteOnce"]. + items: + type: string + type: array resources: description: |- Resources describes the storage resource requirements for a volume. @@ -1008,13 +1340,20 @@ spec: description: Reference to an existing field properties: name: + default: "" description: |- Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string type: object x-kubernetes-map-type: atomic + required: + - mountPath type: object x-kubernetes-validations: - message: One selection is required between ref @@ -1063,10 +1402,15 @@ spec: should be removed. properties: name: + default: "" description: |- Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string type: object x-kubernetes-map-type: atomic @@ -1167,10 +1511,15 @@ spec: the TLS key and cert reside properties: name: + default: "" description: |- Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string type: object x-kubernetes-map-type: atomic @@ -1217,10 +1566,15 @@ spec: the TLS cert resides properties: name: + default: "" description: |- Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string type: object x-kubernetes-map-type: atomic @@ -1280,10 +1634,15 @@ spec: referenced object inside the same namespace. properties: name: + default: "" description: |- Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string type: object x-kubernetes-map-type: atomic @@ -1302,9 +1661,63 @@ spec: pattern: ^[A-Za-z0-9][A-Za-z0-9_]*$ type: string services: - description: FeatureStoreServices defines the desired feast service - deployments. ephemeral registry is deployed by default. + description: FeatureStoreServices defines the desired feast services. + An ephemeral registry is deployed by default. properties: + deploymentStrategy: + description: DeploymentStrategy describes how to replace existing + pods with new ones. + properties: + rollingUpdate: + description: |- + Rolling update config params. Present only if DeploymentStrategyType = + RollingUpdate. + --- + TODO: Update this to follow our convention for oneOf, whatever we decide it + to be. + properties: + maxSurge: + anyOf: + - type: integer + - type: string + description: |- + The maximum number of pods that can be scheduled above the desired number of + pods. + Value can be an absolute number (ex: 5) or a percentage of desired pods (ex: 10%). + This can not be 0 if MaxUnavailable is 0. + Absolute number is calculated from percentage by rounding up. + Defaults to 25%. + Example: when this is set to 30%, the new ReplicaSet can be scaled up immediately when + the rolling update starts, such that the total number of old and new pods do not exceed + 130% of desired pods. Once old pods have been killed, + new ReplicaSet can be scaled up further, ensuring that total number of pods running + at any time during the update is at most 130% of desired pods. + x-kubernetes-int-or-string: true + maxUnavailable: + anyOf: + - type: integer + - type: string + description: |- + The maximum number of pods that can be unavailable during the update. + Value can be an absolute number (ex: 5) or a percentage of desired pods (ex: 10%). + Absolute number is calculated from percentage by rounding down. + This can not be 0 if MaxSurge is 0. + Defaults to 25%. + Example: when this is set to 30%, the old ReplicaSet can be scaled down to 70% of desired pods + immediately when the rolling update starts. Once new pods are ready, old ReplicaSet + can be scaled down further, followed by scaling up the new ReplicaSet, ensuring + that the total number of pods available at all times during the update is at + least 70% of desired pods. + x-kubernetes-int-or-string: true + type: object + type: + description: Type of deployment. Can be "Recreate" or + "RollingUpdate". Default is RollingUpdate. + type: string + type: object + disableInitContainers: + description: Disable the 'feast repo initialization' initContainer + type: boolean offlineStore: description: OfflineStore configures the deployed offline store service @@ -1341,10 +1754,15 @@ spec: description: The key to select. type: string name: + default: "" description: |- Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string optional: description: Specify whether the ConfigMap @@ -1405,10 +1823,15 @@ spec: from. Must be a valid secret key. type: string name: + default: "" description: |- Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string optional: description: Specify whether the Secret @@ -1423,12 +1846,74 @@ spec: - name type: object type: array + envFrom: + items: + description: EnvFromSource represents the source of + a set of ConfigMaps + properties: + configMapRef: + description: The ConfigMap to select from + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. + type: string + optional: + description: Specify whether the ConfigMap must + be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + prefix: + description: An optional identifier to prepend to + each key in the ConfigMap. Must be a C_IDENTIFIER. + type: string + secretRef: + description: The Secret to select from + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. + type: string + optional: + description: Specify whether the Secret must + be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + type: object + type: array image: type: string imagePullPolicy: description: PullPolicy describes a policy for if/when to pull a container image type: string + logLevel: + description: |- + LogLevel sets the logging level for the offline store service + Allowed values: "debug", "info", "warning", "error", "critical". + enum: + - debug + - info + - warning + - error + - critical + type: string persistence: description: OfflineStorePersistence configures the persistence settings for the offline store service @@ -1446,6 +1931,12 @@ spec: create: description: Settings for creating a new PVC properties: + accessModes: + description: AccessModes k8s persistent + volume access modes. Defaults to ["ReadWriteOnce"]. + items: + type: string + type: array resources: description: |- Resources describes the storage resource requirements for a volume. @@ -1497,13 +1988,20 @@ spec: description: Reference to an existing field properties: name: + default: "" description: |- Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string type: object x-kubernetes-map-type: atomic + required: + - mountPath type: object x-kubernetes-validations: - message: One selection is required between ref @@ -1515,6 +2013,7 @@ spec: rule: self.mountPath.matches('^/[^:]*$') type: enum: + - file - dask - duckdb type: string @@ -1534,10 +2033,15 @@ spec: should be removed. properties: name: + default: "" description: |- Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string type: object x-kubernetes-map-type: atomic @@ -1548,8 +2052,10 @@ spec: - redshift - spark - postgres - - feast_trino.trino.TrinoOfflineStore + - trino - redis + - athena + - mssql type: string required: - secretRef @@ -1618,10 +2124,9 @@ spec: type: object type: object tls: - description: OfflineTlsConfigs configures server TLS for - the offline feast service. in an openshift cluster, - this is configured by default using service serving - certificates. + description: TlsConfigs configures server TLS for a feast + service. in an openshift cluster, this is configured + by default using service serving certificates. properties: disable: description: will disable TLS for the feast service. @@ -1644,16 +2149,18 @@ spec: the TLS key and cert reside properties: name: + default: "" description: |- Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string type: object x-kubernetes-map-type: atomic - verifyClient: - description: verify the client TLS certificate. - type: boolean type: object x-kubernetes-validations: - message: '`secretRef` required if `disable` is false.' @@ -1696,10 +2203,15 @@ spec: description: The key to select. type: string name: + default: "" description: |- Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string optional: description: Specify whether the ConfigMap @@ -1760,10 +2272,15 @@ spec: from. Must be a valid secret key. type: string name: + default: "" description: |- Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string optional: description: Specify whether the Secret @@ -1778,12 +2295,74 @@ spec: - name type: object type: array + envFrom: + items: + description: EnvFromSource represents the source of + a set of ConfigMaps + properties: + configMapRef: + description: The ConfigMap to select from + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. + type: string + optional: + description: Specify whether the ConfigMap must + be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + prefix: + description: An optional identifier to prepend to + each key in the ConfigMap. Must be a C_IDENTIFIER. + type: string + secretRef: + description: The Secret to select from + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. + type: string + optional: + description: Specify whether the Secret must + be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + type: object + type: array image: type: string imagePullPolicy: description: PullPolicy describes a policy for if/when to pull a container image type: string + logLevel: + description: |- + LogLevel sets the logging level for the online store service + Allowed values: "debug", "info", "warning", "error", "critical". + enum: + - debug + - info + - warning + - error + - critical + type: string persistence: description: OnlineStorePersistence configures the persistence settings for the online store service @@ -1803,6 +2382,12 @@ spec: create: description: Settings for creating a new PVC properties: + accessModes: + description: AccessModes k8s persistent + volume access modes. Defaults to ["ReadWriteOnce"]. + items: + type: string + type: array resources: description: |- Resources describes the storage resource requirements for a volume. @@ -1854,13 +2439,20 @@ spec: description: Reference to an existing field properties: name: + default: "" description: |- Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string type: object x-kubernetes-map-type: atomic + required: + - mountPath type: object x-kubernetes-validations: - message: One selection is required between ref @@ -1881,8 +2473,8 @@ spec: : true' - message: Online store does not support S3 or GS buckets. - rule: has(self.path) && !self.path.startsWith('s3://') - && !self.path.startsWith('gs://') + rule: 'has(self.path) ? !(self.path.startsWith(''s3://'') + || self.path.startsWith(''gs://'')) : true' store: description: OnlineStoreDBStorePersistence configures the DB store persistence for the offline store service @@ -1898,10 +2490,15 @@ spec: should be removed. properties: name: + default: "" description: |- Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string type: object x-kubernetes-map-type: atomic @@ -1918,6 +2515,11 @@ spec: - mysql - hazelcast - singlestore + - hbase + - elasticsearch + - qdrant + - couchbase + - milvus type: string required: - secretRef @@ -2011,10 +2613,15 @@ spec: the TLS key and cert reside properties: name: + default: "" description: |- Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string type: object x-kubernetes-map-type: atomic @@ -2064,10 +2671,15 @@ spec: description: The key to select. type: string name: + default: "" description: |- Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string optional: description: Specify whether the ConfigMap @@ -2131,10 +2743,15 @@ spec: key. type: string name: + default: "" description: |- Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string optional: description: Specify whether the Secret @@ -2149,12 +2766,74 @@ spec: - name type: object type: array + envFrom: + items: + description: EnvFromSource represents the source + of a set of ConfigMaps + properties: + configMapRef: + description: The ConfigMap to select from + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. + type: string + optional: + description: Specify whether the ConfigMap + must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + prefix: + description: An optional identifier to prepend + to each key in the ConfigMap. Must be a C_IDENTIFIER. + type: string + secretRef: + description: The Secret to select from + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. + type: string + optional: + description: Specify whether the Secret + must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + type: object + type: array image: type: string imagePullPolicy: description: PullPolicy describes a policy for if/when to pull a container image type: string + logLevel: + description: |- + LogLevel sets the logging level for the registry service + Allowed values: "debug", "info", "warning", "error", "critical". + enum: + - debug + - info + - warning + - error + - critical + type: string persistence: description: RegistryPersistence configures the persistence settings for the registry service @@ -2175,6 +2854,13 @@ spec: description: Settings for creating a new PVC properties: + accessModes: + description: AccessModes k8s persistent + volume access modes. Defaults to + ["ReadWriteOnce"]. + items: + type: string + type: array resources: description: |- Resources describes the storage resource requirements for a volume. @@ -2227,13 +2913,20 @@ spec: field properties: name: + default: "" description: |- Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string type: object x-kubernetes-map-type: atomic + required: + - mountPath type: object x-kubernetes-validations: - message: One selection is required between @@ -2282,10 +2975,15 @@ spec: "type" fields should be removed. properties: name: + default: "" description: |- Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string type: object x-kubernetes-map-type: atomic @@ -2387,10 +3085,15 @@ spec: the TLS key and cert reside properties: name: + default: "" description: |- Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string type: object x-kubernetes-map-type: atomic @@ -2437,10 +3140,15 @@ spec: where the TLS cert resides properties: name: + default: "" description: |- Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string type: object x-kubernetes-map-type: atomic @@ -2536,7 +3244,6 @@ spec: type: object type: array feastVersion: - description: Version of feast that's currently deployed type: string phase: type: string diff --git a/infra/feast-operator/config/default/kustomization.yaml b/infra/feast-operator/config/default/kustomization.yaml index 957965b9b35..01534ae5cc8 100644 --- a/infra/feast-operator/config/default/kustomization.yaml +++ b/infra/feast-operator/config/default/kustomization.yaml @@ -25,12 +25,19 @@ resources: #- ../certmanager # [PROMETHEUS] To enable prometheus monitor, uncomment all sections with 'PROMETHEUS'. #- ../prometheus +# [METRICS] Expose the controller manager metrics service. +- metrics_service.yaml +# Uncomment the patches line if you enable Metrics, and/or are using webhooks and cert-manager patches: -# Protect the /metrics endpoint by putting it behind auth. -# If you want your controller-manager to expose the /metrics -# endpoint w/o any authn/z, please comment the following line. -- path: manager_auth_proxy_patch.yaml +- path: manager_related_images_patch.yaml + target: + kind: Deployment +# [METRICS] The following patch will enable the metrics endpoint using HTTPS and the port :8443. +# More info: https://book.kubebuilder.io/reference/metrics +- path: manager_metrics_patch.yaml + target: + kind: Deployment # [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in # crd/kustomization.yaml diff --git a/infra/feast-operator/config/default/manager_auth_proxy_patch.yaml b/infra/feast-operator/config/default/manager_auth_proxy_patch.yaml deleted file mode 100644 index 4c3c27602f5..00000000000 --- a/infra/feast-operator/config/default/manager_auth_proxy_patch.yaml +++ /dev/null @@ -1,39 +0,0 @@ -# This patch inject a sidecar container which is a HTTP proxy for the -# controller manager, it performs RBAC authorization against the Kubernetes API using SubjectAccessReviews. -apiVersion: apps/v1 -kind: Deployment -metadata: - name: controller-manager - namespace: system -spec: - template: - spec: - containers: - - name: kube-rbac-proxy - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - "ALL" - image: gcr.io/kubebuilder/kube-rbac-proxy:v0.16.0 - args: - - "--secure-listen-address=0.0.0.0:8443" - - "--upstream=http://127.0.0.1:8080/" - - "--logtostderr=true" - - "--v=0" - ports: - - containerPort: 8443 - protocol: TCP - name: https - resources: - limits: - cpu: 500m - memory: 128Mi - requests: - cpu: 5m - memory: 64Mi - - name: manager - args: - - "--health-probe-bind-address=:8081" - - "--metrics-bind-address=127.0.0.1:8080" - - "--leader-elect" diff --git a/infra/feast-operator/config/default/manager_metrics_patch.yaml b/infra/feast-operator/config/default/manager_metrics_patch.yaml new file mode 100644 index 00000000000..2aaef6536f4 --- /dev/null +++ b/infra/feast-operator/config/default/manager_metrics_patch.yaml @@ -0,0 +1,4 @@ +# This patch adds the args to allow exposing the metrics endpoint using HTTPS +- op: add + path: /spec/template/spec/containers/0/args/0 + value: --metrics-bind-address=:8443 diff --git a/infra/feast-operator/config/default/manager_related_images_patch.yaml b/infra/feast-operator/config/default/manager_related_images_patch.yaml new file mode 100644 index 00000000000..084de876612 --- /dev/null +++ b/infra/feast-operator/config/default/manager_related_images_patch.yaml @@ -0,0 +1,10 @@ +- op: replace + path: "/spec/template/spec/containers/0/env/0" + value: + name: RELATED_IMAGE_FEATURE_SERVER + value: docker.io/feastdev/feature-server:0.43.0 +- op: replace + path: "/spec/template/spec/containers/0/env/1" + value: + name: RELATED_IMAGE_GRPC_CURL + value: docker.io/fullstorydev/grpcurl:v1.9.1-alpine diff --git a/infra/feast-operator/config/rbac/auth_proxy_service.yaml b/infra/feast-operator/config/default/metrics_service.yaml similarity index 94% rename from infra/feast-operator/config/rbac/auth_proxy_service.yaml rename to infra/feast-operator/config/default/metrics_service.yaml index c2bf4e37939..0207c0469d4 100644 --- a/infra/feast-operator/config/rbac/auth_proxy_service.yaml +++ b/infra/feast-operator/config/default/metrics_service.yaml @@ -12,6 +12,6 @@ spec: - name: https port: 8443 protocol: TCP - targetPort: https + targetPort: 8443 selector: control-plane: controller-manager diff --git a/infra/feast-operator/config/manager/kustomization.yaml b/infra/feast-operator/config/manager/kustomization.yaml index 2e0a046bf15..39b762031d5 100644 --- a/infra/feast-operator/config/manager/kustomization.yaml +++ b/infra/feast-operator/config/manager/kustomization.yaml @@ -5,4 +5,4 @@ kind: Kustomization images: - name: controller newName: feastdev/feast-operator - newTag: 0.42.0 + newTag: 0.43.0 diff --git a/infra/feast-operator/config/manager/manager.yaml b/infra/feast-operator/config/manager/manager.yaml index 90ef7b48635..4259cf8a7e0 100644 --- a/infra/feast-operator/config/manager/manager.yaml +++ b/infra/feast-operator/config/manager/manager.yaml @@ -62,6 +62,7 @@ spec: - /manager args: - --leader-elect + - --health-probe-bind-address=:8081 image: controller:latest name: manager securityContext: @@ -69,6 +70,11 @@ spec: capabilities: drop: - "ALL" + env: + - name: RELATED_IMAGE_FEATURE_SERVER + value: feast:latest + - name: RELATED_IMAGE_GRPC_CURL + value: grpc:latest livenessProbe: httpGet: path: /healthz diff --git a/infra/feast-operator/config/overlays/odh/delete-namespace.yaml b/infra/feast-operator/config/overlays/odh/delete-namespace.yaml new file mode 100644 index 00000000000..9a52c0573de --- /dev/null +++ b/infra/feast-operator/config/overlays/odh/delete-namespace.yaml @@ -0,0 +1,5 @@ +$patch: delete +apiVersion: v1 +kind: Namespace +metadata: + name: system diff --git a/infra/feast-operator/config/overlays/odh/kustomization.yaml b/infra/feast-operator/config/overlays/odh/kustomization.yaml new file mode 100644 index 00000000000..b74a0e4a4d7 --- /dev/null +++ b/infra/feast-operator/config/overlays/odh/kustomization.yaml @@ -0,0 +1,35 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +namespace: opendatahub + + +resources: +- ../../default + + +patches: + # patch to remove default `system` namespace in ../../manager/manager.yaml + - path: delete-namespace.yaml + +configMapGenerator: + - name: feast-operator-parameters + envs: + - params.env + +configurations: + - params.yaml + +replacements: + - source: + kind: ConfigMap + name: feast-operator-parameters + version: v1 + fieldPath: data.odh-feast-operator-controller-image + targets: + - select: + kind: Deployment + name: controller-manager + fieldPaths: + - spec.template.spec.containers.[name=manager].image + diff --git a/infra/feast-operator/config/overlays/odh/params.env b/infra/feast-operator/config/overlays/odh/params.env new file mode 100644 index 00000000000..df62943fbcf --- /dev/null +++ b/infra/feast-operator/config/overlays/odh/params.env @@ -0,0 +1 @@ +odh-feast-operator-controller-image=docker.io/feastdev/feast-operator:0.43.0 diff --git a/infra/feast-operator/config/overlays/odh/params.yaml b/infra/feast-operator/config/overlays/odh/params.yaml new file mode 100644 index 00000000000..43509ff293c --- /dev/null +++ b/infra/feast-operator/config/overlays/odh/params.yaml @@ -0,0 +1,3 @@ +varReference: + - path: spec/template/spec/containers[]/image + kind: Deployment diff --git a/infra/feast-operator/config/prometheus/monitor.yaml b/infra/feast-operator/config/prometheus/monitor.yaml index 55484079677..e76479a1305 100644 --- a/infra/feast-operator/config/prometheus/monitor.yaml +++ b/infra/feast-operator/config/prometheus/monitor.yaml @@ -11,10 +11,19 @@ metadata: spec: endpoints: - path: /metrics - port: https + port: https # Ensure this is the name of the port that exposes HTTPS metrics scheme: https bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token tlsConfig: + # TODO(user): The option insecureSkipVerify: true is not recommended for production since it disables + # certificate verification. This poses a significant security risk by making the system vulnerable to + # man-in-the-middle attacks, where an attacker could intercept and manipulate the communication between + # Prometheus and the monitored services. This could lead to unauthorized access to sensitive metrics data, + # compromising the integrity and confidentiality of the information. + # Please use the following options for secure configurations: + # caFile: /etc/metrics-certs/ca.crt + # certFile: /etc/metrics-certs/tls.crt + # keyFile: /etc/metrics-certs/tls.key insecureSkipVerify: true selector: matchLabels: diff --git a/infra/feast-operator/config/rbac/auth_proxy_role.yaml b/infra/feast-operator/config/rbac/auth_proxy_role.yaml deleted file mode 100644 index 55f87916462..00000000000 --- a/infra/feast-operator/config/rbac/auth_proxy_role.yaml +++ /dev/null @@ -1,20 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - app.kubernetes.io/name: feast-operator - app.kubernetes.io/managed-by: kustomize - name: proxy-role -rules: -- apiGroups: - - authentication.k8s.io - resources: - - tokenreviews - verbs: - - create -- apiGroups: - - authorization.k8s.io - resources: - - subjectaccessreviews - verbs: - - create diff --git a/infra/feast-operator/config/rbac/kustomization.yaml b/infra/feast-operator/config/rbac/kustomization.yaml index 5e4972b5397..d22437a5390 100644 --- a/infra/feast-operator/config/rbac/kustomization.yaml +++ b/infra/feast-operator/config/rbac/kustomization.yaml @@ -9,13 +9,15 @@ resources: - role_binding.yaml - leader_election_role.yaml - leader_election_role_binding.yaml -# Comment the following 4 lines if you want to disable -# the auth proxy (https://github.com/brancz/kube-rbac-proxy) -# which protects your /metrics endpoint. -- auth_proxy_service.yaml -- auth_proxy_role.yaml -- auth_proxy_role_binding.yaml -- auth_proxy_client_clusterrole.yaml +# The following RBAC configurations are used to protect +# the metrics endpoint with authn/authz. These configurations +# ensure that only authorized users and service accounts +# can access the metrics endpoint. Comment the following +# permissions if you want to disable this protection. +# More info: https://book.kubebuilder.io/reference/metrics.html +- metrics_auth_role.yaml +- metrics_auth_role_binding.yaml +- metrics_reader_role.yaml # For each CRD, "Editor" and "Viewer" roles are scaffolded by # default, aiding admins in cluster management. Those roles are # not used by the Project itself. You can comment the following lines diff --git a/infra/feast-operator/config/rbac/metrics_auth_role.yaml b/infra/feast-operator/config/rbac/metrics_auth_role.yaml new file mode 100644 index 00000000000..bee99788cf4 --- /dev/null +++ b/infra/feast-operator/config/rbac/metrics_auth_role.yaml @@ -0,0 +1,20 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/name: feast-operator + app.kubernetes.io/managed-by: kustomize + name: metrics-auth-role +rules: + - apiGroups: + - authentication.k8s.io + resources: + - tokenreviews + verbs: + - create + - apiGroups: + - authorization.k8s.io + resources: + - subjectaccessreviews + verbs: + - create diff --git a/infra/feast-operator/config/rbac/auth_proxy_role_binding.yaml b/infra/feast-operator/config/rbac/metrics_auth_role_binding.yaml similarity index 84% rename from infra/feast-operator/config/rbac/auth_proxy_role_binding.yaml rename to infra/feast-operator/config/rbac/metrics_auth_role_binding.yaml index ffa85c82af6..f84b6c4160c 100644 --- a/infra/feast-operator/config/rbac/auth_proxy_role_binding.yaml +++ b/infra/feast-operator/config/rbac/metrics_auth_role_binding.yaml @@ -4,11 +4,11 @@ metadata: labels: app.kubernetes.io/name: feast-operator app.kubernetes.io/managed-by: kustomize - name: proxy-rolebinding + name: metrics-auth-rolebinding roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole - name: proxy-role + name: metrics-auth-role subjects: - kind: ServiceAccount name: controller-manager diff --git a/infra/feast-operator/config/rbac/auth_proxy_client_clusterrole.yaml b/infra/feast-operator/config/rbac/metrics_reader_role.yaml similarity index 100% rename from infra/feast-operator/config/rbac/auth_proxy_client_clusterrole.yaml rename to infra/feast-operator/config/rbac/metrics_reader_role.yaml diff --git a/infra/feast-operator/config/samples/v1alpha1_featurestore_pvc_persistence.yaml b/infra/feast-operator/config/samples/v1alpha1_featurestore_pvc_persistence.yaml index b7c7412c0f0..15aa46c456c 100644 --- a/infra/feast-operator/config/samples/v1alpha1_featurestore_pvc_persistence.yaml +++ b/infra/feast-operator/config/samples/v1alpha1_featurestore_pvc_persistence.yaml @@ -5,6 +5,7 @@ metadata: spec: feastProject: my_project services: + # demonstrates using a pre-existing PVC onlineStore: persistence: file: @@ -13,6 +14,7 @@ spec: ref: name: online-pvc mountPath: /data/online + # demonstrates specifying a storageClassName and storage size offlineStore: persistence: file: @@ -24,6 +26,7 @@ spec: requests: storage: 5Gi mountPath: /data/offline + # demonstrates letting the Operator create a PVC w/ defaults set registry: local: persistence: @@ -39,7 +42,7 @@ metadata: name: online-pvc spec: accessModes: - - ReadWriteMany + - ReadWriteOnce resources: requests: storage: 5Gi diff --git a/infra/feast-operator/config/samples/v1alpha1_featurestore_services_loglevel.yaml b/infra/feast-operator/config/samples/v1alpha1_featurestore_services_loglevel.yaml new file mode 100644 index 00000000000..7ae96d44f01 --- /dev/null +++ b/infra/feast-operator/config/samples/v1alpha1_featurestore_services_loglevel.yaml @@ -0,0 +1,15 @@ +apiVersion: feast.dev/v1alpha1 +kind: FeatureStore +metadata: + name: sample-services-loglevel +spec: + feastProject: my_project + services: + onlineStore: + logLevel: debug + offlineStore: + logLevel: debug + registry: + local: + logLevel: info + diff --git a/infra/feast-operator/dist/install.yaml b/infra/feast-operator/dist/install.yaml index 18ab82e9ca2..ae9a37d8c9c 100644 --- a/infra/feast-operator/dist/install.yaml +++ b/infra/feast-operator/dist/install.yaml @@ -11,7 +11,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.14.0 + controller-gen.kubebuilder.io/version: v0.15.0 name: featurestores.feast.dev spec: group: feast.dev @@ -88,10 +88,15 @@ spec: referenced object inside the same namespace. properties: name: + default: "" description: |- Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string type: object x-kubernetes-map-type: atomic @@ -109,9 +114,63 @@ spec: pattern: ^[A-Za-z0-9][A-Za-z0-9_]*$ type: string services: - description: FeatureStoreServices defines the desired feast service - deployments. ephemeral registry is deployed by default. + description: FeatureStoreServices defines the desired feast services. + An ephemeral registry is deployed by default. properties: + deploymentStrategy: + description: DeploymentStrategy describes how to replace existing + pods with new ones. + properties: + rollingUpdate: + description: |- + Rolling update config params. Present only if DeploymentStrategyType = + RollingUpdate. + --- + TODO: Update this to follow our convention for oneOf, whatever we decide it + to be. + properties: + maxSurge: + anyOf: + - type: integer + - type: string + description: |- + The maximum number of pods that can be scheduled above the desired number of + pods. + Value can be an absolute number (ex: 5) or a percentage of desired pods (ex: 10%). + This can not be 0 if MaxUnavailable is 0. + Absolute number is calculated from percentage by rounding up. + Defaults to 25%. + Example: when this is set to 30%, the new ReplicaSet can be scaled up immediately when + the rolling update starts, such that the total number of old and new pods do not exceed + 130% of desired pods. Once old pods have been killed, + new ReplicaSet can be scaled up further, ensuring that total number of pods running + at any time during the update is at most 130% of desired pods. + x-kubernetes-int-or-string: true + maxUnavailable: + anyOf: + - type: integer + - type: string + description: |- + The maximum number of pods that can be unavailable during the update. + Value can be an absolute number (ex: 5) or a percentage of desired pods (ex: 10%). + Absolute number is calculated from percentage by rounding down. + This can not be 0 if MaxSurge is 0. + Defaults to 25%. + Example: when this is set to 30%, the old ReplicaSet can be scaled down to 70% of desired pods + immediately when the rolling update starts. Once new pods are ready, old ReplicaSet + can be scaled down further, followed by scaling up the new ReplicaSet, ensuring + that the total number of pods available at all times during the update is at + least 70% of desired pods. + x-kubernetes-int-or-string: true + type: object + type: + description: Type of deployment. Can be "Recreate" or "RollingUpdate". + Default is RollingUpdate. + type: string + type: object + disableInitContainers: + description: Disable the 'feast repo initialization' initContainer + type: boolean offlineStore: description: OfflineStore configures the deployed offline store service @@ -148,10 +207,15 @@ spec: description: The key to select. type: string name: + default: "" description: |- Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string optional: description: Specify whether the ConfigMap or @@ -211,10 +275,15 @@ spec: from. Must be a valid secret key. type: string name: + default: "" description: |- Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string optional: description: Specify whether the Secret or its @@ -229,12 +298,74 @@ spec: - name type: object type: array + envFrom: + items: + description: EnvFromSource represents the source of a set + of ConfigMaps + properties: + configMapRef: + description: The ConfigMap to select from + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. + type: string + optional: + description: Specify whether the ConfigMap must + be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + prefix: + description: An optional identifier to prepend to each + key in the ConfigMap. Must be a C_IDENTIFIER. + type: string + secretRef: + description: The Secret to select from + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. + type: string + optional: + description: Specify whether the Secret must be + defined + type: boolean + type: object + x-kubernetes-map-type: atomic + type: object + type: array image: type: string imagePullPolicy: description: PullPolicy describes a policy for if/when to pull a container image type: string + logLevel: + description: |- + LogLevel sets the logging level for the offline store service + Allowed values: "debug", "info", "warning", "error", "critical". + enum: + - debug + - info + - warning + - error + - critical + type: string persistence: description: OfflineStorePersistence configures the persistence settings for the offline store service @@ -251,6 +382,12 @@ spec: create: description: Settings for creating a new PVC properties: + accessModes: + description: AccessModes k8s persistent volume + access modes. Defaults to ["ReadWriteOnce"]. + items: + type: string + type: array resources: description: |- Resources describes the storage resource requirements for a volume. @@ -302,13 +439,20 @@ spec: description: Reference to an existing field properties: name: + default: "" description: |- Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string type: object x-kubernetes-map-type: atomic + required: + - mountPath type: object x-kubernetes-validations: - message: One selection is required between ref and @@ -320,6 +464,7 @@ spec: rule: self.mountPath.matches('^/[^:]*$') type: enum: + - file - dask - duckdb type: string @@ -338,10 +483,15 @@ spec: key. "registry_type" & "type" fields should be removed. properties: name: + default: "" description: |- Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string type: object x-kubernetes-map-type: atomic @@ -352,8 +502,10 @@ spec: - redshift - spark - postgres - - feast_trino.trino.TrinoOfflineStore + - trino - redis + - athena + - mssql type: string required: - secretRef @@ -420,9 +572,9 @@ spec: type: object type: object tls: - description: OfflineTlsConfigs configures server TLS for the - offline feast service. in an openshift cluster, this is - configured by default using service serving certificates. + description: TlsConfigs configures server TLS for a feast + service. in an openshift cluster, this is configured by + default using service serving certificates. properties: disable: description: will disable TLS for the feast service. useful @@ -445,16 +597,18 @@ spec: TLS key and cert reside properties: name: + default: "" description: |- Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string type: object x-kubernetes-map-type: atomic - verifyClient: - description: verify the client TLS certificate. - type: boolean type: object x-kubernetes-validations: - message: '`secretRef` required if `disable` is false.' @@ -497,10 +651,15 @@ spec: description: The key to select. type: string name: + default: "" description: |- Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string optional: description: Specify whether the ConfigMap or @@ -560,10 +719,15 @@ spec: from. Must be a valid secret key. type: string name: + default: "" description: |- Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string optional: description: Specify whether the Secret or its @@ -578,12 +742,74 @@ spec: - name type: object type: array + envFrom: + items: + description: EnvFromSource represents the source of a set + of ConfigMaps + properties: + configMapRef: + description: The ConfigMap to select from + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. + type: string + optional: + description: Specify whether the ConfigMap must + be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + prefix: + description: An optional identifier to prepend to each + key in the ConfigMap. Must be a C_IDENTIFIER. + type: string + secretRef: + description: The Secret to select from + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. + type: string + optional: + description: Specify whether the Secret must be + defined + type: boolean + type: object + x-kubernetes-map-type: atomic + type: object + type: array image: type: string imagePullPolicy: description: PullPolicy describes a policy for if/when to pull a container image type: string + logLevel: + description: |- + LogLevel sets the logging level for the online store service + Allowed values: "debug", "info", "warning", "error", "critical". + enum: + - debug + - info + - warning + - error + - critical + type: string persistence: description: OnlineStorePersistence configures the persistence settings for the online store service @@ -602,6 +828,12 @@ spec: create: description: Settings for creating a new PVC properties: + accessModes: + description: AccessModes k8s persistent volume + access modes. Defaults to ["ReadWriteOnce"]. + items: + type: string + type: array resources: description: |- Resources describes the storage resource requirements for a volume. @@ -653,13 +885,20 @@ spec: description: Reference to an existing field properties: name: + default: "" description: |- Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string type: object x-kubernetes-map-type: atomic + required: + - mountPath type: object x-kubernetes-validations: - message: One selection is required between ref and @@ -679,8 +918,8 @@ spec: rule: '(has(self.pvc) && has(self.path)) ? !self.path.startsWith(''/'') : true' - message: Online store does not support S3 or GS buckets. - rule: has(self.path) && !self.path.startsWith('s3://') - && !self.path.startsWith('gs://') + rule: 'has(self.path) ? !(self.path.startsWith(''s3://'') + || self.path.startsWith(''gs://'')) : true' store: description: OnlineStoreDBStorePersistence configures the DB store persistence for the offline store service @@ -695,10 +934,15 @@ spec: key. "registry_type" & "type" fields should be removed. properties: name: + default: "" description: |- Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string type: object x-kubernetes-map-type: atomic @@ -715,6 +959,11 @@ spec: - mysql - hazelcast - singlestore + - hbase + - elasticsearch + - qdrant + - couchbase + - milvus type: string required: - secretRef @@ -806,10 +1055,15 @@ spec: TLS key and cert reside properties: name: + default: "" description: |- Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string type: object x-kubernetes-map-type: atomic @@ -859,10 +1113,15 @@ spec: description: The key to select. type: string name: + default: "" description: |- Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string optional: description: Specify whether the ConfigMap @@ -923,10 +1182,15 @@ spec: from. Must be a valid secret key. type: string name: + default: "" description: |- Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string optional: description: Specify whether the Secret @@ -941,12 +1205,74 @@ spec: - name type: object type: array + envFrom: + items: + description: EnvFromSource represents the source of + a set of ConfigMaps + properties: + configMapRef: + description: The ConfigMap to select from + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. + type: string + optional: + description: Specify whether the ConfigMap must + be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + prefix: + description: An optional identifier to prepend to + each key in the ConfigMap. Must be a C_IDENTIFIER. + type: string + secretRef: + description: The Secret to select from + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. + type: string + optional: + description: Specify whether the Secret must + be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + type: object + type: array image: type: string imagePullPolicy: description: PullPolicy describes a policy for if/when to pull a container image type: string + logLevel: + description: |- + LogLevel sets the logging level for the registry service + Allowed values: "debug", "info", "warning", "error", "critical". + enum: + - debug + - info + - warning + - error + - critical + type: string persistence: description: RegistryPersistence configures the persistence settings for the registry service @@ -965,6 +1291,12 @@ spec: create: description: Settings for creating a new PVC properties: + accessModes: + description: AccessModes k8s persistent + volume access modes. Defaults to ["ReadWriteOnce"]. + items: + type: string + type: array resources: description: |- Resources describes the storage resource requirements for a volume. @@ -1016,13 +1348,20 @@ spec: description: Reference to an existing field properties: name: + default: "" description: |- Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string type: object x-kubernetes-map-type: atomic + required: + - mountPath type: object x-kubernetes-validations: - message: One selection is required between ref @@ -1071,10 +1410,15 @@ spec: should be removed. properties: name: + default: "" description: |- Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string type: object x-kubernetes-map-type: atomic @@ -1175,10 +1519,15 @@ spec: the TLS key and cert reside properties: name: + default: "" description: |- Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string type: object x-kubernetes-map-type: atomic @@ -1225,10 +1574,15 @@ spec: the TLS cert resides properties: name: + default: "" description: |- Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string type: object x-kubernetes-map-type: atomic @@ -1288,10 +1642,15 @@ spec: referenced object inside the same namespace. properties: name: + default: "" description: |- Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string type: object x-kubernetes-map-type: atomic @@ -1310,9 +1669,63 @@ spec: pattern: ^[A-Za-z0-9][A-Za-z0-9_]*$ type: string services: - description: FeatureStoreServices defines the desired feast service - deployments. ephemeral registry is deployed by default. + description: FeatureStoreServices defines the desired feast services. + An ephemeral registry is deployed by default. properties: + deploymentStrategy: + description: DeploymentStrategy describes how to replace existing + pods with new ones. + properties: + rollingUpdate: + description: |- + Rolling update config params. Present only if DeploymentStrategyType = + RollingUpdate. + --- + TODO: Update this to follow our convention for oneOf, whatever we decide it + to be. + properties: + maxSurge: + anyOf: + - type: integer + - type: string + description: |- + The maximum number of pods that can be scheduled above the desired number of + pods. + Value can be an absolute number (ex: 5) or a percentage of desired pods (ex: 10%). + This can not be 0 if MaxUnavailable is 0. + Absolute number is calculated from percentage by rounding up. + Defaults to 25%. + Example: when this is set to 30%, the new ReplicaSet can be scaled up immediately when + the rolling update starts, such that the total number of old and new pods do not exceed + 130% of desired pods. Once old pods have been killed, + new ReplicaSet can be scaled up further, ensuring that total number of pods running + at any time during the update is at most 130% of desired pods. + x-kubernetes-int-or-string: true + maxUnavailable: + anyOf: + - type: integer + - type: string + description: |- + The maximum number of pods that can be unavailable during the update. + Value can be an absolute number (ex: 5) or a percentage of desired pods (ex: 10%). + Absolute number is calculated from percentage by rounding down. + This can not be 0 if MaxSurge is 0. + Defaults to 25%. + Example: when this is set to 30%, the old ReplicaSet can be scaled down to 70% of desired pods + immediately when the rolling update starts. Once new pods are ready, old ReplicaSet + can be scaled down further, followed by scaling up the new ReplicaSet, ensuring + that the total number of pods available at all times during the update is at + least 70% of desired pods. + x-kubernetes-int-or-string: true + type: object + type: + description: Type of deployment. Can be "Recreate" or + "RollingUpdate". Default is RollingUpdate. + type: string + type: object + disableInitContainers: + description: Disable the 'feast repo initialization' initContainer + type: boolean offlineStore: description: OfflineStore configures the deployed offline store service @@ -1349,10 +1762,15 @@ spec: description: The key to select. type: string name: + default: "" description: |- Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string optional: description: Specify whether the ConfigMap @@ -1413,10 +1831,15 @@ spec: from. Must be a valid secret key. type: string name: + default: "" description: |- Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string optional: description: Specify whether the Secret @@ -1431,12 +1854,74 @@ spec: - name type: object type: array + envFrom: + items: + description: EnvFromSource represents the source of + a set of ConfigMaps + properties: + configMapRef: + description: The ConfigMap to select from + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. + type: string + optional: + description: Specify whether the ConfigMap must + be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + prefix: + description: An optional identifier to prepend to + each key in the ConfigMap. Must be a C_IDENTIFIER. + type: string + secretRef: + description: The Secret to select from + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. + type: string + optional: + description: Specify whether the Secret must + be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + type: object + type: array image: type: string imagePullPolicy: description: PullPolicy describes a policy for if/when to pull a container image type: string + logLevel: + description: |- + LogLevel sets the logging level for the offline store service + Allowed values: "debug", "info", "warning", "error", "critical". + enum: + - debug + - info + - warning + - error + - critical + type: string persistence: description: OfflineStorePersistence configures the persistence settings for the offline store service @@ -1454,6 +1939,12 @@ spec: create: description: Settings for creating a new PVC properties: + accessModes: + description: AccessModes k8s persistent + volume access modes. Defaults to ["ReadWriteOnce"]. + items: + type: string + type: array resources: description: |- Resources describes the storage resource requirements for a volume. @@ -1505,13 +1996,20 @@ spec: description: Reference to an existing field properties: name: + default: "" description: |- Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string type: object x-kubernetes-map-type: atomic + required: + - mountPath type: object x-kubernetes-validations: - message: One selection is required between ref @@ -1523,6 +2021,7 @@ spec: rule: self.mountPath.matches('^/[^:]*$') type: enum: + - file - dask - duckdb type: string @@ -1542,10 +2041,15 @@ spec: should be removed. properties: name: + default: "" description: |- Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string type: object x-kubernetes-map-type: atomic @@ -1556,8 +2060,10 @@ spec: - redshift - spark - postgres - - feast_trino.trino.TrinoOfflineStore + - trino - redis + - athena + - mssql type: string required: - secretRef @@ -1626,10 +2132,9 @@ spec: type: object type: object tls: - description: OfflineTlsConfigs configures server TLS for - the offline feast service. in an openshift cluster, - this is configured by default using service serving - certificates. + description: TlsConfigs configures server TLS for a feast + service. in an openshift cluster, this is configured + by default using service serving certificates. properties: disable: description: will disable TLS for the feast service. @@ -1652,16 +2157,18 @@ spec: the TLS key and cert reside properties: name: + default: "" description: |- Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string type: object x-kubernetes-map-type: atomic - verifyClient: - description: verify the client TLS certificate. - type: boolean type: object x-kubernetes-validations: - message: '`secretRef` required if `disable` is false.' @@ -1704,10 +2211,15 @@ spec: description: The key to select. type: string name: + default: "" description: |- Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string optional: description: Specify whether the ConfigMap @@ -1768,10 +2280,15 @@ spec: from. Must be a valid secret key. type: string name: + default: "" description: |- Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string optional: description: Specify whether the Secret @@ -1786,12 +2303,74 @@ spec: - name type: object type: array + envFrom: + items: + description: EnvFromSource represents the source of + a set of ConfigMaps + properties: + configMapRef: + description: The ConfigMap to select from + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. + type: string + optional: + description: Specify whether the ConfigMap must + be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + prefix: + description: An optional identifier to prepend to + each key in the ConfigMap. Must be a C_IDENTIFIER. + type: string + secretRef: + description: The Secret to select from + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. + type: string + optional: + description: Specify whether the Secret must + be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + type: object + type: array image: type: string imagePullPolicy: description: PullPolicy describes a policy for if/when to pull a container image type: string + logLevel: + description: |- + LogLevel sets the logging level for the online store service + Allowed values: "debug", "info", "warning", "error", "critical". + enum: + - debug + - info + - warning + - error + - critical + type: string persistence: description: OnlineStorePersistence configures the persistence settings for the online store service @@ -1811,6 +2390,12 @@ spec: create: description: Settings for creating a new PVC properties: + accessModes: + description: AccessModes k8s persistent + volume access modes. Defaults to ["ReadWriteOnce"]. + items: + type: string + type: array resources: description: |- Resources describes the storage resource requirements for a volume. @@ -1862,13 +2447,20 @@ spec: description: Reference to an existing field properties: name: + default: "" description: |- Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string type: object x-kubernetes-map-type: atomic + required: + - mountPath type: object x-kubernetes-validations: - message: One selection is required between ref @@ -1889,8 +2481,8 @@ spec: : true' - message: Online store does not support S3 or GS buckets. - rule: has(self.path) && !self.path.startsWith('s3://') - && !self.path.startsWith('gs://') + rule: 'has(self.path) ? !(self.path.startsWith(''s3://'') + || self.path.startsWith(''gs://'')) : true' store: description: OnlineStoreDBStorePersistence configures the DB store persistence for the offline store service @@ -1906,10 +2498,15 @@ spec: should be removed. properties: name: + default: "" description: |- Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string type: object x-kubernetes-map-type: atomic @@ -1926,6 +2523,11 @@ spec: - mysql - hazelcast - singlestore + - hbase + - elasticsearch + - qdrant + - couchbase + - milvus type: string required: - secretRef @@ -2019,10 +2621,15 @@ spec: the TLS key and cert reside properties: name: + default: "" description: |- Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string type: object x-kubernetes-map-type: atomic @@ -2072,10 +2679,15 @@ spec: description: The key to select. type: string name: + default: "" description: |- Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string optional: description: Specify whether the ConfigMap @@ -2139,10 +2751,15 @@ spec: key. type: string name: + default: "" description: |- Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string optional: description: Specify whether the Secret @@ -2157,12 +2774,74 @@ spec: - name type: object type: array + envFrom: + items: + description: EnvFromSource represents the source + of a set of ConfigMaps + properties: + configMapRef: + description: The ConfigMap to select from + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. + type: string + optional: + description: Specify whether the ConfigMap + must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + prefix: + description: An optional identifier to prepend + to each key in the ConfigMap. Must be a C_IDENTIFIER. + type: string + secretRef: + description: The Secret to select from + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. + type: string + optional: + description: Specify whether the Secret + must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + type: object + type: array image: type: string imagePullPolicy: description: PullPolicy describes a policy for if/when to pull a container image type: string + logLevel: + description: |- + LogLevel sets the logging level for the registry service + Allowed values: "debug", "info", "warning", "error", "critical". + enum: + - debug + - info + - warning + - error + - critical + type: string persistence: description: RegistryPersistence configures the persistence settings for the registry service @@ -2183,6 +2862,13 @@ spec: description: Settings for creating a new PVC properties: + accessModes: + description: AccessModes k8s persistent + volume access modes. Defaults to + ["ReadWriteOnce"]. + items: + type: string + type: array resources: description: |- Resources describes the storage resource requirements for a volume. @@ -2235,13 +2921,20 @@ spec: field properties: name: + default: "" description: |- Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string type: object x-kubernetes-map-type: atomic + required: + - mountPath type: object x-kubernetes-validations: - message: One selection is required between @@ -2290,10 +2983,15 @@ spec: "type" fields should be removed. properties: name: + default: "" description: |- Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string type: object x-kubernetes-map-type: atomic @@ -2395,10 +3093,15 @@ spec: the TLS key and cert reside properties: name: + default: "" description: |- Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string type: object x-kubernetes-map-type: atomic @@ -2445,10 +3148,15 @@ spec: where the TLS cert resides properties: name: + default: "" description: |- Name of the referent. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string type: object x-kubernetes-map-type: atomic @@ -2544,7 +3252,6 @@ spec: type: object type: array feastVersion: - description: Version of feast that's currently deployed type: string phase: type: string @@ -2748,20 +3455,7 @@ metadata: labels: app.kubernetes.io/managed-by: kustomize app.kubernetes.io/name: feast-operator - name: feast-operator-metrics-reader -rules: -- nonResourceURLs: - - /metrics - verbs: - - get ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - app.kubernetes.io/managed-by: kustomize - app.kubernetes.io/name: feast-operator - name: feast-operator-proxy-role + name: feast-operator-metrics-auth-role rules: - apiGroups: - authentication.k8s.io @@ -2777,6 +3471,19 @@ rules: - create --- apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/managed-by: kustomize + app.kubernetes.io/name: feast-operator + name: feast-operator-metrics-reader +rules: +- nonResourceURLs: + - /metrics + verbs: + - get +--- +apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: labels: @@ -2815,11 +3522,11 @@ metadata: labels: app.kubernetes.io/managed-by: kustomize app.kubernetes.io/name: feast-operator - name: feast-operator-proxy-rolebinding + name: feast-operator-metrics-auth-rolebinding roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole - name: feast-operator-proxy-role + name: feast-operator-metrics-auth-role subjects: - kind: ServiceAccount name: feast-operator-controller-manager @@ -2839,7 +3546,7 @@ spec: - name: https port: 8443 protocol: TCP - targetPort: https + targetPort: 8443 selector: control-plane: controller-manager --- @@ -2866,35 +3573,17 @@ spec: spec: containers: - args: - - --secure-listen-address=0.0.0.0:8443 - - --upstream=http://127.0.0.1:8080/ - - --logtostderr=true - - --v=0 - image: gcr.io/kubebuilder/kube-rbac-proxy:v0.16.0 - name: kube-rbac-proxy - ports: - - containerPort: 8443 - name: https - protocol: TCP - resources: - limits: - cpu: 500m - memory: 128Mi - requests: - cpu: 5m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - - args: - - --health-probe-bind-address=:8081 - - --metrics-bind-address=127.0.0.1:8080 + - --metrics-bind-address=:8443 - --leader-elect + - --health-probe-bind-address=:8081 command: - /manager - image: feastdev/feast-operator:0.41.0 + env: + - name: RELATED_IMAGE_FEATURE_SERVER + value: docker.io/feastdev/feature-server:0.42.0 + - name: RELATED_IMAGE_GRPC_CURL + value: docker.io/fullstorydev/grpcurl:v1.9.1-alpine + image: feastdev/feast-operator:0.42.0 livenessProbe: httpGet: path: /healthz diff --git a/infra/feast-operator/go.mod b/infra/feast-operator/go.mod index 4e544d819e4..337cb3e80db 100644 --- a/infra/feast-operator/go.mod +++ b/infra/feast-operator/go.mod @@ -1,25 +1,30 @@ module github.com/feast-dev/feast/infra/feast-operator -go 1.21 +go 1.22.9 require ( - github.com/onsi/ginkgo/v2 v2.14.0 - github.com/onsi/gomega v1.30.0 + github.com/onsi/ginkgo/v2 v2.17.1 + github.com/onsi/gomega v1.32.0 gopkg.in/yaml.v3 v3.0.1 - k8s.io/api v0.29.2 - k8s.io/apimachinery v0.29.2 - k8s.io/client-go v0.29.2 - sigs.k8s.io/controller-runtime v0.17.3 + k8s.io/api v0.30.1 + k8s.io/apimachinery v0.30.1 + k8s.io/client-go v0.30.1 + sigs.k8s.io/controller-runtime v0.18.4 ) require ( + github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20230305170008-8188dc5388df // indirect github.com/beorn7/perks v1.0.1 // indirect + github.com/blang/semver/v4 v4.0.0 // indirect + github.com/cenkalti/backoff/v4 v4.2.1 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/emicklei/go-restful/v3 v3.11.0 // indirect - github.com/evanphx/json-patch/v5 v5.8.0 // indirect + github.com/evanphx/json-patch/v5 v5.9.0 // indirect + github.com/felixge/httpsnoop v1.0.3 // indirect github.com/fsnotify/fsnotify v1.7.0 // indirect github.com/go-logr/logr v1.4.1 // indirect + github.com/go-logr/stdr v1.2.2 // indirect github.com/go-logr/zapr v1.3.0 // indirect github.com/go-openapi/jsonpointer v0.19.6 // indirect github.com/go-openapi/jsonreference v0.20.2 // indirect @@ -27,12 +32,14 @@ require ( github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect - github.com/golang/protobuf v1.5.3 // indirect + github.com/golang/protobuf v1.5.4 // indirect + github.com/google/cel-go v0.17.8 // indirect github.com/google/gnostic-models v0.6.8 // indirect github.com/google/go-cmp v0.6.0 // indirect github.com/google/gofuzz v1.2.0 // indirect github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 // indirect github.com/google/uuid v1.3.0 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0 // indirect github.com/imdario/mergo v0.3.6 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect @@ -47,26 +54,41 @@ require ( github.com/prometheus/common v0.45.0 // indirect github.com/prometheus/procfs v0.12.0 // indirect github.com/spf13/pflag v1.0.5 // indirect + github.com/stoewer/go-strcase v1.2.0 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.44.0 // indirect + go.opentelemetry.io/otel v1.19.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.19.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.19.0 // indirect + go.opentelemetry.io/otel/metric v1.19.0 // indirect + go.opentelemetry.io/otel/sdk v1.19.0 // indirect + go.opentelemetry.io/otel/trace v1.19.0 // indirect + go.opentelemetry.io/proto/otlp v1.0.0 // indirect go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.26.0 // indirect golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e // indirect golang.org/x/net v0.23.0 // indirect golang.org/x/oauth2 v0.12.0 // indirect + golang.org/x/sync v0.6.0 // indirect golang.org/x/sys v0.18.0 // indirect golang.org/x/term v0.18.0 // indirect golang.org/x/text v0.14.0 // indirect golang.org/x/time v0.3.0 // indirect - golang.org/x/tools v0.16.1 // indirect + golang.org/x/tools v0.18.0 // indirect gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect google.golang.org/appengine v1.6.7 // indirect - google.golang.org/protobuf v1.31.0 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20230726155614-23370e0ffb3e // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20230822172742-b8732ec3820d // indirect + google.golang.org/grpc v1.58.3 // indirect + google.golang.org/protobuf v1.33.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect - k8s.io/apiextensions-apiserver v0.29.2 // indirect - k8s.io/component-base v0.29.2 // indirect - k8s.io/klog/v2 v2.110.1 // indirect - k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00 // indirect + k8s.io/apiextensions-apiserver v0.30.1 // indirect + k8s.io/apiserver v0.30.1 // indirect + k8s.io/component-base v0.30.1 // indirect + k8s.io/klog/v2 v2.120.1 // indirect + k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 // indirect k8s.io/utils v0.0.0-20230726121419-3b25d923346b // indirect + sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.29.0 // indirect sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect sigs.k8s.io/yaml v1.4.0 // indirect diff --git a/infra/feast-operator/go.sum b/infra/feast-operator/go.sum index be475e11018..ee6c96863b6 100644 --- a/infra/feast-operator/go.sum +++ b/infra/feast-operator/go.sum @@ -1,5 +1,11 @@ +github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20230305170008-8188dc5388df h1:7RFfzj4SSt6nnvCPbCqijJi1nWCd+TqAT3bYCStRC18= +github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20230305170008-8188dc5388df/go.mod h1:pSwJ0fSY5KhvocuWSx4fz3BA8OrA1bQn+K1Eli3BRwM= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM= +github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= +github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM= +github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= @@ -13,13 +19,17 @@ github.com/emicklei/go-restful/v3 v3.11.0 h1:rAQeMHw1c7zTmncogyy8VvRZwtkmkZ4FxER github.com/emicklei/go-restful/v3 v3.11.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= github.com/evanphx/json-patch v4.12.0+incompatible h1:4onqiflcdA9EOZ4RxV643DvftH5pOlLGNtQ5lPWQu84= github.com/evanphx/json-patch v4.12.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= -github.com/evanphx/json-patch/v5 v5.8.0 h1:lRj6N9Nci7MvzrXuX6HFzU8XjmhPiXPlsKEy1u0KQro= -github.com/evanphx/json-patch/v5 v5.8.0/go.mod h1:VNkHZ/282BpEyt/tObQO8s5CMPmYYq14uClGH4abBuQ= +github.com/evanphx/json-patch/v5 v5.9.0 h1:kcBlZQbplgElYIlo/n1hJbls2z/1awpXxpRi0/FOJfg= +github.com/evanphx/json-patch/v5 v5.9.0/go.mod h1:VNkHZ/282BpEyt/tObQO8s5CMPmYYq14uClGH4abBuQ= +github.com/felixge/httpsnoop v1.0.3 h1:s/nj+GCswXYzN5v2DpNMuMQYe+0DDwt5WVCU6CWBdXk= +github.com/felixge/httpsnoop v1.0.3/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= -github.com/go-logr/logr v1.3.0/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-logr/zapr v1.3.0 h1:XGdV8XW8zdwFiwOA2Dryh1gj2KRQyOOoNmBy4EplIcQ= github.com/go-logr/zapr v1.3.0/go.mod h1:YKepepNBd1u/oyhd/yQmtjVXmm9uML4IXUgMOwR8/Gg= github.com/go-openapi/jsonpointer v0.19.6 h1:eCs3fxoIi3Wh6vtgmLTOjdhSpiqphQ+DaPn38N2ZdrE= @@ -32,15 +42,17 @@ github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEe github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang/glog v1.1.0 h1:/d3pCKDPWNnvIWe0vVUpNP32qc8U3PDVxySP/y360qE= +github.com/golang/glog v1.1.0/go.mod h1:pfYeQZ3JWZoXTV5sFc986z3HTpwQs9At6P4ImfuP3NQ= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= -github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/google/cel-go v0.17.8 h1:j9m730pMZt1Fc4oKhCLUHfjj6527LuhYcYw0Rl8gqto= +github.com/google/cel-go v0.17.8/go.mod h1:HXZKzB0LXqer5lHHgfWAnlYwJaQBDKMjxjulNQzhwhY= github.com/google/gnostic-models v0.6.8 h1:yo/ABAfM5IMRsS1VnXjTBvUb61tFIHozhlYvRgGre9I= github.com/google/gnostic-models v0.6.8/go.mod h1:5n7qKqH0f5wFt+aWF8CW6pZLLNOfYuF5OpfBSENuI8U= -github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= @@ -51,6 +63,8 @@ github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 h1:K6RDEckDVWvDI9JAJY github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0 h1:YBftPWNWd4WwGqtY2yeZL2ef8rHAxPBD8KFhJpmcqms= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0/go.mod h1:YN5jB8ie0yfIUg6VvR9Kz84aCaG7AsGZnLjhHbUqwPg= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/imdario/mergo v0.3.6 h1:xTNEAn+kxVO7dTZGu0CegyqKZmoWFI0rF8UxjlB2d28= github.com/imdario/mergo v0.3.6/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= @@ -78,10 +92,10 @@ github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9G github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= -github.com/onsi/ginkgo/v2 v2.14.0 h1:vSmGj2Z5YPb9JwCWT6z6ihcUvDhuXLc3sJiqd3jMKAY= -github.com/onsi/ginkgo/v2 v2.14.0/go.mod h1:JkUdW7JkN0V6rFvsHcJ478egV3XH9NxpD27Hal/PhZw= -github.com/onsi/gomega v1.30.0 h1:hvMK7xYz4D3HapigLTeGdId/NcfQx1VHMJc60ew99+8= -github.com/onsi/gomega v1.30.0/go.mod h1:9sxs+SwGrKI0+PWe4Fxa9tFQQBG5xSsSbMXOI8PPpoQ= +github.com/onsi/ginkgo/v2 v2.17.1 h1:V++EzdbhI4ZV4ev0UTIj0PzhzOcReJFyJaLjtSF55M8= +github.com/onsi/ginkgo/v2 v2.17.1/go.mod h1:llBI3WDLL9Z6taip6f33H76YcWtJv+7R3HigUjbIBOs= +github.com/onsi/gomega v1.32.0 h1:JRYU78fJ1LPxlckP6Txi/EYqJvjtMrDC04/MM5XRHPk= +github.com/onsi/gomega v1.32.0/go.mod h1:a4x4gW6Pz2yK1MAmvluYme5lvYTn61afQ2ETw/8n4Lg= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= @@ -98,10 +112,13 @@ github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjR github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/stoewer/go-strcase v1.2.0 h1:Z2iHWqGXH00XYgqDmNgQbIBxf3wrNq0F3feEy0ainaU= +github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= @@ -110,6 +127,22 @@ github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcU github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.44.0 h1:KfYpVmrjI7JuToy5k8XV3nkapjWx48k4E4JOtVstzQI= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.44.0/go.mod h1:SeQhzAEccGVZVEy7aH87Nh0km+utSpo1pTv6eMMop48= +go.opentelemetry.io/otel v1.19.0 h1:MuS/TNf4/j4IXsZuJegVzI1cwut7Qc00344rgH7p8bs= +go.opentelemetry.io/otel v1.19.0/go.mod h1:i0QyjOq3UPoTzff0PJB2N66fb4S0+rSbSB15/oyH9fY= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.19.0 h1:Mne5On7VWdx7omSrSSZvM4Kw7cS7NQkOOmLcgscI51U= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.19.0/go.mod h1:IPtUMKL4O3tH5y+iXVyAXqpAwMuzC1IrxVS81rummfE= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.19.0 h1:3d+S281UTjM+AbF31XSOYn1qXn3BgIdWl8HNEpx08Jk= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.19.0/go.mod h1:0+KuTDyKL4gjKCF75pHOX4wuzYDUZYfAQdSu43o+Z2I= +go.opentelemetry.io/otel/metric v1.19.0 h1:aTzpGtV0ar9wlV4Sna9sdJyII5jTVJEvKETPiOKwvpE= +go.opentelemetry.io/otel/metric v1.19.0/go.mod h1:L5rUsV9kM1IxCj1MmSdS+JQAcVm319EUrDVLrt7jqt8= +go.opentelemetry.io/otel/sdk v1.19.0 h1:6USY6zH+L8uMH8L3t1enZPR3WFEmSTADlqldyHtJi3o= +go.opentelemetry.io/otel/sdk v1.19.0/go.mod h1:NedEbbS4w3C6zElbLdPJKOpJQOrGUJ+GfzpjUvI0v1A= +go.opentelemetry.io/otel/trace v1.19.0 h1:DFVQmlVbfVeOuBRrwdtaehRrWiL1JoVs9CPIQ1Dzxpg= +go.opentelemetry.io/otel/trace v1.19.0/go.mod h1:mfaSyvGyEJEI0nyV2I4qhNQnbBOUUmYZpYojqMnX2vo= +go.opentelemetry.io/proto/otlp v1.0.0 h1:T0TX0tmXU8a3CbNXzEKGeU5mIVOdf0oykP+u2lIVU/I= +go.opentelemetry.io/proto/otlp v1.0.0/go.mod h1:Sy6pihPLfYHkr3NkUbEhGHFhINUSI/v80hjKIs5JXpM= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= @@ -135,6 +168,8 @@ golang.org/x/oauth2 v0.12.0/go.mod h1:A74bZ3aGXgCY0qaIC9Ahg6Lglin4AMAco8cIv9baba golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ= +golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -154,8 +189,8 @@ golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGm golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.16.1 h1:TLyB3WofjdOEepBHAU20JdNC1Zbg87elYofWYAY5oZA= -golang.org/x/tools v0.16.1/go.mod h1:kYVVN6I1mBNoB1OX+noeBjbRk4IUEPa7JJ+TJMEooJ0= +golang.org/x/tools v0.18.0 h1:k8NLag8AGHnn+PHbl7g43CtqZAwG60vZkLqgyZgIHgQ= +golang.org/x/tools v0.18.0/go.mod h1:GL7B4CwcLLeo59yx/9UWWuNOW1n3VZ4f5axWfML7Lcg= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -164,39 +199,50 @@ gomodules.xyz/jsonpatch/v2 v2.4.0 h1:Ci3iUJyx9UeRx7CeFN8ARgGbkESwJK+KB9lLcWxY/Zw gomodules.xyz/jsonpatch/v2 v2.4.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuBJT4moAY= google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= -google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= -google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/genproto v0.0.0-20230803162519-f966b187b2e5 h1:L6iMMGrtzgHsWofoFcihmDEMYeDR9KN/ThbPWGrh++g= +google.golang.org/genproto v0.0.0-20230803162519-f966b187b2e5/go.mod h1:oH/ZOT02u4kWEp7oYBGYFFkCdKS/uYR9Z7+0/xuuFp8= +google.golang.org/genproto/googleapis/api v0.0.0-20230726155614-23370e0ffb3e h1:z3vDksarJxsAKM5dmEGv0GHwE2hKJ096wZra71Vs4sw= +google.golang.org/genproto/googleapis/api v0.0.0-20230726155614-23370e0ffb3e/go.mod h1:rsr7RhLuwsDKL7RmgDDCUc6yaGr1iqceVb5Wv6f6YvQ= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230822172742-b8732ec3820d h1:uvYuEyMHKNt+lT4K3bN6fGswmK8qSvcreM3BwjDh+y4= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230822172742-b8732ec3820d/go.mod h1:+Bk1OCOj40wS2hwAMA+aCW9ypzm63QTBBHp6lQ3p+9M= +google.golang.org/grpc v1.58.3 h1:BjnpXut1btbtgN/6sp+brB2Kbm2LjNXnidYujAVbSoQ= +google.golang.org/grpc v1.58.3/go.mod h1:tgX3ZQDlNJGU96V6yHh1T/JeoBQ2TXdr43YbYSsCJk0= +google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= +google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -k8s.io/api v0.29.2 h1:hBC7B9+MU+ptchxEqTNW2DkUosJpp1P+Wn6YncZ474A= -k8s.io/api v0.29.2/go.mod h1:sdIaaKuU7P44aoyyLlikSLayT6Vb7bvJNCX105xZXY0= -k8s.io/apiextensions-apiserver v0.29.2 h1:UK3xB5lOWSnhaCk0RFZ0LUacPZz9RY4wi/yt2Iu+btg= -k8s.io/apiextensions-apiserver v0.29.2/go.mod h1:aLfYjpA5p3OwtqNXQFkhJ56TB+spV8Gc4wfMhUA3/b8= -k8s.io/apimachinery v0.29.2 h1:EWGpfJ856oj11C52NRCHuU7rFDwxev48z+6DSlGNsV8= -k8s.io/apimachinery v0.29.2/go.mod h1:6HVkd1FwxIagpYrHSwJlQqZI3G9LfYWRPAkUvLnXTKU= -k8s.io/client-go v0.29.2 h1:FEg85el1TeZp+/vYJM7hkDlSTFZ+c5nnK44DJ4FyoRg= -k8s.io/client-go v0.29.2/go.mod h1:knlvFZE58VpqbQpJNbCbctTVXcd35mMyAAwBdpt4jrA= -k8s.io/component-base v0.29.2 h1:lpiLyuvPA9yV1aQwGLENYyK7n/8t6l3nn3zAtFTJYe8= -k8s.io/component-base v0.29.2/go.mod h1:BfB3SLrefbZXiBfbM+2H1dlat21Uewg/5qtKOl8degM= -k8s.io/klog/v2 v2.110.1 h1:U/Af64HJf7FcwMcXyKm2RPM22WZzyR7OSpYj5tg3cL0= -k8s.io/klog/v2 v2.110.1/go.mod h1:YGtd1984u+GgbuZ7e08/yBuAfKLSO0+uR1Fhi6ExXjo= -k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00 h1:aVUu9fTY98ivBPKR9Y5w/AuzbMm96cd3YHRTU83I780= -k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00/go.mod h1:AsvuZPBlUDVuCdzJ87iajxtXuR9oktsTctW/R9wwouA= +k8s.io/api v0.30.1 h1:kCm/6mADMdbAxmIh0LBjS54nQBE+U4KmbCfIkF5CpJY= +k8s.io/api v0.30.1/go.mod h1:ddbN2C0+0DIiPntan/bye3SW3PdwLa11/0yqwvuRrJM= +k8s.io/apiextensions-apiserver v0.30.1 h1:4fAJZ9985BmpJG6PkoxVRpXv9vmPUOVzl614xarePws= +k8s.io/apiextensions-apiserver v0.30.1/go.mod h1:R4GuSrlhgq43oRY9sF2IToFh7PVlF1JjfWdoG3pixk4= +k8s.io/apimachinery v0.30.1 h1:ZQStsEfo4n65yAdlGTfP/uSHMQSoYzU/oeEbkmF7P2U= +k8s.io/apimachinery v0.30.1/go.mod h1:iexa2somDaxdnj7bha06bhb43Zpa6eWH8N8dbqVjTUc= +k8s.io/apiserver v0.30.1 h1:BEWEe8bzS12nMtDKXzCF5Q5ovp6LjjYkSp8qOPk8LZ8= +k8s.io/apiserver v0.30.1/go.mod h1:i87ZnQ+/PGAmSbD/iEKM68bm1D5reX8fO4Ito4B01mo= +k8s.io/client-go v0.30.1 h1:uC/Ir6A3R46wdkgCV3vbLyNOYyCJ8oZnjtJGKfytl/Q= +k8s.io/client-go v0.30.1/go.mod h1:wrAqLNs2trwiCH/wxxmT/x3hKVH9PuV0GGW0oDoHVqc= +k8s.io/component-base v0.30.1 h1:bvAtlPh1UrdaZL20D9+sWxsJljMi0QZ3Lmw+kmZAaxQ= +k8s.io/component-base v0.30.1/go.mod h1:e/X9kDiOebwlI41AvBHuWdqFriSRrX50CdwA9TFaHLI= +k8s.io/klog/v2 v2.120.1 h1:QXU6cPEOIslTGvZaXvFWiP9VKyeet3sawzTOvdXb4Vw= +k8s.io/klog/v2 v2.120.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= +k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 h1:BZqlfIlq5YbRMFko6/PM7FjZpUb45WallggurYhKGag= +k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340/go.mod h1:yD4MZYeKMBwQKVht279WycxKyM84kkAx2DPrTXaeb98= k8s.io/utils v0.0.0-20230726121419-3b25d923346b h1:sgn3ZU783SCgtaSJjpcVVlRqd6GSnlTLKgpAAttJvpI= k8s.io/utils v0.0.0-20230726121419-3b25d923346b/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= -sigs.k8s.io/controller-runtime v0.17.3 h1:65QmN7r3FWgTxDMz9fvGnO1kbf2nu+acg9p2R9oYYYk= -sigs.k8s.io/controller-runtime v0.17.3/go.mod h1:N0jpP5Lo7lMTF9aL56Z/B2oWBJjey6StQM0jRbKQXtY= +sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.29.0 h1:/U5vjBbQn3RChhv7P11uhYvCSm5G2GaIi5AIGBS6r4c= +sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.29.0/go.mod h1:z7+wmGM2dfIiLRfrC6jb5kV2Mq/sK1ZP303cxzkV5Y4= +sigs.k8s.io/controller-runtime v0.18.4 h1:87+guW1zhvuPLh1PHybKdYFLU0YJp4FhJRmiHvm5BZw= +sigs.k8s.io/controller-runtime v0.18.4/go.mod h1:TVoGrfdpbA9VRFaRnKgk9P5/atA0pMwq+f+msb9M8Sg= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= sigs.k8s.io/structured-merge-diff/v4 v4.4.1 h1:150L+0vs/8DA78h1u02ooW1/fFq/Lwr+sGiqlzvrtq4= diff --git a/infra/feast-operator/internal/controller/authz/authz.go b/infra/feast-operator/internal/controller/authz/authz.go index efcae23a4b0..8596d993899 100644 --- a/infra/feast-operator/internal/controller/authz/authz.go +++ b/infra/feast-operator/internal/controller/authz/authz.go @@ -134,28 +134,11 @@ func (authz *FeastAuthorization) initFeastRoleBinding() *rbacv1.RoleBinding { func (authz *FeastAuthorization) setFeastRoleBinding(roleBinding *rbacv1.RoleBinding) error { roleBinding.Labels = authz.getLabels() - roleBinding.Subjects = []rbacv1.Subject{} - if authz.Handler.FeatureStore.Status.Applied.Services.OfflineStore != nil { - roleBinding.Subjects = append(roleBinding.Subjects, rbacv1.Subject{ - Kind: rbacv1.ServiceAccountKind, - Name: services.GetFeastServiceName(authz.Handler.FeatureStore, services.OfflineFeastType), - Namespace: authz.Handler.FeatureStore.Namespace, - }) - } - if authz.Handler.FeatureStore.Status.Applied.Services.OnlineStore != nil { - roleBinding.Subjects = append(roleBinding.Subjects, rbacv1.Subject{ - Kind: rbacv1.ServiceAccountKind, - Name: services.GetFeastServiceName(authz.Handler.FeatureStore, services.OnlineFeastType), - Namespace: authz.Handler.FeatureStore.Namespace, - }) - } - if services.IsLocalRegistry(authz.Handler.FeatureStore) { - roleBinding.Subjects = append(roleBinding.Subjects, rbacv1.Subject{ - Kind: rbacv1.ServiceAccountKind, - Name: services.GetFeastServiceName(authz.Handler.FeatureStore, services.RegistryFeastType), - Namespace: authz.Handler.FeatureStore.Namespace, - }) - } + roleBinding.Subjects = append(roleBinding.Subjects, rbacv1.Subject{ + Kind: rbacv1.ServiceAccountKind, + Name: services.GetFeastName(authz.Handler.FeatureStore), + Namespace: authz.Handler.FeatureStore.Namespace, + }) roleBinding.RoleRef = rbacv1.RoleRef{ APIGroup: rbacv1.GroupName, Kind: "Role", diff --git a/infra/feast-operator/internal/controller/featurestore_controller.go b/infra/feast-operator/internal/controller/featurestore_controller.go index 984bb7c9c26..99be6962b56 100644 --- a/infra/feast-operator/internal/controller/featurestore_controller.go +++ b/infra/feast-operator/internal/controller/featurestore_controller.go @@ -52,13 +52,13 @@ type FeatureStoreReconciler struct { Scheme *runtime.Scheme } -//+kubebuilder:rbac:groups=feast.dev,resources=featurestores,verbs=get;list;watch;create;update;patch;delete -//+kubebuilder:rbac:groups=feast.dev,resources=featurestores/status,verbs=get;update;patch -//+kubebuilder:rbac:groups=feast.dev,resources=featurestores/finalizers,verbs=update -//+kubebuilder:rbac:groups=apps,resources=deployments,verbs=get;list;create;update;watch;delete -//+kubebuilder:rbac:groups=core,resources=services;configmaps;persistentvolumeclaims;serviceaccounts,verbs=get;list;create;update;watch;delete -//+kubebuilder:rbac:groups=rbac.authorization.k8s.io,resources=roles;rolebindings,verbs=get;list;create;update;watch;delete -//+kubebuilder:rbac:groups=core,resources=secrets,verbs=get;list +// +kubebuilder:rbac:groups=feast.dev,resources=featurestores,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=feast.dev,resources=featurestores/status,verbs=get;update;patch +// +kubebuilder:rbac:groups=feast.dev,resources=featurestores/finalizers,verbs=update +// +kubebuilder:rbac:groups=apps,resources=deployments,verbs=get;list;create;update;watch;delete +// +kubebuilder:rbac:groups=core,resources=services;configmaps;persistentvolumeclaims;serviceaccounts,verbs=get;list;create;update;watch;delete +// +kubebuilder:rbac:groups=rbac.authorization.k8s.io,resources=roles;rolebindings,verbs=get;list;create;update;watch;delete +// +kubebuilder:rbac:groups=core,resources=secrets,verbs=get;list // Reconcile is part of the main kubernetes reconciliation loop which aims to // move the current state of the cluster closer to the desired state. diff --git a/infra/feast-operator/internal/controller/featurestore_controller_db_store_test.go b/infra/feast-operator/internal/controller/featurestore_controller_db_store_test.go index 60235fe687e..1c9fa4bc1a5 100644 --- a/infra/feast-operator/internal/controller/featurestore_controller_db_store_test.go +++ b/infra/feast-operator/internal/controller/featurestore_controller_db_store_test.go @@ -78,7 +78,7 @@ sqlalchemy_config_kwargs: pool_pre_ping: true ` -var invalidSecretContainingTypeYamlString = ` +var secretContainingValidTypeYamlString = ` type: cassandra hosts: - 192.168.1.1 @@ -202,10 +202,12 @@ var _ = Describe("FeatureStore Controller - db storage services", func() { Expect(k8sClient.Create(ctx, secret)).To(Succeed()) } + createEnvFromSecretAndConfigMap() + By("creating the custom resource for the Kind FeatureStore") err = k8sClient.Get(ctx, typeNamespacedName, featurestore) if err != nil && errors.IsNotFound(err) { - resource := createFeatureStoreResource(resourceName, image, pullPolicy, &[]corev1.EnvVar{}) + resource := createFeatureStoreResource(resourceName, image, pullPolicy, &[]corev1.EnvVar{}, withEnvFrom()) resource.Spec.Services.OfflineStore.Persistence = &feastdevv1alpha1.OfflineStorePersistence{ DBPersistence: &feastdevv1alpha1.OfflineStoreDBStorePersistence{ Type: string(offlineType), @@ -256,6 +258,8 @@ var _ = Describe("FeatureStore Controller - db storage services", func() { err = k8sClient.Get(ctx, typeNamespacedName, resource) Expect(err).NotTo(HaveOccurred()) + deleteEnvFromSecretAndConfigMap() + By("Cleanup the secrets") Expect(k8sClient.Delete(ctx, onlineSecret)).To(Succeed()) Expect(k8sClient.Delete(ctx, offlineSecret)).To(Succeed()) @@ -305,37 +309,12 @@ var _ = Describe("FeatureStore Controller - db storage services", func() { Expect(err.Error()).To(Equal("secret key invalid.secret.key doesn't exist in secret online-store-secret")) - By("Referring to a secret that contains parameter named type") - resource = &feastdevv1alpha1.FeatureStore{} - err = k8sClient.Get(ctx, typeNamespacedName, resource) - Expect(err).NotTo(HaveOccurred()) - - secret := &corev1.Secret{} - err = k8sClient.Get(ctx, onlineSecretNamespacedName, secret) - Expect(err).NotTo(HaveOccurred()) - secret.Data[string(services.OnlineDBPersistenceCassandraConfigType)] = []byte(invalidSecretContainingTypeYamlString) - Expect(k8sClient.Update(ctx, secret)).To(Succeed()) - - resource.Spec.Services.OnlineStore.Persistence.DBPersistence.SecretRef = corev1.LocalObjectReference{Name: "online-store-secret"} - resource.Spec.Services.OnlineStore.Persistence.DBPersistence.SecretKeyName = "" - Expect(k8sClient.Update(ctx, resource)).To(Succeed()) - resource = &feastdevv1alpha1.FeatureStore{} - err = k8sClient.Get(ctx, typeNamespacedName, resource) - Expect(err).NotTo(HaveOccurred()) - - _, err = controllerReconciler.Reconcile(ctx, reconcile.Request{ - NamespacedName: typeNamespacedName, - }) - Expect(err).To(HaveOccurred()) - - Expect(err.Error()).To(Equal("secret key cassandra in secret online-store-secret contains invalid tag named type")) - By("Referring to a secret that contains parameter named type with invalid value") resource = &feastdevv1alpha1.FeatureStore{} err = k8sClient.Get(ctx, typeNamespacedName, resource) Expect(err).NotTo(HaveOccurred()) - secret = &corev1.Secret{} + secret := &corev1.Secret{} err = k8sClient.Get(ctx, onlineSecretNamespacedName, secret) Expect(err).NotTo(HaveOccurred()) secret.Data[string(services.OnlineDBPersistenceCassandraConfigType)] = []byte(invalidSecretTypeYamlString) @@ -353,39 +332,7 @@ var _ = Describe("FeatureStore Controller - db storage services", func() { }) Expect(err).To(HaveOccurred()) - Expect(err.Error()).To(Equal("secret key cassandra in secret online-store-secret contains invalid tag named type")) - - By("Referring to a secret that contains parameter named registry_type") - resource = &feastdevv1alpha1.FeatureStore{} - err = k8sClient.Get(ctx, typeNamespacedName, resource) - Expect(err).NotTo(HaveOccurred()) - - secret = &corev1.Secret{} - err = k8sClient.Get(ctx, onlineSecretNamespacedName, secret) - Expect(err).NotTo(HaveOccurred()) - secret.Data[string(services.OnlineDBPersistenceCassandraConfigType)] = []byte(cassandraYamlString) - Expect(k8sClient.Update(ctx, secret)).To(Succeed()) - - secret = &corev1.Secret{} - err = k8sClient.Get(ctx, registrySecretNamespacedName, secret) - Expect(err).NotTo(HaveOccurred()) - secret.Data["sql_custom_registry_key"] = nil - secret.Data[string(services.RegistryDBPersistenceSQLConfigType)] = []byte(invalidSecretRegistryTypeYamlString) - Expect(k8sClient.Update(ctx, secret)).To(Succeed()) - - resource.Spec.Services.Registry.Local.Persistence.DBPersistence.SecretRef = corev1.LocalObjectReference{Name: "registry-store-secret"} - resource.Spec.Services.Registry.Local.Persistence.DBPersistence.SecretKeyName = "" - Expect(k8sClient.Update(ctx, resource)).To(Succeed()) - resource = &feastdevv1alpha1.FeatureStore{} - err = k8sClient.Get(ctx, typeNamespacedName, resource) - Expect(err).NotTo(HaveOccurred()) - - _, err = controllerReconciler.Reconcile(ctx, reconcile.Request{ - NamespacedName: typeNamespacedName, - }) - Expect(err).To(HaveOccurred()) - - Expect(err.Error()).To(Equal("secret key sql in secret registry-store-secret contains invalid tag named registry_type")) + Expect(err.Error()).To(Equal("secret key cassandra in secret online-store-secret contains tag named type with value wrong")) }) It("should successfully reconcile the resource", func() { @@ -486,16 +433,17 @@ var _ = Describe("FeatureStore Controller - db storage services", func() { Expect(resource.Status.Phase).To(Equal(feastdevv1alpha1.ReadyPhase)) + // check deployment deploy := &appsv1.Deployment{} + objMeta := feast.GetObjectMeta() err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.RegistryFeastType), - Namespace: resource.Namespace, - }, - deploy) + Name: objMeta.Name, + Namespace: objMeta.Namespace, + }, deploy) Expect(err).NotTo(HaveOccurred()) Expect(deploy.Spec.Replicas).To(Equal(&services.DefaultReplicas)) Expect(controllerutil.HasControllerReference(deploy)).To(BeTrue()) - Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(1)) + Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(3)) svc := &corev1.Service{} err = k8sClient.Get(ctx, types.NamespacedName{ @@ -506,6 +454,60 @@ var _ = Describe("FeatureStore Controller - db storage services", func() { Expect(err).NotTo(HaveOccurred()) Expect(controllerutil.HasControllerReference(svc)).To(BeTrue()) Expect(svc.Spec.Ports[0].TargetPort).To(Equal(intstr.FromInt(int(services.FeastServiceConstants[services.RegistryFeastType].TargetHttpPort)))) + + By("Referring to a secret that contains parameter named type") + resource = &feastdevv1alpha1.FeatureStore{} + err = k8sClient.Get(ctx, typeNamespacedName, resource) + Expect(err).NotTo(HaveOccurred()) + + secret := &corev1.Secret{} + err = k8sClient.Get(ctx, onlineSecretNamespacedName, secret) + Expect(err).NotTo(HaveOccurred()) + secret.Data[string(services.OnlineDBPersistenceCassandraConfigType)] = []byte(secretContainingValidTypeYamlString) + Expect(k8sClient.Update(ctx, secret)).To(Succeed()) + + resource.Spec.Services.OnlineStore.Persistence.DBPersistence.SecretRef = corev1.LocalObjectReference{Name: "online-store-secret"} + resource.Spec.Services.OnlineStore.Persistence.DBPersistence.SecretKeyName = "" + Expect(k8sClient.Update(ctx, resource)).To(Succeed()) + resource = &feastdevv1alpha1.FeatureStore{} + err = k8sClient.Get(ctx, typeNamespacedName, resource) + Expect(err).NotTo(HaveOccurred()) + + _, err = controllerReconciler.Reconcile(ctx, reconcile.Request{ + NamespacedName: typeNamespacedName, + }) + + Expect(err).To(Not(HaveOccurred())) + + By("Referring to a secret that contains parameter named registry_type") + resource = &feastdevv1alpha1.FeatureStore{} + err = k8sClient.Get(ctx, typeNamespacedName, resource) + Expect(err).NotTo(HaveOccurred()) + + secret = &corev1.Secret{} + err = k8sClient.Get(ctx, onlineSecretNamespacedName, secret) + Expect(err).NotTo(HaveOccurred()) + secret.Data[string(services.OnlineDBPersistenceCassandraConfigType)] = []byte(cassandraYamlString) + Expect(k8sClient.Update(ctx, secret)).To(Succeed()) + + secret = &corev1.Secret{} + err = k8sClient.Get(ctx, registrySecretNamespacedName, secret) + Expect(err).NotTo(HaveOccurred()) + secret.Data["sql_custom_registry_key"] = nil + secret.Data[string(services.RegistryDBPersistenceSQLConfigType)] = []byte(invalidSecretRegistryTypeYamlString) + Expect(k8sClient.Update(ctx, secret)).To(Succeed()) + + resource.Spec.Services.Registry.Local.Persistence.DBPersistence.SecretRef = corev1.LocalObjectReference{Name: "registry-store-secret"} + resource.Spec.Services.Registry.Local.Persistence.DBPersistence.SecretKeyName = "" + Expect(k8sClient.Update(ctx, resource)).To(Succeed()) + resource = &feastdevv1alpha1.FeatureStore{} + err = k8sClient.Get(ctx, typeNamespacedName, resource) + Expect(err).NotTo(HaveOccurred()) + + _, err = controllerReconciler.Reconcile(ctx, reconcile.Request{ + NamespacedName: typeNamespacedName, + }) + Expect(err).To(Not(HaveOccurred())) }) It("should properly encode a feature_store.yaml config", func() { @@ -531,7 +533,7 @@ var _ = Describe("FeatureStore Controller - db storage services", func() { deployList := appsv1.DeploymentList{} err = k8sClient.List(ctx, &deployList, listOpts) Expect(err).NotTo(HaveOccurred()) - Expect(deployList.Items).To(HaveLen(3)) + Expect(deployList.Items).To(HaveLen(1)) svcList := corev1.ServiceList{} err = k8sClient.List(ctx, &svcList, listOpts) @@ -552,20 +554,20 @@ var _ = Describe("FeatureStore Controller - db storage services", func() { }, } - // check registry config + // check deployment deploy := &appsv1.Deployment{} + objMeta := feast.GetObjectMeta() err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.RegistryFeastType), - Namespace: resource.Namespace, - }, - deploy) + Name: objMeta.Name, + Namespace: objMeta.Namespace, + }, deploy) Expect(err).NotTo(HaveOccurred()) - Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(1)) - Expect(deploy.Spec.Template.Spec.Containers[0].Env).To(HaveLen(1)) - env := getFeatureStoreYamlEnvVar(deploy.Spec.Template.Spec.Containers[0].Env) + registryContainer := services.GetRegistryContainer(*deploy) + Expect(registryContainer.Env).To(HaveLen(1)) + env := getFeatureStoreYamlEnvVar(registryContainer.Env) Expect(env).NotTo(BeNil()) - fsYamlStr, err := feast.GetServiceFeatureStoreYamlBase64(services.RegistryFeastType) + fsYamlStr, err := feast.GetServiceFeatureStoreYamlBase64() Expect(err).NotTo(HaveOccurred()) Expect(fsYamlStr).To(Equal(env.Value)) @@ -581,29 +583,30 @@ var _ = Describe("FeatureStore Controller - db storage services", func() { Project: feastProject, Provider: services.LocalProviderType, EntityKeySerializationVersion: feastdevv1alpha1.SerializationVersion, + OfflineStore: services.OfflineStoreConfig{ + Type: services.OfflineDBPersistenceSnowflakeConfigType, + DBParameters: unmarshallYamlString(snowflakeYamlString), + }, Registry: services.RegistryConfig{ Path: copyMap["path"].(string), RegistryType: services.RegistryDBPersistenceSQLConfigType, DBParameters: dbParametersMap, }, + OnlineStore: services.OnlineStoreConfig{ + Type: onlineType, + DBParameters: unmarshallYamlString(cassandraYamlString), + }, AuthzConfig: noAuthzConfig(), } Expect(repoConfig).To(Equal(testConfig)) - // check offline config - deploy = &appsv1.Deployment{} - err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.OfflineFeastType), - Namespace: resource.Namespace, - }, - deploy) - Expect(err).NotTo(HaveOccurred()) - Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(1)) - Expect(deploy.Spec.Template.Spec.Containers[0].Env).To(HaveLen(1)) - env = getFeatureStoreYamlEnvVar(deploy.Spec.Template.Spec.Containers[0].Env) + offlineContainer := services.GetOfflineContainer(*deploy) + Expect(offlineContainer.Env).To(HaveLen(1)) + assertEnvFrom(*offlineContainer) + env = getFeatureStoreYamlEnvVar(offlineContainer.Env) Expect(env).NotTo(BeNil()) - fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64(services.OfflineFeastType) + fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64() Expect(err).NotTo(HaveOccurred()) Expect(fsYamlStr).To(Equal(env.Value)) @@ -612,38 +615,17 @@ var _ = Describe("FeatureStore Controller - db storage services", func() { repoConfigOffline := &services.RepoConfig{} err = yaml.Unmarshal(envByte, repoConfigOffline) Expect(err).NotTo(HaveOccurred()) - regRemote := services.RegistryConfig{ - RegistryType: services.RegistryRemoteConfigType, - Path: fmt.Sprintf("feast-%s-registry.default.svc.cluster.local:80", resourceName), - } - offlineConfig := &services.RepoConfig{ - Project: feastProject, - Provider: services.LocalProviderType, - EntityKeySerializationVersion: feastdevv1alpha1.SerializationVersion, - OfflineStore: services.OfflineStoreConfig{ - Type: services.OfflineDBPersistenceSnowflakeConfigType, - DBParameters: unmarshallYamlString(snowflakeYamlString), - }, - Registry: regRemote, - AuthzConfig: noAuthzConfig(), - } - Expect(repoConfigOffline).To(Equal(offlineConfig)) + Expect(repoConfigOffline).To(Equal(testConfig)) - // check online config - deploy = &appsv1.Deployment{} - err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.OnlineFeastType), - Namespace: resource.Namespace, - }, - deploy) - Expect(err).NotTo(HaveOccurred()) - Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(1)) - Expect(deploy.Spec.Template.Spec.Containers[0].Env).To(HaveLen(1)) - Expect(deploy.Spec.Template.Spec.Containers[0].ImagePullPolicy).To(Equal(corev1.PullAlways)) - env = getFeatureStoreYamlEnvVar(deploy.Spec.Template.Spec.Containers[0].Env) + onlineContainer := services.GetOnlineContainer(*deploy) + Expect(onlineContainer.VolumeMounts).To(HaveLen(1)) + Expect(onlineContainer.Env).To(HaveLen(1)) + assertEnvFrom(*onlineContainer) + Expect(onlineContainer.ImagePullPolicy).To(Equal(corev1.PullAlways)) + env = getFeatureStoreYamlEnvVar(onlineContainer.Env) Expect(env).NotTo(BeNil()) - fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64(services.OnlineFeastType) + fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64() Expect(err).NotTo(HaveOccurred()) Expect(fsYamlStr).To(Equal(env.Value)) @@ -652,25 +634,9 @@ var _ = Describe("FeatureStore Controller - db storage services", func() { repoConfigOnline := &services.RepoConfig{} err = yaml.Unmarshal(envByte, repoConfigOnline) Expect(err).NotTo(HaveOccurred()) - offlineRemote := services.OfflineStoreConfig{ - Host: fmt.Sprintf("feast-%s-offline.default.svc.cluster.local", resourceName), - Type: services.OfflineRemoteConfigType, - Port: services.HttpPort, - } - onlineConfig := &services.RepoConfig{ - Project: feastProject, - Provider: services.LocalProviderType, - EntityKeySerializationVersion: feastdevv1alpha1.SerializationVersion, - OfflineStore: offlineRemote, - OnlineStore: services.OnlineStoreConfig{ - Type: onlineType, - DBParameters: unmarshallYamlString(cassandraYamlString), - }, - Registry: regRemote, - AuthzConfig: noAuthzConfig(), - } - Expect(repoConfigOnline).To(Equal(onlineConfig)) - Expect(deploy.Spec.Template.Spec.Containers[0].Env).To(HaveLen(1)) + Expect(repoConfigOnline).To(Equal(testConfig)) + onlineContainer = services.GetOnlineContainer(*deploy) + Expect(onlineContainer.Env).To(HaveLen(1)) // check client config cm := &corev1.ConfigMap{} @@ -684,6 +650,15 @@ var _ = Describe("FeatureStore Controller - db storage services", func() { repoConfigClient := &services.RepoConfig{} err = yaml.Unmarshal([]byte(cm.Data[services.FeatureStoreYamlCmKey]), repoConfigClient) Expect(err).NotTo(HaveOccurred()) + offlineRemote := services.OfflineStoreConfig{ + Host: fmt.Sprintf("feast-%s-offline.default.svc.cluster.local", resourceName), + Type: services.OfflineRemoteConfigType, + Port: services.HttpPort, + } + regRemote := services.RegistryConfig{ + RegistryType: services.RegistryRemoteConfigType, + Path: fmt.Sprintf("feast-%s-registry.default.svc.cluster.local:80", resourceName), + } clientConfig := &services.RepoConfig{ Project: feastProject, Provider: services.LocalProviderType, @@ -718,17 +693,16 @@ var _ = Describe("FeatureStore Controller - db storage services", func() { feast.Handler.FeatureStore = resource // check online config - deploy = &appsv1.Deployment{} err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.OnlineFeastType), - Namespace: resource.Namespace, - }, - deploy) + Name: objMeta.Name, + Namespace: objMeta.Namespace, + }, deploy) Expect(err).NotTo(HaveOccurred()) - env = getFeatureStoreYamlEnvVar(deploy.Spec.Template.Spec.Containers[0].Env) + onlineContainer = services.GetOnlineContainer(*deploy) + env = getFeatureStoreYamlEnvVar(onlineContainer.Env) Expect(env).NotTo(BeNil()) - fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64(services.OnlineFeastType) + fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64() Expect(err).NotTo(HaveOccurred()) Expect(fsYamlStr).To(Equal(env.Value)) @@ -738,9 +712,9 @@ var _ = Describe("FeatureStore Controller - db storage services", func() { repoConfigOnline = &services.RepoConfig{} err = yaml.Unmarshal(envByte, repoConfigOnline) Expect(err).NotTo(HaveOccurred()) - onlineConfig.OnlineStore.Type = services.OnlineDBPersistenceSnowflakeConfigType - onlineConfig.OnlineStore.DBParameters = unmarshallYamlString(snowflakeYamlString) - Expect(repoConfigOnline).To(Equal(onlineConfig)) + testConfig.OnlineStore.Type = services.OnlineDBPersistenceSnowflakeConfigType + testConfig.OnlineStore.DBParameters = unmarshallYamlString(snowflakeYamlString) + Expect(repoConfigOnline).To(Equal(testConfig)) }) }) }) diff --git a/infra/feast-operator/internal/controller/featurestore_controller_ephemeral_test.go b/infra/feast-operator/internal/controller/featurestore_controller_ephemeral_test.go index 796de8e5260..381a2d40a1d 100644 --- a/infra/feast-operator/internal/controller/featurestore_controller_ephemeral_test.go +++ b/infra/feast-operator/internal/controller/featurestore_controller_ephemeral_test.go @@ -62,11 +62,12 @@ var _ = Describe("FeatureStore Controller-Ephemeral services", func() { registryPath := "/data/registry.db" BeforeEach(func() { + createEnvFromSecretAndConfigMap() By("creating the custom resource for the Kind FeatureStore") err := k8sClient.Get(ctx, typeNamespacedName, featurestore) if err != nil && errors.IsNotFound(err) { resource := createFeatureStoreResource(resourceName, image, pullPolicy, &[]corev1.EnvVar{{Name: testEnvVarName, Value: testEnvVarValue}, - {Name: "fieldRefName", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{APIVersion: "v1", FieldPath: "metadata.namespace"}}}}) + {Name: "fieldRefName", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{APIVersion: "v1", FieldPath: "metadata.namespace"}}}}, withEnvFrom()) resource.Spec.Services.OfflineStore.Persistence = &feastdevv1alpha1.OfflineStorePersistence{ FilePersistence: &feastdevv1alpha1.OfflineStoreFilePersistence{ Type: offlineType, @@ -97,6 +98,8 @@ var _ = Describe("FeatureStore Controller-Ephemeral services", func() { By("Cleanup the specific resource instance FeatureStore") Expect(k8sClient.Delete(ctx, resource)).To(Succeed()) + + deleteEnvFromSecretAndConfigMap() }) It("should successfully reconcile the resource", func() { @@ -141,6 +144,7 @@ var _ = Describe("FeatureStore Controller-Ephemeral services", func() { Expect(resource.Status.Applied.Services.OnlineStore.Persistence.FilePersistence).NotTo(BeNil()) Expect(resource.Status.Applied.Services.OnlineStore.Persistence.FilePersistence.Path).To(Equal(onlineStorePath)) Expect(resource.Status.Applied.Services.OnlineStore.Env).To(Equal(&[]corev1.EnvVar{{Name: testEnvVarName, Value: testEnvVarValue}, {Name: "fieldRefName", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{APIVersion: "v1", FieldPath: "metadata.namespace"}}}})) + Expect(resource.Status.Applied.Services.OnlineStore.EnvFrom).To(Equal(withEnvFrom())) Expect(resource.Status.Applied.Services.OnlineStore.ImagePullPolicy).To(Equal(&pullPolicy)) Expect(resource.Status.Applied.Services.OnlineStore.Resources).NotTo(BeNil()) Expect(resource.Status.Applied.Services.OnlineStore.Image).To(Equal(&image)) @@ -198,16 +202,17 @@ var _ = Describe("FeatureStore Controller-Ephemeral services", func() { Expect(resource.Status.Phase).To(Equal(feastdevv1alpha1.ReadyPhase)) + // check deployment deploy := &appsv1.Deployment{} + objMeta := feast.GetObjectMeta() err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.RegistryFeastType), - Namespace: resource.Namespace, - }, - deploy) + Name: objMeta.Name, + Namespace: objMeta.Namespace, + }, deploy) Expect(err).NotTo(HaveOccurred()) Expect(deploy.Spec.Replicas).To(Equal(&services.DefaultReplicas)) Expect(controllerutil.HasControllerReference(deploy)).To(BeTrue()) - Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(1)) + Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(3)) svc := &corev1.Service{} err = k8sClient.Get(ctx, types.NamespacedName{ @@ -243,7 +248,7 @@ var _ = Describe("FeatureStore Controller-Ephemeral services", func() { deployList := appsv1.DeploymentList{} err = k8sClient.List(ctx, &deployList, listOpts) Expect(err).NotTo(HaveOccurred()) - Expect(deployList.Items).To(HaveLen(3)) + Expect(deployList.Items).To(HaveLen(1)) svcList := corev1.ServiceList{} err = k8sClient.List(ctx, &svcList, listOpts) @@ -264,20 +269,21 @@ var _ = Describe("FeatureStore Controller-Ephemeral services", func() { }, } - // check registry config + // check deployment deploy := &appsv1.Deployment{} + objMeta := feast.GetObjectMeta() err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.RegistryFeastType), - Namespace: resource.Namespace, - }, - deploy) - Expect(err).NotTo(HaveOccurred()) - Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(1)) - Expect(deploy.Spec.Template.Spec.Containers[0].Env).To(HaveLen(1)) - env := getFeatureStoreYamlEnvVar(deploy.Spec.Template.Spec.Containers[0].Env) + Name: objMeta.Name, + Namespace: objMeta.Namespace, + }, deploy) + Expect(err).NotTo(HaveOccurred()) + Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(3)) + registryContainer := services.GetRegistryContainer(*deploy) + Expect(registryContainer.Env).To(HaveLen(1)) + env := getFeatureStoreYamlEnvVar(registryContainer.Env) Expect(env).NotTo(BeNil()) - fsYamlStr, err := feast.GetServiceFeatureStoreYamlBase64(services.RegistryFeastType) + fsYamlStr, err := feast.GetServiceFeatureStoreYamlBase64() Expect(err).NotTo(HaveOccurred()) Expect(fsYamlStr).To(Equal(env.Value)) @@ -290,28 +296,31 @@ var _ = Describe("FeatureStore Controller-Ephemeral services", func() { Project: feastProject, Provider: services.LocalProviderType, EntityKeySerializationVersion: feastdevv1alpha1.SerializationVersion, + OfflineStore: services.OfflineStoreConfig{ + Type: services.OfflineFilePersistenceDuckDbConfigType, + }, Registry: services.RegistryConfig{ RegistryType: services.RegistryFileConfigType, Path: registryPath, }, + OnlineStore: services.OnlineStoreConfig{ + Path: onlineStorePath, + Type: services.OnlineSqliteConfigType, + }, AuthzConfig: noAuthzConfig(), } Expect(repoConfig).To(Equal(testConfig)) - // check offline config - deploy = &appsv1.Deployment{} - err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.OfflineFeastType), - Namespace: resource.Namespace, - }, - deploy) - Expect(err).NotTo(HaveOccurred()) - Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(1)) - Expect(deploy.Spec.Template.Spec.Containers[0].Env).To(HaveLen(1)) - env = getFeatureStoreYamlEnvVar(deploy.Spec.Template.Spec.Containers[0].Env) + offlineContainer := services.GetOfflineContainer(*deploy) + Expect(offlineContainer.Env).To(HaveLen(1)) + assertEnvFrom(*offlineContainer) + env = getFeatureStoreYamlEnvVar(offlineContainer.Env) Expect(env).NotTo(BeNil()) - fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64(services.OfflineFeastType) + // check envFrom for offlineContainer + assertEnvFrom(*offlineContainer) + + fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64() Expect(err).NotTo(HaveOccurred()) Expect(fsYamlStr).To(Equal(env.Value)) @@ -320,37 +329,15 @@ var _ = Describe("FeatureStore Controller-Ephemeral services", func() { repoConfigOffline := &services.RepoConfig{} err = yaml.Unmarshal(envByte, repoConfigOffline) Expect(err).NotTo(HaveOccurred()) - regRemote := services.RegistryConfig{ - RegistryType: services.RegistryRemoteConfigType, - Path: fmt.Sprintf("feast-%s-registry.default.svc.cluster.local:80", resourceName), - } - offlineConfig := &services.RepoConfig{ - Project: feastProject, - Provider: services.LocalProviderType, - EntityKeySerializationVersion: feastdevv1alpha1.SerializationVersion, - OfflineStore: services.OfflineStoreConfig{ - Type: services.OfflineFilePersistenceDuckDbConfigType, - }, - Registry: regRemote, - AuthzConfig: noAuthzConfig(), - } - Expect(repoConfigOffline).To(Equal(offlineConfig)) + Expect(repoConfigOffline).To(Equal(testConfig)) - // check online config - deploy = &appsv1.Deployment{} - err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.OnlineFeastType), - Namespace: resource.Namespace, - }, - deploy) - Expect(err).NotTo(HaveOccurred()) - Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(1)) - Expect(deploy.Spec.Template.Spec.Containers[0].Env).To(HaveLen(3)) - Expect(deploy.Spec.Template.Spec.Containers[0].ImagePullPolicy).To(Equal(corev1.PullAlways)) - env = getFeatureStoreYamlEnvVar(deploy.Spec.Template.Spec.Containers[0].Env) + onlineContainer := services.GetOnlineContainer(*deploy) + Expect(onlineContainer.Env).To(HaveLen(3)) + Expect(onlineContainer.ImagePullPolicy).To(Equal(corev1.PullAlways)) + env = getFeatureStoreYamlEnvVar(onlineContainer.Env) Expect(env).NotTo(BeNil()) - fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64(services.OnlineFeastType) + fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64() Expect(err).NotTo(HaveOccurred()) Expect(fsYamlStr).To(Equal(env.Value)) @@ -359,25 +346,7 @@ var _ = Describe("FeatureStore Controller-Ephemeral services", func() { repoConfigOnline := &services.RepoConfig{} err = yaml.Unmarshal(envByte, repoConfigOnline) Expect(err).NotTo(HaveOccurred()) - offlineRemote := services.OfflineStoreConfig{ - Host: fmt.Sprintf("feast-%s-offline.default.svc.cluster.local", resourceName), - Type: services.OfflineRemoteConfigType, - Port: services.HttpPort, - } - onlineConfig := &services.RepoConfig{ - Project: feastProject, - Provider: services.LocalProviderType, - EntityKeySerializationVersion: feastdevv1alpha1.SerializationVersion, - OfflineStore: offlineRemote, - OnlineStore: services.OnlineStoreConfig{ - Path: onlineStorePath, - Type: services.OnlineSqliteConfigType, - }, - Registry: regRemote, - AuthzConfig: noAuthzConfig(), - } - Expect(repoConfigOnline).To(Equal(onlineConfig)) - Expect(deploy.Spec.Template.Spec.Containers[0].Env).To(HaveLen(3)) + Expect(repoConfigOnline).To(Equal(testConfig)) // check client config cm := &corev1.ConfigMap{} @@ -395,12 +364,19 @@ var _ = Describe("FeatureStore Controller-Ephemeral services", func() { Project: feastProject, Provider: services.LocalProviderType, EntityKeySerializationVersion: feastdevv1alpha1.SerializationVersion, - OfflineStore: offlineRemote, + OfflineStore: services.OfflineStoreConfig{ + Host: fmt.Sprintf("feast-%s-offline.default.svc.cluster.local", resourceName), + Type: services.OfflineRemoteConfigType, + Port: services.HttpPort, + }, OnlineStore: services.OnlineStoreConfig{ Path: fmt.Sprintf("http://feast-%s-online.default.svc.cluster.local:80", resourceName), Type: services.OnlineRemoteConfigType, }, - Registry: regRemote, + Registry: services.RegistryConfig{ + RegistryType: services.RegistryRemoteConfigType, + Path: fmt.Sprintf("feast-%s-registry.default.svc.cluster.local:80", resourceName), + }, AuthzConfig: noAuthzConfig(), } Expect(repoConfigClient).To(Equal(clientConfig)) @@ -423,17 +399,16 @@ var _ = Describe("FeatureStore Controller-Ephemeral services", func() { Expect(err).NotTo(HaveOccurred()) feast.Handler.FeatureStore = resource - // check registry config - deploy = &appsv1.Deployment{} + // check registry err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.RegistryFeastType), - Namespace: resource.Namespace, - }, - deploy) + Name: objMeta.Name, + Namespace: objMeta.Namespace, + }, deploy) Expect(err).NotTo(HaveOccurred()) - env = getFeatureStoreYamlEnvVar(deploy.Spec.Template.Spec.Containers[0].Env) + registryContainer = services.GetRegistryContainer(*deploy) + env = getFeatureStoreYamlEnvVar(registryContainer.Env) Expect(env).NotTo(BeNil()) - fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64(services.RegistryFeastType) + fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64() Expect(err).NotTo(HaveOccurred()) Expect(fsYamlStr).To(Equal(env.Value)) @@ -442,21 +417,16 @@ var _ = Describe("FeatureStore Controller-Ephemeral services", func() { repoConfig = &services.RepoConfig{} err = yaml.Unmarshal(envByte, repoConfig) Expect(err).NotTo(HaveOccurred()) + testConfig.OnlineStore.Path = newOnlineStorePath testConfig.Registry.Path = newRegistryPath Expect(repoConfig).To(Equal(testConfig)) // check offline config - deploy = &appsv1.Deployment{} - err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.OfflineFeastType), - Namespace: resource.Namespace, - }, - deploy) - Expect(err).NotTo(HaveOccurred()) - env = getFeatureStoreYamlEnvVar(deploy.Spec.Template.Spec.Containers[0].Env) + offlineContainer = services.GetRegistryContainer(*deploy) + env = getFeatureStoreYamlEnvVar(offlineContainer.Env) Expect(env).NotTo(BeNil()) - fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64(services.OfflineFeastType) + fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64() Expect(err).NotTo(HaveOccurred()) Expect(fsYamlStr).To(Equal(env.Value)) @@ -465,20 +435,18 @@ var _ = Describe("FeatureStore Controller-Ephemeral services", func() { repoConfigOffline = &services.RepoConfig{} err = yaml.Unmarshal(envByte, repoConfigOffline) Expect(err).NotTo(HaveOccurred()) - Expect(repoConfigOffline).To(Equal(offlineConfig)) + Expect(repoConfigOffline).To(Equal(testConfig)) // check online config - deploy = &appsv1.Deployment{} - err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.OnlineFeastType), - Namespace: resource.Namespace, - }, - deploy) - Expect(err).NotTo(HaveOccurred()) - env = getFeatureStoreYamlEnvVar(deploy.Spec.Template.Spec.Containers[0].Env) + onlineContainer = services.GetOnlineContainer(*deploy) + env = getFeatureStoreYamlEnvVar(onlineContainer.Env) Expect(env).NotTo(BeNil()) - fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64(services.OnlineFeastType) + // check envFrom + // Validate `envFrom` for ConfigMap and Secret + assertEnvFrom(*onlineContainer) + + fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64() Expect(err).NotTo(HaveOccurred()) Expect(fsYamlStr).To(Equal(env.Value)) @@ -488,8 +456,8 @@ var _ = Describe("FeatureStore Controller-Ephemeral services", func() { repoConfigOnline = &services.RepoConfig{} err = yaml.Unmarshal(envByte, repoConfigOnline) Expect(err).NotTo(HaveOccurred()) - onlineConfig.OnlineStore.Path = newOnlineStorePath - Expect(repoConfigOnline).To(Equal(onlineConfig)) + testConfig.OnlineStore.Path = newOnlineStorePath + Expect(repoConfigOnline).To(Equal(testConfig)) }) }) }) diff --git a/infra/feast-operator/internal/controller/featurestore_controller_kubernetes_auth_test.go b/infra/feast-operator/internal/controller/featurestore_controller_kubernetes_auth_test.go index 4930f3fc590..e3a0e8fa646 100644 --- a/infra/feast-operator/internal/controller/featurestore_controller_kubernetes_auth_test.go +++ b/infra/feast-operator/internal/controller/featurestore_controller_kubernetes_auth_test.go @@ -59,10 +59,12 @@ var _ = Describe("FeatureStore Controller-Kubernetes authorization", func() { roles := []string{"reader", "writer"} BeforeEach(func() { + createEnvFromSecretAndConfigMap() + By("creating the custom resource for the Kind FeatureStore") err := k8sClient.Get(ctx, typeNamespacedName, featurestore) if err != nil && errors.IsNotFound(err) { - resource := createFeatureStoreResource(resourceName, image, pullPolicy, &[]corev1.EnvVar{}) + resource := createFeatureStoreResource(resourceName, image, pullPolicy, &[]corev1.EnvVar{}, withEnvFrom()) resource.Spec.AuthzConfig = &feastdevv1alpha1.AuthzConfig{KubernetesAuthz: &feastdevv1alpha1.KubernetesAuthz{ Roles: roles, }} @@ -75,6 +77,8 @@ var _ = Describe("FeatureStore Controller-Kubernetes authorization", func() { err := k8sClient.Get(ctx, typeNamespacedName, resource) Expect(err).NotTo(HaveOccurred()) + deleteEnvFromSecretAndConfigMap() + By("Cleanup the specific resource instance FeatureStore") Expect(k8sClient.Delete(ctx, resource)).To(Succeed()) }) @@ -124,8 +128,9 @@ var _ = Describe("FeatureStore Controller-Kubernetes authorization", func() { Expect(resource.Status.Applied.Services.OnlineStore).NotTo(BeNil()) Expect(resource.Status.Applied.Services.OnlineStore.Persistence).NotTo(BeNil()) Expect(resource.Status.Applied.Services.OnlineStore.Persistence.FilePersistence).NotTo(BeNil()) - Expect(resource.Status.Applied.Services.OnlineStore.Persistence.FilePersistence.Path).To(Equal(services.DefaultOnlineStoreEphemeralPath)) + Expect(resource.Status.Applied.Services.OnlineStore.Persistence.FilePersistence.Path).To(Equal(services.EphemeralPath + "/" + services.DefaultOnlineStorePath)) Expect(resource.Status.Applied.Services.OnlineStore.Env).To(Equal(&[]corev1.EnvVar{})) + Expect(resource.Status.Applied.Services.OnlineStore.EnvFrom).To(Equal(withEnvFrom())) Expect(resource.Status.Applied.Services.OnlineStore.ImagePullPolicy).To(Equal(&pullPolicy)) Expect(resource.Status.Applied.Services.OnlineStore.Resources).NotTo(BeNil()) Expect(resource.Status.Applied.Services.OnlineStore.Image).To(Equal(&image)) @@ -133,7 +138,7 @@ var _ = Describe("FeatureStore Controller-Kubernetes authorization", func() { Expect(resource.Status.Applied.Services.Registry.Local).NotTo(BeNil()) Expect(resource.Status.Applied.Services.Registry.Local.Persistence).NotTo(BeNil()) Expect(resource.Status.Applied.Services.Registry.Local.Persistence.FilePersistence).NotTo(BeNil()) - Expect(resource.Status.Applied.Services.Registry.Local.Persistence.FilePersistence.Path).To(Equal(services.DefaultRegistryEphemeralPath)) + Expect(resource.Status.Applied.Services.Registry.Local.Persistence.FilePersistence.Path).To(Equal(services.EphemeralPath + "/" + services.DefaultRegistryPath)) Expect(resource.Status.Applied.Services.Registry.Local.ImagePullPolicy).To(BeNil()) Expect(resource.Status.Applied.Services.Registry.Local.Resources).To(BeNil()) Expect(resource.Status.Applied.Services.Registry.Local.Image).To(Equal(&services.DefaultImage)) @@ -187,47 +192,19 @@ var _ = Describe("FeatureStore Controller-Kubernetes authorization", func() { Expect(resource.Status.Phase).To(Equal(feastdevv1alpha1.ReadyPhase)) - // check offline deployment + // check deployment deploy := &appsv1.Deployment{} + objMeta := feast.GetObjectMeta() err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.OfflineFeastType), - Namespace: resource.Namespace, - }, - deploy) - Expect(err).NotTo(HaveOccurred()) - Expect(deploy.Spec.Replicas).To(Equal(&services.DefaultReplicas)) - Expect(controllerutil.HasControllerReference(deploy)).To(BeTrue()) - Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(1)) - Expect(deploy.Spec.Template.Spec.Volumes).To(HaveLen(0)) - Expect(deploy.Spec.Template.Spec.Containers[0].VolumeMounts).To(HaveLen(0)) - - // check online deployment - deploy = &appsv1.Deployment{} - err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.OnlineFeastType), - Namespace: resource.Namespace, - }, - deploy) - Expect(err).NotTo(HaveOccurred()) - Expect(deploy.Spec.Replicas).To(Equal(&services.DefaultReplicas)) - Expect(controllerutil.HasControllerReference(deploy)).To(BeTrue()) - Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(1)) - Expect(deploy.Spec.Template.Spec.Volumes).To(HaveLen(0)) - Expect(deploy.Spec.Template.Spec.Containers[0].VolumeMounts).To(HaveLen(0)) - - // check registry deployment - deploy = &appsv1.Deployment{} - err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.RegistryFeastType), - Namespace: resource.Namespace, - }, - deploy) + Name: objMeta.Name, + Namespace: objMeta.Namespace, + }, deploy) Expect(err).NotTo(HaveOccurred()) Expect(deploy.Spec.Replicas).To(Equal(&services.DefaultReplicas)) Expect(controllerutil.HasControllerReference(deploy)).To(BeTrue()) - Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(1)) - Expect(deploy.Spec.Template.Spec.Volumes).To(HaveLen(0)) - Expect(deploy.Spec.Template.Spec.Containers[0].VolumeMounts).To(HaveLen(0)) + Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(3)) + Expect(deploy.Spec.Template.Spec.Volumes).To(HaveLen(1)) + Expect(deploy.Spec.Template.Spec.Containers[0].VolumeMounts).To(HaveLen(1)) // check configured Roles for _, roleName := range roles { @@ -276,23 +253,21 @@ var _ = Describe("FeatureStore Controller-Kubernetes authorization", func() { Kind: "Role", Name: feastRole.Name, } - for _, serviceType := range []services.FeastServiceType{services.RegistryFeastType, services.OnlineFeastType, services.OfflineFeastType} { - sa := &corev1.ServiceAccount{} - err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(serviceType), - Namespace: resource.Namespace, - }, - sa) - Expect(err).NotTo(HaveOccurred()) + sa := &corev1.ServiceAccount{} + err = k8sClient.Get(ctx, types.NamespacedName{ + Name: services.GetFeastName(feast.Handler.FeatureStore), + Namespace: resource.Namespace, + }, + sa) + Expect(err).NotTo(HaveOccurred()) - expectedSubject := rbacv1.Subject{ - Kind: rbacv1.ServiceAccountKind, - Name: sa.Name, - Namespace: sa.Namespace, - } - Expect(roleBinding.Subjects).To(ContainElement(expectedSubject)) - Expect(roleBinding.RoleRef).To(Equal(expectedRoleRef)) + expectedSubject := rbacv1.Subject{ + Kind: rbacv1.ServiceAccountKind, + Name: sa.Name, + Namespace: sa.Namespace, } + Expect(roleBinding.Subjects).To(ContainElement(expectedSubject)) + Expect(roleBinding.RoleRef).To(Equal(expectedRoleRef)) By("Updating the user roled and reconciling") resourceNew := resource.DeepCopy() @@ -393,7 +368,7 @@ var _ = Describe("FeatureStore Controller-Kubernetes authorization", func() { deployList := appsv1.DeploymentList{} err = k8sClient.List(ctx, &deployList, listOpts) Expect(err).NotTo(HaveOccurred()) - Expect(deployList.Items).To(HaveLen(3)) + Expect(deployList.Items).To(HaveLen(1)) svcList := corev1.ServiceList{} err = k8sClient.List(ctx, &svcList, listOpts) @@ -414,19 +389,19 @@ var _ = Describe("FeatureStore Controller-Kubernetes authorization", func() { }, } - // check registry deployment + // check registry deploy := &appsv1.Deployment{} + objMeta := feast.GetObjectMeta() err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.RegistryFeastType), - Namespace: resource.Namespace, - }, - deploy) + Name: objMeta.Name, + Namespace: objMeta.Namespace, + }, deploy) Expect(err).NotTo(HaveOccurred()) - env := getFeatureStoreYamlEnvVar(deploy.Spec.Template.Spec.Containers[0].Env) + env := getFeatureStoreYamlEnvVar(services.GetRegistryContainer(*deploy).Env) Expect(env).NotTo(BeNil()) // check registry config - fsYamlStr, err := feast.GetServiceFeatureStoreYamlBase64(services.RegistryFeastType) + fsYamlStr, err := feast.GetServiceFeatureStoreYamlBase64() Expect(err).NotTo(HaveOccurred()) Expect(fsYamlStr).To(Equal(env.Value)) @@ -435,34 +410,25 @@ var _ = Describe("FeatureStore Controller-Kubernetes authorization", func() { repoConfig := &services.RepoConfig{} err = yaml.Unmarshal(envByte, repoConfig) Expect(err).NotTo(HaveOccurred()) - testConfig := &services.RepoConfig{ - Project: feastProject, - Provider: services.LocalProviderType, - EntityKeySerializationVersion: feastdevv1alpha1.SerializationVersion, - Registry: services.RegistryConfig{ - RegistryType: services.RegistryFileConfigType, - Path: services.DefaultRegistryEphemeralPath, - S3AdditionalKwargs: nil, - }, - AuthzConfig: services.AuthzConfig{ - Type: services.KubernetesAuthType, - }, + testConfig := feast.GetDefaultRepoConfig() + testConfig.OfflineStore = services.OfflineStoreConfig{ + Type: services.OfflineFilePersistenceDaskConfigType, + } + testConfig.Registry.RegistryType = services.RegistryFileConfigType + testConfig.AuthzConfig = services.AuthzConfig{ + Type: services.KubernetesAuthType, } - Expect(repoConfig).To(Equal(testConfig)) + Expect(repoConfig).To(Equal(&testConfig)) - // check offline deployment - deploy = &appsv1.Deployment{} - err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.OfflineFeastType), - Namespace: resource.Namespace, - }, - deploy) - Expect(err).NotTo(HaveOccurred()) - env = getFeatureStoreYamlEnvVar(deploy.Spec.Template.Spec.Containers[0].Env) + // check offline + offlineContainer := services.GetOfflineContainer(*deploy) + env = getFeatureStoreYamlEnvVar(offlineContainer.Env) Expect(env).NotTo(BeNil()) + assertEnvFrom(*offlineContainer) + // check offline config - fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64(services.OfflineFeastType) + fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64() Expect(err).NotTo(HaveOccurred()) Expect(fsYamlStr).To(Equal(env.Value)) @@ -471,37 +437,17 @@ var _ = Describe("FeatureStore Controller-Kubernetes authorization", func() { repoConfig = &services.RepoConfig{} err = yaml.Unmarshal(envByte, repoConfig) Expect(err).NotTo(HaveOccurred()) - regRemote := services.RegistryConfig{ - RegistryType: services.RegistryRemoteConfigType, - Path: fmt.Sprintf("feast-%s-registry.default.svc.cluster.local:80", resourceName), - } - testConfig = &services.RepoConfig{ - Project: feastProject, - Provider: services.LocalProviderType, - EntityKeySerializationVersion: feastdevv1alpha1.SerializationVersion, - OfflineStore: services.OfflineStoreConfig{ - Type: services.OfflineFilePersistenceDaskConfigType, - }, - Registry: regRemote, - AuthzConfig: services.AuthzConfig{ - Type: services.KubernetesAuthType, - }, - } - Expect(repoConfig).To(Equal(testConfig)) + Expect(repoConfig).To(Equal(&testConfig)) - // check online deployment - deploy = &appsv1.Deployment{} - err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.OnlineFeastType), - Namespace: resource.Namespace, - }, - deploy) - Expect(err).NotTo(HaveOccurred()) - env = getFeatureStoreYamlEnvVar(deploy.Spec.Template.Spec.Containers[0].Env) + // check online + onlineContainer := services.GetOnlineContainer(*deploy) + env = getFeatureStoreYamlEnvVar(onlineContainer.Env) Expect(env).NotTo(BeNil()) + assertEnvFrom(*onlineContainer) + // check online config - fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64(services.OnlineFeastType) + fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64() Expect(err).NotTo(HaveOccurred()) Expect(fsYamlStr).To(Equal(env.Value)) @@ -510,26 +456,7 @@ var _ = Describe("FeatureStore Controller-Kubernetes authorization", func() { repoConfig = &services.RepoConfig{} err = yaml.Unmarshal(envByte, repoConfig) Expect(err).NotTo(HaveOccurred()) - offlineRemote := services.OfflineStoreConfig{ - Host: fmt.Sprintf("feast-%s-offline.default.svc.cluster.local", resourceName), - Type: services.OfflineRemoteConfigType, - Port: services.HttpPort, - } - testConfig = &services.RepoConfig{ - Project: feastProject, - Provider: services.LocalProviderType, - EntityKeySerializationVersion: feastdevv1alpha1.SerializationVersion, - OfflineStore: offlineRemote, - OnlineStore: services.OnlineStoreConfig{ - Path: services.DefaultOnlineStoreEphemeralPath, - Type: services.OnlineSqliteConfigType, - }, - Registry: regRemote, - AuthzConfig: services.AuthzConfig{ - Type: services.KubernetesAuthType, - }, - } - Expect(repoConfig).To(Equal(testConfig)) + Expect(repoConfig).To(Equal(&testConfig)) // check client config cm := &corev1.ConfigMap{} @@ -543,6 +470,15 @@ var _ = Describe("FeatureStore Controller-Kubernetes authorization", func() { repoConfigClient := &services.RepoConfig{} err = yaml.Unmarshal([]byte(cm.Data[services.FeatureStoreYamlCmKey]), repoConfigClient) Expect(err).NotTo(HaveOccurred()) + regRemote := services.RegistryConfig{ + RegistryType: services.RegistryRemoteConfigType, + Path: fmt.Sprintf("feast-%s-registry.default.svc.cluster.local:80", resourceName), + } + offlineRemote := services.OfflineStoreConfig{ + Host: fmt.Sprintf("feast-%s-offline.default.svc.cluster.local", resourceName), + Type: services.OfflineRemoteConfigType, + Port: services.HttpPort, + } clientConfig := &services.RepoConfig{ Project: feastProject, Provider: services.LocalProviderType, @@ -552,10 +488,7 @@ var _ = Describe("FeatureStore Controller-Kubernetes authorization", func() { Path: fmt.Sprintf("http://feast-%s-online.default.svc.cluster.local:80", resourceName), Type: services.OnlineRemoteConfigType, }, - Registry: services.RegistryConfig{ - RegistryType: services.RegistryRemoteConfigType, - Path: fmt.Sprintf("feast-%s-registry.default.svc.cluster.local:80", resourceName), - }, + Registry: regRemote, AuthzConfig: services.AuthzConfig{ Type: services.KubernetesAuthType, }, diff --git a/infra/feast-operator/internal/controller/featurestore_controller_loglevel_test.go b/infra/feast-operator/internal/controller/featurestore_controller_loglevel_test.go new file mode 100644 index 00000000000..f238ca9ee4c --- /dev/null +++ b/infra/feast-operator/internal/controller/featurestore_controller_loglevel_test.go @@ -0,0 +1,235 @@ +/* +Copyright 2024 Feast Community. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller + +import ( + "context" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + appsv1 "k8s.io/api/apps/v1" + "k8s.io/apimachinery/pkg/api/errors" + apimeta "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + "github.com/feast-dev/feast/infra/feast-operator/api/feastversion" + feastdevv1alpha1 "github.com/feast-dev/feast/infra/feast-operator/api/v1alpha1" + "github.com/feast-dev/feast/infra/feast-operator/internal/controller/handler" + "github.com/feast-dev/feast/infra/feast-operator/internal/controller/services" +) + +var _ = Describe("FeatureStore Controller - Feast service LogLevel", func() { + Context("When reconciling a FeatureStore resource", func() { + const resourceName = "test-loglevel" + + ctx := context.Background() + + typeNamespacedName := types.NamespacedName{ + Name: resourceName, + Namespace: "default", + } + featurestore := &feastdevv1alpha1.FeatureStore{} + + BeforeEach(func() { + By("creating the custom resource for the Kind FeatureStore") + err := k8sClient.Get(ctx, typeNamespacedName, featurestore) + if err != nil && errors.IsNotFound(err) { + resource := &feastdevv1alpha1.FeatureStore{ + ObjectMeta: metav1.ObjectMeta{ + Name: resourceName, + Namespace: "default", + }, + Spec: feastdevv1alpha1.FeatureStoreSpec{ + FeastProject: feastProject, + Services: &feastdevv1alpha1.FeatureStoreServices{ + Registry: &feastdevv1alpha1.Registry{ + Local: &feastdevv1alpha1.LocalRegistryConfig{ + LogLevel: "error", + }, + }, + OnlineStore: &feastdevv1alpha1.OnlineStore{ + LogLevel: "debug", + }, + OfflineStore: &feastdevv1alpha1.OfflineStore{ + LogLevel: "info", + }, + }, + }, + } + Expect(k8sClient.Create(ctx, resource)).To(Succeed()) + } + }) + AfterEach(func() { + resource := &feastdevv1alpha1.FeatureStore{} + err := k8sClient.Get(ctx, typeNamespacedName, resource) + Expect(err).NotTo(HaveOccurred()) + + By("Cleanup the specific resource instance FeatureStore") + Expect(k8sClient.Delete(ctx, resource)).To(Succeed()) + }) + + It("should successfully reconcile the resource with logLevel", func() { + By("Reconciling the created resource") + controllerReconciler := &FeatureStoreReconciler{ + Client: k8sClient, + Scheme: k8sClient.Scheme(), + } + + _, err := controllerReconciler.Reconcile(ctx, reconcile.Request{ + NamespacedName: typeNamespacedName, + }) + Expect(err).NotTo(HaveOccurred()) + + resource := &feastdevv1alpha1.FeatureStore{} + err = k8sClient.Get(ctx, typeNamespacedName, resource) + Expect(err).NotTo(HaveOccurred()) + feast := services.FeastServices{ + Handler: handler.FeastHandler{ + Client: controllerReconciler.Client, + Context: ctx, + Scheme: controllerReconciler.Scheme, + FeatureStore: resource, + }, + } + + Expect(resource.Status).NotTo(BeNil()) + Expect(resource.Status.FeastVersion).To(Equal(feastversion.FeastVersion)) + Expect(resource.Status.Applied.FeastProject).To(Equal(resource.Spec.FeastProject)) + Expect(resource.Status.Applied.Services).NotTo(BeNil()) + Expect(resource.Status.Applied.Services.OfflineStore).NotTo(BeNil()) + Expect(resource.Status.Applied.Services.OnlineStore).NotTo(BeNil()) + Expect(resource.Status.Applied.Services.Registry).NotTo(BeNil()) + + Expect(resource.Status.Conditions).NotTo(BeEmpty()) + cond := apimeta.FindStatusCondition(resource.Status.Conditions, feastdevv1alpha1.ReadyType) + Expect(cond).ToNot(BeNil()) + Expect(cond.Status).To(Equal(metav1.ConditionTrue)) + Expect(cond.Reason).To(Equal(feastdevv1alpha1.ReadyReason)) + Expect(cond.Type).To(Equal(feastdevv1alpha1.ReadyType)) + Expect(cond.Message).To(Equal(feastdevv1alpha1.ReadyMessage)) + + cond = apimeta.FindStatusCondition(resource.Status.Conditions, feastdevv1alpha1.RegistryReadyType) + Expect(cond).ToNot(BeNil()) + Expect(cond.Status).To(Equal(metav1.ConditionTrue)) + Expect(cond.Reason).To(Equal(feastdevv1alpha1.ReadyReason)) + Expect(cond.Type).To(Equal(feastdevv1alpha1.RegistryReadyType)) + Expect(cond.Message).To(Equal(feastdevv1alpha1.RegistryReadyMessage)) + + cond = apimeta.FindStatusCondition(resource.Status.Conditions, feastdevv1alpha1.ClientReadyType) + Expect(cond).ToNot(BeNil()) + Expect(cond.Status).To(Equal(metav1.ConditionTrue)) + Expect(cond.Reason).To(Equal(feastdevv1alpha1.ReadyReason)) + Expect(cond.Type).To(Equal(feastdevv1alpha1.ClientReadyType)) + Expect(cond.Message).To(Equal(feastdevv1alpha1.ClientReadyMessage)) + + cond = apimeta.FindStatusCondition(resource.Status.Conditions, feastdevv1alpha1.OfflineStoreReadyType) + Expect(cond).ToNot(BeNil()) + Expect(cond.Status).To(Equal(metav1.ConditionTrue)) + Expect(cond.Reason).To(Equal(feastdevv1alpha1.ReadyReason)) + Expect(cond.Type).To(Equal(feastdevv1alpha1.OfflineStoreReadyType)) + Expect(cond.Message).To(Equal(feastdevv1alpha1.OfflineStoreReadyMessage)) + + cond = apimeta.FindStatusCondition(resource.Status.Conditions, feastdevv1alpha1.OnlineStoreReadyType) + Expect(cond).ToNot(BeNil()) + Expect(cond.Status).To(Equal(metav1.ConditionTrue)) + Expect(cond.Reason).To(Equal(feastdevv1alpha1.ReadyReason)) + Expect(cond.Type).To(Equal(feastdevv1alpha1.OnlineStoreReadyType)) + Expect(cond.Message).To(Equal(feastdevv1alpha1.OnlineStoreReadyMessage)) + Expect(resource.Status.Phase).To(Equal(feastdevv1alpha1.ReadyPhase)) + + // check deployment + deploy := &appsv1.Deployment{} + objMeta := feast.GetObjectMeta() + err = k8sClient.Get(ctx, types.NamespacedName{ + Name: objMeta.Name, + Namespace: objMeta.Namespace, + }, deploy) + Expect(err).NotTo(HaveOccurred()) + Expect(deploy.Spec.Replicas).To(Equal(&services.DefaultReplicas)) + Expect(controllerutil.HasControllerReference(deploy)).To(BeTrue()) + Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(3)) + command := services.GetRegistryContainer(*deploy).Command + Expect(command).To(ContainElement("--log-level")) + Expect(command).To(ContainElement("ERROR")) + + command = services.GetOfflineContainer(*deploy).Command + Expect(command).To(ContainElement("--log-level")) + Expect(command).To(ContainElement("INFO")) + + command = services.GetOnlineContainer(*deploy).Command + Expect(command).To(ContainElement("--log-level")) + Expect(command).To(ContainElement("DEBUG")) + }) + + It("should not include --log-level parameter when logLevel is not specified for any service", func() { + By("Updating the FeatureStore resource without specifying logLevel for any service") + resource := &feastdevv1alpha1.FeatureStore{} + err := k8sClient.Get(ctx, typeNamespacedName, resource) + Expect(err).NotTo(HaveOccurred()) + + resource.Spec.Services = &feastdevv1alpha1.FeatureStoreServices{ + Registry: &feastdevv1alpha1.Registry{ + Local: &feastdevv1alpha1.LocalRegistryConfig{}, + }, + OnlineStore: &feastdevv1alpha1.OnlineStore{}, + OfflineStore: &feastdevv1alpha1.OfflineStore{}, + } + Expect(k8sClient.Update(ctx, resource)).To(Succeed()) + + controllerReconciler := &FeatureStoreReconciler{ + Client: k8sClient, + Scheme: k8sClient.Scheme(), + } + + _, err = controllerReconciler.Reconcile(ctx, reconcile.Request{ + NamespacedName: typeNamespacedName, + }) + Expect(err).NotTo(HaveOccurred()) + + feast := services.FeastServices{ + Handler: handler.FeastHandler{ + Client: controllerReconciler.Client, + Context: ctx, + Scheme: controllerReconciler.Scheme, + FeatureStore: resource, + }, + } + + // check deployment + deploy := &appsv1.Deployment{} + objMeta := feast.GetObjectMeta() + err = k8sClient.Get(ctx, types.NamespacedName{ + Name: objMeta.Name, + Namespace: objMeta.Namespace, + }, deploy) + Expect(err).NotTo(HaveOccurred()) + Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(3)) + command := services.GetRegistryContainer(*deploy).Command + Expect(command).NotTo(ContainElement("--log-level")) + + command = services.GetOfflineContainer(*deploy).Command + Expect(command).NotTo(ContainElement("--log-level")) + + command = services.GetOnlineContainer(*deploy).Command + Expect(command).NotTo(ContainElement("--log-level")) + }) + + }) +}) diff --git a/infra/feast-operator/internal/controller/featurestore_controller_objectstore_test.go b/infra/feast-operator/internal/controller/featurestore_controller_objectstore_test.go index db07418c92b..6e9adbf8eb5 100644 --- a/infra/feast-operator/internal/controller/featurestore_controller_objectstore_test.go +++ b/infra/feast-operator/internal/controller/featurestore_controller_objectstore_test.go @@ -64,11 +64,13 @@ var _ = Describe("FeatureStore Controller-Ephemeral services", func() { } BeforeEach(func() { + createEnvFromSecretAndConfigMap() + By("creating the custom resource for the Kind FeatureStore") err := k8sClient.Get(ctx, typeNamespacedName, featurestore) if err != nil && errors.IsNotFound(err) { resource := createFeatureStoreResource(resourceName, image, pullPolicy, &[]corev1.EnvVar{{Name: testEnvVarName, Value: testEnvVarValue}, - {Name: "fieldRefName", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{APIVersion: "v1", FieldPath: "metadata.namespace"}}}}) + {Name: "fieldRefName", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{APIVersion: "v1", FieldPath: "metadata.namespace"}}}}, withEnvFrom()) resource.Spec.Services.OnlineStore = nil resource.Spec.Services.OfflineStore = nil resource.Spec.Services.Registry = &feastdevv1alpha1.Registry{ @@ -81,7 +83,6 @@ var _ = Describe("FeatureStore Controller-Ephemeral services", func() { }, }, } - Expect(k8sClient.Create(ctx, resource)).To(Succeed()) } }) @@ -90,6 +91,8 @@ var _ = Describe("FeatureStore Controller-Ephemeral services", func() { err := k8sClient.Get(ctx, typeNamespacedName, resource) Expect(err).NotTo(HaveOccurred()) + deleteEnvFromSecretAndConfigMap() + By("Cleanup the specific resource instance FeatureStore") Expect(k8sClient.Delete(ctx, resource)).To(Succeed()) }) @@ -175,39 +178,23 @@ var _ = Describe("FeatureStore Controller-Ephemeral services", func() { Expect(resource.Status.Phase).To(Equal(feastdevv1alpha1.ReadyPhase)) - // check offline deployment + // check deployment deploy := &appsv1.Deployment{} + objMeta := feast.GetObjectMeta() err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.OfflineFeastType), - Namespace: resource.Namespace, - }, - deploy) - Expect(err).To(HaveOccurred()) - Expect(errors.IsNotFound(err)).To(BeTrue()) - - // check online deployment - deploy = &appsv1.Deployment{} - err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.OnlineFeastType), - Namespace: resource.Namespace, - }, - deploy) - Expect(err).To(HaveOccurred()) - Expect(errors.IsNotFound(err)).To(BeTrue()) - - // check registry deployment - deploy = &appsv1.Deployment{} - err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.RegistryFeastType), - Namespace: resource.Namespace, - }, - deploy) + Name: objMeta.Name, + Namespace: objMeta.Namespace, + }, deploy) Expect(err).NotTo(HaveOccurred()) Expect(deploy.Spec.Replicas).To(Equal(&services.DefaultReplicas)) Expect(controllerutil.HasControllerReference(deploy)).To(BeTrue()) + Expect(deploy.Spec.Template.Spec.InitContainers).To(HaveLen(1)) Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(1)) - Expect(deploy.Spec.Template.Spec.Volumes).To(HaveLen(0)) - Expect(deploy.Spec.Template.Spec.Containers[0].VolumeMounts).To(HaveLen(0)) + Expect(services.GetRegistryContainer(*deploy)).NotTo(BeNil()) + Expect(services.GetOnlineContainer(*deploy)).To(BeNil()) + Expect(services.GetOfflineContainer(*deploy)).To(BeNil()) + Expect(deploy.Spec.Template.Spec.Volumes).To(HaveLen(1)) + Expect(deploy.Spec.Template.Spec.Containers[0].VolumeMounts).To(HaveLen(1)) // update S3 additional args and reconcile resourceNew := resource.DeepCopy() @@ -233,16 +220,14 @@ var _ = Describe("FeatureStore Controller-Ephemeral services", func() { Expect(resource.Status.Applied.Services.Registry.Local.Persistence.FilePersistence.S3AdditionalKwargs).To(Equal(&newS3AdditionalKwargs)) // check registry deployment - deploy = &appsv1.Deployment{} err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.RegistryFeastType), - Namespace: resource.Namespace, - }, - deploy) + Name: objMeta.Name, + Namespace: objMeta.Namespace, + }, deploy) Expect(err).NotTo(HaveOccurred()) - Expect(deploy.Spec.Template.Spec.Volumes).To(HaveLen(0)) - Expect(deploy.Spec.Template.Spec.Containers[0].VolumeMounts).To(HaveLen(0)) - + Expect(deploy.Spec.Template.Spec.Volumes).To(HaveLen(1)) + registryContainer := services.GetRegistryContainer(*deploy) + Expect(registryContainer.VolumeMounts).To(HaveLen(1)) }) It("should properly encode a feature_store.yaml config", func() { @@ -289,21 +274,28 @@ var _ = Describe("FeatureStore Controller-Ephemeral services", func() { }, } - // check registry deployment + // check deployment deploy := &appsv1.Deployment{} + objMeta := feast.GetObjectMeta() err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.RegistryFeastType), - Namespace: resource.Namespace, - }, - deploy) + Name: objMeta.Name, + Namespace: objMeta.Namespace, + }, deploy) Expect(err).NotTo(HaveOccurred()) + Expect(deploy.Spec.Replicas).To(Equal(&services.DefaultReplicas)) + Expect(controllerutil.HasControllerReference(deploy)).To(BeTrue()) Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(1)) + Expect(services.GetRegistryContainer(*deploy)).NotTo(BeNil()) + Expect(services.GetOnlineContainer(*deploy)).To(BeNil()) + Expect(services.GetOfflineContainer(*deploy)).To(BeNil()) + Expect(deploy.Spec.Template.Spec.Volumes).To(HaveLen(1)) + Expect(deploy.Spec.Template.Spec.Containers[0].VolumeMounts).To(HaveLen(1)) Expect(deploy.Spec.Template.Spec.Containers[0].Env).To(HaveLen(1)) env := getFeatureStoreYamlEnvVar(deploy.Spec.Template.Spec.Containers[0].Env) Expect(env).NotTo(BeNil()) // check registry config - fsYamlStr, err := feast.GetServiceFeatureStoreYamlBase64(services.RegistryFeastType) + fsYamlStr, err := feast.GetServiceFeatureStoreYamlBase64() Expect(err).NotTo(HaveOccurred()) Expect(fsYamlStr).To(Equal(env.Value)) @@ -312,38 +304,13 @@ var _ = Describe("FeatureStore Controller-Ephemeral services", func() { repoConfig := &services.RepoConfig{} err = yaml.Unmarshal(envByte, repoConfig) Expect(err).NotTo(HaveOccurred()) - testConfig := &services.RepoConfig{ - Project: feastProject, - Provider: services.LocalProviderType, - EntityKeySerializationVersion: feastdevv1alpha1.SerializationVersion, - Registry: services.RegistryConfig{ - RegistryType: services.RegistryFileConfigType, - Path: registryPath, - S3AdditionalKwargs: &s3AdditionalKwargs, - }, - AuthzConfig: noAuthzConfig(), + testConfig := feast.GetDefaultRepoConfig() + testConfig.Registry = services.RegistryConfig{ + RegistryType: services.RegistryFileConfigType, + Path: registryPath, + S3AdditionalKwargs: &s3AdditionalKwargs, } - Expect(repoConfig).To(Equal(testConfig)) - - // check offline deployment - deploy = &appsv1.Deployment{} - err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.OfflineFeastType), - Namespace: resource.Namespace, - }, - deploy) - Expect(err).To(HaveOccurred()) - Expect(errors.IsNotFound(err)).To(BeTrue()) - - // check online deployment - deploy = &appsv1.Deployment{} - err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.OnlineFeastType), - Namespace: resource.Namespace, - }, - deploy) - Expect(err).To(HaveOccurred()) - Expect(errors.IsNotFound(err)).To(BeTrue()) + Expect(repoConfig).To(Equal(&testConfig)) // check client config cm := &corev1.ConfigMap{} @@ -357,17 +324,12 @@ var _ = Describe("FeatureStore Controller-Ephemeral services", func() { repoConfigClient := &services.RepoConfig{} err = yaml.Unmarshal([]byte(cm.Data[services.FeatureStoreYamlCmKey]), repoConfigClient) Expect(err).NotTo(HaveOccurred()) - clientConfig := &services.RepoConfig{ - Project: feastProject, - Provider: services.LocalProviderType, - EntityKeySerializationVersion: feastdevv1alpha1.SerializationVersion, - Registry: services.RegistryConfig{ - RegistryType: services.RegistryRemoteConfigType, - Path: fmt.Sprintf("feast-%s-registry.default.svc.cluster.local:80", resourceName), - }, - AuthzConfig: noAuthzConfig(), + clientConfig := feast.GetInitRepoConfig() + clientConfig.Registry = services.RegistryConfig{ + RegistryType: services.RegistryRemoteConfigType, + Path: fmt.Sprintf("feast-%s-registry.default.svc.cluster.local:80", resourceName), } - Expect(repoConfigClient).To(Equal(clientConfig)) + Expect(repoConfigClient).To(Equal(&clientConfig)) // remove S3 additional keywords and reconcile resourceNew := resource.DeepCopy() @@ -385,16 +347,14 @@ var _ = Describe("FeatureStore Controller-Ephemeral services", func() { feast.Handler.FeatureStore = resource // check registry config - deploy = &appsv1.Deployment{} err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.RegistryFeastType), - Namespace: resource.Namespace, - }, - deploy) + Name: objMeta.Name, + Namespace: objMeta.Namespace, + }, deploy) Expect(err).NotTo(HaveOccurred()) env = getFeatureStoreYamlEnvVar(deploy.Spec.Template.Spec.Containers[0].Env) Expect(env).NotTo(BeNil()) - fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64(services.RegistryFeastType) + fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64() Expect(err).NotTo(HaveOccurred()) Expect(fsYamlStr).To(Equal(env.Value)) @@ -404,27 +364,7 @@ var _ = Describe("FeatureStore Controller-Ephemeral services", func() { err = yaml.Unmarshal(envByte, repoConfig) Expect(err).NotTo(HaveOccurred()) testConfig.Registry.S3AdditionalKwargs = nil - Expect(repoConfig).To(Equal(testConfig)) - - // check offline deployment - deploy = &appsv1.Deployment{} - err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.OfflineFeastType), - Namespace: resource.Namespace, - }, - deploy) - Expect(err).To(HaveOccurred()) - Expect(errors.IsNotFound(err)).To(BeTrue()) - - // check online deployment - deploy = &appsv1.Deployment{} - err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.OnlineFeastType), - Namespace: resource.Namespace, - }, - deploy) - Expect(err).To(HaveOccurred()) - Expect(errors.IsNotFound(err)).To(BeTrue()) + Expect(repoConfig).To(Equal(&testConfig)) }) }) }) diff --git a/infra/feast-operator/internal/controller/featurestore_controller_oidc_auth_test.go b/infra/feast-operator/internal/controller/featurestore_controller_oidc_auth_test.go index c062a573df2..23a4a32702b 100644 --- a/infra/feast-operator/internal/controller/featurestore_controller_oidc_auth_test.go +++ b/infra/feast-operator/internal/controller/featurestore_controller_oidc_auth_test.go @@ -70,10 +70,12 @@ var _ = Describe("FeatureStore Controller-OIDC authorization", func() { Expect(k8sClient.Create(ctx, oidcSecret)).To(Succeed()) } + createEnvFromSecretAndConfigMap() + By("creating the custom resource for the Kind FeatureStore") err = k8sClient.Get(ctx, typeNamespacedName, featurestore) if err != nil && errors.IsNotFound(err) { - resource := createFeatureStoreResource(resourceName, image, pullPolicy, &[]corev1.EnvVar{}) + resource := createFeatureStoreResource(resourceName, image, pullPolicy, &[]corev1.EnvVar{}, withEnvFrom()) resource.Spec.AuthzConfig = &feastdevv1alpha1.AuthzConfig{OidcAuthz: &feastdevv1alpha1.OidcAuthz{ SecretRef: corev1.LocalObjectReference{ Name: oidcSecretName, @@ -82,6 +84,7 @@ var _ = Describe("FeatureStore Controller-OIDC authorization", func() { Expect(k8sClient.Create(ctx, resource)).To(Succeed()) } + }) AfterEach(func() { resource := &feastdevv1alpha1.FeatureStore{} @@ -97,6 +100,8 @@ var _ = Describe("FeatureStore Controller-OIDC authorization", func() { By("Cleanup the specific resource instance FeatureStore") Expect(k8sClient.Delete(ctx, resource)).To(Succeed()) + + deleteEnvFromSecretAndConfigMap() }) It("should successfully reconcile the resource", func() { @@ -146,8 +151,9 @@ var _ = Describe("FeatureStore Controller-OIDC authorization", func() { Expect(resource.Status.Applied.Services.OnlineStore).NotTo(BeNil()) Expect(resource.Status.Applied.Services.OnlineStore.Persistence).NotTo(BeNil()) Expect(resource.Status.Applied.Services.OnlineStore.Persistence.FilePersistence).NotTo(BeNil()) - Expect(resource.Status.Applied.Services.OnlineStore.Persistence.FilePersistence.Path).To(Equal(services.DefaultOnlineStoreEphemeralPath)) + Expect(resource.Status.Applied.Services.OnlineStore.Persistence.FilePersistence.Path).To(Equal(services.EphemeralPath + "/" + services.DefaultOnlineStorePath)) Expect(resource.Status.Applied.Services.OnlineStore.Env).To(Equal(&[]corev1.EnvVar{})) + Expect(resource.Status.Applied.Services.OnlineStore.EnvFrom).To(Equal(withEnvFrom())) Expect(resource.Status.Applied.Services.OnlineStore.ImagePullPolicy).To(Equal(&pullPolicy)) Expect(resource.Status.Applied.Services.OnlineStore.Resources).NotTo(BeNil()) Expect(resource.Status.Applied.Services.OnlineStore.Image).To(Equal(&image)) @@ -155,7 +161,7 @@ var _ = Describe("FeatureStore Controller-OIDC authorization", func() { Expect(resource.Status.Applied.Services.Registry.Local).NotTo(BeNil()) Expect(resource.Status.Applied.Services.Registry.Local.Persistence).NotTo(BeNil()) Expect(resource.Status.Applied.Services.Registry.Local.Persistence.FilePersistence).NotTo(BeNil()) - Expect(resource.Status.Applied.Services.Registry.Local.Persistence.FilePersistence.Path).To(Equal(services.DefaultRegistryEphemeralPath)) + Expect(resource.Status.Applied.Services.Registry.Local.Persistence.FilePersistence.Path).To(Equal(services.EphemeralPath + "/" + services.DefaultRegistryPath)) Expect(resource.Status.Applied.Services.Registry.Local.ImagePullPolicy).To(BeNil()) Expect(resource.Status.Applied.Services.Registry.Local.Resources).To(BeNil()) Expect(resource.Status.Applied.Services.Registry.Local.Image).To(Equal(&services.DefaultImage)) @@ -205,47 +211,25 @@ var _ = Describe("FeatureStore Controller-OIDC authorization", func() { Expect(resource.Status.Phase).To(Equal(feastdevv1alpha1.ReadyPhase)) - // check offline deployment + // check deployment deploy := &appsv1.Deployment{} + objMeta := feast.GetObjectMeta() err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.OfflineFeastType), - Namespace: resource.Namespace, - }, - deploy) + Name: objMeta.Name, + Namespace: objMeta.Namespace, + }, deploy) Expect(err).NotTo(HaveOccurred()) Expect(deploy.Spec.Replicas).To(Equal(&services.DefaultReplicas)) Expect(controllerutil.HasControllerReference(deploy)).To(BeTrue()) - Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(1)) - Expect(deploy.Spec.Template.Spec.Volumes).To(HaveLen(0)) - Expect(deploy.Spec.Template.Spec.Containers[0].VolumeMounts).To(HaveLen(0)) + Expect(deploy.Spec.Template.Spec.InitContainers).To(HaveLen(1)) + Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(3)) + Expect(deploy.Spec.Template.Spec.Volumes).To(HaveLen(1)) + Expect(services.GetOfflineContainer(*deploy).VolumeMounts).To(HaveLen(1)) + Expect(services.GetOnlineContainer(*deploy).VolumeMounts).To(HaveLen(1)) + Expect(services.GetRegistryContainer(*deploy).VolumeMounts).To(HaveLen(1)) - // check online deployment - deploy = &appsv1.Deployment{} - err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.OnlineFeastType), - Namespace: resource.Namespace, - }, - deploy) - Expect(err).NotTo(HaveOccurred()) - Expect(deploy.Spec.Replicas).To(Equal(&services.DefaultReplicas)) - Expect(controllerutil.HasControllerReference(deploy)).To(BeTrue()) - Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(1)) - Expect(deploy.Spec.Template.Spec.Volumes).To(HaveLen(0)) - Expect(deploy.Spec.Template.Spec.Containers[0].VolumeMounts).To(HaveLen(0)) - - // check registry deployment - deploy = &appsv1.Deployment{} - err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.RegistryFeastType), - Namespace: resource.Namespace, - }, - deploy) - Expect(err).NotTo(HaveOccurred()) - Expect(deploy.Spec.Replicas).To(Equal(&services.DefaultReplicas)) - Expect(controllerutil.HasControllerReference(deploy)).To(BeTrue()) - Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(1)) - Expect(deploy.Spec.Template.Spec.Volumes).To(HaveLen(0)) - Expect(deploy.Spec.Template.Spec.Containers[0].VolumeMounts).To(HaveLen(0)) + assertEnvFrom(*services.GetOnlineContainer(*deploy)) + assertEnvFrom(*services.GetOfflineContainer(*deploy)) // check Feast Role feastRole := &rbacv1.Role{} @@ -267,16 +251,14 @@ var _ = Describe("FeatureStore Controller-OIDC authorization", func() { Expect(err).To(HaveOccurred()) Expect(errors.IsNotFound(err)).To(BeTrue()) - // check ServiceAccounts - for _, serviceType := range []services.FeastServiceType{services.RegistryFeastType, services.OnlineFeastType, services.OfflineFeastType} { - sa := &corev1.ServiceAccount{} - err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(serviceType), - Namespace: resource.Namespace, - }, - sa) - Expect(err).NotTo(HaveOccurred()) - } + // check ServiceAccount + sa := &corev1.ServiceAccount{} + err = k8sClient.Get(ctx, types.NamespacedName{ + Name: services.GetFeastName(feast.Handler.FeatureStore), + Namespace: resource.Namespace, + }, + sa) + Expect(err).NotTo(HaveOccurred()) By("Clearing the OIDC authorization and reconciling") resourceNew := resource.DeepCopy() @@ -327,7 +309,7 @@ var _ = Describe("FeatureStore Controller-OIDC authorization", func() { deployList := appsv1.DeploymentList{} err = k8sClient.List(ctx, &deployList, listOpts) Expect(err).NotTo(HaveOccurred()) - Expect(deployList.Items).To(HaveLen(3)) + Expect(deployList.Items).To(HaveLen(1)) svcList := corev1.ServiceList{} err = k8sClient.List(ctx, &svcList, listOpts) @@ -348,19 +330,19 @@ var _ = Describe("FeatureStore Controller-OIDC authorization", func() { }, } - // check registry deployment + // check deployment deploy := &appsv1.Deployment{} + objMeta := feast.GetObjectMeta() err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.RegistryFeastType), - Namespace: resource.Namespace, - }, - deploy) + Name: objMeta.Name, + Namespace: objMeta.Namespace, + }, deploy) Expect(err).NotTo(HaveOccurred()) - env := getFeatureStoreYamlEnvVar(deploy.Spec.Template.Spec.Containers[0].Env) + env := getFeatureStoreYamlEnvVar(services.GetRegistryContainer(*deploy).Env) Expect(env).NotTo(BeNil()) // check registry config - fsYamlStr, err := feast.GetServiceFeatureStoreYamlBase64(services.RegistryFeastType) + fsYamlStr, err := feast.GetServiceFeatureStoreYamlBase64() Expect(err).NotTo(HaveOccurred()) Expect(fsYamlStr).To(Equal(env.Value)) @@ -373,28 +355,27 @@ var _ = Describe("FeatureStore Controller-OIDC authorization", func() { Project: feastProject, Provider: services.LocalProviderType, EntityKeySerializationVersion: feastdevv1alpha1.SerializationVersion, + OfflineStore: services.OfflineStoreConfig{ + Type: services.OfflineFilePersistenceDaskConfigType, + }, Registry: services.RegistryConfig{ - RegistryType: services.RegistryFileConfigType, - Path: services.DefaultRegistryEphemeralPath, - S3AdditionalKwargs: nil, + RegistryType: services.RegistryFileConfigType, + Path: services.EphemeralPath + "/" + services.DefaultRegistryPath, + }, + OnlineStore: services.OnlineStoreConfig{ + Path: services.EphemeralPath + "/" + services.DefaultOnlineStorePath, + Type: services.OnlineSqliteConfigType, }, AuthzConfig: expectedServerOidcAuthorizConfig(), } Expect(repoConfig).To(Equal(testConfig)) - // check offline deployment - deploy = &appsv1.Deployment{} - err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.OfflineFeastType), - Namespace: resource.Namespace, - }, - deploy) - Expect(err).NotTo(HaveOccurred()) - env = getFeatureStoreYamlEnvVar(deploy.Spec.Template.Spec.Containers[0].Env) + // check offline + env = getFeatureStoreYamlEnvVar(services.GetOfflineContainer(*deploy).Env) Expect(env).NotTo(BeNil()) // check offline config - fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64(services.OfflineFeastType) + fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64() Expect(err).NotTo(HaveOccurred()) Expect(fsYamlStr).To(Equal(env.Value)) @@ -403,35 +384,14 @@ var _ = Describe("FeatureStore Controller-OIDC authorization", func() { repoConfig = &services.RepoConfig{} err = yaml.Unmarshal(envByte, repoConfig) Expect(err).NotTo(HaveOccurred()) - regRemote := services.RegistryConfig{ - RegistryType: services.RegistryRemoteConfigType, - Path: fmt.Sprintf("feast-%s-registry.default.svc.cluster.local:80", resourceName), - } - testConfig = &services.RepoConfig{ - Project: feastProject, - Provider: services.LocalProviderType, - EntityKeySerializationVersion: feastdevv1alpha1.SerializationVersion, - OfflineStore: services.OfflineStoreConfig{ - Type: services.OfflineFilePersistenceDaskConfigType, - }, - Registry: regRemote, - AuthzConfig: expectedServerOidcAuthorizConfig(), - } Expect(repoConfig).To(Equal(testConfig)) - // check online deployment - deploy = &appsv1.Deployment{} - err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.OnlineFeastType), - Namespace: resource.Namespace, - }, - deploy) - Expect(err).NotTo(HaveOccurred()) - env = getFeatureStoreYamlEnvVar(deploy.Spec.Template.Spec.Containers[0].Env) + // check online + env = getFeatureStoreYamlEnvVar(services.GetOnlineContainer(*deploy).Env) Expect(env).NotTo(BeNil()) // check online config - fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64(services.OnlineFeastType) + fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64() Expect(err).NotTo(HaveOccurred()) Expect(fsYamlStr).To(Equal(env.Value)) @@ -440,23 +400,6 @@ var _ = Describe("FeatureStore Controller-OIDC authorization", func() { repoConfig = &services.RepoConfig{} err = yaml.Unmarshal(envByte, repoConfig) Expect(err).NotTo(HaveOccurred()) - offlineRemote := services.OfflineStoreConfig{ - Host: fmt.Sprintf("feast-%s-offline.default.svc.cluster.local", resourceName), - Type: services.OfflineRemoteConfigType, - Port: services.HttpPort, - } - testConfig = &services.RepoConfig{ - Project: feastProject, - Provider: services.LocalProviderType, - EntityKeySerializationVersion: feastdevv1alpha1.SerializationVersion, - OfflineStore: offlineRemote, - OnlineStore: services.OnlineStoreConfig{ - Path: services.DefaultOnlineStoreEphemeralPath, - Type: services.OnlineSqliteConfigType, - }, - Registry: regRemote, - AuthzConfig: expectedServerOidcAuthorizConfig(), - } Expect(repoConfig).To(Equal(testConfig)) // check client config @@ -471,6 +414,11 @@ var _ = Describe("FeatureStore Controller-OIDC authorization", func() { repoConfigClient := &services.RepoConfig{} err = yaml.Unmarshal([]byte(cm.Data[services.FeatureStoreYamlCmKey]), repoConfigClient) Expect(err).NotTo(HaveOccurred()) + offlineRemote := services.OfflineStoreConfig{ + Host: fmt.Sprintf("feast-%s-offline.default.svc.cluster.local", resourceName), + Type: services.OfflineRemoteConfigType, + Port: services.HttpPort, + } clientConfig := &services.RepoConfig{ Project: feastProject, Provider: services.LocalProviderType, diff --git a/infra/feast-operator/internal/controller/featurestore_controller_pvc_test.go b/infra/feast-operator/internal/controller/featurestore_controller_pvc_test.go index d0adc62c7c8..c21f702d559 100644 --- a/infra/feast-operator/internal/controller/featurestore_controller_pvc_test.go +++ b/infra/feast-operator/internal/controller/featurestore_controller_pvc_test.go @@ -68,22 +68,26 @@ var _ = Describe("FeatureStore Controller-Ephemeral services", func() { onlineStoreMountPath := "/online" registryMountPath := "/registry" + accessModes := []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce, corev1.ReadWriteMany} storageClassName := "test" onlineStoreMountedPath := path.Join(onlineStoreMountPath, onlineStorePath) registryMountedPath := path.Join(registryMountPath, registryPath) BeforeEach(func() { + createEnvFromSecretAndConfigMap() + By("creating the custom resource for the Kind FeatureStore") err := k8sClient.Get(ctx, typeNamespacedName, featurestore) if err != nil && errors.IsNotFound(err) { resource := createFeatureStoreResource(resourceName, image, pullPolicy, &[]corev1.EnvVar{{Name: testEnvVarName, Value: testEnvVarValue}, - {Name: "fieldRefName", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{APIVersion: "v1", FieldPath: "metadata.namespace"}}}}) + {Name: "fieldRefName", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{APIVersion: "v1", FieldPath: "metadata.namespace"}}}}, withEnvFrom()) resource.Spec.Services.OfflineStore.Persistence = &feastdevv1alpha1.OfflineStorePersistence{ FilePersistence: &feastdevv1alpha1.OfflineStoreFilePersistence{ Type: offlineType, PvcConfig: &feastdevv1alpha1.PvcConfig{ Create: &feastdevv1alpha1.PvcCreate{ + AccessModes: accessModes, StorageClassName: &storageClassName, }, MountPath: offlineStoreMountPath, @@ -123,6 +127,8 @@ var _ = Describe("FeatureStore Controller-Ephemeral services", func() { By("Cleanup the specific resource instance FeatureStore") Expect(k8sClient.Delete(ctx, resource)).To(Succeed()) + + deleteEnvFromSecretAndConfigMap() }) It("should successfully reconcile the resource", func() { @@ -161,6 +167,7 @@ var _ = Describe("FeatureStore Controller-Ephemeral services", func() { Expect(resource.Status.Applied.Services.OfflineStore.Persistence.FilePersistence.Type).To(Equal(offlineType)) Expect(resource.Status.Applied.Services.OfflineStore.Persistence.FilePersistence.PvcConfig).NotTo(BeNil()) Expect(resource.Status.Applied.Services.OfflineStore.Persistence.FilePersistence.PvcConfig.Create).NotTo(BeNil()) + Expect(resource.Status.Applied.Services.OfflineStore.Persistence.FilePersistence.PvcConfig.Create.AccessModes).To(Equal(accessModes)) Expect(resource.Status.Applied.Services.OfflineStore.Persistence.FilePersistence.PvcConfig.Create.StorageClassName).To(Equal(&storageClassName)) expectedResources := corev1.VolumeResourceRequirements{ Requests: corev1.ResourceList{ @@ -178,6 +185,7 @@ var _ = Describe("FeatureStore Controller-Ephemeral services", func() { Expect(resource.Status.Applied.Services.OnlineStore.Persistence.FilePersistence.Path).To(Equal(onlineStorePath)) Expect(resource.Status.Applied.Services.OnlineStore.Persistence.FilePersistence.PvcConfig).NotTo(BeNil()) Expect(resource.Status.Applied.Services.OnlineStore.Persistence.FilePersistence.PvcConfig.Create).NotTo(BeNil()) + Expect(resource.Status.Applied.Services.OnlineStore.Persistence.FilePersistence.PvcConfig.Create.AccessModes).To(Equal(services.DefaultPVCAccessModes)) Expect(resource.Status.Applied.Services.OnlineStore.Persistence.FilePersistence.PvcConfig.Create.StorageClassName).To(BeNil()) expectedResources = corev1.VolumeResourceRequirements{ Requests: corev1.ResourceList{ @@ -187,6 +195,7 @@ var _ = Describe("FeatureStore Controller-Ephemeral services", func() { Expect(resource.Status.Applied.Services.OnlineStore.Persistence.FilePersistence.PvcConfig.Create.Resources).NotTo(BeNil()) Expect(resource.Status.Applied.Services.OnlineStore.Persistence.FilePersistence.PvcConfig.Create.Resources).To(Equal(expectedResources)) Expect(resource.Status.Applied.Services.OnlineStore.Env).To(Equal(&[]corev1.EnvVar{{Name: testEnvVarName, Value: testEnvVarValue}, {Name: "fieldRefName", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{APIVersion: "v1", FieldPath: "metadata.namespace"}}}})) + Expect(resource.Status.Applied.Services.OnlineStore.EnvFrom).To(Equal(withEnvFrom())) Expect(resource.Status.Applied.Services.OnlineStore.ImagePullPolicy).To(Equal(&pullPolicy)) Expect(resource.Status.Applied.Services.OnlineStore.Resources).NotTo(BeNil()) Expect(resource.Status.Applied.Services.OnlineStore.Image).To(Equal(&image)) @@ -197,6 +206,7 @@ var _ = Describe("FeatureStore Controller-Ephemeral services", func() { Expect(resource.Status.Applied.Services.Registry.Local.Persistence.FilePersistence.Path).To(Equal(registryPath)) Expect(resource.Status.Applied.Services.Registry.Local.Persistence.FilePersistence.PvcConfig).NotTo(BeNil()) Expect(resource.Status.Applied.Services.Registry.Local.Persistence.FilePersistence.PvcConfig.Create).NotTo(BeNil()) + Expect(resource.Status.Applied.Services.Registry.Local.Persistence.FilePersistence.PvcConfig.Create.AccessModes).To(Equal(services.DefaultPVCAccessModes)) Expect(resource.Status.Applied.Services.Registry.Local.Persistence.FilePersistence.PvcConfig.Create.StorageClassName).To(BeNil()) expectedResources = corev1.VolumeResourceRequirements{ Requests: corev1.ResourceList{ @@ -254,94 +264,108 @@ var _ = Describe("FeatureStore Controller-Ephemeral services", func() { Expect(resource.Status.Phase).To(Equal(feastdevv1alpha1.ReadyPhase)) - // check offline deployment + ephemeralName := "feast-data" + ephemeralVolume := corev1.Volume{ + Name: ephemeralName, + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{}, + }, + } + ephemeralVolMount := corev1.VolumeMount{ + Name: ephemeralName, + MountPath: "/" + ephemeralName, + } + + // check deployment deploy := &appsv1.Deployment{} + objMeta := feast.GetObjectMeta() err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.OfflineFeastType), - Namespace: resource.Namespace, - }, - deploy) + Name: objMeta.Name, + Namespace: objMeta.Namespace, + }, deploy) Expect(err).NotTo(HaveOccurred()) Expect(deploy.Spec.Replicas).To(Equal(&services.DefaultReplicas)) Expect(controllerutil.HasControllerReference(deploy)).To(BeTrue()) - Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(1)) - Expect(deploy.Spec.Template.Spec.Volumes).To(HaveLen(1)) - Expect(deploy.Spec.Template.Spec.Volumes[0].Name).To(Equal(deploy.Name)) - Expect(deploy.Spec.Template.Spec.Volumes[0].PersistentVolumeClaim.ClaimName).To(Equal(deploy.Name)) - Expect(deploy.Spec.Template.Spec.Containers[0].VolumeMounts).To(HaveLen(1)) - Expect(deploy.Spec.Template.Spec.Containers[0].VolumeMounts[0].MountPath).To(Equal(offlineStoreMountPath)) - Expect(deploy.Spec.Template.Spec.Containers[0].VolumeMounts[0].Name).To(Equal(deploy.Name)) + Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(3)) + Expect(deploy.Spec.Template.Spec.Volumes).To(HaveLen(3)) + Expect(deploy.Spec.Template.Spec.Volumes).NotTo(ContainElement(ephemeralVolume)) + name := feast.GetFeastServiceName(services.RegistryFeastType) + regVol := services.GetRegistryVolume(feast.Handler.FeatureStore, deploy.Spec.Template.Spec.Volumes) + Expect(regVol.Name).To(Equal(name)) + Expect(regVol.PersistentVolumeClaim.ClaimName).To(Equal(name)) + + offlineContainer := services.GetOfflineContainer(*deploy) + Expect(offlineContainer.VolumeMounts).To(HaveLen(3)) + Expect(offlineContainer.VolumeMounts).NotTo(ContainElement(ephemeralVolMount)) + offlineVolMount := services.GetOfflineVolumeMount(feast.Handler.FeatureStore, offlineContainer.VolumeMounts) + Expect(offlineVolMount.MountPath).To(Equal(offlineStoreMountPath)) + offlinePvcName := feast.GetFeastServiceName(services.OfflineFeastType) + Expect(offlineVolMount.Name).To(Equal(offlinePvcName)) + + assertEnvFrom(*offlineContainer) // check offline pvc pvc := &corev1.PersistentVolumeClaim{} err = k8sClient.Get(ctx, types.NamespacedName{ - Name: deploy.Name, + Name: offlinePvcName, Namespace: resource.Namespace, }, pvc) Expect(err).NotTo(HaveOccurred()) - Expect(pvc.Name).To(Equal(deploy.Name)) Expect(pvc.Spec.StorageClassName).To(Equal(&storageClassName)) + Expect(pvc.Spec.AccessModes).To(Equal(accessModes)) Expect(pvc.Spec.Resources.Requests.Storage().String()).To(Equal(services.DefaultOfflineStorageRequest)) Expect(pvc.DeletionTimestamp).To(BeNil()) - // check online deployment - deploy = &appsv1.Deployment{} - err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.OnlineFeastType), - Namespace: resource.Namespace, - }, - deploy) - Expect(err).NotTo(HaveOccurred()) - Expect(deploy.Spec.Replicas).To(Equal(&services.DefaultReplicas)) - Expect(controllerutil.HasControllerReference(deploy)).To(BeTrue()) - Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(1)) - Expect(deploy.Spec.Template.Spec.Volumes).To(HaveLen(1)) - Expect(deploy.Spec.Template.Spec.Volumes[0].Name).To(Equal(deploy.Name)) - Expect(deploy.Spec.Template.Spec.Volumes[0].PersistentVolumeClaim.ClaimName).To(Equal(deploy.Name)) - Expect(deploy.Spec.Template.Spec.Containers[0].VolumeMounts).To(HaveLen(1)) - Expect(deploy.Spec.Template.Spec.Containers[0].VolumeMounts[0].MountPath).To(Equal(onlineStoreMountPath)) - Expect(deploy.Spec.Template.Spec.Containers[0].VolumeMounts[0].Name).To(Equal(deploy.Name)) + // check online + onlinePvcName := feast.GetFeastServiceName(services.OnlineFeastType) + onlineVol := services.GetOnlineVolume(feast.Handler.FeatureStore, deploy.Spec.Template.Spec.Volumes) + Expect(onlineVol.Name).To(Equal(onlinePvcName)) + Expect(onlineVol.PersistentVolumeClaim.ClaimName).To(Equal(onlinePvcName)) + onlineContainer := services.GetOnlineContainer(*deploy) + Expect(onlineContainer.VolumeMounts).To(HaveLen(3)) + Expect(onlineContainer.VolumeMounts).NotTo(ContainElement(ephemeralVolMount)) + onlineVolMount := services.GetOnlineVolumeMount(feast.Handler.FeatureStore, onlineContainer.VolumeMounts) + Expect(onlineVolMount.MountPath).To(Equal(onlineStoreMountPath)) + Expect(onlineVolMount.Name).To(Equal(onlinePvcName)) + + assertEnvFrom(*onlineContainer) // check online pvc pvc = &corev1.PersistentVolumeClaim{} err = k8sClient.Get(ctx, types.NamespacedName{ - Name: deploy.Name, + Name: onlinePvcName, Namespace: resource.Namespace, }, pvc) Expect(err).NotTo(HaveOccurred()) - Expect(pvc.Name).To(Equal(deploy.Name)) + Expect(pvc.Name).To(Equal(onlinePvcName)) + Expect(pvc.Spec.AccessModes).To(Equal(services.DefaultPVCAccessModes)) Expect(pvc.Spec.Resources.Requests.Storage().String()).To(Equal(services.DefaultOnlineStorageRequest)) Expect(pvc.DeletionTimestamp).To(BeNil()) - // check registry deployment - deploy = &appsv1.Deployment{} - err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.RegistryFeastType), - Namespace: resource.Namespace, - }, - deploy) - Expect(err).NotTo(HaveOccurred()) - Expect(deploy.Spec.Replicas).To(Equal(&services.DefaultReplicas)) - Expect(controllerutil.HasControllerReference(deploy)).To(BeTrue()) - Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(1)) - Expect(deploy.Spec.Template.Spec.Volumes).To(HaveLen(1)) - Expect(deploy.Spec.Template.Spec.Volumes[0].Name).To(Equal(deploy.Name)) - Expect(deploy.Spec.Template.Spec.Volumes[0].PersistentVolumeClaim.ClaimName).To(Equal(deploy.Name)) - Expect(deploy.Spec.Template.Spec.Containers[0].VolumeMounts).To(HaveLen(1)) - Expect(deploy.Spec.Template.Spec.Containers[0].VolumeMounts[0].MountPath).To(Equal(registryMountPath)) - Expect(deploy.Spec.Template.Spec.Containers[0].VolumeMounts[0].Name).To(Equal(deploy.Name)) + // check registry + registryPvcName := feast.GetFeastServiceName(services.RegistryFeastType) + registryVol := services.GetRegistryVolume(feast.Handler.FeatureStore, deploy.Spec.Template.Spec.Volumes) + Expect(registryVol.Name).To(Equal(registryPvcName)) + Expect(registryVol.PersistentVolumeClaim.ClaimName).To(Equal(registryPvcName)) + registryContainer := services.GetRegistryContainer(*deploy) + Expect(registryContainer.VolumeMounts).To(HaveLen(3)) + Expect(registryContainer.VolumeMounts).NotTo(ContainElement(ephemeralVolMount)) + registryVolMount := services.GetRegistryVolumeMount(feast.Handler.FeatureStore, registryContainer.VolumeMounts) + Expect(registryVolMount.MountPath).To(Equal(registryMountPath)) + Expect(registryVolMount.Name).To(Equal(registryPvcName)) // check registry pvc pvc = &corev1.PersistentVolumeClaim{} err = k8sClient.Get(ctx, types.NamespacedName{ - Name: deploy.Name, + Name: registryPvcName, Namespace: resource.Namespace, }, pvc) Expect(err).NotTo(HaveOccurred()) - Expect(pvc.Name).To(Equal(deploy.Name)) + Expect(pvc.Name).To(Equal(registryPvcName)) + Expect(pvc.Spec.AccessModes).To(Equal(services.DefaultPVCAccessModes)) Expect(pvc.Spec.Resources.Requests.Storage().String()).To(Equal(services.DefaultRegistryStorageRequest)) Expect(pvc.DeletionTimestamp).To(BeNil()) @@ -363,22 +387,25 @@ var _ = Describe("FeatureStore Controller-Ephemeral services", func() { feast.Handler.FeatureStore = resource Expect(resource.Status.Applied.Services.OnlineStore.Persistence.FilePersistence.PvcConfig).To(BeNil()) - // check online deployment + // check online deployment/container deploy = &appsv1.Deployment{} err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.OnlineFeastType), - Namespace: resource.Namespace, - }, - deploy) + Name: objMeta.Name, + Namespace: objMeta.Namespace, + }, deploy) Expect(err).NotTo(HaveOccurred()) - Expect(deploy.Spec.Template.Spec.Volumes).To(HaveLen(0)) - Expect(deploy.Spec.Template.Spec.Containers[0].VolumeMounts).To(HaveLen(0)) + Expect(deploy.Spec.Template.Spec.Volumes).To(HaveLen(3)) + Expect(deploy.Spec.Template.Spec.Volumes).To(ContainElement(ephemeralVolume)) + Expect(services.GetOnlineContainer(*deploy).VolumeMounts).To(HaveLen(3)) + Expect(services.GetOnlineContainer(*deploy).VolumeMounts).To(ContainElement(ephemeralVolMount)) + Expect(services.GetRegistryContainer(*deploy).VolumeMounts).To(ContainElement(ephemeralVolMount)) + Expect(services.GetOfflineContainer(*deploy).VolumeMounts).To(ContainElement(ephemeralVolMount)) // check online pvc is deleted log.FromContext(feast.Handler.Context).Info("Checking deletion of", "PersistentVolumeClaim", deploy.Name) pvc = &corev1.PersistentVolumeClaim{} err = k8sClient.Get(ctx, types.NamespacedName{ - Name: deploy.Name, + Name: onlinePvcName, Namespace: resource.Namespace, }, pvc) @@ -412,7 +439,7 @@ var _ = Describe("FeatureStore Controller-Ephemeral services", func() { deployList := appsv1.DeploymentList{} err = k8sClient.List(ctx, &deployList, listOpts) Expect(err).NotTo(HaveOccurred()) - Expect(deployList.Items).To(HaveLen(3)) + Expect(deployList.Items).To(HaveLen(1)) svcList := corev1.ServiceList{} err = k8sClient.List(ctx, &svcList, listOpts) @@ -433,21 +460,22 @@ var _ = Describe("FeatureStore Controller-Ephemeral services", func() { }, } - // check registry deployment + // check deployment deploy := &appsv1.Deployment{} + objMeta := feast.GetObjectMeta() err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.RegistryFeastType), - Namespace: resource.Namespace, - }, - deploy) - Expect(err).NotTo(HaveOccurred()) - Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(1)) - Expect(deploy.Spec.Template.Spec.Containers[0].Env).To(HaveLen(1)) - env := getFeatureStoreYamlEnvVar(deploy.Spec.Template.Spec.Containers[0].Env) + Name: objMeta.Name, + Namespace: objMeta.Namespace, + }, deploy) + Expect(err).NotTo(HaveOccurred()) + Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(3)) + registryContainer := services.GetRegistryContainer(*deploy) + Expect(registryContainer.Env).To(HaveLen(1)) + env := getFeatureStoreYamlEnvVar(registryContainer.Env) Expect(env).NotTo(BeNil()) // check registry config - fsYamlStr, err := feast.GetServiceFeatureStoreYamlBase64(services.RegistryFeastType) + fsYamlStr, err := feast.GetServiceFeatureStoreYamlBase64() Expect(err).NotTo(HaveOccurred()) Expect(fsYamlStr).To(Equal(env.Value)) @@ -464,25 +492,24 @@ var _ = Describe("FeatureStore Controller-Ephemeral services", func() { RegistryType: services.RegistryFileConfigType, Path: registryMountedPath, }, + OfflineStore: services.OfflineStoreConfig{ + Type: services.OfflineFilePersistenceDuckDbConfigType, + }, + OnlineStore: services.OnlineStoreConfig{ + Path: onlineStoreMountedPath, + Type: services.OnlineSqliteConfigType, + }, AuthzConfig: noAuthzConfig(), } Expect(repoConfig).To(Equal(testConfig)) - // check offline deployment - deploy = &appsv1.Deployment{} - err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.OfflineFeastType), - Namespace: resource.Namespace, - }, - deploy) - Expect(err).NotTo(HaveOccurred()) - Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(1)) - Expect(deploy.Spec.Template.Spec.Containers[0].Env).To(HaveLen(1)) - env = getFeatureStoreYamlEnvVar(deploy.Spec.Template.Spec.Containers[0].Env) + offlineContainer := services.GetOfflineContainer(*deploy) + Expect(offlineContainer.Env).To(HaveLen(1)) + env = getFeatureStoreYamlEnvVar(offlineContainer.Env) Expect(env).NotTo(BeNil()) // check offline config - fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64(services.OfflineFeastType) + fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64() Expect(err).NotTo(HaveOccurred()) Expect(fsYamlStr).To(Equal(env.Value)) @@ -491,37 +518,16 @@ var _ = Describe("FeatureStore Controller-Ephemeral services", func() { repoConfigOffline := &services.RepoConfig{} err = yaml.Unmarshal(envByte, repoConfigOffline) Expect(err).NotTo(HaveOccurred()) - regRemote := services.RegistryConfig{ - RegistryType: services.RegistryRemoteConfigType, - Path: fmt.Sprintf("feast-%s-registry.default.svc.cluster.local:80", resourceName), - } - offlineConfig := &services.RepoConfig{ - Project: feastProject, - Provider: services.LocalProviderType, - EntityKeySerializationVersion: feastdevv1alpha1.SerializationVersion, - OfflineStore: services.OfflineStoreConfig{ - Type: services.OfflineFilePersistenceDuckDbConfigType, - }, - Registry: regRemote, - AuthzConfig: noAuthzConfig(), - } - Expect(repoConfigOffline).To(Equal(offlineConfig)) + Expect(repoConfigOffline).To(Equal(testConfig)) // check online config - deploy = &appsv1.Deployment{} - err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.OnlineFeastType), - Namespace: resource.Namespace, - }, - deploy) - Expect(err).NotTo(HaveOccurred()) - Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(1)) - Expect(deploy.Spec.Template.Spec.Containers[0].Env).To(HaveLen(3)) - Expect(deploy.Spec.Template.Spec.Containers[0].ImagePullPolicy).To(Equal(corev1.PullAlways)) - env = getFeatureStoreYamlEnvVar(deploy.Spec.Template.Spec.Containers[0].Env) + onlineContainer := services.GetOnlineContainer(*deploy) + Expect(onlineContainer.Env).To(HaveLen(3)) + Expect(onlineContainer.ImagePullPolicy).To(Equal(corev1.PullAlways)) + env = getFeatureStoreYamlEnvVar(onlineContainer.Env) Expect(env).NotTo(BeNil()) - fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64(services.OnlineFeastType) + fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64() Expect(err).NotTo(HaveOccurred()) Expect(fsYamlStr).To(Equal(env.Value)) @@ -530,25 +536,7 @@ var _ = Describe("FeatureStore Controller-Ephemeral services", func() { repoConfigOnline := &services.RepoConfig{} err = yaml.Unmarshal(envByte, repoConfigOnline) Expect(err).NotTo(HaveOccurred()) - offlineRemote := services.OfflineStoreConfig{ - Host: fmt.Sprintf("feast-%s-offline.default.svc.cluster.local", resourceName), - Type: services.OfflineRemoteConfigType, - Port: services.HttpPort, - } - onlineConfig := &services.RepoConfig{ - Project: feastProject, - Provider: services.LocalProviderType, - EntityKeySerializationVersion: feastdevv1alpha1.SerializationVersion, - OfflineStore: offlineRemote, - OnlineStore: services.OnlineStoreConfig{ - Path: onlineStoreMountedPath, - Type: services.OnlineSqliteConfigType, - }, - Registry: regRemote, - AuthzConfig: noAuthzConfig(), - } - Expect(repoConfigOnline).To(Equal(onlineConfig)) - Expect(deploy.Spec.Template.Spec.Containers[0].Env).To(HaveLen(3)) + Expect(repoConfigOnline).To(Equal(testConfig)) // check client config cm := &corev1.ConfigMap{} @@ -562,6 +550,15 @@ var _ = Describe("FeatureStore Controller-Ephemeral services", func() { repoConfigClient := &services.RepoConfig{} err = yaml.Unmarshal([]byte(cm.Data[services.FeatureStoreYamlCmKey]), repoConfigClient) Expect(err).NotTo(HaveOccurred()) + offlineRemote := services.OfflineStoreConfig{ + Host: fmt.Sprintf("feast-%s-offline.default.svc.cluster.local", resourceName), + Type: services.OfflineRemoteConfigType, + Port: services.HttpPort, + } + regRemote := services.RegistryConfig{ + RegistryType: services.RegistryRemoteConfigType, + Path: fmt.Sprintf("feast-%s-registry.default.svc.cluster.local:80", resourceName), + } clientConfig := &services.RepoConfig{ Project: feastProject, Provider: services.LocalProviderType, @@ -601,14 +598,14 @@ var _ = Describe("FeatureStore Controller-Ephemeral services", func() { // check registry config deploy = &appsv1.Deployment{} err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.RegistryFeastType), - Namespace: resource.Namespace, - }, - deploy) + Name: objMeta.Name, + Namespace: objMeta.Namespace, + }, deploy) Expect(err).NotTo(HaveOccurred()) - env = getFeatureStoreYamlEnvVar(deploy.Spec.Template.Spec.Containers[0].Env) + registryContainer = services.GetRegistryContainer(*deploy) + env = getFeatureStoreYamlEnvVar(registryContainer.Env) Expect(env).NotTo(BeNil()) - fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64(services.RegistryFeastType) + fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64() Expect(err).NotTo(HaveOccurred()) Expect(fsYamlStr).To(Equal(env.Value)) @@ -617,21 +614,16 @@ var _ = Describe("FeatureStore Controller-Ephemeral services", func() { repoConfig = &services.RepoConfig{} err = yaml.Unmarshal(envByte, repoConfig) Expect(err).NotTo(HaveOccurred()) + testConfig.OnlineStore.Path = newOnlineStoreMountedPath testConfig.Registry.Path = newRegistryMountedPath Expect(repoConfig).To(Equal(testConfig)) // check offline config - deploy = &appsv1.Deployment{} - err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.OfflineFeastType), - Namespace: resource.Namespace, - }, - deploy) - Expect(err).NotTo(HaveOccurred()) - env = getFeatureStoreYamlEnvVar(deploy.Spec.Template.Spec.Containers[0].Env) + offlineContainer = services.GetOfflineContainer(*deploy) + env = getFeatureStoreYamlEnvVar(offlineContainer.Env) Expect(env).NotTo(BeNil()) - fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64(services.OfflineFeastType) + fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64() Expect(err).NotTo(HaveOccurred()) Expect(fsYamlStr).To(Equal(env.Value)) @@ -640,20 +632,14 @@ var _ = Describe("FeatureStore Controller-Ephemeral services", func() { repoConfigOffline = &services.RepoConfig{} err = yaml.Unmarshal(envByte, repoConfigOffline) Expect(err).NotTo(HaveOccurred()) - Expect(repoConfigOffline).To(Equal(offlineConfig)) + Expect(repoConfigOffline).To(Equal(testConfig)) // check online config - deploy = &appsv1.Deployment{} - err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.OnlineFeastType), - Namespace: resource.Namespace, - }, - deploy) - Expect(err).NotTo(HaveOccurred()) - env = getFeatureStoreYamlEnvVar(deploy.Spec.Template.Spec.Containers[0].Env) + onlineContainer = services.GetOfflineContainer(*deploy) + env = getFeatureStoreYamlEnvVar(onlineContainer.Env) Expect(env).NotTo(BeNil()) - fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64(services.OnlineFeastType) + fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64() Expect(err).NotTo(HaveOccurred()) Expect(fsYamlStr).To(Equal(env.Value)) @@ -663,8 +649,8 @@ var _ = Describe("FeatureStore Controller-Ephemeral services", func() { repoConfigOnline = &services.RepoConfig{} err = yaml.Unmarshal(envByte, repoConfigOnline) Expect(err).NotTo(HaveOccurred()) - onlineConfig.OnlineStore.Path = newOnlineStoreMountedPath - Expect(repoConfigOnline).To(Equal(onlineConfig)) + testConfig.OnlineStore.Path = newOnlineStoreMountedPath + Expect(repoConfigOnline).To(Equal(testConfig)) }) }) }) diff --git a/infra/feast-operator/internal/controller/featurestore_controller_test.go b/infra/feast-operator/internal/controller/featurestore_controller_test.go index 44c81eca59a..9f146bf1f61 100644 --- a/infra/feast-operator/internal/controller/featurestore_controller_test.go +++ b/infra/feast-operator/internal/controller/featurestore_controller_test.go @@ -172,15 +172,16 @@ var _ = Describe("FeatureStore Controller", func() { Expect(resource.Status.Phase).To(Equal(feastdevv1alpha1.ReadyPhase)) deploy := &appsv1.Deployment{} + objMeta := feast.GetObjectMeta() err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.RegistryFeastType), - Namespace: resource.Namespace, - }, - deploy) + Name: objMeta.Name, + Namespace: objMeta.Namespace, + }, deploy) Expect(err).NotTo(HaveOccurred()) Expect(deploy.Spec.Replicas).To(Equal(&services.DefaultReplicas)) Expect(controllerutil.HasControllerReference(deploy)).To(BeTrue()) Expect(deploy.Spec.Template.Spec.ServiceAccountName).To(Equal(deploy.Name)) + Expect(deploy.Spec.Template.Spec.InitContainers).To(HaveLen(1)) Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(1)) svc := &corev1.Service{} @@ -220,19 +221,20 @@ var _ = Describe("FeatureStore Controller", func() { } deploy := &appsv1.Deployment{} + objMeta := feast.GetObjectMeta() err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.RegistryFeastType), - Namespace: resource.Namespace, - }, - deploy) + Name: objMeta.Name, + Namespace: objMeta.Namespace, + }, deploy) Expect(err).NotTo(HaveOccurred()) Expect(deploy.Spec.Template.Spec.ServiceAccountName).To(Equal(deploy.Name)) + Expect(deploy.Spec.Strategy.Type).To(Equal(appsv1.RecreateDeploymentStrategyType)) Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(1)) Expect(deploy.Spec.Template.Spec.Containers[0].Env).To(HaveLen(1)) env := getFeatureStoreYamlEnvVar(deploy.Spec.Template.Spec.Containers[0].Env) Expect(env).NotTo(BeNil()) - fsYamlStr, err := feast.GetServiceFeatureStoreYamlBase64(services.RegistryFeastType) + fsYamlStr, err := feast.GetServiceFeatureStoreYamlBase64() Expect(err).NotTo(HaveOccurred()) Expect(fsYamlStr).To(Equal(env.Value)) @@ -241,17 +243,8 @@ var _ = Describe("FeatureStore Controller", func() { repoConfig := &services.RepoConfig{} err = yaml.Unmarshal(envByte, repoConfig) Expect(err).NotTo(HaveOccurred()) - testConfig := &services.RepoConfig{ - Project: feastProject, - Provider: services.LocalProviderType, - EntityKeySerializationVersion: feastdevv1alpha1.SerializationVersion, - Registry: services.RegistryConfig{ - RegistryType: services.RegistryFileConfigType, - Path: services.DefaultRegistryEphemeralPath, - }, - AuthzConfig: noAuthzConfig(), - } - Expect(repoConfig).To(Equal(testConfig)) + testConfig := feast.GetDefaultRepoConfig() + Expect(repoConfig).To(Equal(&testConfig)) // check client config cm := &corev1.ConfigMap{} @@ -265,21 +258,21 @@ var _ = Describe("FeatureStore Controller", func() { repoConfigClient := &services.RepoConfig{} err = yaml.Unmarshal([]byte(cm.Data[services.FeatureStoreYamlCmKey]), repoConfigClient) Expect(err).NotTo(HaveOccurred()) - clientConfig := &services.RepoConfig{ - Project: feastProject, - Provider: services.LocalProviderType, - EntityKeySerializationVersion: feastdevv1alpha1.SerializationVersion, - Registry: services.RegistryConfig{ - RegistryType: services.RegistryRemoteConfigType, - Path: "feast-test-resource-registry.default.svc.cluster.local:80", - }, - AuthzConfig: noAuthzConfig(), + clientConfig := feast.GetInitRepoConfig() + clientConfig.Registry = services.RegistryConfig{ + RegistryType: services.RegistryRemoteConfigType, + Path: "feast-test-resource-registry.default.svc.cluster.local:80", } - Expect(repoConfigClient).To(Equal(clientConfig)) + Expect(repoConfigClient).To(Equal(&clientConfig)) // change feast project and reconcile resourceNew := resource.DeepCopy() resourceNew.Spec.FeastProject = "changed" + resourceNew.Spec.Services = &feastdevv1alpha1.FeatureStoreServices{ + DeploymentStrategy: &appsv1.DeploymentStrategy{ + Type: appsv1.RollingUpdateDeploymentStrategyType, + }, + } err = k8sClient.Update(ctx, resourceNew) Expect(err).NotTo(HaveOccurred()) _, err = controllerReconciler.Reconcile(ctx, reconcile.Request{ @@ -291,18 +284,19 @@ var _ = Describe("FeatureStore Controller", func() { Expect(err).NotTo(HaveOccurred()) Expect(resource.Spec.FeastProject).To(Equal(resourceNew.Spec.FeastProject)) err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.RegistryFeastType), - Namespace: resource.Namespace, + Name: objMeta.Name, + Namespace: objMeta.Namespace, }, deploy) Expect(err).NotTo(HaveOccurred()) testConfig.Project = resourceNew.Spec.FeastProject + Expect(deploy.Spec.Strategy.Type).To(Equal(appsv1.RollingUpdateDeploymentStrategyType)) Expect(deploy.Spec.Template.Spec.Containers[0].Env).To(HaveLen(1)) env = getFeatureStoreYamlEnvVar(deploy.Spec.Template.Spec.Containers[0].Env) Expect(env).NotTo(BeNil()) - fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64(services.RegistryFeastType) + fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64() Expect(err).NotTo(HaveOccurred()) Expect(fsYamlStr).To(Equal(env.Value)) @@ -310,7 +304,7 @@ var _ = Describe("FeatureStore Controller", func() { Expect(err).NotTo(HaveOccurred()) err = yaml.Unmarshal(envByte, repoConfig) Expect(err).NotTo(HaveOccurred()) - Expect(repoConfig).To(Equal(testConfig)) + Expect(repoConfig).To(Equal(&testConfig)) }) It("should error on reconcile", func() { @@ -339,11 +333,11 @@ var _ = Describe("FeatureStore Controller", func() { } deploy := &appsv1.Deployment{} + objMeta := feast.GetObjectMeta() err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.RegistryFeastType), - Namespace: resource.Namespace, - }, - deploy) + Name: objMeta.Name, + Namespace: objMeta.Namespace, + }, deploy) Expect(err).NotTo(HaveOccurred()) err = controllerutil.RemoveControllerReference(resource, deploy, controllerReconciler.Scheme) @@ -378,17 +372,16 @@ var _ = Describe("FeatureStore Controller", func() { Expect(cond.Type).To(Equal(feastdevv1alpha1.ReadyType)) Expect(cond.Status).To(Equal(metav1.ConditionFalse)) Expect(cond.Reason).To(Equal(feastdevv1alpha1.FailedReason)) - Expect(cond.Message).To(Equal("Error: Object " + resource.Namespace + "/" + name + " is already owned by another Service controller " + name)) + Expect(cond.Message).To(Equal("Error: Object " + resource.Namespace + "/" + deploy.Name + " is already owned by another Service controller " + name)) cond = apimeta.FindStatusCondition(resource.Status.Conditions, feastdevv1alpha1.AuthorizationReadyType) Expect(cond).To(BeNil()) cond = apimeta.FindStatusCondition(resource.Status.Conditions, feastdevv1alpha1.RegistryReadyType) Expect(cond).ToNot(BeNil()) - Expect(cond.Status).To(Equal(metav1.ConditionFalse)) - Expect(cond.Reason).To(Equal(feastdevv1alpha1.RegistryFailedReason)) + Expect(cond.Status).To(Equal(metav1.ConditionTrue)) + Expect(cond.Reason).To(Equal(feastdevv1alpha1.ReadyReason)) Expect(cond.Type).To(Equal(feastdevv1alpha1.RegistryReadyType)) - Expect(cond.Message).To(Equal("Error: Object " + resource.Namespace + "/" + name + " is already owned by another Service controller " + name)) cond = apimeta.FindStatusCondition(resource.Status.Conditions, feastdevv1alpha1.ClientReadyType) Expect(cond).ToNot(BeNil()) @@ -416,11 +409,13 @@ var _ = Describe("FeatureStore Controller", func() { featurestore := &feastdevv1alpha1.FeatureStore{} BeforeEach(func() { + createEnvFromSecretAndConfigMap() + By("creating the custom resource for the Kind FeatureStore") err := k8sClient.Get(ctx, typeNamespacedName, featurestore) if err != nil && errors.IsNotFound(err) { resource := createFeatureStoreResource(resourceName, image, pullPolicy, &[]corev1.EnvVar{{Name: testEnvVarName, Value: testEnvVarValue}, - {Name: "fieldRefName", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{APIVersion: "v1", FieldPath: "metadata.namespace"}}}}) + {Name: "fieldRefName", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{APIVersion: "v1", FieldPath: "metadata.namespace"}}}}, withEnvFrom()) Expect(k8sClient.Create(ctx, resource)).To(Succeed()) } }) @@ -431,6 +426,9 @@ var _ = Describe("FeatureStore Controller", func() { By("Cleanup the specific resource instance FeatureStore") Expect(k8sClient.Delete(ctx, resource)).To(Succeed()) + + // Delete ConfigMap + deleteEnvFromSecretAndConfigMap() }) It("should successfully reconcile the resource", func() { @@ -473,8 +471,9 @@ var _ = Describe("FeatureStore Controller", func() { Expect(resource.Status.Applied.Services.OnlineStore).NotTo(BeNil()) Expect(resource.Status.Applied.Services.OnlineStore.Persistence).NotTo(BeNil()) Expect(resource.Status.Applied.Services.OnlineStore.Persistence.FilePersistence).NotTo(BeNil()) - Expect(resource.Status.Applied.Services.OnlineStore.Persistence.FilePersistence.Path).To(Equal(services.DefaultOnlineStoreEphemeralPath)) + Expect(resource.Status.Applied.Services.OnlineStore.Persistence.FilePersistence.Path).To(Equal(services.EphemeralPath + "/" + services.DefaultOnlineStorePath)) Expect(resource.Status.Applied.Services.OnlineStore.Env).To(Equal(&[]corev1.EnvVar{{Name: testEnvVarName, Value: testEnvVarValue}, {Name: "fieldRefName", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{APIVersion: "v1", FieldPath: "metadata.namespace"}}}})) + Expect(resource.Status.Applied.Services.OnlineStore.EnvFrom).To(Equal(withEnvFrom())) Expect(resource.Status.Applied.Services.OnlineStore.ImagePullPolicy).To(Equal(&pullPolicy)) Expect(resource.Status.Applied.Services.OnlineStore.Resources).NotTo(BeNil()) Expect(resource.Status.Applied.Services.OnlineStore.Image).To(Equal(&image)) @@ -482,7 +481,7 @@ var _ = Describe("FeatureStore Controller", func() { Expect(resource.Status.Applied.Services.Registry.Local).NotTo(BeNil()) Expect(resource.Status.Applied.Services.Registry.Local.Persistence).NotTo(BeNil()) Expect(resource.Status.Applied.Services.Registry.Local.Persistence.FilePersistence).NotTo(BeNil()) - Expect(resource.Status.Applied.Services.Registry.Local.Persistence.FilePersistence.Path).To(Equal(services.DefaultRegistryEphemeralPath)) + Expect(resource.Status.Applied.Services.Registry.Local.Persistence.FilePersistence.Path).To(Equal(services.EphemeralPath + "/" + services.DefaultRegistryPath)) Expect(resource.Status.Applied.Services.Registry.Local.ImagePullPolicy).To(BeNil()) Expect(resource.Status.Applied.Services.Registry.Local.Resources).To(BeNil()) Expect(resource.Status.Applied.Services.Registry.Local.Image).To(Equal(&services.DefaultImage)) @@ -533,16 +532,16 @@ var _ = Describe("FeatureStore Controller", func() { Expect(resource.Status.Phase).To(Equal(feastdevv1alpha1.ReadyPhase)) deploy := &appsv1.Deployment{} + objMeta := feast.GetObjectMeta() err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.RegistryFeastType), - Namespace: resource.Namespace, - }, - deploy) + Name: objMeta.Name, + Namespace: objMeta.Namespace, + }, deploy) Expect(err).NotTo(HaveOccurred()) Expect(deploy.Spec.Replicas).To(Equal(&services.DefaultReplicas)) Expect(controllerutil.HasControllerReference(deploy)).To(BeTrue()) Expect(deploy.Spec.Template.Spec.ServiceAccountName).To(Equal(deploy.Name)) - Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(1)) + Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(3)) svc := &corev1.Service{} err = k8sClient.Get(ctx, types.NamespacedName{ @@ -578,12 +577,12 @@ var _ = Describe("FeatureStore Controller", func() { deployList := appsv1.DeploymentList{} err = k8sClient.List(ctx, &deployList, listOpts) Expect(err).NotTo(HaveOccurred()) - Expect(deployList.Items).To(HaveLen(3)) + Expect(deployList.Items).To(HaveLen(1)) saList := corev1.ServiceAccountList{} err = k8sClient.List(ctx, &saList, listOpts) Expect(err).NotTo(HaveOccurred()) - Expect(saList.Items).To(HaveLen(3)) + Expect(saList.Items).To(HaveLen(1)) svcList := corev1.ServiceList{} err = k8sClient.List(ctx, &svcList, listOpts) @@ -606,19 +605,20 @@ var _ = Describe("FeatureStore Controller", func() { // check registry config deploy := &appsv1.Deployment{} + objMeta := feast.GetObjectMeta() err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.RegistryFeastType), - Namespace: resource.Namespace, - }, - deploy) + Name: objMeta.Name, + Namespace: objMeta.Namespace, + }, deploy) Expect(err).NotTo(HaveOccurred()) Expect(deploy.Spec.Template.Spec.ServiceAccountName).To(Equal(deploy.Name)) - Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(1)) - Expect(deploy.Spec.Template.Spec.Containers[0].Env).To(HaveLen(1)) - env := getFeatureStoreYamlEnvVar(deploy.Spec.Template.Spec.Containers[0].Env) + Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(3)) + registryContainer := services.GetRegistryContainer(*deploy) + Expect(registryContainer.Env).To(HaveLen(1)) + env := getFeatureStoreYamlEnvVar(registryContainer.Env) Expect(env).NotTo(BeNil()) - fsYamlStr, err := feast.GetServiceFeatureStoreYamlBase64(services.RegistryFeastType) + fsYamlStr, err := feast.GetServiceFeatureStoreYamlBase64() Expect(err).NotTo(HaveOccurred()) Expect(fsYamlStr).To(Equal(env.Value)) @@ -627,33 +627,22 @@ var _ = Describe("FeatureStore Controller", func() { repoConfig := &services.RepoConfig{} err = yaml.Unmarshal(envByte, repoConfig) Expect(err).NotTo(HaveOccurred()) - testConfig := &services.RepoConfig{ - Project: feastProject, - Provider: services.LocalProviderType, - EntityKeySerializationVersion: feastdevv1alpha1.SerializationVersion, - Registry: services.RegistryConfig{ - RegistryType: services.RegistryFileConfigType, - Path: services.DefaultRegistryEphemeralPath, - }, - AuthzConfig: noAuthzConfig(), + testConfig := feast.GetDefaultRepoConfig() + testConfig.OfflineStore = services.OfflineStoreConfig{ + Type: services.OfflineFilePersistenceDaskConfigType, } - Expect(repoConfig).To(Equal(testConfig)) + Expect(repoConfig).To(Equal(&testConfig)) // check offline config - deploy = &appsv1.Deployment{} - err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.OfflineFeastType), - Namespace: resource.Namespace, - }, - deploy) - Expect(err).NotTo(HaveOccurred()) Expect(deploy.Spec.Template.Spec.ServiceAccountName).To(Equal(deploy.Name)) - Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(1)) - Expect(deploy.Spec.Template.Spec.Containers[0].Env).To(HaveLen(1)) - env = getFeatureStoreYamlEnvVar(deploy.Spec.Template.Spec.Containers[0].Env) + offlineContainer := services.GetOfflineContainer(*deploy) + Expect(offlineContainer.Env).To(HaveLen(1)) + env = getFeatureStoreYamlEnvVar(offlineContainer.Env) Expect(env).NotTo(BeNil()) - fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64(services.OfflineFeastType) + assertEnvFrom(*offlineContainer) + + fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64() Expect(err).NotTo(HaveOccurred()) Expect(fsYamlStr).To(Equal(env.Value)) @@ -662,38 +651,18 @@ var _ = Describe("FeatureStore Controller", func() { repoConfigOffline := &services.RepoConfig{} err = yaml.Unmarshal(envByte, repoConfigOffline) Expect(err).NotTo(HaveOccurred()) - regRemote := services.RegistryConfig{ - RegistryType: services.RegistryRemoteConfigType, - Path: "feast-services-registry.default.svc.cluster.local:80", - } - offlineConfig := &services.RepoConfig{ - Project: feastProject, - Provider: services.LocalProviderType, - EntityKeySerializationVersion: feastdevv1alpha1.SerializationVersion, - OfflineStore: services.OfflineStoreConfig{ - Type: services.OfflineFilePersistenceDaskConfigType, - }, - Registry: regRemote, - AuthzConfig: noAuthzConfig(), - } - Expect(repoConfigOffline).To(Equal(offlineConfig)) + Expect(repoConfigOffline).To(Equal(&testConfig)) // check online config - deploy = &appsv1.Deployment{} - err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.OnlineFeastType), - Namespace: resource.Namespace, - }, - deploy) - Expect(err).NotTo(HaveOccurred()) - Expect(deploy.Spec.Template.Spec.ServiceAccountName).To(Equal(deploy.Name)) - Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(1)) - Expect(deploy.Spec.Template.Spec.Containers[0].Env).To(HaveLen(3)) - Expect(deploy.Spec.Template.Spec.Containers[0].ImagePullPolicy).To(Equal(corev1.PullAlways)) - env = getFeatureStoreYamlEnvVar(deploy.Spec.Template.Spec.Containers[0].Env) + onlineContainer := services.GetOnlineContainer(*deploy) + Expect(onlineContainer.Env).To(HaveLen(3)) + Expect(onlineContainer.ImagePullPolicy).To(Equal(corev1.PullAlways)) + env = getFeatureStoreYamlEnvVar(onlineContainer.Env) Expect(env).NotTo(BeNil()) - fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64(services.OnlineFeastType) + assertEnvFrom(*onlineContainer) + + fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64() Expect(err).NotTo(HaveOccurred()) Expect(fsYamlStr).To(Equal(env.Value)) @@ -702,25 +671,7 @@ var _ = Describe("FeatureStore Controller", func() { repoConfigOnline := &services.RepoConfig{} err = yaml.Unmarshal(envByte, repoConfigOnline) Expect(err).NotTo(HaveOccurred()) - offlineRemote := services.OfflineStoreConfig{ - Host: "feast-services-offline.default.svc.cluster.local", - Type: services.OfflineRemoteConfigType, - Port: services.HttpPort, - } - onlineConfig := &services.RepoConfig{ - Project: feastProject, - Provider: services.LocalProviderType, - EntityKeySerializationVersion: feastdevv1alpha1.SerializationVersion, - OfflineStore: offlineRemote, - OnlineStore: services.OnlineStoreConfig{ - Path: services.DefaultOnlineStoreEphemeralPath, - Type: services.OnlineSqliteConfigType, - }, - Registry: regRemote, - AuthzConfig: noAuthzConfig(), - } - Expect(repoConfigOnline).To(Equal(onlineConfig)) - Expect(deploy.Spec.Template.Spec.Containers[0].Env).To(HaveLen(3)) + Expect(repoConfigOnline).To(Equal(&testConfig)) // check client config cm := &corev1.ConfigMap{} @@ -734,6 +685,15 @@ var _ = Describe("FeatureStore Controller", func() { repoConfigClient := &services.RepoConfig{} err = yaml.Unmarshal([]byte(cm.Data[services.FeatureStoreYamlCmKey]), repoConfigClient) Expect(err).NotTo(HaveOccurred()) + offlineRemote := services.OfflineStoreConfig{ + Host: "feast-services-offline.default.svc.cluster.local", + Type: services.OfflineRemoteConfigType, + Port: services.HttpPort, + } + regRemote := services.RegistryConfig{ + RegistryType: services.RegistryRemoteConfigType, + Path: "feast-services-registry.default.svc.cluster.local:80", + } clientConfig := &services.RepoConfig{ Project: feastProject, Provider: services.LocalProviderType, @@ -762,8 +722,8 @@ var _ = Describe("FeatureStore Controller", func() { Expect(err).NotTo(HaveOccurred()) Expect(resource.Spec.FeastProject).To(Equal(resourceNew.Spec.FeastProject)) err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.RegistryFeastType), - Namespace: resource.Namespace, + Name: objMeta.Name, + Namespace: objMeta.Namespace, }, deploy) Expect(err).NotTo(HaveOccurred()) @@ -773,7 +733,7 @@ var _ = Describe("FeatureStore Controller", func() { env = getFeatureStoreYamlEnvVar(deploy.Spec.Template.Spec.Containers[0].Env) Expect(env).NotTo(BeNil()) - fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64(services.RegistryFeastType) + fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64() Expect(err).NotTo(HaveOccurred()) Expect(fsYamlStr).To(Equal(env.Value)) @@ -781,7 +741,7 @@ var _ = Describe("FeatureStore Controller", func() { Expect(err).NotTo(HaveOccurred()) err = yaml.Unmarshal(envByte, repoConfig) Expect(err).NotTo(HaveOccurred()) - Expect(repoConfig).To(Equal(testConfig)) + Expect(repoConfig).To(Equal(&testConfig)) }) It("should properly set container env variables", func() { @@ -807,7 +767,7 @@ var _ = Describe("FeatureStore Controller", func() { deployList := appsv1.DeploymentList{} err = k8sClient.List(ctx, &deployList, listOpts) Expect(err).NotTo(HaveOccurred()) - Expect(deployList.Items).To(HaveLen(3)) + Expect(deployList.Items).To(HaveLen(1)) svcList := corev1.ServiceList{} err = k8sClient.List(ctx, &svcList, listOpts) @@ -829,26 +789,27 @@ var _ = Describe("FeatureStore Controller", func() { } fsYamlStr := "" - fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64(services.OnlineFeastType) + fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64() Expect(err).NotTo(HaveOccurred()) // check online config deploy := &appsv1.Deployment{} + objMeta := feast.GetObjectMeta() err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.OnlineFeastType), - Namespace: resource.Namespace, - }, - deploy) + Name: objMeta.Name, + Namespace: objMeta.Namespace, + }, deploy) Expect(err).NotTo(HaveOccurred()) Expect(deploy.Spec.Template.Spec.ServiceAccountName).To(Equal(deploy.Name)) - Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(1)) - Expect(deploy.Spec.Template.Spec.Containers[0].Env).To(HaveLen(3)) - Expect(areEnvVarArraysEqual(deploy.Spec.Template.Spec.Containers[0].Env, []corev1.EnvVar{{Name: testEnvVarName, Value: testEnvVarValue}, {Name: services.FeatureStoreYamlEnvVar, Value: fsYamlStr}, {Name: "fieldRefName", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{APIVersion: "v1", FieldPath: "metadata.namespace"}}}})).To(BeTrue()) - Expect(deploy.Spec.Template.Spec.Containers[0].ImagePullPolicy).To(Equal(corev1.PullAlways)) + Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(3)) + onlineContainer := services.GetOnlineContainer(*deploy) + Expect(onlineContainer.Env).To(HaveLen(3)) + Expect(areEnvVarArraysEqual(onlineContainer.Env, []corev1.EnvVar{{Name: testEnvVarName, Value: testEnvVarValue}, {Name: services.TmpFeatureStoreYamlEnvVar, Value: fsYamlStr}, {Name: "fieldRefName", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{APIVersion: "v1", FieldPath: "metadata.namespace"}}}})).To(BeTrue()) + Expect(onlineContainer.ImagePullPolicy).To(Equal(corev1.PullAlways)) // change feast project and reconcile resourceNew := resource.DeepCopy() - resourceNew.Spec.Services.OnlineStore.Env = &[]corev1.EnvVar{{Name: testEnvVarName, Value: testEnvVarValue + "1"}, {Name: services.FeatureStoreYamlEnvVar, Value: fsYamlStr}, {Name: "fieldRefName", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{FieldPath: "metadata.name"}}}} + resourceNew.Spec.Services.OnlineStore.Env = &[]corev1.EnvVar{{Name: testEnvVarName, Value: testEnvVarValue + "1"}, {Name: services.TmpFeatureStoreYamlEnvVar, Value: fsYamlStr}, {Name: "fieldRefName", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{FieldPath: "metadata.name"}}}} err = k8sClient.Update(ctx, resourceNew) Expect(err).NotTo(HaveOccurred()) _, err = controllerReconciler.Reconcile(ctx, reconcile.Request{ @@ -858,16 +819,16 @@ var _ = Describe("FeatureStore Controller", func() { err = k8sClient.Get(ctx, typeNamespacedName, resource) Expect(err).NotTo(HaveOccurred()) - Expect(areEnvVarArraysEqual(*resource.Status.Applied.Services.OnlineStore.Env, []corev1.EnvVar{{Name: testEnvVarName, Value: testEnvVarValue + "1"}, {Name: services.FeatureStoreYamlEnvVar, Value: fsYamlStr}, {Name: "fieldRefName", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{FieldPath: "metadata.name"}}}})).To(BeTrue()) + Expect(areEnvVarArraysEqual(*resource.Status.Applied.Services.OnlineStore.Env, []corev1.EnvVar{{Name: testEnvVarName, Value: testEnvVarValue + "1"}, {Name: services.TmpFeatureStoreYamlEnvVar, Value: fsYamlStr}, {Name: "fieldRefName", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{FieldPath: "metadata.name"}}}})).To(BeTrue()) err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.OnlineFeastType), - Namespace: resource.Namespace, - }, - deploy) + Name: objMeta.Name, + Namespace: objMeta.Namespace, + }, deploy) Expect(err).NotTo(HaveOccurred()) - Expect(deploy.Spec.Template.Spec.Containers[0].Env).To(HaveLen(3)) - Expect(areEnvVarArraysEqual(deploy.Spec.Template.Spec.Containers[0].Env, []corev1.EnvVar{{Name: testEnvVarName, Value: testEnvVarValue + "1"}, {Name: services.FeatureStoreYamlEnvVar, Value: fsYamlStr}, {Name: "fieldRefName", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{APIVersion: "v1", FieldPath: "metadata.name"}}}})).To(BeTrue()) + onlineContainer = services.GetOnlineContainer(*deploy) + Expect(onlineContainer.Env).To(HaveLen(3)) + Expect(areEnvVarArraysEqual(onlineContainer.Env, []corev1.EnvVar{{Name: testEnvVarName, Value: testEnvVarValue + "1"}, {Name: services.TmpFeatureStoreYamlEnvVar, Value: fsYamlStr}, {Name: "fieldRefName", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{APIVersion: "v1", FieldPath: "metadata.name"}}}})).To(BeTrue()) }) It("Should delete k8s objects owned by the FeatureStore CR", func() { @@ -893,7 +854,7 @@ var _ = Describe("FeatureStore Controller", func() { deployList := appsv1.DeploymentList{} err = k8sClient.List(ctx, &deployList, listOpts) Expect(err).NotTo(HaveOccurred()) - Expect(deployList.Items).To(HaveLen(3)) + Expect(deployList.Items).To(HaveLen(1)) svcList := corev1.ServiceList{} err = k8sClient.List(ctx, &svcList, listOpts) @@ -912,7 +873,7 @@ var _ = Describe("FeatureStore Controller", func() { err = k8sClient.List(ctx, &deployList, listOpts) Expect(err).NotTo(HaveOccurred()) - Expect(deployList.Items).To(HaveLen(2)) + Expect(deployList.Items).To(HaveLen(1)) err = k8sClient.List(ctx, &svcList, listOpts) Expect(err).NotTo(HaveOccurred()) @@ -987,6 +948,7 @@ var _ = Describe("FeatureStore Controller", func() { Expect(err).To(HaveOccurred()) err = k8sClient.Get(ctx, nsName, resource) Expect(err).NotTo(HaveOccurred()) + Expect(resource.Status.Applied.Services.Registry.Remote.FeastRef.Namespace).NotTo(BeEmpty()) Expect(apimeta.FindStatusCondition(resource.Status.Conditions, feastdevv1alpha1.AuthorizationReadyType)).To(BeNil()) Expect(apimeta.FindStatusCondition(resource.Status.Conditions, feastdevv1alpha1.RegistryReadyType)).To(BeNil()) Expect(apimeta.IsStatusConditionTrue(resource.Status.Conditions, feastdevv1alpha1.ReadyType)).To(BeFalse()) @@ -1026,7 +988,6 @@ var _ = Describe("FeatureStore Controller", func() { Expect(apimeta.IsStatusConditionTrue(resource.Status.Conditions, feastdevv1alpha1.ReadyType)).To(BeTrue()) Expect(apimeta.IsStatusConditionTrue(resource.Status.Conditions, feastdevv1alpha1.OnlineStoreReadyType)).To(BeTrue()) Expect(apimeta.IsStatusConditionTrue(resource.Status.Conditions, feastdevv1alpha1.OfflineStoreReadyType)).To(BeTrue()) - Expect(resource.Status.Applied.Services.Registry.Remote.FeastRef.Namespace).To(Equal(resource.Namespace)) Expect(resource.Status.ServiceHostnames.Registry).ToNot(BeEmpty()) Expect(resource.Status.ServiceHostnames.Registry).To(Equal(referencedRegistry.Status.ServiceHostnames.Registry)) feast := services.FeastServices{ @@ -1038,6 +999,16 @@ var _ = Describe("FeatureStore Controller", func() { }, } + // check deployment + deploy := &appsv1.Deployment{} + objMeta := feast.GetObjectMeta() + err = k8sClient.Get(ctx, types.NamespacedName{ + Name: objMeta.Name, + Namespace: objMeta.Namespace, + }, deploy) + Expect(err).NotTo(HaveOccurred()) + Expect(deploy.Spec.Template.Spec.InitContainers).To(HaveLen(2)) + // check client config cm := &corev1.ConfigMap{} err = k8sClient.Get(ctx, types.NamespacedName{ @@ -1069,6 +1040,26 @@ var _ = Describe("FeatureStore Controller", func() { } Expect(repoConfigClient).To(Equal(clientConfig)) + // disable init containers + resource.Spec.Services.DisableInitContainers = true + err = k8sClient.Update(ctx, resource) + Expect(err).NotTo(HaveOccurred()) + + _, err = controllerReconciler.Reconcile(ctx, reconcile.Request{ + NamespacedName: nsName, + }) + Expect(err).NotTo(HaveOccurred()) + + deploy = &appsv1.Deployment{} + objMeta = feast.GetObjectMeta() + err = k8sClient.Get(ctx, types.NamespacedName{ + Name: objMeta.Name, + Namespace: objMeta.Namespace, + }, deploy) + Expect(err).NotTo(HaveOccurred()) + Expect(deploy.Spec.Template.Spec.InitContainers).To(BeEmpty()) + + // break remote reference hostname := "test:80" referencedRegistry.Spec.Services.Registry = &feastdevv1alpha1.Registry{ Remote: &feastdevv1alpha1.RemoteRegistryConfig{ @@ -1126,12 +1117,13 @@ var _ = Describe("FeatureStore Controller", func() { }, } + // check deployment deploy := &appsv1.Deployment{} + objMeta := feast.GetObjectMeta() err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.OfflineFeastType), - Namespace: resource.Namespace, - }, - deploy) + Name: objMeta.Name, + Namespace: objMeta.Namespace, + }, deploy) Expect(err).NotTo(HaveOccurred()) err = controllerutil.RemoveControllerReference(resource, deploy, controllerReconciler.Scheme) @@ -1166,7 +1158,7 @@ var _ = Describe("FeatureStore Controller", func() { Expect(cond.Type).To(Equal(feastdevv1alpha1.ReadyType)) Expect(cond.Status).To(Equal(metav1.ConditionFalse)) Expect(cond.Reason).To(Equal(feastdevv1alpha1.FailedReason)) - Expect(cond.Message).To(Equal("Error: Object " + resource.Namespace + "/" + name + " is already owned by another Service controller " + name)) + Expect(cond.Message).To(Equal("Error: Object " + resource.Namespace + "/" + deploy.Name + " is already owned by another Service controller " + name)) cond = apimeta.FindStatusCondition(resource.Status.Conditions, feastdevv1alpha1.AuthorizationReadyType) Expect(cond).To(BeNil()) @@ -1187,10 +1179,9 @@ var _ = Describe("FeatureStore Controller", func() { cond = apimeta.FindStatusCondition(resource.Status.Conditions, feastdevv1alpha1.OfflineStoreReadyType) Expect(cond).ToNot(BeNil()) - Expect(cond.Status).To(Equal(metav1.ConditionFalse)) - Expect(cond.Reason).To(Equal(feastdevv1alpha1.OfflineStoreFailedReason)) + Expect(cond.Status).To(Equal(metav1.ConditionTrue)) + Expect(cond.Reason).To(Equal(feastdevv1alpha1.ReadyReason)) Expect(cond.Type).To(Equal(feastdevv1alpha1.OfflineStoreReadyType)) - Expect(cond.Message).To(Equal("Error: Object " + resource.Namespace + "/" + name + " is already owned by another Service controller " + name)) cond = apimeta.FindStatusCondition(resource.Status.Conditions, feastdevv1alpha1.OnlineStoreReadyType) Expect(cond).ToNot(BeNil()) @@ -1253,36 +1244,9 @@ var _ = Describe("FeatureStore Controller", func() { }) }) -func createFeatureStoreResource(resourceName string, image string, pullPolicy corev1.PullPolicy, envVars *[]corev1.EnvVar) *feastdevv1alpha1.FeatureStore { - return &feastdevv1alpha1.FeatureStore{ - ObjectMeta: metav1.ObjectMeta{ - Name: resourceName, - Namespace: "default", - }, - Spec: feastdevv1alpha1.FeatureStoreSpec{ - FeastProject: feastProject, - Services: &feastdevv1alpha1.FeatureStoreServices{ - OfflineStore: &feastdevv1alpha1.OfflineStore{}, - OnlineStore: &feastdevv1alpha1.OnlineStore{ - ServiceConfigs: feastdevv1alpha1.ServiceConfigs{ - DefaultConfigs: feastdevv1alpha1.DefaultConfigs{ - Image: &image, - }, - OptionalConfigs: feastdevv1alpha1.OptionalConfigs{ - Env: envVars, - ImagePullPolicy: &pullPolicy, - Resources: &corev1.ResourceRequirements{}, - }, - }, - }, - }, - }, - } -} - func getFeatureStoreYamlEnvVar(envs []corev1.EnvVar) *corev1.EnvVar { for _, e := range envs { - if e.Name == services.FeatureStoreYamlEnvVar { + if e.Name == services.TmpFeatureStoreYamlEnvVar { return &e } } diff --git a/infra/feast-operator/internal/controller/featurestore_controller_test_utils_test.go b/infra/feast-operator/internal/controller/featurestore_controller_test_utils_test.go new file mode 100644 index 00000000000..57fa87ce733 --- /dev/null +++ b/infra/feast-operator/internal/controller/featurestore_controller_test_utils_test.go @@ -0,0 +1,147 @@ +package controller + +import ( + "context" + + . "github.com/onsi/ginkgo/v2" + + feastdevv1alpha1 "github.com/feast-dev/feast/infra/feast-operator/api/v1alpha1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + . "github.com/onsi/gomega" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/types" +) + +func assertEnvFrom(container corev1.Container) { + envFrom := container.EnvFrom + Expect(envFrom).NotTo(BeNil()) + checkEnvFromCounter := 0 + + for _, source := range envFrom { + if source.ConfigMapRef != nil && source.ConfigMapRef.Name == "example-configmap" { + checkEnvFromCounter += 1 + // Simulate retrieval of ConfigMap data and validate + configMap := &corev1.ConfigMap{} + err := k8sClient.Get(context.TODO(), types.NamespacedName{ + Name: source.ConfigMapRef.Name, + Namespace: "default", + }, configMap) + Expect(err).NotTo(HaveOccurred()) + // Validate a specific key-value pair from the ConfigMap + Expect(configMap.Data["example-key"]).To(Equal("example-value")) + } + + if source.SecretRef != nil && source.SecretRef.Name == "example-secret" { + checkEnvFromCounter += 1 + // Simulate retrieval of Secret data and validate + secret := &corev1.Secret{} + err := k8sClient.Get(context.TODO(), types.NamespacedName{ + Name: source.SecretRef.Name, + Namespace: "default", + }, secret) + Expect(err).NotTo(HaveOccurred()) + // Validate a specific key-value pair from the Secret + Expect(string(secret.Data["secret-key"])).To(Equal("secret-value")) + } + } + Expect(checkEnvFromCounter).To(Equal(2)) +} + +func createEnvFromSecretAndConfigMap() { + By("creating the config map and secret for envFrom") + envFromConfigMap := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "example-configmap", + Namespace: "default", + }, + Data: map[string]string{"example-key": "example-value"}, + } + err := k8sClient.Create(context.TODO(), envFromConfigMap) + Expect(err).ToNot(HaveOccurred()) + + envFromSecret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "example-secret", + Namespace: "default", + }, + StringData: map[string]string{"secret-key": "secret-value"}, + } + err = k8sClient.Create(context.TODO(), envFromSecret) + Expect(err).ToNot(HaveOccurred()) +} + +func deleteEnvFromSecretAndConfigMap() { + // Delete ConfigMap + By("Deleting the configmap and secret for envFrom") + configMap := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "example-configmap", + Namespace: "default", + }, + } + err := k8sClient.Delete(context.TODO(), configMap) + Expect(err).ToNot(HaveOccurred()) + + // Delete Secret + secret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "example-secret", + Namespace: "default", + }, + } + err = k8sClient.Delete(context.TODO(), secret) + Expect(err).ToNot(HaveOccurred()) +} + +func createFeatureStoreResource(resourceName string, image string, pullPolicy corev1.PullPolicy, envVars *[]corev1.EnvVar, envFromVar *[]corev1.EnvFromSource) *feastdevv1alpha1.FeatureStore { + return &feastdevv1alpha1.FeatureStore{ + ObjectMeta: metav1.ObjectMeta{ + Name: resourceName, + Namespace: "default", + }, + Spec: feastdevv1alpha1.FeatureStoreSpec{ + FeastProject: feastProject, + Services: &feastdevv1alpha1.FeatureStoreServices{ + OfflineStore: &feastdevv1alpha1.OfflineStore{ + ServiceConfigs: feastdevv1alpha1.ServiceConfigs{ + OptionalConfigs: feastdevv1alpha1.OptionalConfigs{ + EnvFrom: envFromVar, + }, + }, + }, + OnlineStore: &feastdevv1alpha1.OnlineStore{ + ServiceConfigs: feastdevv1alpha1.ServiceConfigs{ + DefaultConfigs: feastdevv1alpha1.DefaultConfigs{ + Image: &image, + }, + OptionalConfigs: feastdevv1alpha1.OptionalConfigs{ + Env: envVars, + EnvFrom: envFromVar, + ImagePullPolicy: &pullPolicy, + Resources: &corev1.ResourceRequirements{}, + }, + }, + }, + }, + }, + } +} + +func withEnvFrom() *[]corev1.EnvFromSource { + + return &[]corev1.EnvFromSource{ + { + ConfigMapRef: &corev1.ConfigMapEnvSource{ + LocalObjectReference: corev1.LocalObjectReference{Name: "example-configmap"}, + }, + }, + { + SecretRef: &corev1.SecretEnvSource{ + LocalObjectReference: corev1.LocalObjectReference{Name: "example-secret"}, + }, + }, + } + +} diff --git a/infra/feast-operator/internal/controller/featurestore_controller_tls_test.go b/infra/feast-operator/internal/controller/featurestore_controller_tls_test.go index 45cda317409..06355c664c3 100644 --- a/infra/feast-operator/internal/controller/featurestore_controller_tls_test.go +++ b/infra/feast-operator/internal/controller/featurestore_controller_tls_test.go @@ -27,7 +27,6 @@ import ( appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" - apierrors "k8s.io/apimachinery/pkg/api/errors" apimeta "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" @@ -56,7 +55,7 @@ var _ = Describe("FeatureStore Controller - Feast service TLS", func() { } featurestore := &feastdevv1alpha1.FeatureStore{} localRef := corev1.LocalObjectReference{Name: "test"} - tlsConfigs := feastdevv1alpha1.TlsConfigs{ + tlsConfigs := &feastdevv1alpha1.TlsConfigs{ SecretRef: &localRef, } BeforeEach(func() { @@ -72,16 +71,14 @@ var _ = Describe("FeatureStore Controller - Feast service TLS", func() { FeastProject: feastProject, Services: &feastdevv1alpha1.FeatureStoreServices{ OnlineStore: &feastdevv1alpha1.OnlineStore{ - TLS: &tlsConfigs, + TLS: tlsConfigs, }, OfflineStore: &feastdevv1alpha1.OfflineStore{ - TLS: &feastdevv1alpha1.OfflineTlsConfigs{ - TlsConfigs: tlsConfigs, - }, + TLS: tlsConfigs, }, Registry: &feastdevv1alpha1.Registry{ Local: &feastdevv1alpha1.LocalRegistryConfig{ - TLS: &tlsConfigs, + TLS: tlsConfigs, }, }, }, @@ -174,16 +171,17 @@ var _ = Describe("FeatureStore Controller - Feast service TLS", func() { Expect(resource.Status.Phase).To(Equal(feastdevv1alpha1.ReadyPhase)) + // check deployment deploy := &appsv1.Deployment{} + objMeta := feast.GetObjectMeta() err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.RegistryFeastType), - Namespace: resource.Namespace, - }, - deploy) + Name: objMeta.Name, + Namespace: objMeta.Namespace, + }, deploy) Expect(err).NotTo(HaveOccurred()) Expect(deploy.Spec.Replicas).To(Equal(&services.DefaultReplicas)) Expect(controllerutil.HasControllerReference(deploy)).To(BeTrue()) - Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(1)) + Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(3)) svc := &corev1.Service{} err = k8sClient.Get(ctx, types.NamespacedName{ @@ -227,7 +225,7 @@ var _ = Describe("FeatureStore Controller - Feast service TLS", func() { deployList := appsv1.DeploymentList{} err = k8sClient.List(ctx, &deployList, listOpts) Expect(err).NotTo(HaveOccurred()) - Expect(deployList.Items).To(HaveLen(3)) + Expect(deployList.Items).To(HaveLen(1)) svcList := corev1.ServiceList{} err = k8sClient.List(ctx, &svcList, listOpts) @@ -239,20 +237,21 @@ var _ = Describe("FeatureStore Controller - Feast service TLS", func() { Expect(err).NotTo(HaveOccurred()) Expect(cmList.Items).To(HaveLen(1)) - // check registry config + // check deployment deploy := &appsv1.Deployment{} + objMeta := feast.GetObjectMeta() err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.RegistryFeastType), - Namespace: resource.Namespace, - }, - deploy) - Expect(err).NotTo(HaveOccurred()) - Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(1)) - Expect(deploy.Spec.Template.Spec.Containers[0].Env).To(HaveLen(1)) - env := getFeatureStoreYamlEnvVar(deploy.Spec.Template.Spec.Containers[0].Env) + Name: objMeta.Name, + Namespace: objMeta.Namespace, + }, deploy) + Expect(err).NotTo(HaveOccurred()) + Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(3)) + registryContainer := services.GetRegistryContainer(*deploy) + Expect(registryContainer.Env).To(HaveLen(1)) + env := getFeatureStoreYamlEnvVar(registryContainer.Env) Expect(env).NotTo(BeNil()) - fsYamlStr, err := feast.GetServiceFeatureStoreYamlBase64(services.RegistryFeastType) + fsYamlStr, err := feast.GetServiceFeatureStoreYamlBase64() Expect(err).NotTo(HaveOccurred()) Expect(fsYamlStr).To(Equal(env.Value)) @@ -261,32 +260,19 @@ var _ = Describe("FeatureStore Controller - Feast service TLS", func() { repoConfig := &services.RepoConfig{} err = yaml.Unmarshal(envByte, repoConfig) Expect(err).NotTo(HaveOccurred()) - testConfig := &services.RepoConfig{ - Project: feastProject, - Provider: services.LocalProviderType, - EntityKeySerializationVersion: feastdevv1alpha1.SerializationVersion, - Registry: services.RegistryConfig{ - RegistryType: services.RegistryFileConfigType, - Path: services.DefaultRegistryEphemeralPath, - }, - AuthzConfig: noAuthzConfig(), + testConfig := feast.GetDefaultRepoConfig() + testConfig.OfflineStore = services.OfflineStoreConfig{ + Type: services.OfflineFilePersistenceDaskConfigType, } - Expect(repoConfig).To(Equal(testConfig)) + Expect(repoConfig).To(Equal(&testConfig)) // check offline config - deploy = &appsv1.Deployment{} - err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.OfflineFeastType), - Namespace: resource.Namespace, - }, - deploy) - Expect(err).NotTo(HaveOccurred()) - Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(1)) - Expect(deploy.Spec.Template.Spec.Containers[0].Env).To(HaveLen(1)) - env = getFeatureStoreYamlEnvVar(deploy.Spec.Template.Spec.Containers[0].Env) + offlineContainer := services.GetOfflineContainer(*deploy) + Expect(offlineContainer.Env).To(HaveLen(1)) + env = getFeatureStoreYamlEnvVar(offlineContainer.Env) Expect(env).NotTo(BeNil()) - fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64(services.OfflineFeastType) + fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64() Expect(err).NotTo(HaveOccurred()) Expect(fsYamlStr).To(Equal(env.Value)) @@ -295,37 +281,15 @@ var _ = Describe("FeatureStore Controller - Feast service TLS", func() { repoConfigOffline := &services.RepoConfig{} err = yaml.Unmarshal(envByte, repoConfigOffline) Expect(err).NotTo(HaveOccurred()) - regRemote := services.RegistryConfig{ - RegistryType: services.RegistryRemoteConfigType, - Path: fmt.Sprintf("feast-%s-registry.default.svc.cluster.local:443", resourceName), - Cert: services.GetTlsPath(services.RegistryFeastType) + "tls.crt", - } - offlineConfig := &services.RepoConfig{ - Project: feastProject, - Provider: services.LocalProviderType, - EntityKeySerializationVersion: feastdevv1alpha1.SerializationVersion, - OfflineStore: services.OfflineStoreConfig{ - Type: services.OfflineFilePersistenceDaskConfigType, - }, - Registry: regRemote, - AuthzConfig: noAuthzConfig(), - } - Expect(repoConfigOffline).To(Equal(offlineConfig)) + Expect(repoConfigOffline).To(Equal(&testConfig)) // check online config - deploy = &appsv1.Deployment{} - err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.OnlineFeastType), - Namespace: resource.Namespace, - }, - deploy) - Expect(err).NotTo(HaveOccurred()) - Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(1)) - Expect(deploy.Spec.Template.Spec.Containers[0].Env).To(HaveLen(1)) - env = getFeatureStoreYamlEnvVar(deploy.Spec.Template.Spec.Containers[0].Env) + onlineContainer := services.GetOnlineContainer(*deploy) + Expect(onlineContainer.Env).To(HaveLen(1)) + env = getFeatureStoreYamlEnvVar(onlineContainer.Env) Expect(env).NotTo(BeNil()) - fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64(services.OnlineFeastType) + fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64() Expect(err).NotTo(HaveOccurred()) Expect(fsYamlStr).To(Equal(env.Value)) @@ -334,26 +298,7 @@ var _ = Describe("FeatureStore Controller - Feast service TLS", func() { repoConfigOnline := &services.RepoConfig{} err = yaml.Unmarshal(envByte, repoConfigOnline) Expect(err).NotTo(HaveOccurred()) - offlineRemote := services.OfflineStoreConfig{ - Host: fmt.Sprintf("feast-%s-offline.default.svc.cluster.local", resourceName), - Type: services.OfflineRemoteConfigType, - Port: services.HttpsPort, - Scheme: services.HttpsScheme, - Cert: services.GetTlsPath(services.OfflineFeastType) + "tls.crt", - } - onlineConfig := &services.RepoConfig{ - Project: feastProject, - Provider: services.LocalProviderType, - EntityKeySerializationVersion: feastdevv1alpha1.SerializationVersion, - OfflineStore: offlineRemote, - OnlineStore: services.OnlineStoreConfig{ - Path: services.DefaultOnlineStoreEphemeralPath, - Type: services.OnlineSqliteConfigType, - }, - Registry: regRemote, - AuthzConfig: noAuthzConfig(), - } - Expect(repoConfigOnline).To(Equal(onlineConfig)) + Expect(repoConfigOnline).To(Equal(&testConfig)) Expect(deploy.Spec.Template.Spec.Containers[0].Env).To(HaveLen(1)) // check client config @@ -368,6 +313,18 @@ var _ = Describe("FeatureStore Controller - Feast service TLS", func() { repoConfigClient := &services.RepoConfig{} err = yaml.Unmarshal([]byte(cm.Data[services.FeatureStoreYamlCmKey]), repoConfigClient) Expect(err).NotTo(HaveOccurred()) + offlineRemote := services.OfflineStoreConfig{ + Host: fmt.Sprintf("feast-%s-offline.default.svc.cluster.local", resourceName), + Type: services.OfflineRemoteConfigType, + Port: services.HttpsPort, + Scheme: services.HttpsScheme, + Cert: services.GetTlsPath(services.OfflineFeastType) + "tls.crt", + } + regRemote := services.RegistryConfig{ + RegistryType: services.RegistryRemoteConfigType, + Path: fmt.Sprintf("feast-%s-registry.default.svc.cluster.local:443", resourceName), + Cert: services.GetTlsPath(services.RegistryFeastType) + "tls.crt", + } clientConfig := &services.RepoConfig{ Project: feastProject, Provider: services.LocalProviderType, @@ -396,9 +353,7 @@ var _ = Describe("FeatureStore Controller - Feast service TLS", func() { }, }, OfflineStore: &feastdevv1alpha1.OfflineStore{ - TLS: &feastdevv1alpha1.OfflineTlsConfigs{ - TlsConfigs: tlsConfigs, - }, + TLS: tlsConfigs, }, Registry: &feastdevv1alpha1.Registry{ Remote: &feastdevv1alpha1.RemoteRegistryConfig{ @@ -426,25 +381,18 @@ var _ = Describe("FeatureStore Controller - Feast service TLS", func() { // check registry deploy = &appsv1.Deployment{} err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.RegistryFeastType), - Namespace: resource.Namespace, - }, - deploy) - Expect(err).To(HaveOccurred()) - Expect(apierrors.IsNotFound(err)).To(BeTrue()) + Name: objMeta.Name, + Namespace: objMeta.Namespace, + }, deploy) + Expect(err).NotTo(HaveOccurred()) + Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(2)) // check offline config - deploy = &appsv1.Deployment{} - err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.OfflineFeastType), - Namespace: resource.Namespace, - }, - deploy) - Expect(err).NotTo(HaveOccurred()) - env = getFeatureStoreYamlEnvVar(deploy.Spec.Template.Spec.Containers[0].Env) + offlineContainer = services.GetOfflineContainer(*deploy) + env = getFeatureStoreYamlEnvVar(offlineContainer.Env) Expect(env).NotTo(BeNil()) - fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64(services.OfflineFeastType) + fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64() Expect(err).NotTo(HaveOccurred()) Expect(fsYamlStr).To(Equal(env.Value)) @@ -458,21 +406,15 @@ var _ = Describe("FeatureStore Controller - Feast service TLS", func() { Path: remoteRegHost, Cert: services.GetTlsPath(services.RegistryFeastType) + "remote.crt", } - offlineConfig.Registry = regRemote - Expect(repoConfigOffline).To(Equal(offlineConfig)) + testConfig.Registry = regRemote + Expect(repoConfigOffline).To(Equal(&testConfig)) // check online config - deploy = &appsv1.Deployment{} - err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.OnlineFeastType), - Namespace: resource.Namespace, - }, - deploy) - Expect(err).NotTo(HaveOccurred()) - env = getFeatureStoreYamlEnvVar(deploy.Spec.Template.Spec.Containers[0].Env) + onlineContainer = services.GetOnlineContainer(*deploy) + env = getFeatureStoreYamlEnvVar(onlineContainer.Env) Expect(env).NotTo(BeNil()) - fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64(services.OnlineFeastType) + fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64() Expect(err).NotTo(HaveOccurred()) Expect(fsYamlStr).To(Equal(env.Value)) @@ -482,8 +424,8 @@ var _ = Describe("FeatureStore Controller - Feast service TLS", func() { repoConfigOnline = &services.RepoConfig{} err = yaml.Unmarshal(envByte, repoConfigOnline) Expect(err).NotTo(HaveOccurred()) - onlineConfig.Registry = regRemote - Expect(repoConfigOnline).To(Equal(onlineConfig)) + testConfig.Registry = regRemote + Expect(repoConfigOnline).To(Equal(&testConfig)) }) }) }) diff --git a/infra/feast-operator/internal/controller/services/client.go b/infra/feast-operator/internal/controller/services/client.go index d4b78e2611e..89e22f7be6d 100644 --- a/infra/feast-operator/internal/controller/services/client.go +++ b/infra/feast-operator/internal/controller/services/client.go @@ -32,7 +32,7 @@ func (feast *FeastServices) deployClient() error { func (feast *FeastServices) createClientConfigMap() error { logger := log.FromContext(feast.Handler.Context) cm := &corev1.ConfigMap{ - ObjectMeta: feast.GetObjectMeta(ClientFeastType), + ObjectMeta: feast.GetObjectMetaType(ClientFeastType), } cm.SetGroupVersionKind(corev1.SchemeGroupVersion.WithKind("ConfigMap")) if op, err := controllerutil.CreateOrUpdate(feast.Handler.Context, feast.Handler.Client, cm, controllerutil.MutateFn(func() error { @@ -46,7 +46,7 @@ func (feast *FeastServices) createClientConfigMap() error { } func (feast *FeastServices) setClientConfigMap(cm *corev1.ConfigMap) error { - cm.Labels = feast.getLabels(ClientFeastType) + cm.Labels = feast.getFeastTypeLabels(ClientFeastType) clientYaml, err := feast.getClientFeatureStoreYaml(feast.extractConfigFromSecret) if err != nil { return err @@ -81,7 +81,7 @@ func (feast *FeastServices) setCaConfigMap(cm *corev1.ConfigMap) error { func (feast *FeastServices) initCaConfigMap() *corev1.ConfigMap { cm := &corev1.ConfigMap{ - ObjectMeta: feast.GetObjectMeta(ClientCaFeastType), + ObjectMeta: feast.GetObjectMetaType(ClientCaFeastType), } cm.SetGroupVersionKind(corev1.SchemeGroupVersion.WithKind("ConfigMap")) return cm diff --git a/infra/feast-operator/internal/controller/services/repo_config.go b/infra/feast-operator/internal/controller/services/repo_config.go index 22052aa724d..0a5dd11544e 100644 --- a/infra/feast-operator/internal/controller/services/repo_config.go +++ b/infra/feast-operator/internal/controller/services/repo_config.go @@ -27,41 +27,79 @@ import ( ) // GetServiceFeatureStoreYamlBase64 returns a base64 encoded feature_store.yaml config for the feast service -func (feast *FeastServices) GetServiceFeatureStoreYamlBase64(feastType FeastServiceType) (string, error) { - fsYaml, err := feast.getServiceFeatureStoreYaml(feastType) +func (feast *FeastServices) GetServiceFeatureStoreYamlBase64() (string, error) { + fsYaml, err := feast.getServiceFeatureStoreYaml() if err != nil { return "", err } return base64.StdEncoding.EncodeToString(fsYaml), nil } -func (feast *FeastServices) getServiceFeatureStoreYaml(feastType FeastServiceType) ([]byte, error) { - repoConfig, err := feast.getServiceRepoConfig(feastType) +func (feast *FeastServices) getServiceFeatureStoreYaml() ([]byte, error) { + repoConfig, err := feast.getServiceRepoConfig() if err != nil { return nil, err } return yaml.Marshal(repoConfig) } -func (feast *FeastServices) getServiceRepoConfig(feastType FeastServiceType) (RepoConfig, error) { - return getServiceRepoConfig(feastType, feast.Handler.FeatureStore, feast.extractConfigFromSecret) +func (feast *FeastServices) getServiceRepoConfig() (RepoConfig, error) { + return getServiceRepoConfig(feast.Handler.FeatureStore, feast.extractConfigFromSecret) } func getServiceRepoConfig( - feastType FeastServiceType, featureStore *feastdevv1alpha1.FeatureStore, - secretExtractionFunc func(secretRef string, secretKeyName string) (map[string]interface{}, error)) (RepoConfig, error) { + secretExtractionFunc func(storeType string, secretRef string, secretKeyName string) (map[string]interface{}, error)) (RepoConfig, error) { + repoConfig, err := getBaseServiceRepoConfig(featureStore, secretExtractionFunc) + if err != nil { + return repoConfig, err + } + appliedSpec := featureStore.Status.Applied + if appliedSpec.Services != nil { + services := appliedSpec.Services + if services.OfflineStore != nil { + err := setRepoConfigOffline(services, secretExtractionFunc, &repoConfig) + if err != nil { + return repoConfig, err + } + } + if services.OnlineStore != nil { + err := setRepoConfigOnline(services, secretExtractionFunc, &repoConfig) + if err != nil { + return repoConfig, err + } + } + if IsLocalRegistry(featureStore) { + err := setRepoConfigRegistry(services, secretExtractionFunc, &repoConfig) + if err != nil { + return repoConfig, err + } + } + } - repoConfig, err := getClientRepoConfig(featureStore, secretExtractionFunc) + return repoConfig, nil +} + +func getBaseServiceRepoConfig( + featureStore *feastdevv1alpha1.FeatureStore, + secretExtractionFunc func(storeType string, secretRef string, secretKeyName string) (map[string]interface{}, error)) (RepoConfig, error) { + + repoConfig := defaultRepoConfig(featureStore) + clientRepoConfig, err := getClientRepoConfig(featureStore, secretExtractionFunc) if err != nil { return repoConfig, err } + if isRemoteRegistry(featureStore) { + repoConfig.Registry = clientRepoConfig.Registry + } + repoConfig.AuthzConfig = clientRepoConfig.AuthzConfig + appliedSpec := featureStore.Status.Applied if appliedSpec.AuthzConfig != nil && appliedSpec.AuthzConfig.OidcAuthz != nil { - propertiesMap, err := secretExtractionFunc(appliedSpec.AuthzConfig.OidcAuthz.SecretRef.Name, "") - if err != nil { - return repoConfig, err + propertiesMap, authSecretErr := secretExtractionFunc("", appliedSpec.AuthzConfig.OidcAuthz.SecretRef.Name, "") + if authSecretErr != nil { + return repoConfig, authSecretErr } oidcServerProperties := map[string]interface{}{} @@ -75,43 +113,10 @@ func getServiceRepoConfig( repoConfig.AuthzConfig.OidcParameters = oidcServerProperties } - if appliedSpec.Services != nil { - services := appliedSpec.Services - - switch feastType { - case OfflineFeastType: - // Offline server has an `offline_store` section and a remote `registry` - if services.OfflineStore != nil { - err := setRepoConfigOffline(services, secretExtractionFunc, &repoConfig) - if err != nil { - return repoConfig, err - } - } - case OnlineFeastType: - // Online server has an `online_store` section, a remote `registry` and a remote `offline_store` - if services.OnlineStore != nil { - err := setRepoConfigOnline(services, secretExtractionFunc, &repoConfig) - if err != nil { - return repoConfig, err - } - } - case RegistryFeastType: - // Registry server only has a `registry` section - if IsLocalRegistry(featureStore) { - err := setRepoConfigRegistry(services, secretExtractionFunc, &repoConfig) - if err != nil { - return repoConfig, err - } - } - } - } - return repoConfig, nil } -func setRepoConfigRegistry(services *feastdevv1alpha1.FeatureStoreServices, secretExtractionFunc func(secretRef string, secretKeyName string) (map[string]interface{}, error), repoConfig *RepoConfig) error { - repoConfig.Registry = RegistryConfig{} - repoConfig.Registry.Path = DefaultRegistryEphemeralPath +func setRepoConfigRegistry(services *feastdevv1alpha1.FeatureStoreServices, secretExtractionFunc func(storeType string, secretRef string, secretKeyName string) (map[string]interface{}, error), repoConfig *RepoConfig) error { registryPersistence := services.Registry.Local.Persistence if registryPersistence != nil { @@ -129,7 +134,7 @@ func setRepoConfigRegistry(services *feastdevv1alpha1.FeatureStoreServices, secr if len(secretKeyName) == 0 { secretKeyName = string(repoConfig.Registry.RegistryType) } - parametersMap, err := secretExtractionFunc(dbPersistence.SecretRef.Name, secretKeyName) + parametersMap, err := secretExtractionFunc(dbPersistence.Type, dbPersistence.SecretRef.Name, secretKeyName) if err != nil { return err } @@ -142,18 +147,10 @@ func setRepoConfigRegistry(services *feastdevv1alpha1.FeatureStoreServices, secr repoConfig.Registry.DBParameters = parametersMap } } - - repoConfig.OfflineStore = OfflineStoreConfig{} - repoConfig.OnlineStore = OnlineStoreConfig{} - return nil } -func setRepoConfigOnline(services *feastdevv1alpha1.FeatureStoreServices, secretExtractionFunc func(secretRef string, secretKeyName string) (map[string]interface{}, error), repoConfig *RepoConfig) error { - repoConfig.OnlineStore = OnlineStoreConfig{} - - repoConfig.OnlineStore.Path = DefaultOnlineStoreEphemeralPath - repoConfig.OnlineStore.Type = OnlineSqliteConfigType +func setRepoConfigOnline(services *feastdevv1alpha1.FeatureStoreServices, secretExtractionFunc func(storeType string, secretRef string, secretKeyName string) (map[string]interface{}, error), repoConfig *RepoConfig) error { onlineStorePersistence := services.OnlineStore.Persistence if onlineStorePersistence != nil { @@ -170,7 +167,7 @@ func setRepoConfigOnline(services *feastdevv1alpha1.FeatureStoreServices, secret secretKeyName = string(repoConfig.OnlineStore.Type) } - parametersMap, err := secretExtractionFunc(dbPersistence.SecretRef.Name, secretKeyName) + parametersMap, err := secretExtractionFunc(dbPersistence.Type, dbPersistence.SecretRef.Name, secretKeyName) if err != nil { return err } @@ -183,13 +180,11 @@ func setRepoConfigOnline(services *feastdevv1alpha1.FeatureStoreServices, secret repoConfig.OnlineStore.DBParameters = parametersMap } } - return nil } -func setRepoConfigOffline(services *feastdevv1alpha1.FeatureStoreServices, secretExtractionFunc func(secretRef string, secretKeyName string) (map[string]interface{}, error), repoConfig *RepoConfig) error { - repoConfig.OfflineStore = OfflineStoreConfig{} - repoConfig.OfflineStore.Type = OfflineFilePersistenceDaskConfigType +func setRepoConfigOffline(services *feastdevv1alpha1.FeatureStoreServices, secretExtractionFunc func(storeType string, secretRef string, secretKeyName string) (map[string]interface{}, error), repoConfig *RepoConfig) error { + repoConfig.OfflineStore = defaultOfflineStoreConfig offlineStorePersistence := services.OfflineStore.Persistence if offlineStorePersistence != nil { @@ -205,7 +200,7 @@ func setRepoConfigOffline(services *feastdevv1alpha1.FeatureStoreServices, secre secretKeyName = string(repoConfig.OfflineStore.Type) } - parametersMap, err := secretExtractionFunc(dbPersistence.SecretRef.Name, secretKeyName) + parametersMap, err := secretExtractionFunc(dbPersistence.Type, dbPersistence.SecretRef.Name, secretKeyName) if err != nil { return err } @@ -218,13 +213,10 @@ func setRepoConfigOffline(services *feastdevv1alpha1.FeatureStoreServices, secre repoConfig.OfflineStore.DBParameters = parametersMap } } - - repoConfig.OnlineStore = OnlineStoreConfig{} - return nil } -func (feast *FeastServices) getClientFeatureStoreYaml(secretExtractionFunc func(secretRef string, secretKeyName string) (map[string]interface{}, error)) ([]byte, error) { +func (feast *FeastServices) getClientFeatureStoreYaml(secretExtractionFunc func(storeType string, secretRef string, secretKeyName string) (map[string]interface{}, error)) ([]byte, error) { clientRepo, err := getClientRepoConfig(feast.Handler.FeatureStore, secretExtractionFunc) if err != nil { return []byte{}, err @@ -234,13 +226,12 @@ func (feast *FeastServices) getClientFeatureStoreYaml(secretExtractionFunc func( func getClientRepoConfig( featureStore *feastdevv1alpha1.FeatureStore, - secretExtractionFunc func(secretRef string, secretKeyName string) (map[string]interface{}, error)) (RepoConfig, error) { + secretExtractionFunc func(storeType string, secretRef string, secretKeyName string) (map[string]interface{}, error)) (RepoConfig, error) { status := featureStore.Status appliedServices := status.Applied.Services - clientRepoConfig := RepoConfig{ - Project: status.Applied.FeastProject, - Provider: LocalProviderType, - EntityKeySerializationVersion: feastdevv1alpha1.SerializationVersion, + clientRepoConfig, err := getRepoConfig(featureStore, secretExtractionFunc) + if err != nil { + return clientRepoConfig, err } if len(status.ServiceHostnames.OfflineStore) > 0 { clientRepoConfig.OfflineStore = OfflineStoreConfig{ @@ -248,9 +239,8 @@ func getClientRepoConfig( Host: strings.Split(status.ServiceHostnames.OfflineStore, ":")[0], Port: HttpPort, } - if appliedServices.OfflineStore != nil && appliedServices.OfflineStore.TLS != nil && - (&appliedServices.OfflineStore.TLS.TlsConfigs).IsTLS() { - clientRepoConfig.OfflineStore.Cert = GetTlsPath(OfflineFeastType) + appliedServices.OfflineStore.TLS.TlsConfigs.SecretKeyNames.TlsCrt + if appliedServices.OfflineStore != nil && appliedServices.OfflineStore.TLS.IsTLS() { + clientRepoConfig.OfflineStore.Cert = GetTlsPath(OfflineFeastType) + appliedServices.OfflineStore.TLS.SecretKeyNames.TlsCrt clientRepoConfig.OfflineStore.Port = HttpsPort clientRepoConfig.OfflineStore.Scheme = HttpsScheme } @@ -278,23 +268,27 @@ func getClientRepoConfig( } } - if status.Applied.AuthzConfig == nil { - clientRepoConfig.AuthzConfig = AuthzConfig{ - Type: NoAuthAuthType, - } - } else { + return clientRepoConfig, nil +} + +func getRepoConfig( + featureStore *feastdevv1alpha1.FeatureStore, + secretExtractionFunc func(storeType string, secretRef string, secretKeyName string) (map[string]interface{}, error)) (RepoConfig, error) { + status := featureStore.Status + repoConfig := initRepoConfig(status.Applied.FeastProject) + if status.Applied.AuthzConfig != nil { if status.Applied.AuthzConfig.KubernetesAuthz != nil { - clientRepoConfig.AuthzConfig = AuthzConfig{ + repoConfig.AuthzConfig = AuthzConfig{ Type: KubernetesAuthType, } } else if status.Applied.AuthzConfig.OidcAuthz != nil { - clientRepoConfig.AuthzConfig = AuthzConfig{ + repoConfig.AuthzConfig = AuthzConfig{ Type: OidcAuthType, } - propertiesMap, err := secretExtractionFunc(status.Applied.AuthzConfig.OidcAuthz.SecretRef.Name, "") + propertiesMap, err := secretExtractionFunc("", status.Applied.AuthzConfig.OidcAuthz.SecretRef.Name, "") if err != nil { - return clientRepoConfig, err + return repoConfig, err } oidcClientProperties := map[string]interface{}{} @@ -302,13 +296,13 @@ func getClientRepoConfig( if val, exists := propertiesMap[string(oidcClientProperty)]; exists { oidcClientProperties[string(oidcClientProperty)] = val } else { - return clientRepoConfig, missingOidcSecretProperty(oidcClientProperty) + return repoConfig, missingOidcSecretProperty(oidcClientProperty) } } - clientRepoConfig.AuthzConfig.OidcParameters = oidcClientProperties + repoConfig.AuthzConfig.OidcParameters = oidcClientProperties } } - return clientRepoConfig, nil + return repoConfig, nil } func getActualPath(filePath string, pvcConfig *feastdevv1alpha1.PvcConfig) string { @@ -318,7 +312,7 @@ func getActualPath(filePath string, pvcConfig *feastdevv1alpha1.PvcConfig) strin return path.Join(pvcConfig.MountPath, filePath) } -func (feast *FeastServices) extractConfigFromSecret(secretRef string, secretKeyName string) (map[string]interface{}, error) { +func (feast *FeastServices) extractConfigFromSecret(storeType string, secretRef string, secretKeyName string) (map[string]interface{}, error) { secret, err := feast.getSecret(secretRef) if err != nil { return nil, err @@ -330,18 +324,20 @@ func (feast *FeastServices) extractConfigFromSecret(secretRef string, secretKeyN if !exists { return nil, fmt.Errorf("secret key %s doesn't exist in secret %s", secretKeyName, secretRef) } + err = yaml.Unmarshal(val, ¶meters) if err != nil { return nil, fmt.Errorf("secret %s contains invalid value", secretKeyName) } - _, exists = parameters["type"] - if exists { - return nil, fmt.Errorf("secret key %s in secret %s contains invalid tag named type", secretKeyName, secretRef) + + typeVal, typeExists := parameters["type"] + if typeExists && storeType != typeVal { + return nil, fmt.Errorf("secret key %s in secret %s contains tag named type with value %s", secretKeyName, secretRef, typeVal) } - _, exists = parameters["registry_type"] - if exists { - return nil, fmt.Errorf("secret key %s in secret %s contains invalid tag named registry_type", secretKeyName, secretRef) + typeVal, typeExists = parameters["registry_type"] + if typeExists && storeType != typeVal { + return nil, fmt.Errorf("secret key %s in secret %s contains tag named registry_type with value %s", secretKeyName, secretRef, typeVal) } } else { for k, v := range secret.Data { @@ -371,3 +367,49 @@ func mergeStructWithDBParametersMap(parametersMap *map[string]interface{}, s int return nil } + +func (feast *FeastServices) GetDefaultRepoConfig() RepoConfig { + return defaultRepoConfig(feast.Handler.FeatureStore) +} + +func defaultRepoConfig(featureStore *feastdevv1alpha1.FeatureStore) RepoConfig { + repoConfig := initRepoConfig(featureStore.Status.Applied.FeastProject) + repoConfig.OnlineStore = defaultOnlineStoreConfig(featureStore) + repoConfig.Registry = defaultRegistryConfig(featureStore) + return repoConfig +} + +func (feast *FeastServices) GetInitRepoConfig() RepoConfig { + return initRepoConfig(feast.Handler.FeatureStore.Status.Applied.FeastProject) +} + +func initRepoConfig(feastProject string) RepoConfig { + return RepoConfig{ + Project: feastProject, + Provider: LocalProviderType, + EntityKeySerializationVersion: feastdevv1alpha1.SerializationVersion, + AuthzConfig: defaultAuthzConfig, + } +} + +func defaultOnlineStoreConfig(featureStore *feastdevv1alpha1.FeatureStore) OnlineStoreConfig { + return OnlineStoreConfig{ + Type: OnlineSqliteConfigType, + Path: defaultOnlineStorePath(featureStore), + } +} + +func defaultRegistryConfig(featureStore *feastdevv1alpha1.FeatureStore) RegistryConfig { + return RegistryConfig{ + RegistryType: RegistryFileConfigType, + Path: defaultRegistryPath(featureStore), + } +} + +var defaultOfflineStoreConfig = OfflineStoreConfig{ + Type: OfflineFilePersistenceDaskConfigType, +} + +var defaultAuthzConfig = AuthzConfig{ + Type: NoAuthAuthType, +} diff --git a/infra/feast-operator/internal/controller/services/repo_config_test.go b/infra/feast-operator/internal/controller/services/repo_config_test.go index b148f904706..1e5909b8892 100644 --- a/infra/feast-operator/internal/controller/services/repo_config_test.go +++ b/infra/feast-operator/internal/controller/services/repo_config_test.go @@ -32,75 +32,76 @@ var projectName = "test-project" var _ = Describe("Repo Config", func() { Context("When creating the RepoConfig of a FeatureStore", func() { - It("should successfully create the repo configs", func() { By("Having the minimal created resource") featureStore := minimalFeatureStore() ApplyDefaultsToStatus(featureStore) - var repoConfig RepoConfig - repoConfig, err := getServiceRepoConfig(OfflineFeastType, featureStore, emptyMockExtractConfigFromSecret) - Expect(err).NotTo(HaveOccurred()) - Expect(repoConfig.AuthzConfig.Type).To(Equal(NoAuthAuthType)) - Expect(repoConfig.OfflineStore).To(Equal(emptyOfflineStoreConfig())) - Expect(repoConfig.OnlineStore).To(Equal(emptyOnlineStoreConfig())) - Expect(repoConfig.Registry).To(Equal(emptyRegistryConfig())) - - repoConfig, err = getServiceRepoConfig(OnlineFeastType, featureStore, emptyMockExtractConfigFromSecret) - Expect(err).NotTo(HaveOccurred()) - Expect(repoConfig.AuthzConfig.Type).To(Equal(NoAuthAuthType)) - Expect(repoConfig.OfflineStore).To(Equal(emptyOfflineStoreConfig())) - Expect(repoConfig.OnlineStore).To(Equal(emptyOnlineStoreConfig())) - Expect(repoConfig.Registry).To(Equal(emptyRegistryConfig())) - repoConfig, err = getServiceRepoConfig(RegistryFeastType, featureStore, emptyMockExtractConfigFromSecret) - Expect(err).NotTo(HaveOccurred()) - Expect(repoConfig.AuthzConfig.Type).To(Equal(NoAuthAuthType)) - Expect(repoConfig.OfflineStore).To(Equal(emptyOfflineStoreConfig())) - Expect(repoConfig.OnlineStore).To(Equal(emptyOnlineStoreConfig())) expectedRegistryConfig := RegistryConfig{ RegistryType: "file", - Path: DefaultRegistryEphemeralPath, + Path: EphemeralPath + "/" + DefaultRegistryPath, } + expectedOnlineConfig := OnlineStoreConfig{ + Type: "sqlite", + Path: EphemeralPath + "/" + DefaultOnlineStorePath, + } + + repoConfig, err := getServiceRepoConfig(featureStore, emptyMockExtractConfigFromSecret) + Expect(err).NotTo(HaveOccurred()) + Expect(repoConfig.AuthzConfig.Type).To(Equal(NoAuthAuthType)) + Expect(repoConfig.OfflineStore).To(Equal(emptyOfflineStoreConfig)) + Expect(repoConfig.OnlineStore).To(Equal(expectedOnlineConfig)) Expect(repoConfig.Registry).To(Equal(expectedRegistryConfig)) By("Having the local registry resource") featureStore = minimalFeatureStore() + testPath := "/test/file.db" featureStore.Spec.Services = &feastdevv1alpha1.FeatureStoreServices{ Registry: &feastdevv1alpha1.Registry{ Local: &feastdevv1alpha1.LocalRegistryConfig{ Persistence: &feastdevv1alpha1.RegistryPersistence{ FilePersistence: &feastdevv1alpha1.RegistryFilePersistence{ - Path: "file.db", + Path: testPath, }, }, }, }, } ApplyDefaultsToStatus(featureStore) - repoConfig, err = getServiceRepoConfig(OfflineFeastType, featureStore, emptyMockExtractConfigFromSecret) - Expect(err).NotTo(HaveOccurred()) - Expect(repoConfig.AuthzConfig.Type).To(Equal(NoAuthAuthType)) - Expect(repoConfig.OfflineStore).To(Equal(emptyOfflineStoreConfig())) - Expect(repoConfig.OnlineStore).To(Equal(emptyOnlineStoreConfig())) - Expect(repoConfig.Registry).To(Equal(emptyRegistryConfig())) - repoConfig, err = getServiceRepoConfig(OnlineFeastType, featureStore, emptyMockExtractConfigFromSecret) + expectedRegistryConfig = RegistryConfig{ + RegistryType: "file", + Path: testPath, + } + + repoConfig, err = getServiceRepoConfig(featureStore, emptyMockExtractConfigFromSecret) Expect(err).NotTo(HaveOccurred()) Expect(repoConfig.AuthzConfig.Type).To(Equal(NoAuthAuthType)) - Expect(repoConfig.OfflineStore).To(Equal(emptyOfflineStoreConfig())) - Expect(repoConfig.OnlineStore).To(Equal(emptyOnlineStoreConfig())) - Expect(repoConfig.Registry).To(Equal(emptyRegistryConfig())) + Expect(repoConfig.OfflineStore).To(Equal(emptyOfflineStoreConfig)) + Expect(repoConfig.OnlineStore).To(Equal(expectedOnlineConfig)) + Expect(repoConfig.Registry).To(Equal(expectedRegistryConfig)) + + By("Adding an offlineStore with PVC") + featureStore.Spec.Services.OfflineStore = &feastdevv1alpha1.OfflineStore{ + Persistence: &feastdevv1alpha1.OfflineStorePersistence{ + FilePersistence: &feastdevv1alpha1.OfflineStoreFilePersistence{ + PvcConfig: &feastdevv1alpha1.PvcConfig{ + MountPath: "/testing", + }, + }, + }, + } + ApplyDefaultsToStatus(featureStore) + appliedServices := featureStore.Status.Applied.Services + Expect(appliedServices.OnlineStore).To(BeNil()) + Expect(appliedServices.OnlineStore).To(BeNil()) - repoConfig, err = getServiceRepoConfig(RegistryFeastType, featureStore, emptyMockExtractConfigFromSecret) + repoConfig, err = getServiceRepoConfig(featureStore, emptyMockExtractConfigFromSecret) Expect(err).NotTo(HaveOccurred()) + Expect(repoConfig.OfflineStore).To(Equal(defaultOfflineStoreConfig)) Expect(repoConfig.AuthzConfig.Type).To(Equal(NoAuthAuthType)) - Expect(repoConfig.OfflineStore).To(Equal(emptyOfflineStoreConfig())) - Expect(repoConfig.OnlineStore).To(Equal(emptyOnlineStoreConfig())) - expectedRegistryConfig = RegistryConfig{ - RegistryType: "file", - Path: "file.db", - } Expect(repoConfig.Registry).To(Equal(expectedRegistryConfig)) + Expect(repoConfig.OnlineStore).To(Equal(expectedOnlineConfig)) By("Having the remote registry resource") featureStore = minimalFeatureStore() @@ -108,33 +109,18 @@ var _ = Describe("Repo Config", func() { Registry: &feastdevv1alpha1.Registry{ Remote: &feastdevv1alpha1.RemoteRegistryConfig{ FeastRef: &feastdevv1alpha1.FeatureStoreRef{ - Name: "registry", - Namespace: "remoteNS", + Name: "registry", }, }, }, } ApplyDefaultsToStatus(featureStore) - repoConfig, err = getServiceRepoConfig(OfflineFeastType, featureStore, emptyMockExtractConfigFromSecret) - Expect(err).NotTo(HaveOccurred()) - Expect(repoConfig.AuthzConfig.Type).To(Equal(NoAuthAuthType)) - Expect(repoConfig.OfflineStore).To(Equal(emptyOfflineStoreConfig())) - Expect(repoConfig.OnlineStore).To(Equal(emptyOnlineStoreConfig())) - Expect(repoConfig.Registry).To(Equal(emptyRegistryConfig())) - - repoConfig, err = getServiceRepoConfig(OnlineFeastType, featureStore, emptyMockExtractConfigFromSecret) + repoConfig, err = getServiceRepoConfig(featureStore, emptyMockExtractConfigFromSecret) Expect(err).NotTo(HaveOccurred()) Expect(repoConfig.AuthzConfig.Type).To(Equal(NoAuthAuthType)) - Expect(repoConfig.OfflineStore).To(Equal(emptyOfflineStoreConfig())) - Expect(repoConfig.OnlineStore).To(Equal(emptyOnlineStoreConfig())) - Expect(repoConfig.Registry).To(Equal(emptyRegistryConfig())) - - repoConfig, err = getServiceRepoConfig(RegistryFeastType, featureStore, emptyMockExtractConfigFromSecret) - Expect(err).NotTo(HaveOccurred()) - Expect(repoConfig.AuthzConfig.Type).To(Equal(NoAuthAuthType)) - Expect(repoConfig.OfflineStore).To(Equal(emptyOfflineStoreConfig())) - Expect(repoConfig.OnlineStore).To(Equal(emptyOnlineStoreConfig())) - Expect(repoConfig.Registry).To(Equal(emptyRegistryConfig())) + Expect(repoConfig.OfflineStore).To(Equal(emptyOfflineStoreConfig)) + Expect(repoConfig.OnlineStore).To(Equal(expectedOnlineConfig)) + Expect(repoConfig.Registry).To(Equal(emptyRegistryConfig)) By("Having the all the file services") featureStore = minimalFeatureStore() @@ -164,36 +150,24 @@ var _ = Describe("Repo Config", func() { }, } ApplyDefaultsToStatus(featureStore) - repoConfig, err = getServiceRepoConfig(OfflineFeastType, featureStore, emptyMockExtractConfigFromSecret) - Expect(err).NotTo(HaveOccurred()) - Expect(repoConfig.AuthzConfig.Type).To(Equal(NoAuthAuthType)) + expectedOfflineConfig := OfflineStoreConfig{ Type: "duckdb", } - Expect(repoConfig.OfflineStore).To(Equal(expectedOfflineConfig)) - Expect(repoConfig.OnlineStore).To(Equal(emptyOnlineStoreConfig())) - Expect(repoConfig.Registry).To(Equal(emptyRegistryConfig())) - - repoConfig, err = getServiceRepoConfig(OnlineFeastType, featureStore, emptyMockExtractConfigFromSecret) - Expect(err).NotTo(HaveOccurred()) - Expect(repoConfig.AuthzConfig.Type).To(Equal(NoAuthAuthType)) - Expect(repoConfig.OfflineStore).To(Equal(emptyOfflineStoreConfig())) - expectedOnlineConfig := OnlineStoreConfig{ + expectedRegistryConfig = RegistryConfig{ + RegistryType: "file", + Path: "/data/registry.db", + } + expectedOnlineConfig = OnlineStoreConfig{ Type: "sqlite", Path: "/data/online.db", } - Expect(repoConfig.OnlineStore).To(Equal(expectedOnlineConfig)) - Expect(repoConfig.Registry).To(Equal(emptyRegistryConfig())) - repoConfig, err = getServiceRepoConfig(RegistryFeastType, featureStore, emptyMockExtractConfigFromSecret) + repoConfig, err = getServiceRepoConfig(featureStore, emptyMockExtractConfigFromSecret) Expect(err).NotTo(HaveOccurred()) Expect(repoConfig.AuthzConfig.Type).To(Equal(NoAuthAuthType)) - Expect(repoConfig.OfflineStore).To(Equal(emptyOfflineStoreConfig())) - Expect(repoConfig.OnlineStore).To(Equal(emptyOnlineStoreConfig())) - expectedRegistryConfig = RegistryConfig{ - RegistryType: "file", - Path: "/data/registry.db", - } + Expect(repoConfig.OfflineStore).To(Equal(expectedOfflineConfig)) + Expect(repoConfig.OnlineStore).To(Equal(expectedOnlineConfig)) Expect(repoConfig.Registry).To(Equal(expectedRegistryConfig)) By("Having kubernetes authorization") @@ -202,56 +176,24 @@ var _ = Describe("Repo Config", func() { KubernetesAuthz: &feastdevv1alpha1.KubernetesAuthz{}, } featureStore.Spec.Services = &feastdevv1alpha1.FeatureStoreServices{ - OfflineStore: &feastdevv1alpha1.OfflineStore{ - Persistence: &feastdevv1alpha1.OfflineStorePersistence{ - FilePersistence: &feastdevv1alpha1.OfflineStoreFilePersistence{}, - }, - }, - OnlineStore: &feastdevv1alpha1.OnlineStore{ - Persistence: &feastdevv1alpha1.OnlineStorePersistence{ - FilePersistence: &feastdevv1alpha1.OnlineStoreFilePersistence{}, - }, - }, + OfflineStore: &feastdevv1alpha1.OfflineStore{}, + OnlineStore: &feastdevv1alpha1.OnlineStore{}, Registry: &feastdevv1alpha1.Registry{ - Local: &feastdevv1alpha1.LocalRegistryConfig{ - Persistence: &feastdevv1alpha1.RegistryPersistence{ - FilePersistence: &feastdevv1alpha1.RegistryFilePersistence{}, - }, - }, + Local: &feastdevv1alpha1.LocalRegistryConfig{}, }, } ApplyDefaultsToStatus(featureStore) - repoConfig, err = getServiceRepoConfig(OfflineFeastType, featureStore, mockExtractConfigFromSecret) - Expect(err).NotTo(HaveOccurred()) - Expect(repoConfig.AuthzConfig.Type).To(Equal(KubernetesAuthType)) + expectedOfflineConfig = OfflineStoreConfig{ Type: "dask", } - Expect(repoConfig.OfflineStore).To(Equal(expectedOfflineConfig)) - Expect(repoConfig.OnlineStore).To(Equal(emptyOnlineStoreConfig())) - Expect(repoConfig.Registry).To(Equal(emptyRegistryConfig())) - repoConfig, err = getServiceRepoConfig(OnlineFeastType, featureStore, mockExtractConfigFromSecret) + repoConfig, err = getServiceRepoConfig(featureStore, mockExtractConfigFromSecret) Expect(err).NotTo(HaveOccurred()) Expect(repoConfig.AuthzConfig.Type).To(Equal(KubernetesAuthType)) - Expect(repoConfig.OfflineStore).To(Equal(emptyOfflineStoreConfig())) - expectedOnlineConfig = OnlineStoreConfig{ - Type: "sqlite", - Path: DefaultOnlineStoreEphemeralPath, - } - Expect(repoConfig.OnlineStore).To(Equal(expectedOnlineConfig)) - Expect(repoConfig.Registry).To(Equal(emptyRegistryConfig())) - - repoConfig, err = getServiceRepoConfig(RegistryFeastType, featureStore, mockExtractConfigFromSecret) - Expect(err).NotTo(HaveOccurred()) - Expect(repoConfig.AuthzConfig.Type).To(Equal(KubernetesAuthType)) - Expect(repoConfig.OfflineStore).To(Equal(emptyOfflineStoreConfig())) - Expect(repoConfig.OnlineStore).To(Equal(emptyOnlineStoreConfig())) - expectedRegistryConfig = RegistryConfig{ - RegistryType: "file", - Path: DefaultRegistryEphemeralPath, - } - Expect(repoConfig.Registry).To(Equal(expectedRegistryConfig)) + Expect(repoConfig.OfflineStore).To(Equal(expectedOfflineConfig)) + Expect(repoConfig.OnlineStore).To(Equal(defaultOnlineStoreConfig(featureStore))) + Expect(repoConfig.Registry).To(Equal(defaultRegistryConfig(featureStore))) By("Having oidc authorization") featureStore.Spec.AuthzConfig = &feastdevv1alpha1.AuthzConfig{ @@ -269,46 +211,15 @@ var _ = Describe("Repo Config", func() { string(OidcClientSecret): "client-secret", string(OidcUsername): "username", string(OidcPassword): "password"}) - repoConfig, err = getServiceRepoConfig(OfflineFeastType, featureStore, secretExtractionFunc) + repoConfig, err = getServiceRepoConfig(featureStore, secretExtractionFunc) Expect(err).NotTo(HaveOccurred()) Expect(repoConfig.AuthzConfig.Type).To(Equal(OidcAuthType)) Expect(repoConfig.AuthzConfig.OidcParameters).To(HaveLen(2)) Expect(repoConfig.AuthzConfig.OidcParameters).To(HaveKey(string(OidcClientId))) Expect(repoConfig.AuthzConfig.OidcParameters).To(HaveKey(string(OidcAuthDiscoveryUrl))) - expectedOfflineConfig = OfflineStoreConfig{ - Type: "dask", - } Expect(repoConfig.OfflineStore).To(Equal(expectedOfflineConfig)) - Expect(repoConfig.OnlineStore).To(Equal(emptyOnlineStoreConfig())) - Expect(repoConfig.Registry).To(Equal(emptyRegistryConfig())) - - repoConfig, err = getServiceRepoConfig(OnlineFeastType, featureStore, secretExtractionFunc) - Expect(err).NotTo(HaveOccurred()) - Expect(repoConfig.AuthzConfig.Type).To(Equal(OidcAuthType)) - Expect(repoConfig.AuthzConfig.OidcParameters).To(HaveLen(2)) - Expect(repoConfig.AuthzConfig.OidcParameters).To(HaveKey(string(OidcClientId))) - Expect(repoConfig.AuthzConfig.OidcParameters).To(HaveKey(string(OidcAuthDiscoveryUrl))) - Expect(repoConfig.OfflineStore).To(Equal(emptyOfflineStoreConfig())) - expectedOnlineConfig = OnlineStoreConfig{ - Type: "sqlite", - Path: DefaultOnlineStoreEphemeralPath, - } - Expect(repoConfig.OnlineStore).To(Equal(expectedOnlineConfig)) - Expect(repoConfig.Registry).To(Equal(emptyRegistryConfig())) - - repoConfig, err = getServiceRepoConfig(RegistryFeastType, featureStore, secretExtractionFunc) - Expect(err).NotTo(HaveOccurred()) - Expect(repoConfig.AuthzConfig.Type).To(Equal(OidcAuthType)) - Expect(repoConfig.AuthzConfig.OidcParameters).To(HaveLen(2)) - Expect(repoConfig.AuthzConfig.OidcParameters).To(HaveKey(string(OidcClientId))) - Expect(repoConfig.AuthzConfig.OidcParameters).To(HaveKey(string(OidcAuthDiscoveryUrl))) - Expect(repoConfig.OfflineStore).To(Equal(emptyOfflineStoreConfig())) - Expect(repoConfig.OnlineStore).To(Equal(emptyOnlineStoreConfig())) - expectedRegistryConfig = RegistryConfig{ - RegistryType: "file", - Path: DefaultRegistryEphemeralPath, - } - Expect(repoConfig.Registry).To(Equal(expectedRegistryConfig)) + Expect(repoConfig.OnlineStore).To(Equal(defaultOnlineStoreConfig(featureStore))) + Expect(repoConfig.Registry).To(Equal(defaultRegistryConfig(featureStore))) repoConfig, err = getClientRepoConfig(featureStore, secretExtractionFunc) Expect(err).NotTo(HaveOccurred()) @@ -359,7 +270,7 @@ var _ = Describe("Repo Config", func() { featureStore.Spec.Services.OfflineStore.Persistence.FilePersistence = nil featureStore.Spec.Services.OnlineStore.Persistence.FilePersistence = nil featureStore.Spec.Services.Registry.Local.Persistence.FilePersistence = nil - repoConfig, err = getServiceRepoConfig(OfflineFeastType, featureStore, mockExtractConfigFromSecret) + repoConfig, err = getServiceRepoConfig(featureStore, mockExtractConfigFromSecret) Expect(err).NotTo(HaveOccurred()) newMap := CopyMap(parameterMap) port := parameterMap["port"].(int) @@ -369,29 +280,16 @@ var _ = Describe("Repo Config", func() { Port: port, DBParameters: newMap, } - Expect(repoConfig.OfflineStore).To(Equal(expectedOfflineConfig)) - Expect(repoConfig.OnlineStore).To(Equal(emptyOnlineStoreConfig())) - Expect(repoConfig.Registry).To(Equal(emptyRegistryConfig())) - - repoConfig, err = getServiceRepoConfig(OnlineFeastType, featureStore, mockExtractConfigFromSecret) - Expect(err).NotTo(HaveOccurred()) - Expect(repoConfig.OfflineStore).To(Equal(emptyOfflineStoreConfig())) - newMap = CopyMap(parameterMap) expectedOnlineConfig = OnlineStoreConfig{ Type: OnlineDBPersistenceSnowflakeConfigType, - DBParameters: newMap, + DBParameters: CopyMap(parameterMap), } - Expect(repoConfig.OnlineStore).To(Equal(expectedOnlineConfig)) - Expect(repoConfig.Registry).To(Equal(emptyRegistryConfig())) - - repoConfig, err = getServiceRepoConfig(RegistryFeastType, featureStore, mockExtractConfigFromSecret) - Expect(err).NotTo(HaveOccurred()) - Expect(repoConfig.OfflineStore).To(Equal(emptyOfflineStoreConfig())) - Expect(repoConfig.OnlineStore).To(Equal(emptyOnlineStoreConfig())) expectedRegistryConfig = RegistryConfig{ RegistryType: RegistryDBPersistenceSnowflakeConfigType, DBParameters: parameterMap, } + Expect(repoConfig.OfflineStore).To(Equal(expectedOfflineConfig)) + Expect(repoConfig.OnlineStore).To(Equal(expectedOnlineConfig)) Expect(repoConfig.Registry).To(Equal(expectedRegistryConfig)) }) }) @@ -413,13 +311,13 @@ var _ = Describe("Repo Config", func() { string(OidcClientSecret): "client-secret", string(OidcUsername): "username", string(OidcPassword): "password"}) - _, err := getServiceRepoConfig(OfflineFeastType, featureStore, secretExtractionFunc) + _, err := getServiceRepoConfig(featureStore, secretExtractionFunc) Expect(err).To(HaveOccurred()) Expect(err.Error()).To(ContainSubstring("missing OIDC secret")) - _, err = getServiceRepoConfig(OnlineFeastType, featureStore, secretExtractionFunc) + _, err = getServiceRepoConfig(featureStore, secretExtractionFunc) Expect(err).To(HaveOccurred()) Expect(err.Error()).To(ContainSubstring("missing OIDC secret")) - _, err = getServiceRepoConfig(RegistryFeastType, featureStore, secretExtractionFunc) + _, err = getServiceRepoConfig(featureStore, secretExtractionFunc) Expect(err).To(HaveOccurred()) Expect(err.Error()).To(ContainSubstring("missing OIDC secret")) _, err = getClientRepoConfig(featureStore, secretExtractionFunc) @@ -440,13 +338,13 @@ var _ = Describe("Repo Config", func() { string(OidcClientId): "client-id", string(OidcUsername): "username", string(OidcPassword): "password"}) - _, err = getServiceRepoConfig(OfflineFeastType, featureStore, secretExtractionFunc) + _, err = getServiceRepoConfig(featureStore, secretExtractionFunc) Expect(err).To(HaveOccurred()) Expect(err.Error()).To(ContainSubstring("missing OIDC secret")) - _, err = getServiceRepoConfig(OnlineFeastType, featureStore, secretExtractionFunc) + _, err = getServiceRepoConfig(featureStore, secretExtractionFunc) Expect(err).To(HaveOccurred()) Expect(err.Error()).To(ContainSubstring("missing OIDC secret")) - _, err = getServiceRepoConfig(RegistryFeastType, featureStore, secretExtractionFunc) + _, err = getServiceRepoConfig(featureStore, secretExtractionFunc) Expect(err).To(HaveOccurred()) Expect(err.Error()).To(ContainSubstring("missing OIDC secret")) _, err = getClientRepoConfig(featureStore, secretExtractionFunc) @@ -455,17 +353,8 @@ var _ = Describe("Repo Config", func() { }) }) -func emptyOnlineStoreConfig() OnlineStoreConfig { - return OnlineStoreConfig{} -} - -func emptyOfflineStoreConfig() OfflineStoreConfig { - return OfflineStoreConfig{} -} - -func emptyRegistryConfig() RegistryConfig { - return RegistryConfig{} -} +var emptyOfflineStoreConfig = OfflineStoreConfig{} +var emptyRegistryConfig = RegistryConfig{} func minimalFeatureStore() *feastdevv1alpha1.FeatureStore { return &feastdevv1alpha1.FeatureStore{ @@ -486,17 +375,17 @@ func minimalFeatureStoreWithAllServices() *feastdevv1alpha1.FeatureStore { return feast } -func emptyMockExtractConfigFromSecret(secretRef string, secretKeyName string) (map[string]interface{}, error) { +func emptyMockExtractConfigFromSecret(storeType string, secretRef string, secretKeyName string) (map[string]interface{}, error) { return map[string]interface{}{}, nil } -func mockExtractConfigFromSecret(secretRef string, secretKeyName string) (map[string]interface{}, error) { +func mockExtractConfigFromSecret(storeType string, secretRef string, secretKeyName string) (map[string]interface{}, error) { return createParameterMap(), nil } func mockOidcConfigFromSecret( - oidcProperties map[string]interface{}) func(secretRef string, secretKeyName string) (map[string]interface{}, error) { - return func(secretRef string, secretKeyName string) (map[string]interface{}, error) { + oidcProperties map[string]interface{}) func(storeType string, secretRef string, secretKeyName string) (map[string]interface{}, error) { + return func(storeType string, secretRef string, secretKeyName string) (map[string]interface{}, error) { return oidcProperties, nil } } diff --git a/infra/feast-operator/internal/controller/services/services.go b/infra/feast-operator/internal/controller/services/services.go index b1878ee00ae..1c8a7d26e53 100644 --- a/infra/feast-operator/internal/controller/services/services.go +++ b/infra/feast-operator/internal/controller/services/services.go @@ -49,6 +49,9 @@ func (feast *FeastServices) ApplyDefaults() error { // Deploy the feast services func (feast *FeastServices) Deploy() error { + if feast.noLocalServiceConfigured() { + return errors.New("At least one local service must be configured. e.g. registry / online / offline.") + } openshiftTls, err := feast.checkOpenshiftTls() if err != nil { return err @@ -106,7 +109,12 @@ func (feast *FeastServices) Deploy() error { return err } } - + if err := feast.createServiceAccount(); err != nil { + return err + } + if err := feast.createDeployment(); err != nil { + return err + } if err := feast.deployClient(); err != nil { return err } @@ -194,12 +202,6 @@ func (feast *FeastServices) deployFeastServiceByType(feastType FeastServiceType) if err := feast.createService(feastType); err != nil { return feast.setFeastServiceCondition(err, feastType) } - if err := feast.createServiceAccount(feastType); err != nil { - return feast.setFeastServiceCondition(err, feastType) - } - if err := feast.createDeployment(feastType); err != nil { - return feast.setFeastServiceCondition(err, feastType) - } return feast.setFeastServiceCondition(nil, feastType) } @@ -207,12 +209,6 @@ func (feast *FeastServices) removeFeastServiceByType(feastType FeastServiceType) if err := feast.Handler.DeleteOwnedFeastObj(feast.initFeastSvc(feastType)); err != nil { return err } - if err := feast.Handler.DeleteOwnedFeastObj(feast.initFeastDeploy(feastType)); err != nil { - return err - } - if err := feast.Handler.DeleteOwnedFeastObj(feast.initFeastSA(feastType)); err != nil { - return err - } if err := feast.Handler.DeleteOwnedFeastObj(feast.initPVC(feastType)); err != nil { return err } @@ -233,11 +229,11 @@ func (feast *FeastServices) createService(feastType FeastServiceType) error { return nil } -func (feast *FeastServices) createServiceAccount(feastType FeastServiceType) error { +func (feast *FeastServices) createServiceAccount() error { logger := log.FromContext(feast.Handler.Context) - sa := feast.initFeastSA(feastType) + sa := feast.initFeastSA() if op, err := controllerutil.CreateOrUpdate(feast.Handler.Context, feast.Handler.Client, sa, controllerutil.MutateFn(func() error { - return feast.setServiceAccount(sa, feastType) + return feast.setServiceAccount(sa) })); err != nil { return err } else if op == controllerutil.OperationResultCreated || op == controllerutil.OperationResultUpdated { @@ -246,11 +242,11 @@ func (feast *FeastServices) createServiceAccount(feastType FeastServiceType) err return nil } -func (feast *FeastServices) createDeployment(feastType FeastServiceType) error { +func (feast *FeastServices) createDeployment() error { logger := log.FromContext(feast.Handler.Context) - deploy := feast.initFeastDeploy(feastType) + deploy := feast.initFeastDeploy() if op, err := controllerutil.CreateOrUpdate(feast.Handler.Context, feast.Handler.Client, deploy, controllerutil.MutateFn(func() error { - return feast.setDeployment(deploy, feastType) + return feast.setDeployment(deploy) })); err != nil { return err } else if op == controllerutil.OperationResultCreated || op == controllerutil.OperationResultUpdated { @@ -280,134 +276,185 @@ func (feast *FeastServices) createPVC(pvcCreate *feastdevv1alpha1.PvcCreate, fea return nil } -func (feast *FeastServices) setDeployment(deploy *appsv1.Deployment, feastType FeastServiceType) error { - fsYamlB64, err := feast.GetServiceFeatureStoreYamlBase64(feastType) - if err != nil { - return err - } - deploy.Labels = feast.getLabels(feastType) - sa := feast.initFeastSA(feastType) - tls := feast.getTlsConfigs(feastType) - serviceConfigs := feast.getServiceConfigs(feastType) - defaultServiceConfigs := serviceConfigs.DefaultConfigs - probeHandler := getProbeHandler(feastType, tls) - +func (feast *FeastServices) setDeployment(deploy *appsv1.Deployment) error { + deploy.Labels = feast.getLabels() deploy.Spec = appsv1.DeploymentSpec{ Replicas: &DefaultReplicas, Selector: metav1.SetAsLabelSelector(deploy.GetLabels()), + Strategy: feast.getDeploymentStrategy(), Template: corev1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ Labels: deploy.GetLabels(), }, Spec: corev1.PodSpec{ - ServiceAccountName: sa.Name, - Containers: []corev1.Container{ - { - Name: string(feastType), - Image: *defaultServiceConfigs.Image, - Command: feast.getContainerCommand(feastType), - Ports: []corev1.ContainerPort{ - { - Name: string(feastType), - ContainerPort: getTargetPort(feastType, tls), - Protocol: corev1.ProtocolTCP, - }, - }, - Env: []corev1.EnvVar{ - { - Name: FeatureStoreYamlEnvVar, - Value: fsYamlB64, - }, - }, - LivenessProbe: &corev1.Probe{ - ProbeHandler: probeHandler, - InitialDelaySeconds: 30, - PeriodSeconds: 30, - }, - ReadinessProbe: &corev1.Probe{ - ProbeHandler: probeHandler, - InitialDelaySeconds: 20, - PeriodSeconds: 30, - }, - }, - }, + ServiceAccountName: feast.initFeastSA().Name, }, }, } + if err := feast.setPod(&deploy.Spec.Template.Spec); err != nil { + return err + } + return controllerutil.SetControllerReference(feast.Handler.FeatureStore, deploy, feast.Handler.Scheme) +} - // configs are applied here - podSpec := &deploy.Spec.Template.Spec - applyOptionalContainerConfigs(&podSpec.Containers[0], serviceConfigs.OptionalConfigs) - feast.mountTlsConfig(feastType, podSpec) - if pvcConfig, hasPvcConfig := hasPvcConfig(feast.Handler.FeatureStore, feastType); hasPvcConfig { - mountPvcConfig(podSpec, pvcConfig, deploy.Name) +func (feast *FeastServices) setPod(podSpec *corev1.PodSpec) error { + if err := feast.setContainers(podSpec); err != nil { + return err } + feast.setRegistryClientInitContainer(podSpec) + feast.mountTlsConfigs(podSpec) + feast.mountPvcConfigs(podSpec) + feast.mountEmptyDirVolumes(podSpec) - switch feastType { - case OfflineFeastType: - feast.registryClientPodConfigs(podSpec) - case OnlineFeastType: - feast.registryClientPodConfigs(podSpec) - feast.offlineClientPodConfigs(podSpec) + return nil +} + +func (feast *FeastServices) setContainers(podSpec *corev1.PodSpec) error { + fsYamlB64, err := feast.GetServiceFeatureStoreYamlBase64() + if err != nil { + return err } - return controllerutil.SetControllerReference(feast.Handler.FeatureStore, deploy, feast.Handler.Scheme) + feast.setInitContainer(podSpec, fsYamlB64) + + if feast.isLocalRegistry() { + feast.setContainer(&podSpec.Containers, RegistryFeastType, fsYamlB64) + } + if feast.isOfflinStore() { + feast.setContainer(&podSpec.Containers, OfflineFeastType, fsYamlB64) + } + if feast.isOnlinStore() { + feast.setContainer(&podSpec.Containers, OnlineFeastType, fsYamlB64) + } + return nil +} + +func (feast *FeastServices) setContainer(containers *[]corev1.Container, feastType FeastServiceType, fsYamlB64 string) { + tls := feast.getTlsConfigs(feastType) + serviceConfigs := feast.getServiceConfigs(feastType) + defaultServiceConfigs := serviceConfigs.DefaultConfigs + probeHandler := getProbeHandler(feastType, tls) + container := &corev1.Container{ + Name: string(feastType), + Image: *defaultServiceConfigs.Image, + WorkingDir: getOfflineMountPath(feast.Handler.FeatureStore) + "/" + feast.Handler.FeatureStore.Status.Applied.FeastProject + FeatureRepoDir, + Command: feast.getContainerCommand(feastType), + Ports: []corev1.ContainerPort{ + { + Name: string(feastType), + ContainerPort: getTargetPort(feastType, tls), + Protocol: corev1.ProtocolTCP, + }, + }, + Env: []corev1.EnvVar{ + { + Name: TmpFeatureStoreYamlEnvVar, + Value: fsYamlB64, + }, + }, + StartupProbe: &corev1.Probe{ + ProbeHandler: probeHandler, + PeriodSeconds: 3, + FailureThreshold: 40, + }, + LivenessProbe: &corev1.Probe{ + ProbeHandler: probeHandler, + PeriodSeconds: 20, + FailureThreshold: 6, + }, + ReadinessProbe: &corev1.Probe{ + ProbeHandler: probeHandler, + PeriodSeconds: 10, + }, + } + applyOptionalContainerConfigs(container, serviceConfigs.OptionalConfigs) + *containers = append(*containers, *container) } func (feast *FeastServices) getContainerCommand(feastType FeastServiceType) []string { + baseCommand := "feast" + options := []string{} + logLevel := feast.getLogLevelForType(feastType) + if logLevel != nil { + options = append(options, "--log-level", strings.ToUpper(*logLevel)) + } + deploySettings := FeastServiceConstants[feastType] targetPort := deploySettings.TargetHttpPort tls := feast.getTlsConfigs(feastType) if tls.IsTLS() { targetPort = deploySettings.TargetHttpsPort feastTlsPath := GetTlsPath(feastType) - deploySettings.Command = append(deploySettings.Command, []string{"--key", feastTlsPath + tls.SecretKeyNames.TlsKey, + deploySettings.Args = append(deploySettings.Args, []string{"--key", feastTlsPath + tls.SecretKeyNames.TlsKey, "--cert", feastTlsPath + tls.SecretKeyNames.TlsCrt}...) } - deploySettings.Command = append(deploySettings.Command, []string{"-p", strconv.Itoa(int(targetPort))}...) + deploySettings.Args = append(deploySettings.Args, []string{"-p", strconv.Itoa(int(targetPort))}...) - if feastType == OfflineFeastType { - if tls.IsTLS() && feast.Handler.FeatureStore.Status.Applied.Services.OfflineStore.TLS.VerifyClient != nil { - deploySettings.Command = append(deploySettings.Command, - []string{"--verify_client", strconv.FormatBool(*feast.Handler.FeatureStore.Status.Applied.Services.OfflineStore.TLS.VerifyClient)}...) - } - } + // Combine base command, options, and arguments + feastCommand := append([]string{baseCommand}, options...) + feastCommand = append(feastCommand, deploySettings.Args...) - return deploySettings.Command + return feastCommand } -func (feast *FeastServices) offlineClientPodConfigs(podSpec *corev1.PodSpec) { - feast.mountTlsConfig(OfflineFeastType, podSpec) +func (feast *FeastServices) getDeploymentStrategy() appsv1.DeploymentStrategy { + if feast.Handler.FeatureStore.Status.Applied.Services.DeploymentStrategy != nil { + return *feast.Handler.FeatureStore.Status.Applied.Services.DeploymentStrategy + } + return appsv1.DeploymentStrategy{ + Type: appsv1.RecreateDeploymentStrategyType, + } } -func (feast *FeastServices) registryClientPodConfigs(podSpec *corev1.PodSpec) { - feast.setRegistryClientInitContainer(podSpec) - feast.mountRegistryClientTls(podSpec) +func (feast *FeastServices) setInitContainer(podSpec *corev1.PodSpec, fsYamlB64 string) { + if !feast.Handler.FeatureStore.Status.Applied.Services.DisableInitContainers { + feastProject := feast.Handler.FeatureStore.Status.Applied.FeastProject + feastRepoDir := feastProject + FeatureRepoDir + workingDir := getOfflineMountPath(feast.Handler.FeatureStore) + podSpec.InitContainers = append(podSpec.InitContainers, corev1.Container{ + Name: "feast-init", + Image: getFeatureServerImage(), + Env: []corev1.EnvVar{ + { + Name: TmpFeatureStoreYamlEnvVar, + Value: fsYamlB64, + }, + }, + Command: []string{"/bin/sh", "-c"}, + Args: []string{"echo \"Starting feast initialization job...\";\n[ -d " + + feastRepoDir + " ] || feast init " + feastProject + ";\necho $" + + TmpFeatureStoreYamlEnvVar + " | base64 -d \u003e " + workingDir + "/" + feastRepoDir + + "/feature_store.yaml;\necho \"Feast initialization complete\";\n"}, + WorkingDir: workingDir, + }) + } } +// add grpc init container if remote registry reference (feastRef) is configured func (feast *FeastServices) setRegistryClientInitContainer(podSpec *corev1.PodSpec) { - hostname := feast.Handler.FeatureStore.Status.ServiceHostnames.Registry - if len(hostname) > 0 { - grpcurlFlag := "-plaintext" - hostSplit := strings.Split(hostname, ":") - if len(hostSplit) > 1 && hostSplit[1] == "443" { - grpcurlFlag = "-insecure" - } - podSpec.InitContainers = []corev1.Container{ - { + if !feast.Handler.FeatureStore.Status.Applied.Services.DisableInitContainers { + hostname := feast.Handler.FeatureStore.Status.ServiceHostnames.Registry + if len(hostname) > 0 && feast.IsRemoteRefRegistry() { + grpcurlFlag := "-plaintext" + hostSplit := strings.Split(hostname, ":") + if len(hostSplit) > 1 && hostSplit[1] == "443" { + grpcurlFlag = "-insecure" + } + podSpec.InitContainers = append(podSpec.InitContainers, corev1.Container{ Name: "init-registry", - Image: "fullstorydev/grpcurl:v1.9.1-alpine", + Image: getGrpcCurlImage(), Command: []string{ "sh", "-c", - "until grpcurl " + grpcurlFlag + " -d '' -format text " + hostname + " grpc.health.v1.Health/Check; do echo waiting for registry; sleep 2; done", + "until grpcurl -H \"authorization: Bearer $(cat /var/run/secrets/kubernetes.io/serviceaccount/token)\" " + + grpcurlFlag + " -d '' -format text " + hostname + " grpc.health.v1.Health/Check; do echo waiting for registry; sleep 2; done", }, - }, + }) } } } func (feast *FeastServices) setService(svc *corev1.Service, feastType FeastServiceType) error { - svc.Labels = feast.getLabels(feastType) + svc.Labels = feast.getFeastTypeLabels(feastType) if feast.isOpenShiftTls(feastType) { svc.Annotations = map[string]string{ "service.beta.openshift.io/serving-cert-secret-name": svc.Name + tlsNameSuffix, @@ -422,7 +469,7 @@ func (feast *FeastServices) setService(svc *corev1.Service, feastType FeastServi scheme = HttpsScheme } svc.Spec = corev1.ServiceSpec{ - Selector: svc.GetLabels(), + Selector: feast.getLabels(), Type: corev1.ServiceTypeClusterIP, Ports: []corev1.ServicePort{ { @@ -437,8 +484,8 @@ func (feast *FeastServices) setService(svc *corev1.Service, feastType FeastServi return controllerutil.SetControllerReference(feast.Handler.FeatureStore, svc, feast.Handler.Scheme) } -func (feast *FeastServices) setServiceAccount(sa *corev1.ServiceAccount, feastType FeastServiceType) error { - sa.Labels = feast.getLabels(feastType) +func (feast *FeastServices) setServiceAccount(sa *corev1.ServiceAccount) error { + sa.Labels = feast.getLabels() return controllerutil.SetControllerReference(feast.Handler.FeatureStore, sa, feast.Handler.Scheme) } @@ -446,7 +493,7 @@ func (feast *FeastServices) createNewPVC(pvcCreate *feastdevv1alpha1.PvcCreate, pvc := feast.initPVC(feastType) pvc.Spec = corev1.PersistentVolumeClaimSpec{ - AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}, + AccessModes: pvcCreate.AccessModes, Resources: pvcCreate.Resources, } if pvcCreate.StorageClassName != nil { @@ -474,8 +521,32 @@ func (feast *FeastServices) getServiceConfigs(feastType FeastServiceType) feastd return feastdevv1alpha1.ServiceConfigs{} } -// GetObjectMeta returns the feast k8s object metadata -func (feast *FeastServices) GetObjectMeta(feastType FeastServiceType) metav1.ObjectMeta { +func (feast *FeastServices) getLogLevelForType(feastType FeastServiceType) *string { + services := feast.Handler.FeatureStore.Status.Applied.Services + switch feastType { + case OfflineFeastType: + if services.OfflineStore != nil && services.OfflineStore.LogLevel != "" { + return &services.OfflineStore.LogLevel + } + case OnlineFeastType: + if services.OnlineStore != nil && services.OnlineStore.LogLevel != "" { + return &services.OnlineStore.LogLevel + } + case RegistryFeastType: + if services.Registry != nil && services.Registry.Local.LogLevel != "" { + return &services.Registry.Local.LogLevel + } + } + return nil +} + +// GetObjectMeta returns the feast k8s object metadata with type +func (feast *FeastServices) GetObjectMeta() metav1.ObjectMeta { + return metav1.ObjectMeta{Name: GetFeastName(feast.Handler.FeatureStore), Namespace: feast.Handler.FeatureStore.Namespace} +} + +// GetObjectMeta returns the feast k8s object metadata with type +func (feast *FeastServices) GetObjectMetaType(feastType FeastServiceType) metav1.ObjectMeta { return metav1.ObjectMeta{Name: feast.GetFeastServiceName(feastType), Namespace: feast.Handler.FeatureStore.Namespace} } @@ -492,10 +563,15 @@ func GetFeastName(featureStore *feastdevv1alpha1.FeatureStore) string { return handler.FeastPrefix + featureStore.Name } -func (feast *FeastServices) getLabels(feastType FeastServiceType) map[string]string { +func (feast *FeastServices) getFeastTypeLabels(feastType FeastServiceType) map[string]string { + labels := feast.getLabels() + labels[ServiceTypeLabelKey] = string(feastType) + return labels +} + +func (feast *FeastServices) getLabels() map[string]string { return map[string]string{ - NameLabelKey: feast.Handler.FeatureStore.Name, - ServiceTypeLabelKey: string(feastType), + NameLabelKey: feast.Handler.FeatureStore.Name, } } @@ -503,20 +579,17 @@ func (feast *FeastServices) setServiceHostnames() error { feast.Handler.FeatureStore.Status.ServiceHostnames = feastdevv1alpha1.ServiceHostnames{} domain := svcDomain + ":" if feast.isOfflinStore() { - objMeta := feast.GetObjectMeta(OfflineFeastType) - port := strconv.Itoa(HttpPort) - if feast.offlineTls() { - port = strconv.Itoa(HttpsPort) - } - feast.Handler.FeatureStore.Status.ServiceHostnames.OfflineStore = objMeta.Name + "." + objMeta.Namespace + domain + port + objMeta := feast.initFeastSvc(OfflineFeastType) + feast.Handler.FeatureStore.Status.ServiceHostnames.OfflineStore = objMeta.Name + "." + objMeta.Namespace + domain + + getPortStr(feast.Handler.FeatureStore.Status.Applied.Services.OfflineStore.TLS) } if feast.isOnlinStore() { - objMeta := feast.GetObjectMeta(OnlineFeastType) + objMeta := feast.initFeastSvc(OnlineFeastType) feast.Handler.FeatureStore.Status.ServiceHostnames.OnlineStore = objMeta.Name + "." + objMeta.Namespace + domain + getPortStr(feast.Handler.FeatureStore.Status.Applied.Services.OnlineStore.TLS) } if feast.isLocalRegistry() { - objMeta := feast.GetObjectMeta(RegistryFeastType) + objMeta := feast.initFeastSvc(RegistryFeastType) feast.Handler.FeatureStore.Status.ServiceHostnames.Registry = objMeta.Name + "." + objMeta.Namespace + domain + getPortStr(feast.Handler.FeatureStore.Status.Applied.Services.Registry.Local.TLS) } else if feast.isRemoteRegistry() { @@ -563,10 +636,6 @@ func (feast *FeastServices) setRemoteRegistryURL() error { func (feast *FeastServices) getRemoteRegistryFeastHandler() (*FeastServices, error) { if feast.IsRemoteRefRegistry() { feastRemoteRef := feast.Handler.FeatureStore.Status.Applied.Services.Registry.Remote.FeastRef - // default to FeatureStore namespace if not set - if len(feastRemoteRef.Namespace) == 0 { - feastRemoteRef.Namespace = feast.Handler.FeatureStore.Namespace - } nsName := types.NamespacedName{Name: feastRemoteRef.Name, Namespace: feastRemoteRef.Namespace} crNsName := client.ObjectKeyFromObject(feast.Handler.FeatureStore) if nsName == crNsName { @@ -579,6 +648,9 @@ func (feast *FeastServices) getRemoteRegistryFeastHandler() (*FeastServices, err } return nil, err } + if feast.Handler.FeatureStore.Status.Applied.FeastProject != remoteFeastObj.Status.Applied.FeastProject { + return nil, errors.New("FeatureStore '" + remoteFeastObj.Name + "' is using a different feast project than '" + feast.Handler.FeatureStore.Status.Applied.FeastProject + "'. Project names must match.") + } return &FeastServices{ Handler: handler.FeastHandler{ Client: feast.Handler.Client, @@ -600,19 +672,13 @@ func (feast *FeastServices) isRemoteRegistry() bool { } func (feast *FeastServices) IsRemoteRefRegistry() bool { - if feast.isRemoteRegistry() { - remote := feast.Handler.FeatureStore.Status.Applied.Services.Registry.Remote - return remote != nil && remote.FeastRef != nil - } - return false + return feast.isRemoteRegistry() && + feast.Handler.FeatureStore.Status.Applied.Services.Registry.Remote.FeastRef != nil } func (feast *FeastServices) isRemoteHostnameRegistry() bool { - if feast.isRemoteRegistry() { - remote := feast.Handler.FeatureStore.Status.Applied.Services.Registry.Remote - return remote != nil && remote.Hostname != nil - } - return false + return feast.isRemoteRegistry() && + feast.Handler.FeatureStore.Status.Applied.Services.Registry.Remote.Hostname != nil } func (feast *FeastServices) isOfflinStore() bool { @@ -625,9 +691,13 @@ func (feast *FeastServices) isOnlinStore() bool { return appliedServices != nil && appliedServices.OnlineStore != nil } -func (feast *FeastServices) initFeastDeploy(feastType FeastServiceType) *appsv1.Deployment { +func (feast *FeastServices) noLocalServiceConfigured() bool { + return !(feast.isLocalRegistry() || feast.isOnlinStore() || feast.isOfflinStore()) +} + +func (feast *FeastServices) initFeastDeploy() *appsv1.Deployment { deploy := &appsv1.Deployment{ - ObjectMeta: feast.GetObjectMeta(feastType), + ObjectMeta: feast.GetObjectMeta(), } deploy.SetGroupVersionKind(appsv1.SchemeGroupVersion.WithKind("Deployment")) return deploy @@ -635,15 +705,15 @@ func (feast *FeastServices) initFeastDeploy(feastType FeastServiceType) *appsv1. func (feast *FeastServices) initFeastSvc(feastType FeastServiceType) *corev1.Service { svc := &corev1.Service{ - ObjectMeta: feast.GetObjectMeta(feastType), + ObjectMeta: feast.GetObjectMetaType(feastType), } svc.SetGroupVersionKind(corev1.SchemeGroupVersion.WithKind("Service")) return svc } -func (feast *FeastServices) initFeastSA(feastType FeastServiceType) *corev1.ServiceAccount { +func (feast *FeastServices) initFeastSA() *corev1.ServiceAccount { sa := &corev1.ServiceAccount{ - ObjectMeta: feast.GetObjectMeta(feastType), + ObjectMeta: feast.GetObjectMeta(), } sa.SetGroupVersionKind(corev1.SchemeGroupVersion.WithKind("ServiceAccount")) return sa @@ -651,7 +721,7 @@ func (feast *FeastServices) initFeastSA(feastType FeastServiceType) *corev1.Serv func (feast *FeastServices) initPVC(feastType FeastServiceType) *corev1.PersistentVolumeClaim { pvc := &corev1.PersistentVolumeClaim{ - ObjectMeta: feast.GetObjectMeta(feastType), + ObjectMeta: feast.GetObjectMetaType(feastType), } pvc.SetGroupVersionKind(corev1.SchemeGroupVersion.WithKind("PersistentVolumeClaim")) return pvc @@ -661,6 +731,9 @@ func applyOptionalContainerConfigs(container *corev1.Container, optionalConfigs if optionalConfigs.Env != nil { container.Env = envOverride(container.Env, *optionalConfigs.Env) } + if optionalConfigs.EnvFrom != nil { + container.EnvFrom = *optionalConfigs.EnvFrom + } if optionalConfigs.ImagePullPolicy != nil { container.ImagePullPolicy = *optionalConfigs.ImagePullPolicy } @@ -669,28 +742,73 @@ func applyOptionalContainerConfigs(container *corev1.Container, optionalConfigs } } -func mountPvcConfig(podSpec *corev1.PodSpec, pvcConfig *feastdevv1alpha1.PvcConfig, deployName string) { +func (feast *FeastServices) mountPvcConfigs(podSpec *corev1.PodSpec) { + for _, feastType := range feastServerTypes { + if pvcConfig, hasPvcConfig := hasPvcConfig(feast.Handler.FeatureStore, feastType); hasPvcConfig { + feast.mountPvcConfig(podSpec, pvcConfig, feastType) + } + } +} + +func (feast *FeastServices) mountPvcConfig(podSpec *corev1.PodSpec, pvcConfig *feastdevv1alpha1.PvcConfig, feastType FeastServiceType) { if podSpec != nil && pvcConfig != nil { - container := &podSpec.Containers[0] - var pvcName string - if pvcConfig.Create != nil { - pvcName = deployName - } else { + volName := feast.initPVC(feastType).Name + pvcName := volName + if pvcConfig.Ref != nil { pvcName = pvcConfig.Ref.Name } - podSpec.Volumes = append(podSpec.Volumes, corev1.Volume{ - Name: pvcName, + Name: volName, VolumeSource: corev1.VolumeSource{ PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ ClaimName: pvcName, }, }, }) - container.VolumeMounts = append(container.VolumeMounts, corev1.VolumeMount{ - Name: pvcName, - MountPath: pvcConfig.MountPath, + if feastType == OfflineFeastType { + for i := range podSpec.InitContainers { + podSpec.InitContainers[i].VolumeMounts = append(podSpec.InitContainers[i].VolumeMounts, corev1.VolumeMount{ + Name: volName, + MountPath: pvcConfig.MountPath, + }) + } + } + for i := range podSpec.Containers { + podSpec.Containers[i].VolumeMounts = append(podSpec.Containers[i].VolumeMounts, corev1.VolumeMount{ + Name: volName, + MountPath: pvcConfig.MountPath, + }) + } + } +} + +func (feast *FeastServices) mountEmptyDirVolumes(podSpec *corev1.PodSpec) { + if shouldMountEmptyDir(feast.Handler.FeatureStore) { + mountEmptyDirVolume(podSpec) + } +} + +func mountEmptyDirVolume(podSpec *corev1.PodSpec) { + if podSpec != nil { + volName := strings.TrimPrefix(EphemeralPath, "/") + podSpec.Volumes = append(podSpec.Volumes, corev1.Volume{ + Name: volName, + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{}, + }, }) + for i := range podSpec.InitContainers { + podSpec.InitContainers[i].VolumeMounts = append(podSpec.InitContainers[i].VolumeMounts, corev1.VolumeMount{ + Name: volName, + MountPath: EphemeralPath, + }) + } + for i := range podSpec.Containers { + podSpec.Containers[i].VolumeMounts = append(podSpec.Containers[i].VolumeMounts, corev1.VolumeMount{ + Name: volName, + MountPath: EphemeralPath, + }) + } } } diff --git a/infra/feast-operator/internal/controller/services/services_types.go b/infra/feast-operator/internal/controller/services/services_types.go index 2c454459d88..55a5a42e2a7 100644 --- a/infra/feast-operator/internal/controller/services/services_types.go +++ b/infra/feast-operator/internal/controller/services/services_types.go @@ -20,17 +20,18 @@ import ( "github.com/feast-dev/feast/infra/feast-operator/api/feastversion" feastdevv1alpha1 "github.com/feast-dev/feast/infra/feast-operator/api/v1alpha1" handler "github.com/feast-dev/feast/infra/feast-operator/internal/controller/handler" + corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) const ( - FeatureStoreYamlEnvVar = "FEATURE_STORE_YAML_BASE64" - FeatureStoreYamlCmKey = "feature_store.yaml" - DefaultRegistryEphemeralPath = "/tmp/registry.db" - DefaultRegistryPvcPath = "registry.db" - DefaultOnlineStoreEphemeralPath = "/tmp/online_store.db" - DefaultOnlineStorePvcPath = "online_store.db" - svcDomain = ".svc.cluster.local" + TmpFeatureStoreYamlEnvVar = "TMP_FEATURE_STORE_YAML_BASE64" + FeatureStoreYamlCmKey = "feature_store.yaml" + EphemeralPath = "/feast-data" + FeatureRepoDir = "/feature_repo" + DefaultRegistryPath = "registry.db" + DefaultOnlineStorePath = "online_store.db" + svcDomain = ".svc.cluster.local" HttpPort = 80 HttpsPort = 443 @@ -80,24 +81,25 @@ const ( ) var ( - DefaultImage = "feastdev/feature-server:" + feastversion.FeastVersion - DefaultReplicas = int32(1) - NameLabelKey = feastdevv1alpha1.GroupVersion.Group + "/name" - ServiceTypeLabelKey = feastdevv1alpha1.GroupVersion.Group + "/service-type" + DefaultImage = "feastdev/feature-server:" + feastversion.FeastVersion + DefaultReplicas = int32(1) + DefaultPVCAccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce} + NameLabelKey = feastdevv1alpha1.GroupVersion.Group + "/name" + ServiceTypeLabelKey = feastdevv1alpha1.GroupVersion.Group + "/service-type" FeastServiceConstants = map[FeastServiceType]deploymentSettings{ OfflineFeastType: { - Command: []string{"feast", "serve_offline", "-h", "0.0.0.0"}, + Args: []string{"serve_offline", "-h", "0.0.0.0"}, TargetHttpPort: 8815, TargetHttpsPort: 8816, }, OnlineFeastType: { - Command: []string{"feast", "serve", "-h", "0.0.0.0"}, + Args: []string{"serve", "-h", "0.0.0.0"}, TargetHttpPort: 6566, TargetHttpsPort: 6567, }, RegistryFeastType: { - Command: []string{"feast", "serve_registry"}, + Args: []string{"serve_registry"}, TargetHttpPort: 6570, TargetHttpsPort: 6571, }, @@ -162,6 +164,13 @@ var ( OidcClientProperties = []OidcPropertyType{OidcClientSecret, OidcUsername, OidcPassword} ) +// feast server types, not the client types +var feastServerTypes = []FeastServiceType{ + RegistryFeastType, + OfflineFeastType, + OnlineFeastType, +} + // AuthzType defines the authorization type type AuthzType string @@ -234,7 +243,7 @@ type AuthzConfig struct { } type deploymentSettings struct { - Command []string + Args []string TargetHttpPort int32 TargetHttpsPort int32 } diff --git a/infra/feast-operator/internal/controller/services/suite_test.go b/infra/feast-operator/internal/controller/services/suite_test.go index e1e485f1bf6..5e922bc7e4a 100644 --- a/infra/feast-operator/internal/controller/services/suite_test.go +++ b/infra/feast-operator/internal/controller/services/suite_test.go @@ -32,7 +32,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/log/zap" feastdevv1alpha1 "github.com/feast-dev/feast/infra/feast-operator/api/v1alpha1" - //+kubebuilder:scaffold:imports + // +kubebuilder:scaffold:imports ) // These tests use Ginkgo (BDD-style Go testing framework). Refer to @@ -71,7 +71,7 @@ var _ = BeforeSuite(func() { err = feastdevv1alpha1.AddToScheme(scheme.Scheme) Expect(err).NotTo(HaveOccurred()) - //+kubebuilder:scaffold:scheme + // +kubebuilder:scaffold:scheme k8sClient, err = client.New(cfg, client.Options{Scheme: scheme.Scheme}) Expect(err).NotTo(HaveOccurred()) diff --git a/infra/feast-operator/internal/controller/services/tls.go b/infra/feast-operator/internal/controller/services/tls.go index c92c4d8de23..c1b07e7e1e6 100644 --- a/infra/feast-operator/internal/controller/services/tls.go +++ b/infra/feast-operator/internal/controller/services/tls.go @@ -29,7 +29,7 @@ func (feast *FeastServices) setTlsDefaults() error { } appliedServices := feast.Handler.FeatureStore.Status.Applied.Services if feast.isOfflinStore() && appliedServices.OfflineStore.TLS != nil { - tlsDefaults(&appliedServices.OfflineStore.TLS.TlsConfigs) + tlsDefaults(appliedServices.OfflineStore.TLS) } if feast.isOnlinStore() { tlsDefaults(appliedServices.OnlineStore.TLS) @@ -43,11 +43,9 @@ func (feast *FeastServices) setTlsDefaults() error { func (feast *FeastServices) setOpenshiftTls() error { appliedServices := feast.Handler.FeatureStore.Status.Applied.Services if feast.offlineOpenshiftTls() { - appliedServices.OfflineStore.TLS = &feastdevv1alpha1.OfflineTlsConfigs{ - TlsConfigs: feastdevv1alpha1.TlsConfigs{ - SecretRef: &corev1.LocalObjectReference{ - Name: feast.initFeastSvc(OfflineFeastType).Name + tlsNameSuffix, - }, + appliedServices.OfflineStore.TLS = &feastdevv1alpha1.TlsConfigs{ + SecretRef: &corev1.LocalObjectReference{ + Name: feast.initFeastSvc(OfflineFeastType).Name + tlsNameSuffix, }, } } @@ -103,8 +101,8 @@ func (feast *FeastServices) getTlsConfigs(feastType FeastServiceType) (tls *feas appliedServices := feast.Handler.FeatureStore.Status.Applied.Services switch feastType { case OfflineFeastType: - if feast.isOfflinStore() && appliedServices.OfflineStore.TLS != nil { - tls = &appliedServices.OfflineStore.TLS.TlsConfigs + if feast.isOfflinStore() { + tls = appliedServices.OfflineStore.TLS } case OnlineFeastType: if feast.isOnlinStore() { @@ -154,12 +152,6 @@ func (feast *FeastServices) remoteRegistryOpenshiftTls() (bool, error) { return false, nil } -func (feast *FeastServices) offlineTls() bool { - return feast.isOfflinStore() && - feast.Handler.FeatureStore.Status.Applied.Services.OfflineStore.TLS != nil && - (&feast.Handler.FeatureStore.Status.Applied.Services.OfflineStore.TLS.TlsConfigs).IsTLS() -} - func (feast *FeastServices) localRegistryTls() bool { return localRegistryTls(feast.Handler.FeatureStore) } @@ -173,12 +165,19 @@ func (feast *FeastServices) mountRegistryClientTls(podSpec *corev1.PodSpec) { if feast.localRegistryTls() { feast.mountTlsConfig(RegistryFeastType, podSpec) } else if feast.remoteRegistryTls() { - mountTlsRemoteRegistryConfig(RegistryFeastType, podSpec, + mountTlsRemoteRegistryConfig(podSpec, feast.Handler.FeatureStore.Status.Applied.Services.Registry.Remote.TLS) } } } +func (feast *FeastServices) mountTlsConfigs(podSpec *corev1.PodSpec) { + // how deal w/ client deployment tls mounts when the time comes? new function? + feast.mountRegistryClientTls(podSpec) + feast.mountTlsConfig(OfflineFeastType, podSpec) + feast.mountTlsConfig(OnlineFeastType, podSpec) +} + func (feast *FeastServices) mountTlsConfig(feastType FeastServiceType, podSpec *corev1.PodSpec) { tls := feast.getTlsConfigs(feastType) if tls.IsTLS() && podSpec != nil { @@ -191,18 +190,19 @@ func (feast *FeastServices) mountTlsConfig(feastType FeastServiceType, podSpec * }, }, }) - container := &podSpec.Containers[0] - container.VolumeMounts = append(container.VolumeMounts, corev1.VolumeMount{ - Name: volName, - MountPath: GetTlsPath(feastType), - ReadOnly: true, - }) + if i, container := getContainerByType(feastType, *podSpec); container != nil { + podSpec.Containers[i].VolumeMounts = append(podSpec.Containers[i].VolumeMounts, corev1.VolumeMount{ + Name: volName, + MountPath: GetTlsPath(feastType), + ReadOnly: true, + }) + } } } -func mountTlsRemoteRegistryConfig(feastType FeastServiceType, podSpec *corev1.PodSpec, tls *feastdevv1alpha1.TlsRemoteRegistryConfigs) { +func mountTlsRemoteRegistryConfig(podSpec *corev1.PodSpec, tls *feastdevv1alpha1.TlsRemoteRegistryConfigs) { if tls != nil { - volName := string(feastType) + tlsNameSuffix + volName := string(RegistryFeastType) + tlsNameSuffix podSpec.Volumes = append(podSpec.Volumes, corev1.Volume{ Name: volName, VolumeSource: corev1.VolumeSource{ @@ -211,12 +211,13 @@ func mountTlsRemoteRegistryConfig(feastType FeastServiceType, podSpec *corev1.Po }, }, }) - container := &podSpec.Containers[0] - container.VolumeMounts = append(container.VolumeMounts, corev1.VolumeMount{ - Name: volName, - MountPath: GetTlsPath(feastType), - ReadOnly: true, - }) + for i := range podSpec.Containers { + podSpec.Containers[i].VolumeMounts = append(podSpec.Containers[i].VolumeMounts, corev1.VolumeMount{ + Name: volName, + MountPath: GetTlsPath(RegistryFeastType), + ReadOnly: true, + }) + } } } diff --git a/infra/feast-operator/internal/controller/services/tls_test.go b/infra/feast-operator/internal/controller/services/tls_test.go index 2a66d8a4fdd..6c964084204 100644 --- a/infra/feast-operator/internal/controller/services/tls_test.go +++ b/infra/feast-operator/internal/controller/services/tls_test.go @@ -51,28 +51,27 @@ var _ = Describe("TLS Config", func() { }, } err := feast.ApplyDefaults() - Expect(err).To(BeNil()) + Expect(err).ToNot(HaveOccurred()) tls := feast.getTlsConfigs(RegistryFeastType) Expect(tls).To(BeNil()) Expect(tls.IsTLS()).To(BeFalse()) Expect(getPortStr(tls)).To(Equal("80")) - Expect(feast.offlineTls()).To(BeFalse()) Expect(feast.remoteRegistryTls()).To(BeFalse()) Expect(feast.localRegistryTls()).To(BeFalse()) Expect(feast.isOpenShiftTls(OfflineFeastType)).To(BeFalse()) Expect(feast.isOpenShiftTls(OnlineFeastType)).To(BeFalse()) Expect(feast.isOpenShiftTls(RegistryFeastType)).To(BeFalse()) openshiftTls, err := feast.checkOpenshiftTls() - Expect(err).To(BeNil()) + Expect(err).ToNot(HaveOccurred()) Expect(openshiftTls).To(BeFalse()) // registry service w/ openshift tls testSetIsOpenShift() feast.Handler.FeatureStore = minimalFeatureStore() err = feast.ApplyDefaults() - Expect(err).To(BeNil()) + Expect(err).ToNot(HaveOccurred()) tls = feast.getTlsConfigs(OfflineFeastType) Expect(tls).To(BeNil()) @@ -87,20 +86,19 @@ var _ = Describe("TLS Config", func() { Expect(getPortStr(tls)).To(Equal("443")) Expect(GetTlsPath(RegistryFeastType)).To(Equal("/tls/registry/")) - Expect(feast.offlineTls()).To(BeFalse()) Expect(feast.remoteRegistryTls()).To(BeFalse()) Expect(feast.localRegistryTls()).To(BeTrue()) Expect(feast.isOpenShiftTls(OfflineFeastType)).To(BeFalse()) Expect(feast.isOpenShiftTls(OnlineFeastType)).To(BeFalse()) Expect(feast.isOpenShiftTls(RegistryFeastType)).To(BeTrue()) openshiftTls, err = feast.checkOpenshiftTls() - Expect(err).To(BeNil()) + Expect(err).ToNot(HaveOccurred()) Expect(openshiftTls).To(BeTrue()) // all services w/ openshift tls feast.Handler.FeatureStore = minimalFeatureStoreWithAllServices() err = feast.ApplyDefaults() - Expect(err).To(BeNil()) + Expect(err).ToNot(HaveOccurred()) repoConfig, err := getClientRepoConfig(feast.Handler.FeatureStore, emptyMockExtractConfigFromSecret) Expect(err).NotTo(HaveOccurred()) @@ -127,34 +125,25 @@ var _ = Describe("TLS Config", func() { Expect(tls.SecretKeyNames).To(Equal(secretKeyNames)) Expect(tls.IsTLS()).To(BeTrue()) - Expect(feast.offlineTls()).To(BeTrue()) Expect(feast.remoteRegistryTls()).To(BeFalse()) Expect(feast.localRegistryTls()).To(BeTrue()) Expect(feast.isOpenShiftTls(OfflineFeastType)).To(BeTrue()) Expect(feast.isOpenShiftTls(OnlineFeastType)).To(BeTrue()) Expect(feast.isOpenShiftTls(RegistryFeastType)).To(BeTrue()) openshiftTls, err = feast.checkOpenshiftTls() - Expect(err).To(BeNil()) + Expect(err).ToNot(HaveOccurred()) Expect(openshiftTls).To(BeTrue()) // check k8s deployment objects - offlineDeploy := feast.initFeastDeploy(OfflineFeastType) - err = feast.setDeployment(offlineDeploy, OfflineFeastType) - Expect(err).To(BeNil()) - Expect(offlineDeploy.Spec.Template.Spec.InitContainers).To(HaveLen(1)) - Expect(offlineDeploy.Spec.Template.Spec.InitContainers[0].Command).To(ContainElements(ContainSubstring("-insecure"))) - Expect(offlineDeploy.Spec.Template.Spec.Containers).To(HaveLen(1)) - Expect(offlineDeploy.Spec.Template.Spec.Containers[0].Command).To(ContainElements(ContainSubstring("--key"))) - Expect(offlineDeploy.Spec.Template.Spec.Volumes).To(HaveLen(2)) - onlineDeploy := feast.initFeastDeploy(OnlineFeastType) - err = feast.setDeployment(onlineDeploy, OnlineFeastType) - Expect(err).To(BeNil()) - Expect(onlineDeploy.Spec.Template.Spec.InitContainers).To(HaveLen(1)) - Expect(onlineDeploy.Spec.Template.Spec.InitContainers[0].Command).To(ContainElements(ContainSubstring("-insecure"))) - Expect(onlineDeploy.Spec.Template.Spec.Containers).To(HaveLen(1)) - Expect(onlineDeploy.Spec.Template.Spec.Containers).To(HaveLen(1)) - Expect(onlineDeploy.Spec.Template.Spec.Containers[0].Command).To(ContainElements(ContainSubstring("--key"))) - Expect(onlineDeploy.Spec.Template.Spec.Volumes).To(HaveLen(3)) + feastDeploy := feast.initFeastDeploy() + err = feast.setDeployment(feastDeploy) + Expect(err).ToNot(HaveOccurred()) + Expect(feastDeploy.Spec.Template.Spec.InitContainers).To(HaveLen(1)) + Expect(feastDeploy.Spec.Template.Spec.Containers).To(HaveLen(3)) + Expect(feastDeploy.Spec.Template.Spec.Containers[0].Command).To(ContainElements(ContainSubstring("--key"))) + Expect(feastDeploy.Spec.Template.Spec.Containers[1].Command).To(ContainElements(ContainSubstring("--key"))) + Expect(feastDeploy.Spec.Template.Spec.Containers[2].Command).To(ContainElements(ContainSubstring("--key"))) + Expect(feastDeploy.Spec.Template.Spec.Volumes).To(HaveLen(4)) // registry service w/ tls and in an openshift cluster feast.Handler.FeatureStore = minimalFeatureStore() @@ -174,7 +163,7 @@ var _ = Describe("TLS Config", func() { }, } err = feast.ApplyDefaults() - Expect(err).To(BeNil()) + Expect(err).ToNot(HaveOccurred()) tls = feast.getTlsConfigs(OfflineFeastType) Expect(tls).To(BeNil()) @@ -189,14 +178,13 @@ var _ = Describe("TLS Config", func() { Expect(getPortStr(tls)).To(Equal("443")) Expect(GetTlsPath(RegistryFeastType)).To(Equal("/tls/registry/")) - Expect(feast.offlineTls()).To(BeFalse()) Expect(feast.remoteRegistryTls()).To(BeFalse()) Expect(feast.localRegistryTls()).To(BeTrue()) Expect(feast.isOpenShiftTls(OfflineFeastType)).To(BeFalse()) Expect(feast.isOpenShiftTls(OnlineFeastType)).To(BeFalse()) Expect(feast.isOpenShiftTls(RegistryFeastType)).To(BeFalse()) openshiftTls, err = feast.checkOpenshiftTls() - Expect(err).To(BeNil()) + Expect(err).ToNot(HaveOccurred()) Expect(openshiftTls).To(BeFalse()) // all services w/ tls and in an openshift cluster @@ -213,7 +201,7 @@ var _ = Describe("TLS Config", func() { }, } err = feast.ApplyDefaults() - Expect(err).To(BeNil()) + Expect(err).ToNot(HaveOccurred()) repoConfig, err = getClientRepoConfig(feast.Handler.FeatureStore, emptyMockExtractConfigFromSecret) Expect(err).NotTo(HaveOccurred()) @@ -238,47 +226,43 @@ var _ = Describe("TLS Config", func() { Expect(getPortStr(tls)).To(Equal("80")) Expect(GetTlsPath(RegistryFeastType)).To(Equal("/tls/registry/")) - Expect(feast.offlineTls()).To(BeTrue()) Expect(feast.remoteRegistryTls()).To(BeFalse()) Expect(feast.localRegistryTls()).To(BeFalse()) Expect(feast.isOpenShiftTls(OfflineFeastType)).To(BeTrue()) Expect(feast.isOpenShiftTls(OnlineFeastType)).To(BeFalse()) Expect(feast.isOpenShiftTls(RegistryFeastType)).To(BeFalse()) openshiftTls, err = feast.checkOpenshiftTls() - Expect(err).To(BeNil()) + Expect(err).ToNot(HaveOccurred()) Expect(openshiftTls).To(BeTrue()) // check k8s service objects offlineSvc := feast.initFeastSvc(OfflineFeastType) Expect(offlineSvc.Annotations).To(BeEmpty()) err = feast.setService(offlineSvc, OfflineFeastType) - Expect(err).To(BeNil()) + Expect(err).ToNot(HaveOccurred()) Expect(offlineSvc.Annotations).NotTo(BeEmpty()) Expect(offlineSvc.Spec.Ports[0].Name).To(Equal(HttpsScheme)) onlineSvc := feast.initFeastSvc(OnlineFeastType) err = feast.setService(onlineSvc, OnlineFeastType) - Expect(err).To(BeNil()) + Expect(err).ToNot(HaveOccurred()) Expect(onlineSvc.Annotations).To(BeEmpty()) Expect(onlineSvc.Spec.Ports[0].Name).To(Equal(HttpScheme)) // check k8s deployment objects - offlineDeploy = feast.initFeastDeploy(OfflineFeastType) - err = feast.setDeployment(offlineDeploy, OfflineFeastType) - Expect(err).To(BeNil()) - Expect(offlineDeploy.Spec.Template.Spec.InitContainers).To(HaveLen(1)) - Expect(offlineDeploy.Spec.Template.Spec.InitContainers[0].Command).To(ContainElements(ContainSubstring("-plaintext"))) - Expect(offlineDeploy.Spec.Template.Spec.Containers).To(HaveLen(1)) - Expect(offlineDeploy.Spec.Template.Spec.Containers[0].Command).To(ContainElements(ContainSubstring("--key"))) - Expect(offlineDeploy.Spec.Template.Spec.Volumes).To(HaveLen(1)) - onlineDeploy = feast.initFeastDeploy(OnlineFeastType) - err = feast.setDeployment(onlineDeploy, OnlineFeastType) - Expect(err).To(BeNil()) - Expect(onlineDeploy.Spec.Template.Spec.InitContainers).To(HaveLen(1)) - Expect(onlineDeploy.Spec.Template.Spec.InitContainers[0].Command).To(ContainElements(ContainSubstring("-plaintext"))) - Expect(onlineDeploy.Spec.Template.Spec.Containers).To(HaveLen(1)) - Expect(onlineDeploy.Spec.Template.Spec.Containers[0].Command).NotTo(ContainElements(ContainSubstring("--key"))) - Expect(onlineDeploy.Spec.Template.Spec.Volumes).To(HaveLen(1)) + feastDeploy = feast.initFeastDeploy() + err = feast.setDeployment(feastDeploy) + Expect(err).ToNot(HaveOccurred()) + Expect(feastDeploy.Spec.Template.Spec.Containers).To(HaveLen(3)) + Expect(GetOfflineContainer(*feastDeploy)).NotTo(BeNil()) + Expect(feastDeploy.Spec.Template.Spec.Volumes).To(HaveLen(2)) + + Expect(GetRegistryContainer(*feastDeploy).Command).NotTo(ContainElements(ContainSubstring("--key"))) + Expect(GetRegistryContainer(*feastDeploy).VolumeMounts).To(HaveLen(1)) + Expect(GetOfflineContainer(*feastDeploy).Command).To(ContainElements(ContainSubstring("--key"))) + Expect(GetOfflineContainer(*feastDeploy).VolumeMounts).To(HaveLen(2)) + Expect(GetOnlineContainer(*feastDeploy).Command).NotTo(ContainElements(ContainSubstring("--key"))) + Expect(GetOnlineContainer(*feastDeploy).VolumeMounts).To(HaveLen(1)) }) }) }) diff --git a/infra/feast-operator/internal/controller/services/util.go b/infra/feast-operator/internal/controller/services/util.go index 85bd02e653a..b39a851c14b 100644 --- a/infra/feast-operator/internal/controller/services/util.go +++ b/infra/feast-operator/internal/controller/services/util.go @@ -2,12 +2,14 @@ package services import ( "fmt" + "os" "reflect" "slices" "strings" "github.com/feast-dev/feast/infra/feast-operator/api/feastversion" feastdevv1alpha1 "github.com/feast-dev/feast/infra/feast-operator/api/v1alpha1" + appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" v1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -32,20 +34,25 @@ func isRemoteRegistry(featureStore *feastdevv1alpha1.FeatureStore) bool { } func hasPvcConfig(featureStore *feastdevv1alpha1.FeatureStore, feastType FeastServiceType) (*feastdevv1alpha1.PvcConfig, bool) { + var pvcConfig *feastdevv1alpha1.PvcConfig services := featureStore.Status.Applied.Services - var pvcConfig *feastdevv1alpha1.PvcConfig = nil - switch feastType { - case OnlineFeastType: - if services.OnlineStore != nil && services.OnlineStore.Persistence.FilePersistence != nil { - pvcConfig = services.OnlineStore.Persistence.FilePersistence.PvcConfig - } - case OfflineFeastType: - if services.OfflineStore != nil && services.OfflineStore.Persistence.FilePersistence != nil { - pvcConfig = services.OfflineStore.Persistence.FilePersistence.PvcConfig - } - case RegistryFeastType: - if IsLocalRegistry(featureStore) && services.Registry.Local.Persistence.FilePersistence != nil { - pvcConfig = services.Registry.Local.Persistence.FilePersistence.PvcConfig + if services != nil { + switch feastType { + case OnlineFeastType: + if services.OnlineStore != nil && services.OnlineStore.Persistence != nil && + services.OnlineStore.Persistence.FilePersistence != nil { + pvcConfig = services.OnlineStore.Persistence.FilePersistence.PvcConfig + } + case OfflineFeastType: + if services.OfflineStore != nil && services.OfflineStore.Persistence != nil && + services.OfflineStore.Persistence.FilePersistence != nil { + pvcConfig = services.OfflineStore.Persistence.FilePersistence.PvcConfig + } + case RegistryFeastType: + if IsLocalRegistry(featureStore) && services.Registry.Local.Persistence != nil && + services.Registry.Local.Persistence.FilePersistence != nil { + pvcConfig = services.Registry.Local.Persistence.FilePersistence.PvcConfig + } } } return pvcConfig, pvcConfig != nil @@ -58,10 +65,28 @@ func shouldCreatePvc(featureStore *feastdevv1alpha1.FeatureStore, feastType Feas return nil, false } +func shouldMountEmptyDir(featureStore *feastdevv1alpha1.FeatureStore) bool { + for _, feastType := range feastServerTypes { + if _, ok := hasPvcConfig(featureStore, feastType); !ok { + return true + } + } + return false +} + +func getOfflineMountPath(featureStore *feastdevv1alpha1.FeatureStore) string { + if pvcConfig, ok := hasPvcConfig(featureStore, OfflineFeastType); ok { + return pvcConfig.MountPath + } + return EphemeralPath +} + func ApplyDefaultsToStatus(cr *feastdevv1alpha1.FeatureStore) { + // overwrite status.applied with every reconcile + cr.Spec.DeepCopyInto(&cr.Status.Applied) cr.Status.FeastVersion = feastversion.FeastVersion - applied := cr.Spec.DeepCopy() + applied := &cr.Status.Applied if applied.Services == nil { applied.Services = &feastdevv1alpha1.FeatureStoreServices{} } @@ -87,19 +112,17 @@ func ApplyDefaultsToStatus(cr *feastdevv1alpha1.FeatureStore) { } if len(services.Registry.Local.Persistence.FilePersistence.Path) == 0 { - services.Registry.Local.Persistence.FilePersistence.Path = defaultRegistryPath(services.Registry.Local.Persistence.FilePersistence) + services.Registry.Local.Persistence.FilePersistence.Path = defaultRegistryPath(cr) } - if services.Registry.Local.Persistence.FilePersistence.PvcConfig != nil { - pvc := services.Registry.Local.Persistence.FilePersistence.PvcConfig - if pvc.Create != nil { - ensureRequestedStorage(&pvc.Create.Resources, DefaultRegistryStorageRequest) - } - } + ensurePVCDefaults(services.Registry.Local.Persistence.FilePersistence.PvcConfig, RegistryFeastType) } setServiceDefaultConfigs(&services.Registry.Local.ServiceConfigs.DefaultConfigs) + } else if services.Registry.Remote.FeastRef != nil && len(services.Registry.Remote.FeastRef.Namespace) == 0 { + services.Registry.Remote.FeastRef.Namespace = cr.Namespace } + if services.OfflineStore != nil { if services.OfflineStore.Persistence == nil { services.OfflineStore.Persistence = &feastdevv1alpha1.OfflineStorePersistence{} @@ -114,12 +137,7 @@ func ApplyDefaultsToStatus(cr *feastdevv1alpha1.FeatureStore) { services.OfflineStore.Persistence.FilePersistence.Type = string(OfflineFilePersistenceDaskConfigType) } - if services.OfflineStore.Persistence.FilePersistence.PvcConfig != nil { - pvc := services.OfflineStore.Persistence.FilePersistence.PvcConfig - if pvc.Create != nil { - ensureRequestedStorage(&pvc.Create.Resources, DefaultOfflineStorageRequest) - } - } + ensurePVCDefaults(services.OfflineStore.Persistence.FilePersistence.PvcConfig, OfflineFeastType) } setServiceDefaultConfigs(&services.OfflineStore.ServiceConfigs.DefaultConfigs) @@ -136,27 +154,35 @@ func ApplyDefaultsToStatus(cr *feastdevv1alpha1.FeatureStore) { } if len(services.OnlineStore.Persistence.FilePersistence.Path) == 0 { - services.OnlineStore.Persistence.FilePersistence.Path = defaultOnlineStorePath(services.OnlineStore.Persistence.FilePersistence) + services.OnlineStore.Persistence.FilePersistence.Path = defaultOnlineStorePath(cr) } - if services.OnlineStore.Persistence.FilePersistence.PvcConfig != nil { - pvc := services.OnlineStore.Persistence.FilePersistence.PvcConfig - if pvc.Create != nil { - ensureRequestedStorage(&pvc.Create.Resources, DefaultOnlineStorageRequest) - } - } + ensurePVCDefaults(services.OnlineStore.Persistence.FilePersistence.PvcConfig, OnlineFeastType) } setServiceDefaultConfigs(&services.OnlineStore.ServiceConfigs.DefaultConfigs) } - // overwrite status.applied with every reconcile - applied.DeepCopyInto(&cr.Status.Applied) } func setServiceDefaultConfigs(defaultConfigs *feastdevv1alpha1.DefaultConfigs) { if defaultConfigs.Image == nil { - defaultConfigs.Image = &DefaultImage + img := getFeatureServerImage() + defaultConfigs.Image = &img + } +} + +func getFeatureServerImage() string { + if img, exists := os.LookupEnv("RELATED_IMAGE_FEATURE_SERVER"); exists { + return img + } + return DefaultImage +} + +func getGrpcCurlImage() string { + if img, exists := os.LookupEnv("RELATED_IMAGE_GRPC_CURL"); exists { + return img } + return "fullstorydev/grpcurl:v1.9.1-alpine" } func checkOfflineStoreFilePersistenceType(value string) error { @@ -175,18 +201,40 @@ func ensureRequestedStorage(resources *v1.VolumeResourceRequirements, requestedS } } -func defaultOnlineStorePath(persistence *feastdevv1alpha1.OnlineStoreFilePersistence) string { - if persistence.PvcConfig == nil { - return DefaultOnlineStoreEphemeralPath +func ensurePVCDefaults(pvc *feastdevv1alpha1.PvcConfig, feastType FeastServiceType) { + if pvc != nil { + var storageRequest string + switch feastType { + case OnlineFeastType: + storageRequest = DefaultOnlineStorageRequest + case OfflineFeastType: + storageRequest = DefaultOfflineStorageRequest + case RegistryFeastType: + storageRequest = DefaultRegistryStorageRequest + } + if pvc.Create != nil { + ensureRequestedStorage(&pvc.Create.Resources, storageRequest) + if pvc.Create.AccessModes == nil { + pvc.Create.AccessModes = DefaultPVCAccessModes + } + } } - return DefaultOnlineStorePvcPath } -func defaultRegistryPath(persistence *feastdevv1alpha1.RegistryFilePersistence) string { - if persistence.PvcConfig == nil { - return DefaultRegistryEphemeralPath +func defaultOnlineStorePath(featureStore *feastdevv1alpha1.FeatureStore) string { + if _, ok := hasPvcConfig(featureStore, OnlineFeastType); ok { + return DefaultOnlineStorePath } - return DefaultRegistryPvcPath + // if pvc not set, use the ephemeral mount path. + return EphemeralPath + "/" + DefaultOnlineStorePath +} + +func defaultRegistryPath(featureStore *feastdevv1alpha1.FeatureStore) string { + if _, ok := hasPvcConfig(featureStore, RegistryFeastType); ok { + return DefaultRegistryPath + } + // if pvc not set, use the ephemeral mount path. + return EphemeralPath + "/" + DefaultRegistryPath } func checkOfflineStoreDBStorePersistenceType(value string) error { @@ -211,15 +259,14 @@ func checkRegistryDBStorePersistenceType(value string) error { } func (feast *FeastServices) getSecret(secretRef string) (*corev1.Secret, error) { + logger := log.FromContext(feast.Handler.Context) secret := &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: secretRef, Namespace: feast.Handler.FeatureStore.Namespace}} objectKey := client.ObjectKeyFromObject(secret) if err := feast.Handler.Client.Get(feast.Handler.Context, objectKey, secret); err != nil { - if apierrors.IsNotFound(err) || err != nil { - logger := log.FromContext(feast.Handler.Context) + if apierrors.IsNotFound(err) { logger.Error(err, "invalid secret "+secretRef+" for offline store") - - return nil, err } + return nil, err } return secret, nil @@ -338,3 +385,69 @@ func envOverride(dst, src []corev1.EnvVar) []corev1.EnvVar { } return dst } + +func GetRegistryContainer(deployment appsv1.Deployment) *corev1.Container { + _, container := getContainerByType(RegistryFeastType, deployment.Spec.Template.Spec) + return container +} + +func GetOfflineContainer(deployment appsv1.Deployment) *corev1.Container { + _, container := getContainerByType(OfflineFeastType, deployment.Spec.Template.Spec) + return container +} + +func GetOnlineContainer(deployment appsv1.Deployment) *corev1.Container { + _, container := getContainerByType(OnlineFeastType, deployment.Spec.Template.Spec) + return container +} + +func getContainerByType(feastType FeastServiceType, podSpec corev1.PodSpec) (int, *corev1.Container) { + for i, c := range podSpec.Containers { + if c.Name == string(feastType) { + return i, &c + } + } + return -1, nil +} + +func GetRegistryVolume(featureStore *feastdevv1alpha1.FeatureStore, volumes []corev1.Volume) *corev1.Volume { + return getVolumeByType(RegistryFeastType, featureStore, volumes) +} + +func GetOnlineVolume(featureStore *feastdevv1alpha1.FeatureStore, volumes []corev1.Volume) *corev1.Volume { + return getVolumeByType(OnlineFeastType, featureStore, volumes) +} + +func GetOfflineVolume(featureStore *feastdevv1alpha1.FeatureStore, volumes []corev1.Volume) *corev1.Volume { + return getVolumeByType(OfflineFeastType, featureStore, volumes) +} + +func getVolumeByType(feastType FeastServiceType, featureStore *feastdevv1alpha1.FeatureStore, volumes []corev1.Volume) *corev1.Volume { + for _, v := range volumes { + if v.Name == GetFeastServiceName(featureStore, feastType) { + return &v + } + } + return nil +} + +func GetRegistryVolumeMount(featureStore *feastdevv1alpha1.FeatureStore, volumeMounts []corev1.VolumeMount) *corev1.VolumeMount { + return getVolumeMountByType(RegistryFeastType, featureStore, volumeMounts) +} + +func GetOnlineVolumeMount(featureStore *feastdevv1alpha1.FeatureStore, volumeMounts []corev1.VolumeMount) *corev1.VolumeMount { + return getVolumeMountByType(OnlineFeastType, featureStore, volumeMounts) +} + +func GetOfflineVolumeMount(featureStore *feastdevv1alpha1.FeatureStore, volumeMounts []corev1.VolumeMount) *corev1.VolumeMount { + return getVolumeMountByType(OfflineFeastType, featureStore, volumeMounts) +} + +func getVolumeMountByType(feastType FeastServiceType, featureStore *feastdevv1alpha1.FeatureStore, volumeMounts []corev1.VolumeMount) *corev1.VolumeMount { + for _, vm := range volumeMounts { + if vm.Name == GetFeastServiceName(featureStore, feastType) { + return &vm + } + } + return nil +} diff --git a/infra/feast-operator/internal/controller/suite_test.go b/infra/feast-operator/internal/controller/suite_test.go index 38da27cc9c5..51208d6dbb0 100644 --- a/infra/feast-operator/internal/controller/suite_test.go +++ b/infra/feast-operator/internal/controller/suite_test.go @@ -32,7 +32,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/log/zap" feastdevv1alpha1 "github.com/feast-dev/feast/infra/feast-operator/api/v1alpha1" - //+kubebuilder:scaffold:imports + // +kubebuilder:scaffold:imports ) // These tests use Ginkgo (BDD-style Go testing framework). Refer to @@ -71,7 +71,7 @@ var _ = BeforeSuite(func() { err = feastdevv1alpha1.AddToScheme(scheme.Scheme) Expect(err).NotTo(HaveOccurred()) - //+kubebuilder:scaffold:scheme + // +kubebuilder:scaffold:scheme k8sClient, err = client.New(cfg, client.Options{Scheme: scheme.Scheme}) Expect(err).NotTo(HaveOccurred()) diff --git a/infra/feast-operator/test/api/featurestore_types_test.go b/infra/feast-operator/test/api/featurestore_types_test.go index 302abef9384..1049436fec9 100644 --- a/infra/feast-operator/test/api/featurestore_types_test.go +++ b/infra/feast-operator/test/api/featurestore_types_test.go @@ -34,7 +34,7 @@ func attemptInvalidCreationAndAsserts(ctx context.Context, featurestore *feastde logger.Info("Creating", "FeatureStore", featurestore) err := k8sClient.Create(ctx, featurestore) logger.Info("Got", "err", err) - Expect(err).ToNot(BeNil()) + Expect(err).To(HaveOccurred()) Expect(err.Error()).Should(ContainSubstring(matcher)) } @@ -350,12 +350,12 @@ func initContext() (context.Context, *feastdevv1alpha1.FeatureStore) { BeforeEach(func() { By("verifying the custom resource FeatureStore is not there") err := k8sClient.Get(ctx, typeNamespacedName, featurestore) - Expect(err != nil && errors.IsNotFound(err)) + Expect(err != nil && errors.IsNotFound(err)).To(BeTrue()) }) AfterEach(func() { By("verifying the custom resource FeatureStore is not there") err := k8sClient.Get(ctx, typeNamespacedName, featurestore) - Expect(err != nil && errors.IsNotFound(err)) + Expect(err != nil && errors.IsNotFound(err)).To(BeTrue()) }) return ctx, featurestore @@ -377,7 +377,7 @@ var _ = Describe("FeatureStore API", func() { }) It("should fail when db persistence type is invalid", func() { - attemptInvalidCreationAndAsserts(ctx, onlineStoreWithDBPersistenceType("invalid", featurestore), "Unsupported value: \"invalid\": supported values: \"snowflake.online\", \"redis\", \"ikv\", \"datastore\", \"dynamodb\", \"bigtable\", \"postgres\", \"cassandra\", \"mysql\", \"hazelcast\", \"singlestore\"") + attemptInvalidCreationAndAsserts(ctx, onlineStoreWithDBPersistenceType("invalid", featurestore), "Unsupported value: \"invalid\": supported values: \"snowflake.online\", \"redis\", \"ikv\", \"datastore\", \"dynamodb\", \"bigtable\", \"postgres\", \"cassandra\", \"mysql\", \"hazelcast\", \"singlestore\", \"hbase\", \"elasticsearch\", \"qdrant\", \"couchbase\"") }) }) @@ -388,7 +388,7 @@ var _ = Describe("FeatureStore API", func() { attemptInvalidCreationAndAsserts(ctx, offlineStoreWithUnmanagedFileType(featurestore), "Unsupported value") }) It("should fail when db persistence type is invalid", func() { - attemptInvalidCreationAndAsserts(ctx, offlineStoreWithDBPersistenceType("invalid", featurestore), "Unsupported value: \"invalid\": supported values: \"snowflake.offline\", \"bigquery\", \"redshift\", \"spark\", \"postgres\", \"feast_trino.trino.TrinoOfflineStore\", \"redis\"") + attemptInvalidCreationAndAsserts(ctx, offlineStoreWithDBPersistenceType("invalid", featurestore), "Unsupported value: \"invalid\": supported values: \"snowflake.offline\", \"bigquery\", \"redshift\", \"spark\", \"postgres\", \"trino\", \"redis\", \"athena\", \"mssql\"") }) }) diff --git a/infra/feast-operator/test/api/suite_test.go b/infra/feast-operator/test/api/suite_test.go index 270742760e7..e8c46a240c1 100644 --- a/infra/feast-operator/test/api/suite_test.go +++ b/infra/feast-operator/test/api/suite_test.go @@ -32,7 +32,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/envtest" logf "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/log/zap" - //+kubebuilder:scaffold:imports + // +kubebuilder:scaffold:imports ) // These tests use Ginkgo (BDD-style Go testing framework). Refer to @@ -74,7 +74,7 @@ var _ = BeforeSuite(func() { err = feastdevv1alpha1.AddToScheme(scheme.Scheme) Expect(err).NotTo(HaveOccurred()) - //+kubebuilder:scaffold:scheme + // +kubebuilder:scaffold:scheme k8sClient, err = client.New(cfg, client.Options{Scheme: scheme.Scheme}) Expect(err).NotTo(HaveOccurred()) diff --git a/infra/feast-operator/test/e2e/e2e_suite_test.go b/infra/feast-operator/test/e2e/e2e_suite_test.go index 8e46d8a5063..c45853a0073 100644 --- a/infra/feast-operator/test/e2e/e2e_suite_test.go +++ b/infra/feast-operator/test/e2e/e2e_suite_test.go @@ -27,6 +27,6 @@ import ( // Run e2e tests using the Ginkgo runner. func TestE2E(t *testing.T) { RegisterFailHandler(Fail) - fmt.Fprintf(GinkgoWriter, "Starting feast-operator suite\n") + _, _ = fmt.Fprintf(GinkgoWriter, "Starting feast-operator suite\n") RunSpecs(t, "e2e suite") } diff --git a/infra/feast-operator/test/e2e/e2e_test.go b/infra/feast-operator/test/e2e/e2e_test.go index 7d9fb9af056..e55ac0156a7 100644 --- a/infra/feast-operator/test/e2e/e2e_test.go +++ b/infra/feast-operator/test/e2e/e2e_test.go @@ -27,146 +27,196 @@ import ( "github.com/feast-dev/feast/infra/feast-operator/test/utils" ) -const feastControllerNamespace = "feast-operator-system" +const ( + feastControllerNamespace = "feast-operator-system" + timeout = 2 * time.Minute + controllerDeploymentName = "feast-operator-controller-manager" + feastPrefix = "feast-" +) var _ = Describe("controller", Ordered, func() { BeforeAll(func() { By("creating manager namespace") cmd := exec.Command("kubectl", "create", "ns", feastControllerNamespace) _, _ = utils.Run(cmd) + var err error + // projectimage stores the name of the image used in the example + var projectimage = "localhost/feast-operator:v0.0.1" + + By("building the manager(Operator) image") + cmd = exec.Command("make", "docker-build", fmt.Sprintf("IMG=%s", projectimage)) + _, err = utils.Run(cmd) + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + + By("loading the the manager(Operator) image on Kind") + err = utils.LoadImageToKindClusterWithName(projectimage) + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + + By("building the feast image") + cmd = exec.Command("make", "feast-ci-dev-docker-img") + _, err = utils.Run(cmd) + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + // this image will be built in above make target. + var feastImage = "feastdev/feature-server:dev" + var feastLocalImage = "localhost/feastdev/feature-server:dev" + + By("Tag the local feast image for the integration tests") + cmd = exec.Command("docker", "image", "tag", feastImage, feastLocalImage) + _, err = utils.Run(cmd) + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + + By("loading the the feast image on Kind cluster") + err = utils.LoadImageToKindClusterWithName(feastLocalImage) + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + + By("installing CRDs") + cmd = exec.Command("make", "install") + _, err = utils.Run(cmd) + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + + By("deploying the controller-manager") + cmd = exec.Command("make", "deploy", fmt.Sprintf("IMG=%s", projectimage)) + _, err = utils.Run(cmd) + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + + By("Validating that the controller-manager deployment is in available state") + err = checkIfDeploymentExistsAndAvailable(feastControllerNamespace, controllerDeploymentName, timeout) + Expect(err).ToNot(HaveOccurred(), fmt.Sprintf( + "Deployment %s is not available but expected to be available. \nError: %v\n", + controllerDeploymentName, err, + )) + fmt.Printf("Feast Control Manager Deployment %s is available\n", controllerDeploymentName) }) AfterAll(func() { - //Add any post clean up code here. + // Add any post clean up code here. + By("Uninstalling the feast CRD") + cmd := exec.Command("kubectl", "delete", "deployment", controllerDeploymentName, "-n", feastControllerNamespace) + _, err := utils.Run(cmd) + ExpectWithOffset(1, err).NotTo(HaveOccurred()) }) - Context("Operator", func() { + Context("Operator E2E Tests", func() { It("Should be able to deploy and run a default feature store CR successfully", func() { - //var controllerPodName string - var err error - - // projectimage stores the name of the image used in the example - var projectimage = "localhost/feast-operator:v0.0.1" - - By("building the manager(Operator) image") - cmd := exec.Command("make", "docker-build", fmt.Sprintf("IMG=%s", projectimage)) - _, err = utils.Run(cmd) - ExpectWithOffset(1, err).NotTo(HaveOccurred()) - - By("loading the the manager(Operator) image on Kind") - err = utils.LoadImageToKindClusterWithName(projectimage) - ExpectWithOffset(1, err).NotTo(HaveOccurred()) - - By("building the feast image") - cmd = exec.Command("make", "feast-ci-dev-docker-img") - _, err = utils.Run(cmd) - ExpectWithOffset(1, err).NotTo(HaveOccurred()) - // this image will be built in above make target. - var feastImage = "feastdev/feature-server:dev" - var feastLocalImage = "localhost/feastdev/feature-server:dev" - - By("Tag the local feast image for the integration tests") - cmd = exec.Command("docker", "image", "tag", feastImage, feastLocalImage) - _, err = utils.Run(cmd) - ExpectWithOffset(1, err).NotTo(HaveOccurred()) - - By("loading the the feast image on Kind cluster") - err = utils.LoadImageToKindClusterWithName(feastLocalImage) - ExpectWithOffset(1, err).NotTo(HaveOccurred()) - - By("installing CRDs") - cmd = exec.Command("make", "install") - _, err = utils.Run(cmd) - ExpectWithOffset(1, err).NotTo(HaveOccurred()) - - By("deploying the controller-manager") - cmd = exec.Command("make", "deploy", fmt.Sprintf("IMG=%s", projectimage)) - _, err = utils.Run(cmd) - ExpectWithOffset(1, err).NotTo(HaveOccurred()) - - timeout := 2 * time.Minute - - controllerDeploymentName := "feast-operator-controller-manager" - By("Validating that the controller-manager deployment is in available state") - err = checkIfDeploymentExistsAndAvailable(feastControllerNamespace, controllerDeploymentName, timeout) - Expect(err).To(BeNil(), fmt.Sprintf( - "Deployment %s is not available but expected to be available. \nError: %v\n", - controllerDeploymentName, err, - )) - fmt.Printf("Feast Control Manager Deployment %s is available\n", controllerDeploymentName) - By("deploying the Simple Feast Custom Resource to Kubernetes") - cmd = exec.Command("kubectl", "apply", "-f", - "test/testdata/feast_integration_test_crs/v1alpha1_default_featurestore.yaml") + namespace := "default" + cmd := exec.Command("kubectl", "apply", "-f", + "test/testdata/feast_integration_test_crs/v1alpha1_default_featurestore.yaml", "-n", namespace) _, cmdOutputerr := utils.Run(cmd) ExpectWithOffset(1, cmdOutputerr).NotTo(HaveOccurred()) - namespace := "default" - - deploymentNames := [3]string{"feast-simple-feast-setup-registry", "feast-simple-feast-setup-online", - "feast-simple-feast-setup-offline"} - for _, deploymentName := range deploymentNames { - By(fmt.Sprintf("validate the feast deployment: %s is up and in availability state.", deploymentName)) - err = checkIfDeploymentExistsAndAvailable(namespace, deploymentName, timeout) - Expect(err).To(BeNil(), fmt.Sprintf( - "Deployment %s is not available but expected to be available. \nError: %v\n", - deploymentName, err, - )) - fmt.Printf("Feast Deployment %s is available\n", deploymentName) - } - - By("Check if the feast client - kubernetes config map exists.") - configMapName := "feast-simple-feast-setup-client" - err = checkIfConfigMapExists(namespace, configMapName) - Expect(err).To(BeNil(), fmt.Sprintf( - "config map %s is not available but expected to be available. \nError: %v\n", - configMapName, err, - )) - fmt.Printf("Feast Deployment %s is available\n", configMapName) - - serviceAccountNames := [3]string{"feast-simple-feast-setup-registry", "feast-simple-feast-setup-online", - "feast-simple-feast-setup-offline"} - for _, serviceAccountName := range serviceAccountNames { - By(fmt.Sprintf("validate the feast service account: %s is available.", serviceAccountName)) - err = checkIfServiceAccountExists(namespace, serviceAccountName) - Expect(err).To(BeNil(), fmt.Sprintf( - "Service account %s does not exist in namespace %s. Error: %v", - serviceAccountName, namespace, err, - )) - fmt.Printf("Service account %s exists in namespace %s\n", serviceAccountName, namespace) - } - - serviceNames := [3]string{"feast-simple-feast-setup-registry", "feast-simple-feast-setup-online", - "feast-simple-feast-setup-offline"} - for _, serviceName := range serviceNames { - By(fmt.Sprintf("validate the kubernetes service name: %s is available.", serviceName)) - err = checkIfKubernetesServiceExists(namespace, serviceName) - Expect(err).To(BeNil(), fmt.Sprintf( - "kubernetes service %s is not available but expected to be available. \nError: %v\n", - serviceName, err, - )) - fmt.Printf("kubernetes service %s is available\n", serviceName) - } - - By(fmt.Sprintf("Checking FeatureStore customer resource: %s is in Ready Status.", "simple-feast-setup")) - err = checkIfFeatureStoreCustomResourceConditionsInReady("simple-feast-setup", namespace) - Expect(err).To(BeNil(), fmt.Sprintf( - "FeatureStore custom resource %s all conditions are not in ready state. \nError: %v\n", - "simple-feast-setup", err, - )) - fmt.Printf("FeatureStore customer resource %s conditions are in Ready State\n", "simple-feast-setup") + featureStoreName := "simple-feast-setup" + validateTheFeatureStoreCustomResource(namespace, featureStoreName, timeout) By("deleting the feast deployment") cmd = exec.Command("kubectl", "delete", "-f", "test/testdata/feast_integration_test_crs/v1alpha1_default_featurestore.yaml") _, cmdOutputerr = utils.Run(cmd) ExpectWithOffset(1, cmdOutputerr).NotTo(HaveOccurred()) + }) - By("Uninstalling the feast CRD") - cmd = exec.Command("kubectl", "delete", "deployment", controllerDeploymentName, "-n", feastControllerNamespace) - _, err = utils.Run(cmd) - ExpectWithOffset(1, err).NotTo(HaveOccurred()) + It("Should be able to deploy and run a feature store with remote registry CR successfully", func() { + By("deploying the Simple Feast Custom Resource to Kubernetes") + namespace := "default" + cmd := exec.Command("kubectl", "apply", "-f", + "test/testdata/feast_integration_test_crs/v1alpha1_default_featurestore.yaml", "-n", namespace) + _, cmdOutputerr := utils.Run(cmd) + ExpectWithOffset(1, cmdOutputerr).NotTo(HaveOccurred()) + featureStoreName := "simple-feast-setup" + validateTheFeatureStoreCustomResource(namespace, featureStoreName, timeout) + + var remoteRegistryNs = "remote-registry" + By(fmt.Sprintf("Creating the remote registry namespace=%s", remoteRegistryNs)) + cmd = exec.Command("kubectl", "create", "ns", remoteRegistryNs) + _, _ = utils.Run(cmd) + + By("deploying the Simple Feast remote registry Custom Resource on Kubernetes") + cmd = exec.Command("kubectl", "apply", "-f", + "test/testdata/feast_integration_test_crs/v1alpha1_remote_registry_featurestore.yaml", "-n", remoteRegistryNs) + _, cmdOutputerr = utils.Run(cmd) + ExpectWithOffset(1, cmdOutputerr).NotTo(HaveOccurred()) + + remoteFeatureStoreName := "simple-feast-remote-setup" + + validateTheFeatureStoreCustomResource(remoteRegistryNs, remoteFeatureStoreName, timeout) + + By("deleting the feast remote registry deployment") + cmd = exec.Command("kubectl", "delete", "-f", + "test/testdata/feast_integration_test_crs/v1alpha1_remote_registry_featurestore.yaml", "-n", remoteRegistryNs) + _, cmdOutputerr = utils.Run(cmd) + ExpectWithOffset(1, cmdOutputerr).NotTo(HaveOccurred()) + + By("deleting the feast deployment") + cmd = exec.Command("kubectl", "delete", "-f", + "test/testdata/feast_integration_test_crs/v1alpha1_default_featurestore.yaml", "-n", namespace) + _, cmdOutputerr = utils.Run(cmd) + ExpectWithOffset(1, cmdOutputerr).NotTo(HaveOccurred()) }) }) }) + +func validateTheFeatureStoreCustomResource(namespace string, featureStoreName string, timeout time.Duration) { + hasRemoteRegistry, err := isFeatureStoreHavingRemoteRegistry(namespace, featureStoreName) + Expect(err).ToNot(HaveOccurred(), fmt.Sprintf( + "Error occurred while checking FeatureStore %s is having remote registry or not. \nError: %v\n", + featureStoreName, err)) + + feastResourceName := feastPrefix + featureStoreName + k8sResourceNames := []string{feastResourceName} + feastK8sResourceNames := []string{ + feastResourceName + "-online", + feastResourceName + "-offline", + } + + if !hasRemoteRegistry { + feastK8sResourceNames = append(feastK8sResourceNames, feastResourceName+"-registry") + } + + for _, deploymentName := range k8sResourceNames { + By(fmt.Sprintf("validate the feast deployment: %s is up and in availability state.", deploymentName)) + err = checkIfDeploymentExistsAndAvailable(namespace, deploymentName, timeout) + Expect(err).ToNot(HaveOccurred(), fmt.Sprintf( + "Deployment %s is not available but expected to be available. \nError: %v\n", + deploymentName, err, + )) + fmt.Printf("Feast Deployment %s is available\n", deploymentName) + } + + By("Check if the feast client - kubernetes config map exists.") + configMapName := feastResourceName + "-client" + err = checkIfConfigMapExists(namespace, configMapName) + Expect(err).ToNot(HaveOccurred(), fmt.Sprintf( + "config map %s is not available but expected to be available. \nError: %v\n", + configMapName, err, + )) + fmt.Printf("Feast Deployment client config map %s is available\n", configMapName) + + for _, serviceAccountName := range k8sResourceNames { + By(fmt.Sprintf("validate the feast service account: %s is available.", serviceAccountName)) + err = checkIfServiceAccountExists(namespace, serviceAccountName) + Expect(err).ToNot(HaveOccurred(), fmt.Sprintf( + "Service account %s does not exist in namespace %s. Error: %v", + serviceAccountName, namespace, err, + )) + fmt.Printf("Service account %s exists in namespace %s\n", serviceAccountName, namespace) + } + + for _, serviceName := range feastK8sResourceNames { + By(fmt.Sprintf("validate the kubernetes service name: %s is available.", serviceName)) + err = checkIfKubernetesServiceExists(namespace, serviceName) + Expect(err).ToNot(HaveOccurred(), fmt.Sprintf( + "kubernetes service %s is not available but expected to be available. \nError: %v\n", + serviceName, err, + )) + fmt.Printf("kubernetes service %s is available\n", serviceName) + } + + By(fmt.Sprintf("Checking FeatureStore customer resource: %s is in Ready Status.", featureStoreName)) + err = checkIfFeatureStoreCustomResourceConditionsInReady(featureStoreName, namespace) + Expect(err).ToNot(HaveOccurred(), fmt.Sprintf( + "FeatureStore custom resource %s all conditions are not in ready state. \nError: %v\n", + featureStoreName, err, + )) + fmt.Printf("FeatureStore custom resource %s conditions are in Ready State\n", featureStoreName) +} diff --git a/infra/feast-operator/test/e2e/test_util.go b/infra/feast-operator/test/e2e/test_util.go index f30d8cbebf5..017690a0ec9 100644 --- a/infra/feast-operator/test/e2e/test_util.go +++ b/infra/feast-operator/test/e2e/test_util.go @@ -3,10 +3,15 @@ package e2e import ( "bytes" "encoding/json" + "errors" "fmt" "os/exec" "strings" "time" + + appsv1 "k8s.io/api/apps/v1" + + "github.com/feast-dev/feast/infra/feast-operator/api/v1alpha1" ) // dynamically checks if all conditions of custom resource featurestore are in "Ready" state. @@ -23,36 +28,17 @@ func checkIfFeatureStoreCustomResourceConditionsInReady(featureStoreName, namesp featureStoreName, namespace, err, stderr.String()) } - // Parse the JSON into a generic map - var resource map[string]interface{} + // Parse the JSON into FeatureStore + var resource v1alpha1.FeatureStore if err := json.Unmarshal(out.Bytes(), &resource); err != nil { return fmt.Errorf("failed to parse the resource JSON. Error: %v", err) } - // Traverse the JSON structure to extract conditions - status, ok := resource["status"].(map[string]interface{}) - if !ok { - return fmt.Errorf("status field is missing or invalid in the resource JSON") - } - - conditions, ok := status["conditions"].([]interface{}) - if !ok { - return fmt.Errorf("conditions field is missing or invalid in the status section") - } - // Validate all conditions - for _, condition := range conditions { - conditionMap, ok := condition.(map[string]interface{}) - if !ok { - return fmt.Errorf("invalid condition format") - } - - conditionType := conditionMap["type"].(string) - conditionStatus := conditionMap["status"].(string) - - if conditionStatus != "True" { + for _, condition := range resource.Status.Conditions { + if condition.Status != "True" { return fmt.Errorf(" FeatureStore=%s condition '%s' is not in 'Ready' state. Status: %s", - featureStoreName, conditionType, conditionStatus) + featureStoreName, condition.Type, condition.Status) } } @@ -84,30 +70,15 @@ func checkIfDeploymentExistsAndAvailable(namespace string, deploymentName string continue } - // Parse the JSON output into a map - var result map[string]interface{} + // Parse the JSON output into Deployment + var result appsv1.Deployment if err := json.Unmarshal(output.Bytes(), &result); err != nil { return fmt.Errorf("failed to parse deployment JSON: %v", err) } - // Navigate to status.conditions - status, ok := result["status"].(map[string]interface{}) - if !ok { - return fmt.Errorf("failed to get status field from deployment JSON") - } - - conditions, ok := status["conditions"].([]interface{}) - if !ok { - return fmt.Errorf("failed to get conditions field from deployment JSON") - } - // Check for Available condition - for _, condition := range conditions { - cond, ok := condition.(map[string]interface{}) - if !ok { - continue - } - if cond["type"] == "Available" && cond["status"] == "True" { + for _, condition := range result.Status.Conditions { + if condition.Type == "Available" && condition.Status == "True" { return nil // Deployment is available } } @@ -184,3 +155,42 @@ func checkIfKubernetesServiceExists(namespace, serviceName string) error { return nil } + +func isFeatureStoreHavingRemoteRegistry(namespace, featureStoreName string) (bool, error) { + cmd := exec.Command("kubectl", "get", "featurestore", featureStoreName, "-n", namespace, + "-o=jsonpath='{.spec.services.registry}'") + + // Capture the output + output, err := cmd.Output() + if err != nil { + return false, err // Return false on command execution failure + } + + // Convert output to string and trim any extra spaces + result := strings.TrimSpace(string(output)) + + // Remove single quotes if present + if strings.HasPrefix(result, "'") && strings.HasSuffix(result, "'") { + result = strings.Trim(result, "'") + } + + if result == "" { + return false, errors.New("kubectl get featurestore command returned empty output") + } + + // Parse the JSON into a map + var registryConfig v1alpha1.Registry + if err := json.Unmarshal([]byte(result), ®istryConfig); err != nil { + return false, err // Return false on JSON parsing failure + } + + if registryConfig.Remote == nil { + return false, nil + } + + hasHostname := registryConfig.Remote.Hostname != nil + hasValidFeastRef := registryConfig.Remote.FeastRef != nil && + registryConfig.Remote.FeastRef.Name != "" + + return hasHostname || hasValidFeastRef, nil +} diff --git a/infra/feast-operator/test/testdata/feast_integration_test_crs/v1alpha1_remote_registry_featurestore.yaml b/infra/feast-operator/test/testdata/feast_integration_test_crs/v1alpha1_remote_registry_featurestore.yaml new file mode 100644 index 00000000000..61c010f0576 --- /dev/null +++ b/infra/feast-operator/test/testdata/feast_integration_test_crs/v1alpha1_remote_registry_featurestore.yaml @@ -0,0 +1,16 @@ +apiVersion: feast.dev/v1alpha1 +kind: FeatureStore +metadata: + name: simple-feast-remote-setup +spec: + feastProject: my_project + services: + onlineStore: + image: 'localhost/feastdev/feature-server:dev' + offlineStore: + image: 'localhost/feastdev/feature-server:dev' + registry: + remote: + feastRef: + name: simple-feast-setup + namespace: default \ No newline at end of file diff --git a/infra/feast-operator/test/utils/utils.go b/infra/feast-operator/test/utils/utils.go index 1027041273c..9b57f9af61c 100644 --- a/infra/feast-operator/test/utils/utils.go +++ b/infra/feast-operator/test/utils/utils.go @@ -35,7 +35,7 @@ const ( ) func warnError(err error) { - fmt.Fprintf(GinkgoWriter, "warning: %v\n", err) + _, _ = fmt.Fprintf(GinkgoWriter, "warning: %v\n", err) } // InstallPrometheusOperator installs the prometheus Operator to be used to export the enabled metrics. @@ -52,12 +52,12 @@ func Run(cmd *exec.Cmd) ([]byte, error) { cmd.Dir = dir if err := os.Chdir(cmd.Dir); err != nil { - fmt.Fprintf(GinkgoWriter, "chdir dir: %s\n", err) + _, _ = fmt.Fprintf(GinkgoWriter, "chdir dir: %s\n", err) } cmd.Env = append(os.Environ(), "GO111MODULE=on") command := strings.Join(cmd.Args, " ") - fmt.Fprintf(GinkgoWriter, "running: %s\n", command) + _, _ = fmt.Fprintf(GinkgoWriter, "running: %s\n", command) output, err := cmd.CombinedOutput() if err != nil { return output, fmt.Errorf("%s failed with error: (%v) %s", command, err, string(output)) diff --git a/infra/scripts/release/files_to_bump.txt b/infra/scripts/release/files_to_bump.txt index 652bc3cad10..0e0ac0bce6d 100644 --- a/infra/scripts/release/files_to_bump.txt +++ b/infra/scripts/release/files_to_bump.txt @@ -14,6 +14,8 @@ infra/feast-helm-operator/Makefile 6 infra/feast-helm-operator/config/manager/kustomization.yaml 8 infra/feast-operator/Makefile 6 infra/feast-operator/config/manager/kustomization.yaml 8 +infra/feast-operator/config/default/manager_related_images_patch.yaml 5 +infra/feast-operator/config/overlays/odh/params.env 1 infra/feast-operator/api/feastversion/version.go 20 java/pom.xml 38 ui/package.json 3 diff --git a/java/datatypes/pom.xml b/java/datatypes/pom.xml index b0ba049c575..967262d0e01 100644 --- a/java/datatypes/pom.xml +++ b/java/datatypes/pom.xml @@ -118,6 +118,11 @@ grpc-stub ${grpc.version} + + io.grpc + grpc-api + ${grpc.version} + javax.annotation javax.annotation-api diff --git a/java/pom.xml b/java/pom.xml index 6b18588923a..8173385e0c9 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -35,15 +35,15 @@ - 0.42.0 + 0.43.0 https://github.com/feast-dev/feast UTF-8 UTF-8 - 1.30.2 + 1.63.0 3.12.2 - 3.16.1 + 3.25.5 1.111.1 0.8.0 1.9.10 @@ -61,15 +61,15 @@ 1.5.24 3.14.7 3.10 - 2.14.0 + 2.15.0 2.3.1 1.3.2 2.0.1.Final 0.21.0 1.6.6 - 30.1-jre + 32.0.0-jre 3.4.34 - 4.1.101.Final + 4.1.96.Final src/main/java/**/BatchLoadsWithResult.java - + @@ -365,7 +365,7 @@ [11.0,) - + @@ -376,7 +376,7 @@ - + diff --git a/java/serving-client/pom.xml b/java/serving-client/pom.xml index 7b8838a009c..dc611b4a76e 100644 --- a/java/serving-client/pom.xml +++ b/java/serving-client/pom.xml @@ -50,6 +50,11 @@ grpc-testing ${grpc.version} + + io.grpc + grpc-api + ${grpc.version} + com.google.protobuf protobuf-java-util diff --git a/java/serving/pom.xml b/java/serving/pom.xml index ca7f8a73b5f..1be4da1b622 100644 --- a/java/serving/pom.xml +++ b/java/serving/pom.xml @@ -126,7 +126,7 @@ com.azure azure-storage-blob - 12.25.2 + 12.26.1 com.azure @@ -164,6 +164,11 @@ grpc-stub ${grpc.version} + + io.grpc + grpc-api + ${grpc.version} + io.grpc grpc-netty-shaded @@ -192,7 +197,7 @@ io.jaegertracing jaeger-client - 1.3.2 + 1.8.1 io.opentracing @@ -240,7 +245,7 @@ com.google.cloud google-cloud-storage - 1.118.0 + 2.43.1 @@ -253,13 +258,13 @@ com.amazonaws aws-java-sdk-s3 - 1.12.261 + 1.12.546 com.amazonaws aws-java-sdk-sts - 1.12.476 + 1.12.546 @@ -378,7 +383,7 @@ io.lettuce lettuce-core - 6.0.2.RELEASE + 6.5.1.RELEASE org.apache.commons diff --git a/protos/feast/core/Feature.proto b/protos/feast/core/Feature.proto index 882de47eb9c..8a56d67905a 100644 --- a/protos/feast/core/Feature.proto +++ b/protos/feast/core/Feature.proto @@ -35,6 +35,11 @@ message FeatureSpecV2 { map tags = 3; // Description of the feature. - string description = 4; + + // Field indicating the vector will be indexed for vector similarity search + bool vector_index = 5; + + // Metric used for vector similarity search. + string vector_search_metric = 6; } diff --git a/sdk/python/docs/source/feast.infra.online_stores.cassandra_online_store.rst b/sdk/python/docs/source/feast.infra.online_stores.cassandra_online_store.rst new file mode 100644 index 00000000000..7c5c3d371a7 --- /dev/null +++ b/sdk/python/docs/source/feast.infra.online_stores.cassandra_online_store.rst @@ -0,0 +1,29 @@ +feast.infra.online\_stores.cassandra\_online\_store package +=========================================================== + +Submodules +---------- + +feast.infra.online\_stores.cassandra\_online\_store.cassandra\_online\_store module +----------------------------------------------------------------------------------- + +.. automodule:: feast.infra.online_stores.cassandra_online_store.cassandra_online_store + :members: + :undoc-members: + :show-inheritance: + +feast.infra.online\_stores.cassandra\_online\_store.cassandra\_repo\_configuration module +----------------------------------------------------------------------------------------- + +.. automodule:: feast.infra.online_stores.cassandra_online_store.cassandra_repo_configuration + :members: + :undoc-members: + :show-inheritance: + +Module contents +--------------- + +.. automodule:: feast.infra.online_stores.cassandra_online_store + :members: + :undoc-members: + :show-inheritance: diff --git a/sdk/python/docs/source/feast.infra.online_stores.couchbase_online_store.rst b/sdk/python/docs/source/feast.infra.online_stores.couchbase_online_store.rst new file mode 100644 index 00000000000..29d51304928 --- /dev/null +++ b/sdk/python/docs/source/feast.infra.online_stores.couchbase_online_store.rst @@ -0,0 +1,29 @@ +feast.infra.online\_stores.couchbase\_online\_store package +=========================================================== + +Submodules +---------- + +feast.infra.online\_stores.couchbase\_online\_store.couchbase module +-------------------------------------------------------------------- + +.. automodule:: feast.infra.online_stores.couchbase_online_store.couchbase + :members: + :undoc-members: + :show-inheritance: + +feast.infra.online\_stores.couchbase\_online\_store.couchbase\_repo\_configuration module +----------------------------------------------------------------------------------------- + +.. automodule:: feast.infra.online_stores.couchbase_online_store.couchbase_repo_configuration + :members: + :undoc-members: + :show-inheritance: + +Module contents +--------------- + +.. automodule:: feast.infra.online_stores.couchbase_online_store + :members: + :undoc-members: + :show-inheritance: diff --git a/sdk/python/docs/source/feast.infra.online_stores.elasticsearch_online_store.rst b/sdk/python/docs/source/feast.infra.online_stores.elasticsearch_online_store.rst new file mode 100644 index 00000000000..d470e3301d0 --- /dev/null +++ b/sdk/python/docs/source/feast.infra.online_stores.elasticsearch_online_store.rst @@ -0,0 +1,29 @@ +feast.infra.online\_stores.elasticsearch\_online\_store package +=============================================================== + +Submodules +---------- + +feast.infra.online\_stores.elasticsearch\_online\_store.elasticsearch module +---------------------------------------------------------------------------- + +.. automodule:: feast.infra.online_stores.elasticsearch_online_store.elasticsearch + :members: + :undoc-members: + :show-inheritance: + +feast.infra.online\_stores.elasticsearch\_online\_store.elasticsearch\_repo\_configuration module +------------------------------------------------------------------------------------------------- + +.. automodule:: feast.infra.online_stores.elasticsearch_online_store.elasticsearch_repo_configuration + :members: + :undoc-members: + :show-inheritance: + +Module contents +--------------- + +.. automodule:: feast.infra.online_stores.elasticsearch_online_store + :members: + :undoc-members: + :show-inheritance: diff --git a/sdk/python/docs/source/feast.infra.online_stores.hazelcast_online_store.rst b/sdk/python/docs/source/feast.infra.online_stores.hazelcast_online_store.rst new file mode 100644 index 00000000000..9cb565ca132 --- /dev/null +++ b/sdk/python/docs/source/feast.infra.online_stores.hazelcast_online_store.rst @@ -0,0 +1,29 @@ +feast.infra.online\_stores.hazelcast\_online\_store package +=========================================================== + +Submodules +---------- + +feast.infra.online\_stores.hazelcast\_online\_store.hazelcast\_online\_store module +----------------------------------------------------------------------------------- + +.. automodule:: feast.infra.online_stores.hazelcast_online_store.hazelcast_online_store + :members: + :undoc-members: + :show-inheritance: + +feast.infra.online\_stores.hazelcast\_online\_store.hazelcast\_repo\_configuration module +----------------------------------------------------------------------------------------- + +.. automodule:: feast.infra.online_stores.hazelcast_online_store.hazelcast_repo_configuration + :members: + :undoc-members: + :show-inheritance: + +Module contents +--------------- + +.. automodule:: feast.infra.online_stores.hazelcast_online_store + :members: + :undoc-members: + :show-inheritance: diff --git a/sdk/python/docs/source/feast.infra.online_stores.hbase_online_store.rst b/sdk/python/docs/source/feast.infra.online_stores.hbase_online_store.rst new file mode 100644 index 00000000000..50ad80e0a9e --- /dev/null +++ b/sdk/python/docs/source/feast.infra.online_stores.hbase_online_store.rst @@ -0,0 +1,29 @@ +feast.infra.online\_stores.hbase\_online\_store package +======================================================= + +Submodules +---------- + +feast.infra.online\_stores.hbase\_online\_store.hbase module +------------------------------------------------------------ + +.. automodule:: feast.infra.online_stores.hbase_online_store.hbase + :members: + :undoc-members: + :show-inheritance: + +feast.infra.online\_stores.hbase\_online\_store.hbase\_repo\_configuration module +--------------------------------------------------------------------------------- + +.. automodule:: feast.infra.online_stores.hbase_online_store.hbase_repo_configuration + :members: + :undoc-members: + :show-inheritance: + +Module contents +--------------- + +.. automodule:: feast.infra.online_stores.hbase_online_store + :members: + :undoc-members: + :show-inheritance: diff --git a/sdk/python/docs/source/feast.infra.online_stores.ikv_online_store.rst b/sdk/python/docs/source/feast.infra.online_stores.ikv_online_store.rst new file mode 100644 index 00000000000..391af17024f --- /dev/null +++ b/sdk/python/docs/source/feast.infra.online_stores.ikv_online_store.rst @@ -0,0 +1,21 @@ +feast.infra.online\_stores.ikv\_online\_store package +===================================================== + +Submodules +---------- + +feast.infra.online\_stores.ikv\_online\_store.ikv module +-------------------------------------------------------- + +.. automodule:: feast.infra.online_stores.ikv_online_store.ikv + :members: + :undoc-members: + :show-inheritance: + +Module contents +--------------- + +.. automodule:: feast.infra.online_stores.ikv_online_store + :members: + :undoc-members: + :show-inheritance: diff --git a/sdk/python/docs/source/feast.infra.online_stores.milvus_online_store.rst b/sdk/python/docs/source/feast.infra.online_stores.milvus_online_store.rst new file mode 100644 index 00000000000..5ae3015bf37 --- /dev/null +++ b/sdk/python/docs/source/feast.infra.online_stores.milvus_online_store.rst @@ -0,0 +1,29 @@ +feast.infra.online\_stores.milvus\_online\_store package +======================================================== + +Submodules +---------- + +feast.infra.online\_stores.milvus\_online\_store.milvus module +-------------------------------------------------------------- + +.. automodule:: feast.infra.online_stores.milvus_online_store.milvus + :members: + :undoc-members: + :show-inheritance: + +feast.infra.online\_stores.milvus\_online\_store.milvus\_repo\_configuration module +----------------------------------------------------------------------------------- + +.. automodule:: feast.infra.online_stores.milvus_online_store.milvus_repo_configuration + :members: + :undoc-members: + :show-inheritance: + +Module contents +--------------- + +.. automodule:: feast.infra.online_stores.milvus_online_store + :members: + :undoc-members: + :show-inheritance: diff --git a/sdk/python/docs/source/feast.infra.online_stores.mysql_online_store.rst b/sdk/python/docs/source/feast.infra.online_stores.mysql_online_store.rst new file mode 100644 index 00000000000..b1a9ea4f802 --- /dev/null +++ b/sdk/python/docs/source/feast.infra.online_stores.mysql_online_store.rst @@ -0,0 +1,29 @@ +feast.infra.online\_stores.mysql\_online\_store package +======================================================= + +Submodules +---------- + +feast.infra.online\_stores.mysql\_online\_store.mysql module +------------------------------------------------------------ + +.. automodule:: feast.infra.online_stores.mysql_online_store.mysql + :members: + :undoc-members: + :show-inheritance: + +feast.infra.online\_stores.mysql\_online\_store.mysql\_repo\_configuration module +--------------------------------------------------------------------------------- + +.. automodule:: feast.infra.online_stores.mysql_online_store.mysql_repo_configuration + :members: + :undoc-members: + :show-inheritance: + +Module contents +--------------- + +.. automodule:: feast.infra.online_stores.mysql_online_store + :members: + :undoc-members: + :show-inheritance: diff --git a/sdk/python/docs/source/feast.infra.online_stores.postgres_online_store.rst b/sdk/python/docs/source/feast.infra.online_stores.postgres_online_store.rst new file mode 100644 index 00000000000..9dfd200a4e1 --- /dev/null +++ b/sdk/python/docs/source/feast.infra.online_stores.postgres_online_store.rst @@ -0,0 +1,37 @@ +feast.infra.online\_stores.postgres\_online\_store package +========================================================== + +Submodules +---------- + +feast.infra.online\_stores.postgres\_online\_store.pgvector\_repo\_configuration module +--------------------------------------------------------------------------------------- + +.. automodule:: feast.infra.online_stores.postgres_online_store.pgvector_repo_configuration + :members: + :undoc-members: + :show-inheritance: + +feast.infra.online\_stores.postgres\_online\_store.postgres module +------------------------------------------------------------------ + +.. automodule:: feast.infra.online_stores.postgres_online_store.postgres + :members: + :undoc-members: + :show-inheritance: + +feast.infra.online\_stores.postgres\_online\_store.postgres\_repo\_configuration module +--------------------------------------------------------------------------------------- + +.. automodule:: feast.infra.online_stores.postgres_online_store.postgres_repo_configuration + :members: + :undoc-members: + :show-inheritance: + +Module contents +--------------- + +.. automodule:: feast.infra.online_stores.postgres_online_store + :members: + :undoc-members: + :show-inheritance: diff --git a/sdk/python/docs/source/feast.infra.online_stores.qdrant_online_store.rst b/sdk/python/docs/source/feast.infra.online_stores.qdrant_online_store.rst new file mode 100644 index 00000000000..5c210d4124d --- /dev/null +++ b/sdk/python/docs/source/feast.infra.online_stores.qdrant_online_store.rst @@ -0,0 +1,29 @@ +feast.infra.online\_stores.qdrant\_online\_store package +======================================================== + +Submodules +---------- + +feast.infra.online\_stores.qdrant\_online\_store.qdrant module +-------------------------------------------------------------- + +.. automodule:: feast.infra.online_stores.qdrant_online_store.qdrant + :members: + :undoc-members: + :show-inheritance: + +feast.infra.online\_stores.qdrant\_online\_store.qdrant\_repo\_configuration module +----------------------------------------------------------------------------------- + +.. automodule:: feast.infra.online_stores.qdrant_online_store.qdrant_repo_configuration + :members: + :undoc-members: + :show-inheritance: + +Module contents +--------------- + +.. automodule:: feast.infra.online_stores.qdrant_online_store + :members: + :undoc-members: + :show-inheritance: diff --git a/sdk/python/docs/source/feast.infra.online_stores.rst b/sdk/python/docs/source/feast.infra.online_stores.rst index ea714e45c5b..c07c7e0c279 100644 --- a/sdk/python/docs/source/feast.infra.online_stores.rst +++ b/sdk/python/docs/source/feast.infra.online_stores.rst @@ -7,7 +7,16 @@ Subpackages .. toctree:: :maxdepth: 4 - feast.infra.online_stores + feast.infra.online_stores.cassandra_online_store + feast.infra.online_stores.couchbase_online_store + feast.infra.online_stores.elasticsearch_online_store + feast.infra.online_stores.hazelcast_online_store + feast.infra.online_stores.hbase_online_store + feast.infra.online_stores.ikv_online_store + feast.infra.online_stores.milvus_online_store + feast.infra.online_stores.mysql_online_store + feast.infra.online_stores.postgres_online_store + feast.infra.online_stores.qdrant_online_store Submodules ---------- @@ -36,6 +45,14 @@ feast.infra.online\_stores.dynamodb module :undoc-members: :show-inheritance: +feast.infra.online\_stores.faiss\_online\_store module +------------------------------------------------------ + +.. automodule:: feast.infra.online_stores.faiss_online_store + :members: + :undoc-members: + :show-inheritance: + feast.infra.online\_stores.helpers module ----------------------------------------- diff --git a/sdk/python/docs/source/feast.infra.rst b/sdk/python/docs/source/feast.infra.rst index b0046a2719e..791a4ace832 100644 --- a/sdk/python/docs/source/feast.infra.rst +++ b/sdk/python/docs/source/feast.infra.rst @@ -51,6 +51,14 @@ feast.infra.provider module :undoc-members: :show-inheritance: +feast.infra.supported\_async\_methods module +-------------------------------------------- + +.. automodule:: feast.infra.supported_async_methods + :members: + :undoc-members: + :show-inheritance: + Module contents --------------- diff --git a/sdk/python/feast/cli.py b/sdk/python/feast/cli.py index a02013b11f9..165677a843a 100644 --- a/sdk/python/feast/cli.py +++ b/sdk/python/feast/cli.py @@ -865,6 +865,7 @@ def materialize_incremental_command(ctx: click.Context, end_ts: str, views: List "cassandra", "hazelcast", "ikv", + "couchbase", ], case_sensitive=False, ), @@ -981,7 +982,6 @@ def serve_command( raise click.BadParameter( "Please pass --cert and --key args to start the feature server in TLS mode." ) - store = create_feature_store(ctx) store.serve( @@ -1132,15 +1132,6 @@ def serve_registry_command( show_default=False, help="path to TLS certificate public key. You need to pass --key as well to start server in TLS mode", ) -@click.option( - "--verify_client", - "-v", - "tls_verify_client", - type=click.BOOL, - default="True", - show_default=True, - help="Verify the client or not for the TLS client certificate.", -) @click.pass_context def serve_offline_command( ctx: click.Context, @@ -1148,7 +1139,6 @@ def serve_offline_command( port: int, tls_key_path: str, tls_cert_path: str, - tls_verify_client: bool, ): """Start a remote server locally on a given host, port.""" if (tls_key_path and not tls_cert_path) or (not tls_key_path and tls_cert_path): @@ -1157,7 +1147,7 @@ def serve_offline_command( ) store = create_feature_store(ctx) - store.serve_offline(host, port, tls_key_path, tls_cert_path, tls_verify_client) + store.serve_offline(host, port, tls_key_path, tls_cert_path) @cli.command("validate") diff --git a/sdk/python/feast/entity.py b/sdk/python/feast/entity.py index 290e6307a42..9c529115c8e 100644 --- a/sdk/python/feast/entity.py +++ b/sdk/python/feast/entity.py @@ -11,6 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +import warnings from datetime import datetime from typing import Dict, List, Optional @@ -79,6 +80,13 @@ def __init__( ValueError: Parameters are specified incorrectly. """ self.name = name + if value_type is None: + warnings.warn( + "Entity value_type will be mandatory in the next release. " + "Please specify a value_type for entity '%s'." % name, + DeprecationWarning, + stacklevel=2, + ) self.value_type = value_type or ValueType.UNKNOWN if join_keys and len(join_keys) > 1: diff --git a/sdk/python/feast/feature_server.py b/sdk/python/feast/feature_server.py index 1f4918fe7a5..e22ff43b9c2 100644 --- a/sdk/python/feast/feature_server.py +++ b/sdk/python/feast/feature_server.py @@ -76,6 +76,38 @@ class GetOnlineFeaturesRequest(BaseModel): full_feature_names: bool = False +def _get_features(request: GetOnlineFeaturesRequest, store: "feast.FeatureStore"): + if request.feature_service: + feature_service = store.get_feature_service( + request.feature_service, allow_cache=True + ) + assert_permissions( + resource=feature_service, actions=[AuthzedAction.READ_ONLINE] + ) + features = feature_service # type: ignore + else: + all_feature_views, all_on_demand_feature_views = ( + utils._get_feature_views_to_use( + store.registry, + store.project, + request.features, + allow_cache=True, + hide_dummy_entity=False, + ) + ) + for feature_view in all_feature_views: + assert_permissions( + resource=feature_view, actions=[AuthzedAction.READ_ONLINE] + ) + for od_feature_view in all_on_demand_feature_views: + assert_permissions( + resource=od_feature_view, actions=[AuthzedAction.READ_ONLINE] + ) + features = request.features # type: ignore + + return features + + def get_app( store: "feast.FeatureStore", registry_ttl_sec: int = DEFAULT_FEATURE_SERVER_REGISTRY_TTL, @@ -121,33 +153,7 @@ async def lifespan(app: FastAPI): ) async def get_online_features(request: GetOnlineFeaturesRequest) -> Dict[str, Any]: # Initialize parameters for FeatureStore.get_online_features(...) call - if request.feature_service: - feature_service = store.get_feature_service( - request.feature_service, allow_cache=True - ) - assert_permissions( - resource=feature_service, actions=[AuthzedAction.READ_ONLINE] - ) - features = feature_service # type: ignore - else: - all_feature_views, all_on_demand_feature_views = ( - utils._get_feature_views_to_use( - store.registry, - store.project, - request.features, - allow_cache=True, - hide_dummy_entity=False, - ) - ) - for feature_view in all_feature_views: - assert_permissions( - resource=feature_view, actions=[AuthzedAction.READ_ONLINE] - ) - for od_feature_view in all_on_demand_feature_views: - assert_permissions( - resource=od_feature_view, actions=[AuthzedAction.READ_ONLINE] - ) - features = request.features # type: ignore + features = await run_in_threadpool(_get_features, request, store) read_params = dict( features=features, @@ -163,9 +169,13 @@ async def get_online_features(request: GetOnlineFeaturesRequest) -> Dict[str, An ) # Convert the Protobuf object to JSON and return it - return MessageToDict( - response.proto, preserving_proto_field_name=True, float_precision=18 + response_dict = await run_in_threadpool( + MessageToDict, + response.proto, + preserving_proto_field_name=True, + float_precision=18, ) + return response_dict @app.post("/push", dependencies=[Depends(inject_user_details)]) async def push(request: PushFeaturesRequest) -> None: diff --git a/sdk/python/feast/feature_store.py b/sdk/python/feast/feature_store.py index 79a0d752efb..98db710d7f3 100644 --- a/sdk/python/feast/feature_store.py +++ b/sdk/python/feast/feature_store.py @@ -86,6 +86,7 @@ from feast.repo_config import RepoConfig, load_repo_config from feast.repo_contents import RepoContents from feast.saved_dataset import SavedDataset, SavedDatasetStorage, ValidationReference +from feast.ssl_ca_trust_store_setup import configure_ca_trust_store_env_variables from feast.stream_feature_view import StreamFeatureView from feast.utils import _utc_now @@ -129,6 +130,8 @@ def __init__( if fs_yaml_file is not None and config is not None: raise ValueError("You cannot specify both fs_yaml_file and config.") + configure_ca_trust_store_env_variables() + if repo_path: self.repo_path = Path(repo_path) else: @@ -1750,10 +1753,11 @@ async def get_online_features_async( def retrieve_online_documents( self, - feature: str, + feature: Optional[str], query: Union[str, List[float]], top_k: int, - distance_metric: Optional[str] = None, + features: Optional[List[str]] = None, + distance_metric: Optional[str] = "L2", ) -> OnlineResponse: """ Retrieves the top k closest document features. Note, embeddings are a subset of features. @@ -1762,6 +1766,7 @@ def retrieve_online_documents( feature: The list of document features that should be retrieved from the online document store. These features can be specified either as a list of string document feature references or as a feature service. String feature references must have format "feature_view:feature", e.g, "document_fv:document_embeddings". + features: The list of features that should be retrieved from the online store. query: The query to retrieve the closest document features for. top_k: The number of closest document features to retrieve. distance_metric: The distance metric to use for retrieval. @@ -1770,18 +1775,44 @@ def retrieve_online_documents( raise ValueError( "Using embedding functionality is not supported for document retrieval. Please embed the query before calling retrieve_online_documents." ) + feature_list: List[str] = ( + features + if features is not None + else ([feature] if feature is not None else []) + ) + ( available_feature_views, _, ) = utils._get_feature_views_to_use( registry=self._registry, project=self.project, - features=[feature], + features=feature_list, allow_cache=True, hide_dummy_entity=False, ) + if features: + feature_view_set = set() + for feature in features: + feature_view_name = feature.split(":")[0] + feature_view = self.get_feature_view(feature_view_name) + feature_view_set.add(feature_view.name) + if len(feature_view_set) > 1: + raise ValueError( + "Document retrieval only supports a single feature view." + ) + requested_feature = None + requested_features = [ + f.split(":")[1] for f in features if isinstance(f, str) and ":" in f + ] + else: + requested_feature = ( + feature.split(":")[1] if isinstance(feature, str) else feature + ) + requested_features = [requested_feature] if requested_feature else [] + requested_feature_view_name = ( - feature.split(":")[0] if isinstance(feature, str) else feature + feature.split(":")[0] if feature else list(feature_view_set)[0] ) for feature_view in available_feature_views: if feature_view.name == requested_feature_view_name: @@ -1790,14 +1821,15 @@ def retrieve_online_documents( raise ValueError( f"Feature view {requested_feature_view} not found in the registry." ) - requested_feature = ( - feature.split(":")[1] if isinstance(feature, str) else feature - ) + + requested_feature_view = available_feature_views[0] + provider = self._get_provider() document_features = self._retrieve_from_online_store( provider, requested_feature_view, requested_feature, + requested_features, query, top_k, distance_metric, @@ -1819,6 +1851,7 @@ def retrieve_online_documents( document_feature_vals = [feature[4] for feature in document_features] document_feature_distance_vals = [feature[5] for feature in document_features] online_features_response = GetOnlineFeaturesResponse(results=[]) + requested_feature = requested_feature or requested_features[0] utils._populate_result_rows_from_columnar( online_features_response=online_features_response, data={ @@ -1833,7 +1866,8 @@ def _retrieve_from_online_store( self, provider: Provider, table: FeatureView, - requested_feature: str, + requested_feature: Optional[str], + requested_features: Optional[List[str]], query: List[float], top_k: int, distance_metric: Optional[str], @@ -1849,6 +1883,7 @@ def _retrieve_from_online_store( config=self.config, table=table, requested_feature=requested_feature, + requested_features=requested_features, query=query, top_k=top_k, distance_metric=distance_metric, @@ -1964,14 +1999,11 @@ def serve_offline( port: int, tls_key_path: str = "", tls_cert_path: str = "", - tls_verify_client: bool = True, ) -> None: """Start offline server locally on a given port.""" from feast import offline_server - offline_server.start_server( - self, host, port, tls_key_path, tls_cert_path, tls_verify_client - ) + offline_server.start_server(self, host, port, tls_key_path, tls_cert_path) def serve_transformations(self, port: int) -> None: """Start the feature transformation server locally on a given port.""" diff --git a/sdk/python/feast/field.py b/sdk/python/feast/field.py index a41dcf5d5e6..fda1fbffe54 100644 --- a/sdk/python/feast/field.py +++ b/sdk/python/feast/field.py @@ -32,12 +32,16 @@ class Field: dtype: The type of the field, such as string or float. description: A human-readable description. tags: User-defined metadata in dictionary form. + vector_index: If set to True the field will be indexed for vector similarity search. + vector_search_metric: The metric used for vector similarity search. """ name: str dtype: FeastType description: str tags: Dict[str, str] + vector_index: bool + vector_search_metric: Optional[str] def __init__( self, @@ -46,6 +50,8 @@ def __init__( dtype: FeastType, description: str = "", tags: Optional[Dict[str, str]] = None, + vector_index: bool = False, + vector_search_metric: Optional[str] = None, ): """ Creates a Field object. @@ -55,11 +61,15 @@ def __init__( dtype: The type of the field, such as string or float. description (optional): A human-readable description. tags (optional): User-defined metadata in dictionary form. + vector_index (optional): If set to True the field will be indexed for vector similarity search. + vector_search_metric (optional): The metric used for vector similarity search. """ self.name = name self.dtype = dtype self.description = description self.tags = tags or {} + self.vector_index = vector_index + self.vector_search_metric = vector_search_metric def __eq__(self, other): if type(self) != type(other): @@ -70,6 +80,8 @@ def __eq__(self, other): or self.dtype != other.dtype or self.description != other.description or self.tags != other.tags + # or self.vector_index != other.vector_index + # or self.vector_search_metric != other.vector_search_metric ): return False return True @@ -87,6 +99,8 @@ def __repr__(self): f" dtype={self.dtype!r},\n" f" description={self.description!r},\n" f" tags={self.tags!r}\n" + f" vector_index={self.vector_index!r}\n" + f" vector_search_metric={self.vector_search_metric!r}\n" f")" ) @@ -96,11 +110,14 @@ def __str__(self): def to_proto(self) -> FieldProto: """Converts a Field object to its protobuf representation.""" value_type = self.dtype.to_value_type() + vector_search_metric = self.vector_search_metric or "" return FieldProto( name=self.name, value_type=value_type.value, description=self.description, tags=self.tags, + vector_index=self.vector_index, + vector_search_metric=vector_search_metric, ) @classmethod @@ -112,11 +129,15 @@ def from_proto(cls, field_proto: FieldProto): field_proto: FieldProto protobuf object """ value_type = ValueType(field_proto.value_type) + vector_search_metric = getattr(field_proto, "vector_search_metric", "") + vector_index = getattr(field_proto, "vector_index", False) return cls( name=field_proto.name, dtype=from_value_type(value_type=value_type), tags=dict(field_proto.tags), description=field_proto.description, + vector_index=vector_index, + vector_search_metric=vector_search_metric, ) @classmethod diff --git a/sdk/python/feast/infra/feature_servers/multicloud/Dockerfile b/sdk/python/feast/infra/feature_servers/multicloud/Dockerfile index f6bcbae8cd0..fd3c258d892 100644 --- a/sdk/python/feast/infra/feature_servers/multicloud/Dockerfile +++ b/sdk/python/feast/infra/feature_servers/multicloud/Dockerfile @@ -1,18 +1,7 @@ -FROM python:3.11-slim-bullseye +FROM registry.access.redhat.com/ubi9/python-311:9.5 -RUN pip install --no-cache-dir pip --upgrade -RUN pip install --no-cache-dir "feast[aws,gcp,snowflake,redis,go,mysql,postgres,opentelemetry,grpcio]" - - -RUN apt update && apt install -y -V ca-certificates lsb-release wget && \ - wget https://apache.jfrog.io/artifactory/arrow/$(lsb_release --id --short | tr 'A-Z' 'a-z')/apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb && \ - apt install -y -V ./apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb && apt update && \ - apt install -y \ - jq \ - libarrow-dev && \ - apt remove -y lsb-release wget && \ - apt-get clean && rm -rf /var/cache/apt/lists +ARG VERSION +RUN pip install "feast[aws,gcp,snowflake,redis,go,mysql,postgres,opentelemetry,grpcio,k8s,duckdb,milvus]"==${VERSION} # modify permissions to support running with a random uid -RUN mkdir -m 775 /.cache -RUN chmod g+w $(python3 -c "import feast.ui as _; print(_.__path__)" | tr -d "[']")/build/projects-list.json +RUN chmod g+w $(python -c "import feast.ui as ui; print(ui.__path__)" | tr -d "[']")/build/projects-list.json diff --git a/sdk/python/feast/infra/feature_servers/multicloud/Dockerfile.dev b/sdk/python/feast/infra/feature_servers/multicloud/Dockerfile.dev index 7d31fc3600b..add170fb1dc 100644 --- a/sdk/python/feast/infra/feature_servers/multicloud/Dockerfile.dev +++ b/sdk/python/feast/infra/feature_servers/multicloud/Dockerfile.dev @@ -1,28 +1,13 @@ -FROM python:3.11-slim-bullseye +FROM registry.access.redhat.com/ubi9/python-311:9.5 -RUN pip install --no-cache-dir pip --upgrade -RUN pip install --no-cache-dir pip-tools +COPY --chown=default . ${APP_ROOT}/src +RUN pip install --no-cache-dir pip-tools && \ + make install-python-ci-dependencies && \ + pip uninstall -y pip-tools -RUN apt update && apt install -y -V ca-certificates lsb-release wget make git curl gcc && \ - curl -sL https://deb.nodesource.com/setup_20.x | bash - && \ - wget https://apache.jfrog.io/artifactory/arrow/$(lsb_release --id --short | tr 'A-Z' 'a-z')/apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb && \ - apt install -y -V ./apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb && apt update && \ - apt install -y \ - jq \ - nodejs \ - libarrow-dev && \ - npm install -g yarn && \ - apt remove -y lsb-release wget && \ - apt-get clean && rm -rf /var/cache/apt/lists - -COPY . /feast -WORKDIR /feast -RUN make install-python-ci-dependencies && pip cache purge -ENV NPM_TOKEN '//registry.npmjs.org/:_authToken' -RUN make build-ui && yarn cache clean - -WORKDIR / +RUN npm install -S yarn +ENV PATH ${PATH}:${APP_ROOT}/src/node_modules/yarn/bin +RUN make build-ui && yarn cache clean --all # modify permissions to support running with a random uid -RUN mkdir -m 775 /.cache -RUN chmod g+w $(python3 -c "import feast.ui as _; print(_.__path__)" | tr -d "[']")/build/projects-list.json +RUN chmod g+w $(python -c "import feast.ui as ui; print(ui.__path__)" | tr -d "[']")/build/projects-list.json diff --git a/sdk/python/feast/infra/key_encoding_utils.py b/sdk/python/feast/infra/key_encoding_utils.py index 1f9ffeef140..18127896bd5 100644 --- a/sdk/python/feast/infra/key_encoding_utils.py +++ b/sdk/python/feast/infra/key_encoding_utils.py @@ -1,5 +1,7 @@ import struct -from typing import List, Tuple +from typing import List, Tuple, Union + +from google.protobuf.internal.containers import RepeatedScalarFieldContainer from feast.protos.feast.types.EntityKey_pb2 import EntityKey as EntityKeyProto from feast.protos.feast.types.Value_pb2 import Value as ValueProto @@ -163,3 +165,16 @@ def get_list_val_str(val): if val.HasField(accept_type): return str(getattr(val, accept_type).val) return None + + +def serialize_f32( + vector: Union[RepeatedScalarFieldContainer[float], List[float]], vector_length: int +) -> bytes: + """serializes a list of floats into a compact "raw bytes" format""" + return struct.pack(f"{vector_length}f", *vector) + + +def deserialize_f32(byte_vector: bytes, vector_length: int) -> List[float]: + """deserializes a list of floats from a compact "raw bytes" format""" + num_floats = vector_length // 4 # 4 bytes per float + return list(struct.unpack(f"{num_floats}f", byte_vector)) diff --git a/sdk/python/feast/infra/offline_stores/bigquery.py b/sdk/python/feast/infra/offline_stores/bigquery.py index 23f80d79ff2..f0516b594ee 100644 --- a/sdk/python/feast/infra/offline_stores/bigquery.py +++ b/sdk/python/feast/infra/offline_stores/bigquery.py @@ -901,7 +901,10 @@ def arrow_schema_to_bq_schema(arrow_schema: pyarrow.Schema) -> List[SchemaField] {{ featureview.created_timestamp_column ~ ' as created_timestamp,' if featureview.created_timestamp_column else '' }} {{ featureview.entity_selections | join(', ')}}{% if featureview.entity_selections %},{% else %}{% endif %} {% for feature in featureview.features %} - {{ feature }} as {% if full_feature_names %}{{ featureview.name }}__{{featureview.field_mapping.get(feature, feature)}}{% else %}{{ featureview.field_mapping.get(feature, feature) }}{% endif %}{% if loop.last %}{% else %}, {% endif %} + {{ feature | backticks }} as {% if full_feature_names %} + {{ featureview.name }}__{{featureview.field_mapping.get(feature, feature)}}{% else %} + {{ featureview.field_mapping.get(feature, feature) | backticks }}{% endif %} + {% if loop.last %}{% else %}, {% endif %} {% endfor %} FROM {{ featureview.table_subquery }} WHERE {{ featureview.timestamp_field }} <= '{{ featureview.max_event_timestamp }}' @@ -995,14 +998,14 @@ def arrow_schema_to_bq_schema(arrow_schema: pyarrow.Schema) -> List[SchemaField] The entity_dataframe dataset being our source of truth here. */ -SELECT {{ final_output_feature_names | join(', ')}} +SELECT {{ final_output_feature_names | backticks | join(', ')}} FROM entity_dataframe {% for featureview in featureviews %} LEFT JOIN ( SELECT {{featureview.name}}__entity_row_unique_id {% for feature in featureview.features %} - ,{% if full_feature_names %}{{ featureview.name }}__{{featureview.field_mapping.get(feature, feature)}}{% else %}{{ featureview.field_mapping.get(feature, feature) }}{% endif %} + ,{% if full_feature_names %}{{ featureview.name }}__{{featureview.field_mapping.get(feature, feature)}}{% else %}{{ featureview.field_mapping.get(feature, feature) | backticks }}{% endif %} {% endfor %} FROM {{ featureview.name }}__cleaned ) USING ({{featureview.name}}__entity_row_unique_id) diff --git a/sdk/python/feast/infra/offline_stores/contrib/spark_offline_store/spark.py b/sdk/python/feast/infra/offline_stores/contrib/spark_offline_store/spark.py index aeb9e3cd68b..4b501886327 100644 --- a/sdk/python/feast/infra/offline_stores/contrib/spark_offline_store/spark.py +++ b/sdk/python/feast/infra/offline_stores/contrib/spark_offline_store/spark.py @@ -99,6 +99,8 @@ def pull_latest_from_table_or_query( fields_as_string = ", ".join(fields_with_aliases) aliases_as_string = ", ".join(aliases) + date_partition_column = data_source.date_partition_column + start_date_str = _format_datetime(start_date) end_date_str = _format_datetime(end_date) query = f""" @@ -109,7 +111,7 @@ def pull_latest_from_table_or_query( SELECT {fields_as_string}, ROW_NUMBER() OVER({partition_by_join_key_string} ORDER BY {timestamp_desc_string}) AS feast_row_ FROM {from_expression} t1 - WHERE {timestamp_field} BETWEEN TIMESTAMP('{start_date_str}') AND TIMESTAMP('{end_date_str}') + WHERE {timestamp_field} BETWEEN TIMESTAMP('{start_date_str}') AND TIMESTAMP('{end_date_str}'){" AND "+date_partition_column+" >= '"+start_date.strftime('%Y-%m-%d')+"' AND "+date_partition_column+" <= '"+end_date.strftime('%Y-%m-%d')+"' " if date_partition_column != "" and date_partition_column is not None else ''} ) t2 WHERE feast_row_ = 1 """ @@ -641,8 +643,15 @@ def _cast_data_frame( {% endfor %} FROM {{ featureview.table_subquery }} WHERE {{ featureview.timestamp_field }} <= '{{ featureview.max_event_timestamp }}' + {% if featureview.date_partition_column != "" and featureview.date_partition_column is not none %} + AND {{ featureview.date_partition_column }} <= '{{ featureview.max_event_timestamp[:10] }}' + {% endif %} + {% if featureview.ttl == 0 %}{% else %} AND {{ featureview.timestamp_field }} >= '{{ featureview.min_event_timestamp }}' + {% if featureview.date_partition_column != "" and featureview.date_partition_column is not none %} + AND {{ featureview.date_partition_column }} >= '{{ featureview.min_event_timestamp[:10] }}' + {% endif %} {% endif %} ), diff --git a/sdk/python/feast/infra/offline_stores/contrib/spark_offline_store/spark_source.py b/sdk/python/feast/infra/offline_stores/contrib/spark_offline_store/spark_source.py index 209e3b87e8b..7ad331239ff 100644 --- a/sdk/python/feast/infra/offline_stores/contrib/spark_offline_store/spark_source.py +++ b/sdk/python/feast/infra/offline_stores/contrib/spark_offline_store/spark_source.py @@ -45,6 +45,7 @@ def __init__( tags: Optional[Dict[str, str]] = None, owner: Optional[str] = "", timestamp_field: Optional[str] = None, + date_partition_column: Optional[str] = None, ): """Creates a SparkSource object. @@ -64,6 +65,8 @@ def __init__( maintainer. timestamp_field: Event timestamp field used for point-in-time joins of feature values. + date_partition_column: The column to partition the data on for faster + retrieval. This is useful for large tables and will limit the number ofi """ # If no name, use the table as the default name. if name is None and table is None: @@ -77,6 +80,7 @@ def __init__( created_timestamp_column=created_timestamp_column, field_mapping=field_mapping, description=description, + date_partition_column=date_partition_column, tags=tags, owner=owner, ) @@ -135,6 +139,7 @@ def from_proto(data_source: DataSourceProto) -> Any: query=spark_options.query, path=spark_options.path, file_format=spark_options.file_format, + date_partition_column=data_source.date_partition_column, timestamp_field=data_source.timestamp_field, created_timestamp_column=data_source.created_timestamp_column, description=data_source.description, @@ -148,6 +153,7 @@ def to_proto(self) -> DataSourceProto: type=DataSourceProto.BATCH_SPARK, data_source_class_type="feast.infra.offline_stores.contrib.spark_offline_store.spark_source.SparkSource", field_mapping=self.field_mapping, + date_partition_column=self.date_partition_column, spark_options=self.spark_options.to_proto(), description=self.description, tags=self.tags, diff --git a/sdk/python/feast/infra/offline_stores/offline_utils.py b/sdk/python/feast/infra/offline_stores/offline_utils.py index 2d4fa268e40..2076f977acc 100644 --- a/sdk/python/feast/infra/offline_stores/offline_utils.py +++ b/sdk/python/feast/infra/offline_stores/offline_utils.py @@ -186,7 +186,9 @@ def build_point_in_time_query( full_feature_names: bool = False, ) -> str: """Build point-in-time query between each feature view table and the entity dataframe for Bigquery and Redshift""" - template = Environment(loader=BaseLoader()).from_string(source=query_template) + env = Environment(loader=BaseLoader()) + env.filters["backticks"] = enclose_in_backticks + template = env.from_string(source=query_template) final_output_feature_names = list(entity_df_columns) final_output_feature_names.extend( @@ -252,3 +254,11 @@ def get_pyarrow_schema_from_batch_source( column_names.append(column_name) return pa.schema(pa_schema), column_names + + +def enclose_in_backticks(value): + # Check if the input is a list + if isinstance(value, list): + return [f"`{v}`" for v in value] + else: + return f"`{value}`" diff --git a/sdk/python/feast/infra/offline_stores/remote.py b/sdk/python/feast/infra/offline_stores/remote.py index 6f26e06c6ba..d11fb4673db 100644 --- a/sdk/python/feast/infra/offline_stores/remote.py +++ b/sdk/python/feast/infra/offline_stores/remote.py @@ -74,7 +74,7 @@ def build_arrow_flight_client( scheme: str, host: str, port, auth_config: AuthConfig, cert: str = "" ): arrow_scheme = "grpc+tcp" - if cert: + if scheme == "https": logger.info( "Scheme is https so going to connect offline server in SSL(TLS) mode." ) diff --git a/sdk/python/feast/infra/offline_stores/snowflake.py b/sdk/python/feast/infra/offline_stores/snowflake.py index 3d23682769b..101685cec6f 100644 --- a/sdk/python/feast/infra/offline_stores/snowflake.py +++ b/sdk/python/feast/infra/offline_stores/snowflake.py @@ -716,8 +716,8 @@ def _get_entity_df_event_timestamp_range( MULTIPLE_FEATURE_VIEW_POINT_IN_TIME_JOIN = """ /* - Compute a deterministic hash for the `left_table_query_string` that will be used throughout - all the logic as the field to GROUP BY the data + 0. Compute a deterministic hash for the `left_table_query_string` that will be used throughout + all the logic as the field to GROUP BY the data. */ WITH "entity_dataframe" AS ( SELECT *, @@ -739,6 +739,10 @@ def _get_entity_df_event_timestamp_range( {% for featureview in featureviews %} +/* + 1. Only select the required columns with entities of the featureview. +*/ + "{{ featureview.name }}__entity_dataframe" AS ( SELECT {{ featureview.entities | map('tojson') | join(', ')}}{% if featureview.entities %},{% else %}{% endif %} @@ -752,20 +756,7 @@ def _get_entity_df_event_timestamp_range( ), /* - This query template performs the point-in-time correctness join for a single feature set table - to the provided entity table. - - 1. We first join the current feature_view to the entity dataframe that has been passed. - This JOIN has the following logic: - - For each row of the entity dataframe, only keep the rows where the `timestamp_field` - is less than the one provided in the entity dataframe - - If there a TTL for the current feature_view, also keep the rows where the `timestamp_field` - is higher the the one provided minus the TTL - - For each row, Join on the entity key and retrieve the `entity_row_unique_id` that has been - computed previously - - The output of this CTE will contain all the necessary information and already filtered out most - of the data that is not relevant. +2. Use subquery to prepare event_timestamp, created_timestamp, entity columns and feature columns. */ "{{ featureview.name }}__subquery" AS ( @@ -777,94 +768,61 @@ def _get_entity_df_event_timestamp_range( "{{ feature }}" as {% if full_feature_names %}"{{ featureview.name }}__{{featureview.field_mapping.get(feature, feature)}}"{% else %}"{{ featureview.field_mapping.get(feature, feature) }}"{% endif %}{% if loop.last %}{% else %}, {% endif %} {% endfor %} FROM {{ featureview.table_subquery }} - WHERE "{{ featureview.timestamp_field }}" <= '{{ featureview.max_event_timestamp }}' - {% if featureview.ttl == 0 %}{% else %} - AND "{{ featureview.timestamp_field }}" >= '{{ featureview.min_event_timestamp }}' - {% endif %} -), - -"{{ featureview.name }}__base" AS ( - SELECT - "subquery".*, - "entity_dataframe"."entity_timestamp", - "entity_dataframe"."{{featureview.name}}__entity_row_unique_id" - FROM "{{ featureview.name }}__subquery" AS "subquery" - INNER JOIN "{{ featureview.name }}__entity_dataframe" AS "entity_dataframe" - ON TRUE - AND "subquery"."event_timestamp" <= "entity_dataframe"."entity_timestamp" - - {% if featureview.ttl == 0 %}{% else %} - AND "subquery"."event_timestamp" >= TIMESTAMPADD(second,-{{ featureview.ttl }},"entity_dataframe"."entity_timestamp") - {% endif %} - - {% for entity in featureview.entities %} - AND "subquery"."{{ entity }}" = "entity_dataframe"."{{ entity }}" - {% endfor %} ), /* - 2. If the `created_timestamp_column` has been set, we need to - deduplicate the data first. This is done by calculating the - `MAX(created_at_timestamp)` for each event_timestamp. - We then join the data on the next CTE +3. If the `created_timestamp_column` has been set, we need to +deduplicate the data first. This is done by calculating the +`MAX(created_at_timestamp)` for each event_timestamp and joining back on the subquery. +Otherwise, the ASOF JOIN can have unstable side effects +https://docs.snowflake.com/en/sql-reference/constructs/asof-join#expected-behavior-when-ties-exist-in-the-right-table */ + {% if featureview.created_timestamp_column %} "{{ featureview.name }}__dedup" AS ( - SELECT - "{{featureview.name}}__entity_row_unique_id", - "event_timestamp", - MAX("created_timestamp") AS "created_timestamp" - FROM "{{ featureview.name }}__base" - GROUP BY "{{featureview.name}}__entity_row_unique_id", "event_timestamp" + SELECT * + FROM "{{ featureview.name }}__subquery" + INNER JOIN ( + SELECT + {{ featureview.entities | map('tojson') | join(', ')}}{% if featureview.entities %},{% else %}{% endif %} + "event_timestamp", + MAX("created_timestamp") AS "created_timestamp" + FROM "{{ featureview.name }}__subquery" + GROUP BY {{ featureview.entities | map('tojson') | join(', ')}}{% if featureview.entities %},{% else %}{% endif %} "event_timestamp" + ) + USING({{ featureview.entities | map('tojson') | join(', ')}}{% if featureview.entities %},{% else %}{% endif %} "event_timestamp", "created_timestamp") ), {% endif %} /* - 3. The data has been filtered during the first CTE "*__base" - Thus we only need to compute the latest timestamp of each feature. +4. Make ASOF JOIN of deduplicated feature CTE on reduced entity dataframe. */ -"{{ featureview.name }}__latest" AS ( + +"{{ featureview.name }}__asof_join" AS ( SELECT - "event_timestamp", - {% if featureview.created_timestamp_column %}"created_timestamp",{% endif %} - "{{featureview.name}}__entity_row_unique_id" - FROM - ( - SELECT *, - ROW_NUMBER() OVER( - PARTITION BY "{{featureview.name}}__entity_row_unique_id" - ORDER BY "event_timestamp" DESC{% if featureview.created_timestamp_column %},"created_timestamp" DESC{% endif %} - ) AS "row_number" - FROM "{{ featureview.name }}__base" - {% if featureview.created_timestamp_column %} - INNER JOIN "{{ featureview.name }}__dedup" - USING ("{{featureview.name}}__entity_row_unique_id", "event_timestamp", "created_timestamp") - {% endif %} - ) - WHERE "row_number" = 1 + e.*, + v.* + FROM "{{ featureview.name }}__entity_dataframe" e + ASOF JOIN {% if featureview.created_timestamp_column %}"{{ featureview.name }}__dedup"{% else %}"{{ featureview.name }}__subquery"{% endif %} v + MATCH_CONDITION (e."entity_timestamp" >= v."event_timestamp") + {% if featureview.entities %} USING({{ featureview.entities | map('tojson') | join(', ')}}) {% endif %} ), /* - 4. Once we know the latest value of each feature for a given timestamp, - we can join again the data back to the original "base" dataset +5. If TTL is configured filter the CTE to remove rows where the feature values are older than the configured ttl. */ -"{{ featureview.name }}__cleaned" AS ( - SELECT "base".* - FROM "{{ featureview.name }}__base" AS "base" - INNER JOIN "{{ featureview.name }}__latest" - USING( - "{{featureview.name}}__entity_row_unique_id", - "event_timestamp" - {% if featureview.created_timestamp_column %} - ,"created_timestamp" - {% endif %} - ) -){% if loop.last %}{% else %}, {% endif %} +"{{ featureview.name }}__ttl" AS ( + SELECT * + FROM "{{ featureview.name }}__asof_join" + {% if featureview.ttl == 0 %}{% else %} + WHERE "event_timestamp" >= TIMESTAMPADD(second,-{{ featureview.ttl }},"entity_timestamp") + {% endif %} +){% if loop.last %}{% else %}, {% endif %} {% endfor %} /* - Joins the outputs of multiple time travel joins to a single table. + Join the outputs of multiple time travel joins to a single table. The entity_dataframe dataset being our source of truth here. */ @@ -877,7 +835,7 @@ def _get_entity_df_event_timestamp_range( {% for feature in featureview.features %} ,{% if full_feature_names %}"{{ featureview.name }}__{{featureview.field_mapping.get(feature, feature)}}"{% else %}"{{ featureview.field_mapping.get(feature, feature) }}"{% endif %} {% endfor %} - FROM "{{ featureview.name }}__cleaned" -) "{{ featureview.name }}__cleaned" USING ("{{featureview.name}}__entity_row_unique_id") + FROM "{{ featureview.name }}__ttl" +) "{{ featureview.name }}__ttl" USING ("{{featureview.name}}__entity_row_unique_id") {% endfor %} """ diff --git a/sdk/python/feast/infra/online_stores/elasticsearch_online_store/elasticsearch.py b/sdk/python/feast/infra/online_stores/elasticsearch_online_store/elasticsearch.py index 0152ca330c9..af328141520 100644 --- a/sdk/python/feast/infra/online_stores/elasticsearch_online_store/elasticsearch.py +++ b/sdk/python/feast/infra/online_stores/elasticsearch_online_store/elasticsearch.py @@ -213,7 +213,8 @@ def retrieve_online_documents( self, config: RepoConfig, table: FeatureView, - requested_feature: str, + requested_feature: Optional[str], + requested_features: Optional[List[str]], embedding: List[float], top_k: int, *args, diff --git a/sdk/python/feast/infra/online_stores/faiss_online_store.py b/sdk/python/feast/infra/online_stores/faiss_online_store.py index cc2e75800e6..fd4d6768abd 100644 --- a/sdk/python/feast/infra/online_stores/faiss_online_store.py +++ b/sdk/python/feast/infra/online_stores/faiss_online_store.py @@ -176,7 +176,8 @@ def retrieve_online_documents( self, config: RepoConfig, table: FeatureView, - requested_feature: str, + requested_feature: Optional[str], + requested_featres: Optional[List[str]], embedding: List[float], top_k: int, distance_metric: Optional[str] = None, diff --git a/sdk/python/feast/infra/online_stores/milvus_online_store/__init__.py b/sdk/python/feast/infra/online_stores/milvus_online_store/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/sdk/python/feast/infra/online_stores/milvus_online_store/milvus.py b/sdk/python/feast/infra/online_stores/milvus_online_store/milvus.py new file mode 100644 index 00000000000..7e840622a8a --- /dev/null +++ b/sdk/python/feast/infra/online_stores/milvus_online_store/milvus.py @@ -0,0 +1,469 @@ +from datetime import datetime +from pathlib import Path +from typing import Any, Callable, Dict, List, Literal, Optional, Sequence, Tuple, Union + +from pydantic import StrictStr +from pymilvus import ( + Collection, + CollectionSchema, + DataType, + FieldSchema, + MilvusClient, +) + +from feast import Entity +from feast.feature_view import FeatureView +from feast.infra.infra_object import InfraObject +from feast.infra.key_encoding_utils import ( + serialize_entity_key, +) +from feast.infra.online_stores.online_store import OnlineStore +from feast.infra.online_stores.vector_store import VectorStoreConfig +from feast.protos.feast.core.InfraObject_pb2 import InfraObject as InfraObjectProto +from feast.protos.feast.core.Registry_pb2 import Registry as RegistryProto +from feast.protos.feast.types.EntityKey_pb2 import EntityKey as EntityKeyProto +from feast.protos.feast.types.Value_pb2 import Value as ValueProto +from feast.repo_config import FeastConfigBaseModel, RepoConfig +from feast.type_map import PROTO_VALUE_TO_VALUE_TYPE_MAP +from feast.types import ( + VALUE_TYPES_TO_FEAST_TYPES, + Array, + ComplexFeastType, + PrimitiveFeastType, + ValueType, +) +from feast.utils import ( + _build_retrieve_online_document_record, + _serialize_vector_to_float_list, + to_naive_utc, +) + +PROTO_TO_MILVUS_TYPE_MAPPING: Dict[ValueType, DataType] = { + PROTO_VALUE_TO_VALUE_TYPE_MAP["bytes_val"]: DataType.VARCHAR, + PROTO_VALUE_TO_VALUE_TYPE_MAP["bool_val"]: DataType.BOOL, + PROTO_VALUE_TO_VALUE_TYPE_MAP["string_val"]: DataType.VARCHAR, + PROTO_VALUE_TO_VALUE_TYPE_MAP["float_val"]: DataType.FLOAT, + PROTO_VALUE_TO_VALUE_TYPE_MAP["double_val"]: DataType.DOUBLE, + PROTO_VALUE_TO_VALUE_TYPE_MAP["int32_val"]: DataType.INT32, + PROTO_VALUE_TO_VALUE_TYPE_MAP["int64_val"]: DataType.INT64, + PROTO_VALUE_TO_VALUE_TYPE_MAP["float_list_val"]: DataType.FLOAT_VECTOR, + PROTO_VALUE_TO_VALUE_TYPE_MAP["int32_list_val"]: DataType.FLOAT_VECTOR, + PROTO_VALUE_TO_VALUE_TYPE_MAP["int64_list_val"]: DataType.FLOAT_VECTOR, + PROTO_VALUE_TO_VALUE_TYPE_MAP["double_list_val"]: DataType.FLOAT_VECTOR, + PROTO_VALUE_TO_VALUE_TYPE_MAP["bool_list_val"]: DataType.BINARY_VECTOR, +} + +FEAST_PRIMITIVE_TO_MILVUS_TYPE_MAPPING: Dict[ + Union[PrimitiveFeastType, Array, ComplexFeastType], DataType +] = {} + +for value_type, feast_type in VALUE_TYPES_TO_FEAST_TYPES.items(): + if isinstance(feast_type, PrimitiveFeastType): + milvus_type = PROTO_TO_MILVUS_TYPE_MAPPING.get(value_type) + if milvus_type: + FEAST_PRIMITIVE_TO_MILVUS_TYPE_MAPPING[feast_type] = milvus_type + elif isinstance(feast_type, Array): + base_type = feast_type.base_type + base_value_type = base_type.to_value_type() + if base_value_type in [ + ValueType.INT32, + ValueType.INT64, + ValueType.FLOAT, + ValueType.DOUBLE, + ]: + FEAST_PRIMITIVE_TO_MILVUS_TYPE_MAPPING[feast_type] = DataType.FLOAT_VECTOR + elif base_value_type == ValueType.STRING: + FEAST_PRIMITIVE_TO_MILVUS_TYPE_MAPPING[feast_type] = DataType.VARCHAR + elif base_value_type == ValueType.BOOL: + FEAST_PRIMITIVE_TO_MILVUS_TYPE_MAPPING[feast_type] = DataType.BINARY_VECTOR + + +class MilvusOnlineStoreConfig(FeastConfigBaseModel, VectorStoreConfig): + """ + Configuration for the Milvus online store. + NOTE: The class *must* end with the `OnlineStoreConfig` suffix. + """ + + type: Literal["milvus"] = "milvus" + path: Optional[StrictStr] = "data/online_store.db" + host: Optional[StrictStr] = "localhost" + port: Optional[int] = 19530 + index_type: Optional[str] = "FLAT" + metric_type: Optional[str] = "L2" + embedding_dim: Optional[int] = 128 + vector_enabled: Optional[bool] = True + nlist: Optional[int] = 128 + username: Optional[StrictStr] = "" + password: Optional[StrictStr] = "" + + +class MilvusOnlineStore(OnlineStore): + """ + Milvus implementation of the online store interface. + + Attributes: + _collections: Dictionary to cache Milvus collections. + """ + + client: Optional[MilvusClient] = None + _collections: Dict[str, Any] = {} + + def _get_db_path(self, config: RepoConfig) -> str: + assert ( + config.online_store.type == "milvus" + or config.online_store.type.endswith("MilvusOnlineStore") + ) + + if config.repo_path and not Path(config.online_store.path).is_absolute(): + db_path = str(config.repo_path / config.online_store.path) + else: + db_path = config.online_store.path + return db_path + + def _connect(self, config: RepoConfig) -> MilvusClient: + if not self.client: + if config.provider == "local": + db_path = self._get_db_path(config) + print(f"Connecting to Milvus in local mode using {db_path}") + self.client = MilvusClient(db_path) + else: + self.client = MilvusClient( + url=f"{config.online_store.host}:{config.online_store.port}", + token=f"{config.online_store.username}:{config.online_store.password}" + if config.online_store.username and config.online_store.password + else "", + ) + return self.client + + def _get_collection(self, config: RepoConfig, table: FeatureView) -> Dict[str, Any]: + self.client = self._connect(config) + collection_name = _table_id(config.project, table) + if collection_name not in self._collections: + # Create a composite key by combining entity fields + composite_key_name = ( + "_".join([field.name for field in table.entity_columns]) + "_pk" + ) + + fields = [ + FieldSchema( + name=composite_key_name, + dtype=DataType.VARCHAR, + max_length=512, + is_primary=True, + ), + FieldSchema(name="event_ts", dtype=DataType.INT64), + FieldSchema(name="created_ts", dtype=DataType.INT64), + ] + fields_to_exclude = [ + "event_ts", + "created_ts", + ] + fields_to_add = [f for f in table.schema if f.name not in fields_to_exclude] + for field in fields_to_add: + dtype = FEAST_PRIMITIVE_TO_MILVUS_TYPE_MAPPING.get(field.dtype) + if dtype: + if dtype == DataType.FLOAT_VECTOR: + fields.append( + FieldSchema( + name=field.name, + dtype=dtype, + dim=config.online_store.embedding_dim, + ) + ) + elif dtype == DataType.VARCHAR: + fields.append( + FieldSchema( + name=field.name, + dtype=dtype, + max_length=512, + ) + ) + else: + fields.append(FieldSchema(name=field.name, dtype=dtype)) + + schema = CollectionSchema( + fields=fields, description="Feast feature view data" + ) + collection_exists = self.client.has_collection( + collection_name=collection_name + ) + if not collection_exists: + self.client.create_collection( + collection_name=collection_name, + dimension=config.online_store.embedding_dim, + schema=schema, + ) + index_params = self.client.prepare_index_params() + for vector_field in schema.fields: + if vector_field.dtype in [ + DataType.FLOAT_VECTOR, + DataType.BINARY_VECTOR, + ]: + index_params.add_index( + collection_name=collection_name, + field_name=vector_field.name, + metric_type=config.online_store.metric_type, + index_type=config.online_store.index_type, + index_name=f"vector_index_{vector_field.name}", + params={"nlist": config.online_store.nlist}, + ) + self.client.create_index( + collection_name=collection_name, + index_params=index_params, + ) + else: + self.client.load_collection(collection_name) + self._collections[collection_name] = self.client.describe_collection( + collection_name + ) + return self._collections[collection_name] + + def online_write_batch( + self, + config: RepoConfig, + table: FeatureView, + data: List[ + Tuple[ + EntityKeyProto, + Dict[str, ValueProto], + datetime, + Optional[datetime], + ] + ], + progress: Optional[Callable[[int], Any]], + ) -> None: + self.client = self._connect(config) + collection = self._get_collection(config, table) + entity_batch_to_insert = [] + for entity_key, values_dict, timestamp, created_ts in data: + # need to construct the composite primary key also need to handle the fact that entities are a list + entity_key_str = serialize_entity_key( + entity_key, + entity_key_serialization_version=config.entity_key_serialization_version, + ).hex() + composite_key_name = ( + "_".join([str(value) for value in entity_key.join_keys]) + "_pk" + ) + timestamp_int = int(to_naive_utc(timestamp).timestamp() * 1e6) + created_ts_int = ( + int(to_naive_utc(created_ts).timestamp() * 1e6) if created_ts else 0 + ) + values_dict = _extract_proto_values_to_dict(values_dict) + entity_dict = _extract_proto_values_to_dict( + dict(zip(entity_key.join_keys, entity_key.entity_values)) + ) + values_dict.update(entity_dict) + + single_entity_record = { + composite_key_name: entity_key_str, + "event_ts": timestamp_int, + "created_ts": created_ts_int, + } + single_entity_record.update(values_dict) + entity_batch_to_insert.append(single_entity_record) + + if progress: + progress(1) + + self.client.insert( + collection_name=collection["collection_name"], + data=entity_batch_to_insert, + ) + + def online_read( + self, + config: RepoConfig, + table: FeatureView, + entity_keys: List[EntityKeyProto], + requested_features: Optional[List[str]] = None, + ) -> List[Tuple[Optional[datetime], Optional[Dict[str, ValueProto]]]]: + raise NotImplementedError + + def update( + self, + config: RepoConfig, + tables_to_delete: Sequence[FeatureView], + tables_to_keep: Sequence[FeatureView], + entities_to_delete: Sequence[Entity], + entities_to_keep: Sequence[Entity], + partial: bool, + ): + self.client = self._connect(config) + for table in tables_to_keep: + self._collections = self._get_collection(config, table) + + for table in tables_to_delete: + collection_name = _table_id(config.project, table) + if self._collections.get(collection_name, None): + self.client.drop_collection(collection_name) + self._collections.pop(collection_name, None) + + def plan( + self, config: RepoConfig, desired_registry_proto: RegistryProto + ) -> List[InfraObject]: + raise NotImplementedError + + def teardown( + self, + config: RepoConfig, + tables: Sequence[FeatureView], + entities: Sequence[Entity], + ): + self.client = self._connect(config) + for table in tables: + collection_name = _table_id(config.project, table) + if self._collections.get(collection_name, None): + self.client.drop_collection(collection_name) + self._collections.pop(collection_name, None) + + def retrieve_online_documents( + self, + config: RepoConfig, + table: FeatureView, + requested_feature: Optional[str], + requested_features: Optional[List[str]], + embedding: List[float], + top_k: int, + distance_metric: Optional[str] = None, + ) -> List[ + Tuple[ + Optional[datetime], + Optional[EntityKeyProto], + Optional[ValueProto], + Optional[ValueProto], + Optional[ValueProto], + ] + ]: + self.client = self._connect(config) + collection_name = _table_id(config.project, table) + collection = self._get_collection(config, table) + if not config.online_store.vector_enabled: + raise ValueError("Vector search is not enabled in the online store config") + + search_params = { + "metric_type": distance_metric or config.online_store.metric_type, + "params": {"nprobe": 10}, + } + expr = f"feature_name == '{requested_feature}'" + + composite_key_name = ( + "_".join([str(field.name) for field in table.entity_columns]) + "_pk" + ) + if requested_features: + features_str = ", ".join([f"'{f}'" for f in requested_features]) + expr += f" && feature_name in [{features_str}]" + + output_fields = ( + [composite_key_name] + + (requested_features if requested_features else []) + + ["created_ts", "event_ts"] + ) + assert all( + field in [f["name"] for f in collection["fields"]] + for field in output_fields + ), f"field(s) [{[field for field in output_fields if field not in [f['name'] for f in collection['fields']]]}] not found in collection schema" + # Note we choose the first vector field as the field to search on. Not ideal but it's something. + ann_search_field = None + for field in collection["fields"]: + if ( + field["type"] in [DataType.FLOAT_VECTOR, DataType.BINARY_VECTOR] + and field["name"] in output_fields + ): + ann_search_field = field["name"] + break + + self.client.load_collection(collection_name) + results = self.client.search( + collection_name=collection_name, + data=[embedding], + anns_field=ann_search_field, + search_params=search_params, + limit=top_k, + output_fields=output_fields, + ) + + result_list = [] + for hits in results: + for hit in hits: + single_record = {} + for field in output_fields: + single_record[field] = hit.get("entity", {}).get(field, None) + + entity_key_bytes = bytes.fromhex( + hit.get("entity", {}).get(composite_key_name, None) + ) + embedding = hit.get("entity", {}).get(ann_search_field) + serialized_embedding = _serialize_vector_to_float_list(embedding) + distance = hit.get("distance", None) + event_ts = datetime.fromtimestamp( + hit.get("entity", {}).get("event_ts") / 1e6 + ) + prepared_result = _build_retrieve_online_document_record( + entity_key_bytes, + # This may have a bug + serialized_embedding.SerializeToString(), + embedding, + distance, + event_ts, + config.entity_key_serialization_version, + ) + result_list.append(prepared_result) + return result_list + + +def _table_id(project: str, table: FeatureView) -> str: + return f"{project}_{table.name}" + + +def _extract_proto_values_to_dict(input_dict: Dict[str, Any]) -> Dict[str, Any]: + numeric_vector_list_types = [ + k + for k in PROTO_VALUE_TO_VALUE_TYPE_MAP.keys() + if k is not None and "list" in k and "string" not in k + ] + output_dict = {} + for feature_name, feature_values in input_dict.items(): + for proto_val_type in PROTO_VALUE_TO_VALUE_TYPE_MAP: + if feature_values.HasField(proto_val_type): + if proto_val_type in numeric_vector_list_types: + vector_values = getattr(feature_values, proto_val_type).val + else: + vector_values = getattr(feature_values, proto_val_type) + output_dict[feature_name] = vector_values + return output_dict + + +class MilvusTable(InfraObject): + """ + A Milvus collection managed by Feast. + + Attributes: + host: The host of the Milvus server. + port: The port of the Milvus server. + name: The name of the collection. + """ + + host: str + port: int + + def __init__(self, host: str, port: int, name: str): + super().__init__(name) + self.host = host + self.port = port + self._connect() + + def _connect(self): + raise NotImplementedError + + def to_infra_object_proto(self) -> InfraObjectProto: + # Implement serialization if needed + raise NotImplementedError + + def update(self): + # Implement update logic if needed + raise NotImplementedError + + def teardown(self): + collection = Collection(name=self.name) + if collection.exists(): + collection.drop() diff --git a/sdk/python/feast/infra/online_stores/milvus_online_store/milvus_repo_configuration.py b/sdk/python/feast/infra/online_stores/milvus_online_store/milvus_repo_configuration.py new file mode 100644 index 00000000000..8e8402862cb --- /dev/null +++ b/sdk/python/feast/infra/online_stores/milvus_online_store/milvus_repo_configuration.py @@ -0,0 +1,12 @@ +from tests.integration.feature_repos.integration_test_repo_config import ( + IntegrationTestRepoConfig, +) +from tests.integration.feature_repos.universal.online_store.milvus import ( + MilvusOnlineStoreCreator, +) + +FULL_REPO_CONFIGS = [ + IntegrationTestRepoConfig( + online_store="milvus", online_store_creator=MilvusOnlineStoreCreator + ), +] diff --git a/sdk/python/feast/infra/online_stores/online_store.py b/sdk/python/feast/infra/online_stores/online_store.py index 789885f82bc..be3128562dc 100644 --- a/sdk/python/feast/infra/online_stores/online_store.py +++ b/sdk/python/feast/infra/online_stores/online_store.py @@ -390,7 +390,8 @@ def retrieve_online_documents( self, config: RepoConfig, table: FeatureView, - requested_feature: str, + requested_feature: Optional[str], + requested_features: Optional[List[str]], embedding: List[float], top_k: int, distance_metric: Optional[str] = None, @@ -411,6 +412,7 @@ def retrieve_online_documents( config: The config for the current feature store. table: The feature view whose feature values should be read. requested_feature: The name of the feature whose embeddings should be used for retrieval. + requested_features: The list of features whose embeddings should be used for retrieval. embedding: The embeddings to use for retrieval. top_k: The number of documents to retrieve. @@ -419,6 +421,10 @@ def retrieve_online_documents( where the first item is the event timestamp for the row, and the second item is a dict of feature name to embeddings. """ + if not requested_feature and not requested_features: + raise ValueError( + "Either requested_feature or requested_features must be specified" + ) raise NotImplementedError( f"Online store {self.__class__.__name__} does not support online retrieval" ) diff --git a/sdk/python/feast/infra/online_stores/postgres_online_store/postgres.py b/sdk/python/feast/infra/online_stores/postgres_online_store/postgres.py index 7c099c80ecc..f43247a5457 100644 --- a/sdk/python/feast/infra/online_stores/postgres_online_store/postgres.py +++ b/sdk/python/feast/infra/online_stores/postgres_online_store/postgres.py @@ -347,7 +347,8 @@ def retrieve_online_documents( self, config: RepoConfig, table: FeatureView, - requested_feature: str, + requested_feature: Optional[str], + requested_features: Optional[List[str]], embedding: List[float], top_k: int, distance_metric: Optional[str] = "L2", @@ -366,6 +367,7 @@ def retrieve_online_documents( config: Feast configuration object table: FeatureView object as the table to search requested_feature: The requested feature as the column to search + requested_features: The list of features whose embeddings should be used for retrieval. embedding: The query embedding to search for top_k: The number of items to return distance_metric: The distance metric to use for the search.G diff --git a/sdk/python/feast/infra/online_stores/qdrant_online_store/qdrant.py b/sdk/python/feast/infra/online_stores/qdrant_online_store/qdrant.py index 074c52ba5e8..cdbef95348d 100644 --- a/sdk/python/feast/infra/online_stores/qdrant_online_store/qdrant.py +++ b/sdk/python/feast/infra/online_stores/qdrant_online_store/qdrant.py @@ -248,7 +248,8 @@ def retrieve_online_documents( self, config: RepoConfig, table: FeatureView, - requested_feature: str, + requested_feature: Optional[str], + requested_features: Optional[List[str]], embedding: List[float], top_k: int, distance_metric: Optional[str] = "cosine", diff --git a/sdk/python/feast/infra/online_stores/sqlite.py b/sdk/python/feast/infra/online_stores/sqlite.py index e2eeb038d00..23b4f6db3a3 100644 --- a/sdk/python/feast/infra/online_stores/sqlite.py +++ b/sdk/python/feast/infra/online_stores/sqlite.py @@ -15,19 +15,20 @@ import logging import os import sqlite3 -import struct import sys from datetime import date, datetime from pathlib import Path -from typing import Any, Callable, Dict, List, Literal, Optional, Sequence, Tuple, Union +from typing import Any, Callable, Dict, List, Literal, Optional, Sequence, Tuple -from google.protobuf.internal.containers import RepeatedScalarFieldContainer from pydantic import StrictStr from feast import Entity from feast.feature_view import FeatureView from feast.infra.infra_object import SQLITE_INFRA_OBJECT_CLASS_TYPE, InfraObject -from feast.infra.key_encoding_utils import serialize_entity_key +from feast.infra.key_encoding_utils import ( + serialize_entity_key, + serialize_f32, +) from feast.infra.online_stores.online_store import OnlineStore from feast.infra.online_stores.vector_store import VectorStoreConfig from feast.protos.feast.core.InfraObject_pb2 import InfraObject as InfraObjectProto @@ -330,7 +331,8 @@ def retrieve_online_documents( self, config: RepoConfig, table: FeatureView, - requested_feature: str, + requested_feature: Optional[str], + requested_featuers: Optional[List[str]], embedding: List[float], top_k: int, distance_metric: Optional[str] = None, @@ -432,6 +434,7 @@ def retrieve_online_documents( _build_retrieve_online_document_record( entity_key, string_value if string_value else b"", + # This may be a bug embedding, distance, event_ts, @@ -459,19 +462,6 @@ def _table_id(project: str, table: FeatureView) -> str: return f"{project}_{table.name}" -def serialize_f32( - vector: Union[RepeatedScalarFieldContainer[float], List[float]], vector_length: int -) -> bytes: - """serializes a list of floats into a compact "raw bytes" format""" - return struct.pack(f"{vector_length}f", *vector) - - -def deserialize_f32(byte_vector: bytes, vector_length: int) -> List[float]: - """deserializes a list of floats from a compact "raw bytes" format""" - num_floats = vector_length // 4 # 4 bytes per float - return list(struct.unpack(f"{num_floats}f", byte_vector)) - - class SqliteTable(InfraObject): """ A Sqlite table managed by Feast. diff --git a/sdk/python/feast/infra/passthrough_provider.py b/sdk/python/feast/infra/passthrough_provider.py index 215b175eb2e..57aa122ae8a 100644 --- a/sdk/python/feast/infra/passthrough_provider.py +++ b/sdk/python/feast/infra/passthrough_provider.py @@ -294,7 +294,8 @@ def retrieve_online_documents( self, config: RepoConfig, table: FeatureView, - requested_feature: str, + requested_feature: Optional[str], + requested_features: Optional[List[str]], query: List[float], top_k: int, distance_metric: Optional[str] = None, @@ -305,6 +306,7 @@ def retrieve_online_documents( config, table, requested_feature, + requested_features, query, top_k, distance_metric, diff --git a/sdk/python/feast/infra/provider.py b/sdk/python/feast/infra/provider.py index 8351f389ad9..efc806ba2f0 100644 --- a/sdk/python/feast/infra/provider.py +++ b/sdk/python/feast/infra/provider.py @@ -419,7 +419,8 @@ def retrieve_online_documents( self, config: RepoConfig, table: FeatureView, - requested_feature: str, + requested_feature: Optional[str], + requested_features: Optional[List[str]], query: List[float], top_k: int, distance_metric: Optional[str] = None, @@ -440,6 +441,7 @@ def retrieve_online_documents( config: The config for the current feature store. table: The feature view whose embeddings should be searched. requested_feature: the requested document feature name. + requested_features: the requested document feature names. query: The query embedding to search for. top_k: The number of documents to return. diff --git a/sdk/python/feast/infra/registry/remote.py b/sdk/python/feast/infra/registry/remote.py index 6cc80d5dad1..590c0454b73 100644 --- a/sdk/python/feast/infra/registry/remote.py +++ b/sdk/python/feast/infra/registry/remote.py @@ -1,3 +1,4 @@ +import os from datetime import datetime from pathlib import Path from typing import List, Optional, Union @@ -59,6 +60,12 @@ class RemoteRegistryConfig(RegistryConfig): """ str: Path to the public certificate when the registry server starts in TLS(SSL) mode. This may be needed if the registry server started with a self-signed certificate, typically this file ends with `*.crt`, `*.cer`, or `*.pem`. If registry_type is 'remote', then this configuration is needed to connect to remote registry server in TLS mode. If the remote registry started in non-tls mode then this configuration is not needed.""" + is_tls: bool = False + """ bool: Set to `True` if you plan to connect to a registry server running in TLS (SSL) mode. + If you intend to add the public certificate to the trust store instead of passing it via the `cert` parameter, this field must be set to `True`. + If you are planning to add the public certificate as part of the trust store instead of passing it as a `cert` parameters then setting this field to `true` is mandatory. + """ + class RemoteRegistry(BaseRegistry): def __init__( @@ -70,20 +77,32 @@ def __init__( ): self.auth_config = auth_config assert isinstance(registry_config, RemoteRegistryConfig) - if registry_config.cert: - with open(registry_config.cert, "rb") as cert_file: - trusted_certs = cert_file.read() - tls_credentials = grpc.ssl_channel_credentials( - root_certificates=trusted_certs - ) - self.channel = grpc.secure_channel(registry_config.path, tls_credentials) - else: - self.channel = grpc.insecure_channel(registry_config.path) + self.channel = self._create_grpc_channel(registry_config) auth_header_interceptor = GrpcClientAuthHeaderInterceptor(auth_config) self.channel = grpc.intercept_channel(self.channel, auth_header_interceptor) self.stub = RegistryServer_pb2_grpc.RegistryServerStub(self.channel) + def _create_grpc_channel(self, registry_config): + assert isinstance(registry_config, RemoteRegistryConfig) + if registry_config.cert or registry_config.is_tls: + cafile = os.getenv("SSL_CERT_FILE") or os.getenv("REQUESTS_CA_BUNDLE") + if not cafile and not registry_config.cert: + raise EnvironmentError( + "SSL_CERT_FILE or REQUESTS_CA_BUNDLE environment variable must be set to use secure TLS or set the cert parameter in feature_Store.yaml file under remote registry configuration." + ) + with open( + registry_config.cert if registry_config.cert else cafile, "rb" + ) as cert_file: + trusted_certs = cert_file.read() + tls_credentials = grpc.ssl_channel_credentials( + root_certificates=trusted_certs + ) + return grpc.secure_channel(registry_config.path, tls_credentials) + else: + # Create an insecure gRPC channel + return grpc.insecure_channel(registry_config.path) + def close(self): if self.channel: self.channel.close() diff --git a/sdk/python/feast/offline_server.py b/sdk/python/feast/offline_server.py index 8774dea8aed..1b714a45c7e 100644 --- a/sdk/python/feast/offline_server.py +++ b/sdk/python/feast/offline_server.py @@ -45,7 +45,6 @@ def __init__( location: str, host: str = "localhost", tls_certificates: List = [], - verify_client=False, **kwargs, ): super(OfflineServer, self).__init__( @@ -54,7 +53,7 @@ def __init__( str_to_auth_manager_type(store.config.auth_config.type) ), tls_certificates=tls_certificates, - verify_client=verify_client, + verify_client=False, # this is needed for when we don't need mTLS **kwargs, ) self._location = location @@ -568,7 +567,6 @@ def start_server( port: int, tls_key_path: str = "", tls_cert_path: str = "", - tls_verify_client: bool = True, ): _init_auth_manager(store) @@ -591,7 +589,6 @@ def start_server( location=location, host=host, tls_certificates=tls_certificates, - verify_client=tls_verify_client, ) try: logger.info(f"Offline store server serving at: {location}") diff --git a/sdk/python/feast/protos/feast/core/DataSource_pb2.py b/sdk/python/feast/protos/feast/core/DataSource_pb2.py index b58c33a3830..68bee8d7609 100644 --- a/sdk/python/feast/protos/feast/core/DataSource_pb2.py +++ b/sdk/python/feast/protos/feast/core/DataSource_pb2.py @@ -19,7 +19,7 @@ from feast.protos.feast.core import Feature_pb2 as feast_dot_core_dot_Feature__pb2 -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x1b\x66\x65\x61st/core/DataSource.proto\x12\nfeast.core\x1a\x1egoogle/protobuf/duration.proto\x1a\x1fgoogle/protobuf/timestamp.proto\x1a\x1b\x66\x65\x61st/core/DataFormat.proto\x1a\x17\x66\x65\x61st/types/Value.proto\x1a\x18\x66\x65\x61st/core/Feature.proto\"\xc0\x16\n\nDataSource\x12\x0c\n\x04name\x18\x14 \x01(\t\x12\x0f\n\x07project\x18\x15 \x01(\t\x12\x13\n\x0b\x64\x65scription\x18\x17 \x01(\t\x12.\n\x04tags\x18\x18 \x03(\x0b\x32 .feast.core.DataSource.TagsEntry\x12\r\n\x05owner\x18\x19 \x01(\t\x12/\n\x04type\x18\x01 \x01(\x0e\x32!.feast.core.DataSource.SourceType\x12?\n\rfield_mapping\x18\x02 \x03(\x0b\x32(.feast.core.DataSource.FieldMappingEntry\x12\x17\n\x0ftimestamp_field\x18\x03 \x01(\t\x12\x1d\n\x15\x64\x61te_partition_column\x18\x04 \x01(\t\x12 \n\x18\x63reated_timestamp_column\x18\x05 \x01(\t\x12\x1e\n\x16\x64\x61ta_source_class_type\x18\x11 \x01(\t\x12,\n\x0c\x62\x61tch_source\x18\x1a \x01(\x0b\x32\x16.feast.core.DataSource\x12/\n\x04meta\x18\x32 \x01(\x0b\x32!.feast.core.DataSource.SourceMeta\x12:\n\x0c\x66ile_options\x18\x0b \x01(\x0b\x32\".feast.core.DataSource.FileOptionsH\x00\x12\x42\n\x10\x62igquery_options\x18\x0c \x01(\x0b\x32&.feast.core.DataSource.BigQueryOptionsH\x00\x12<\n\rkafka_options\x18\r \x01(\x0b\x32#.feast.core.DataSource.KafkaOptionsH\x00\x12@\n\x0fkinesis_options\x18\x0e \x01(\x0b\x32%.feast.core.DataSource.KinesisOptionsH\x00\x12\x42\n\x10redshift_options\x18\x0f \x01(\x0b\x32&.feast.core.DataSource.RedshiftOptionsH\x00\x12I\n\x14request_data_options\x18\x12 \x01(\x0b\x32).feast.core.DataSource.RequestDataOptionsH\x00\x12\x44\n\x0e\x63ustom_options\x18\x10 \x01(\x0b\x32*.feast.core.DataSource.CustomSourceOptionsH\x00\x12\x44\n\x11snowflake_options\x18\x13 \x01(\x0b\x32\'.feast.core.DataSource.SnowflakeOptionsH\x00\x12:\n\x0cpush_options\x18\x16 \x01(\x0b\x32\".feast.core.DataSource.PushOptionsH\x00\x12<\n\rspark_options\x18\x1b \x01(\x0b\x32#.feast.core.DataSource.SparkOptionsH\x00\x12<\n\rtrino_options\x18\x1e \x01(\x0b\x32#.feast.core.DataSource.TrinoOptionsH\x00\x12>\n\x0e\x61thena_options\x18# \x01(\x0b\x32$.feast.core.DataSource.AthenaOptionsH\x00\x1a+\n\tTagsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\x1a\x33\n\x11\x46ieldMappingEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\x1a\x82\x01\n\nSourceMeta\x12:\n\x16\x65\x61rliestEventTimestamp\x18\x01 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x38\n\x14latestEventTimestamp\x18\x02 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x1a\x65\n\x0b\x46ileOptions\x12+\n\x0b\x66ile_format\x18\x01 \x01(\x0b\x32\x16.feast.core.FileFormat\x12\x0b\n\x03uri\x18\x02 \x01(\t\x12\x1c\n\x14s3_endpoint_override\x18\x03 \x01(\t\x1a/\n\x0f\x42igQueryOptions\x12\r\n\x05table\x18\x01 \x01(\t\x12\r\n\x05query\x18\x02 \x01(\t\x1a,\n\x0cTrinoOptions\x12\r\n\x05table\x18\x01 \x01(\t\x12\r\n\x05query\x18\x02 \x01(\t\x1a\xae\x01\n\x0cKafkaOptions\x12\x1f\n\x17kafka_bootstrap_servers\x18\x01 \x01(\t\x12\r\n\x05topic\x18\x02 \x01(\t\x12\x30\n\x0emessage_format\x18\x03 \x01(\x0b\x32\x18.feast.core.StreamFormat\x12<\n\x19watermark_delay_threshold\x18\x04 \x01(\x0b\x32\x19.google.protobuf.Duration\x1a\x66\n\x0eKinesisOptions\x12\x0e\n\x06region\x18\x01 \x01(\t\x12\x13\n\x0bstream_name\x18\x02 \x01(\t\x12/\n\rrecord_format\x18\x03 \x01(\x0b\x32\x18.feast.core.StreamFormat\x1aQ\n\x0fRedshiftOptions\x12\r\n\x05table\x18\x01 \x01(\t\x12\r\n\x05query\x18\x02 \x01(\t\x12\x0e\n\x06schema\x18\x03 \x01(\t\x12\x10\n\x08\x64\x61tabase\x18\x04 \x01(\t\x1aT\n\rAthenaOptions\x12\r\n\x05table\x18\x01 \x01(\t\x12\r\n\x05query\x18\x02 \x01(\t\x12\x10\n\x08\x64\x61tabase\x18\x03 \x01(\t\x12\x13\n\x0b\x64\x61ta_source\x18\x04 \x01(\t\x1aX\n\x10SnowflakeOptions\x12\r\n\x05table\x18\x01 \x01(\t\x12\r\n\x05query\x18\x02 \x01(\t\x12\x0e\n\x06schema\x18\x03 \x01(\t\x12\x10\n\x08\x64\x61tabase\x18\x04 \x01(\tJ\x04\x08\x05\x10\x06\x1aO\n\x0cSparkOptions\x12\r\n\x05table\x18\x01 \x01(\t\x12\r\n\x05query\x18\x02 \x01(\t\x12\x0c\n\x04path\x18\x03 \x01(\t\x12\x13\n\x0b\x66ile_format\x18\x04 \x01(\t\x1a,\n\x13\x43ustomSourceOptions\x12\x15\n\rconfiguration\x18\x01 \x01(\x0c\x1a\xf7\x01\n\x12RequestDataOptions\x12Z\n\x11\x64\x65precated_schema\x18\x02 \x03(\x0b\x32?.feast.core.DataSource.RequestDataOptions.DeprecatedSchemaEntry\x12)\n\x06schema\x18\x03 \x03(\x0b\x32\x19.feast.core.FeatureSpecV2\x1aT\n\x15\x44\x65precatedSchemaEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12*\n\x05value\x18\x02 \x01(\x0e\x32\x1b.feast.types.ValueType.Enum:\x02\x38\x01J\x04\x08\x01\x10\x02\x1a\x13\n\x0bPushOptionsJ\x04\x08\x01\x10\x02\"\xf8\x01\n\nSourceType\x12\x0b\n\x07INVALID\x10\x00\x12\x0e\n\nBATCH_FILE\x10\x01\x12\x13\n\x0f\x42\x41TCH_SNOWFLAKE\x10\x08\x12\x12\n\x0e\x42\x41TCH_BIGQUERY\x10\x02\x12\x12\n\x0e\x42\x41TCH_REDSHIFT\x10\x05\x12\x10\n\x0cSTREAM_KAFKA\x10\x03\x12\x12\n\x0eSTREAM_KINESIS\x10\x04\x12\x11\n\rCUSTOM_SOURCE\x10\x06\x12\x12\n\x0eREQUEST_SOURCE\x10\x07\x12\x0f\n\x0bPUSH_SOURCE\x10\t\x12\x0f\n\x0b\x42\x41TCH_TRINO\x10\n\x12\x0f\n\x0b\x42\x41TCH_SPARK\x10\x0b\x12\x10\n\x0c\x42\x41TCH_ATHENA\x10\x0c\x42\t\n\x07optionsJ\x04\x08\x06\x10\x0b\x42T\n\x10\x66\x65\x61st.proto.coreB\x0f\x44\x61taSourceProtoZ/github.com/feast-dev/feast/go/protos/feast/coreb\x06proto3') +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x1b\x66\x65\x61st/core/DataSource.proto\x12\nfeast.core\x1a\x1egoogle/protobuf/duration.proto\x1a\x1fgoogle/protobuf/timestamp.proto\x1a\x1b\x66\x65\x61st/core/DataFormat.proto\x1a\x17\x66\x65\x61st/types/Value.proto\x1a\x18\x66\x65\x61st/core/Feature.proto\"\xc0\x16\n\nDataSource\x12\x0c\n\x04name\x18\x14 \x01(\t\x12\x0f\n\x07project\x18\x15 \x01(\t\x12\x13\n\x0b\x64\x65scription\x18\x17 \x01(\t\x12.\n\x04tags\x18\x18 \x03(\x0b\x32 .feast.core.DataSource.TagsEntry\x12\r\n\x05owner\x18\x19 \x01(\t\x12/\n\x04type\x18\x01 \x01(\x0e\x32!.feast.core.DataSource.SourceType\x12?\n\rfield_mapping\x18\x02 \x03(\x0b\x32(.feast.core.DataSource.FieldMappingEntry\x12\x17\n\x0ftimestamp_field\x18\x03 \x01(\t\x12\x1d\n\x15\x64\x61te_partition_column\x18\x04 \x01(\t\x12 \n\x18\x63reated_timestamp_column\x18\x05 \x01(\t\x12\x1e\n\x16\x64\x61ta_source_class_type\x18\x11 \x01(\t\x12,\n\x0c\x62\x61tch_source\x18\x1a \x01(\x0b\x32\x16.feast.core.DataSource\x12/\n\x04meta\x18\x32 \x01(\x0b\x32!.feast.core.DataSource.SourceMeta\x12:\n\x0c\x66ile_options\x18\x0b \x01(\x0b\x32\".feast.core.DataSource.FileOptionsH\x00\x12\x42\n\x10\x62igquery_options\x18\x0c \x01(\x0b\x32&.feast.core.DataSource.BigQueryOptionsH\x00\x12<\n\rkafka_options\x18\r \x01(\x0b\x32#.feast.core.DataSource.KafkaOptionsH\x00\x12@\n\x0fkinesis_options\x18\x0e \x01(\x0b\x32%.feast.core.DataSource.KinesisOptionsH\x00\x12\x42\n\x10redshift_options\x18\x0f \x01(\x0b\x32&.feast.core.DataSource.RedshiftOptionsH\x00\x12I\n\x14request_data_options\x18\x12 \x01(\x0b\x32).feast.core.DataSource.RequestDataOptionsH\x00\x12\x44\n\x0e\x63ustom_options\x18\x10 \x01(\x0b\x32*.feast.core.DataSource.CustomSourceOptionsH\x00\x12\x44\n\x11snowflake_options\x18\x13 \x01(\x0b\x32\'.feast.core.DataSource.SnowflakeOptionsH\x00\x12:\n\x0cpush_options\x18\x16 \x01(\x0b\x32\".feast.core.DataSource.PushOptionsH\x00\x12<\n\rspark_options\x18\x1b \x01(\x0b\x32#.feast.core.DataSource.SparkOptionsH\x00\x12<\n\rtrino_options\x18\x1e \x01(\x0b\x32#.feast.core.DataSource.TrinoOptionsH\x00\x12>\n\x0e\x61thena_options\x18# \x01(\x0b\x32$.feast.core.DataSource.AthenaOptionsH\x00\x1a+\n\tTagsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\x1a\x33\n\x11\x46ieldMappingEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\x1a\x82\x01\n\nSourceMeta\x12:\n\x16\x65\x61rliestEventTimestamp\x18\x01 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x38\n\x14latestEventTimestamp\x18\x02 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x1a\x65\n\x0b\x46ileOptions\x12+\n\x0b\x66ile_format\x18\x01 \x01(\x0b\x32\x16.feast.core.FileFormat\x12\x0b\n\x03uri\x18\x02 \x01(\t\x12\x1c\n\x14s3_endpoint_override\x18\x03 \x01(\t\x1a/\n\x0f\x42igQueryOptions\x12\r\n\x05table\x18\x01 \x01(\t\x12\r\n\x05query\x18\x02 \x01(\t\x1a,\n\x0cTrinoOptions\x12\r\n\x05table\x18\x01 \x01(\t\x12\r\n\x05query\x18\x02 \x01(\t\x1a\xae\x01\n\x0cKafkaOptions\x12\x1f\n\x17kafka_bootstrap_servers\x18\x01 \x01(\t\x12\r\n\x05topic\x18\x02 \x01(\t\x12\x30\n\x0emessage_format\x18\x03 \x01(\x0b\x32\x18.feast.core.StreamFormat\x12<\n\x19watermark_delay_threshold\x18\x04 \x01(\x0b\x32\x19.google.protobuf.Duration\x1a\x66\n\x0eKinesisOptions\x12\x0e\n\x06region\x18\x01 \x01(\t\x12\x13\n\x0bstream_name\x18\x02 \x01(\t\x12/\n\rrecord_format\x18\x03 \x01(\x0b\x32\x18.feast.core.StreamFormat\x1aQ\n\x0fRedshiftOptions\x12\r\n\x05table\x18\x01 \x01(\t\x12\r\n\x05query\x18\x02 \x01(\t\x12\x0e\n\x06schema\x18\x03 \x01(\t\x12\x10\n\x08\x64\x61tabase\x18\x04 \x01(\t\x1aT\n\rAthenaOptions\x12\r\n\x05table\x18\x01 \x01(\t\x12\r\n\x05query\x18\x02 \x01(\t\x12\x10\n\x08\x64\x61tabase\x18\x03 \x01(\t\x12\x13\n\x0b\x64\x61ta_source\x18\x04 \x01(\t\x1aX\n\x10SnowflakeOptions\x12\r\n\x05table\x18\x01 \x01(\t\x12\r\n\x05query\x18\x02 \x01(\t\x12\x0e\n\x06schema\x18\x03 \x01(\t\x12\x10\n\x08\x64\x61tabase\x18\x04 \x01(\tJ\x04\x08\x05\x10\x06\x1aO\n\x0cSparkOptions\x12\r\n\x05table\x18\x01 \x01(\t\x12\r\n\x05query\x18\x02 \x01(\t\x12\x0c\n\x04path\x18\x03 \x01(\t\x12\x13\n\x0b\x66ile_format\x18\x04 \x01(\t\x1a,\n\x13\x43ustomSourceOptions\x12\x15\n\rconfiguration\x18\x01 \x01(\x0c\x1a\xf7\x01\n\x12RequestDataOptions\x12Z\n\x11\x64\x65precated_schema\x18\x02 \x03(\x0b\x32?.feast.core.DataSource.RequestDataOptions.DeprecatedSchemaEntry\x12)\n\x06schema\x18\x03 \x03(\x0b\x32\x19.feast.core.FeatureSpecV2\x1aT\n\x15\x44\x65precatedSchemaEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12*\n\x05value\x18\x02 \x01(\x0e\x32\x1b.feast.types.ValueType.Enum:\x02\x38\x01J\x04\x08\x01\x10\x02\x1a\x13\n\x0bPushOptionsJ\x04\x08\x01\x10\x02\"\xf8\x01\n\nSourceType\x12\x0b\n\x07INVALID\x10\x00\x12\x0e\n\nBATCH_FILE\x10\x01\x12\x13\n\x0f\x42\x41TCH_SNOWFLAKE\x10\x08\x12\x12\n\x0e\x42\x41TCH_BIGQUERY\x10\x02\x12\x12\n\x0e\x42\x41TCH_REDSHIFT\x10\x05\x12\x10\n\x0cSTREAM_KAFKA\x10\x03\x12\x12\n\x0eSTREAM_KINESIS\x10\x04\x12\x11\n\rCUSTOM_SOURCE\x10\x06\x12\x12\n\x0eREQUEST_SOURCE\x10\x07\x12\x0f\n\x0bPUSH_SOURCE\x10\t\x12\x0f\n\x0b\x42\x41TCH_TRINO\x10\n\x12\x0f\n\x0b\x42\x41TCH_SPARK\x10\x0b\x12\x10\n\x0c\x42\x41TCH_ATHENA\x10\x0c\x42\t\n\x07optionsJ\x04\x08\x06\x10\x0b\"=\n\x0e\x44\x61taSourceList\x12+\n\x0b\x64\x61tasources\x18\x01 \x03(\x0b\x32\x16.feast.core.DataSourceBT\n\x10\x66\x65\x61st.proto.coreB\x0f\x44\x61taSourceProtoZ/github.com/feast-dev/feast/go/protos/feast/coreb\x06proto3') _globals = globals() _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) @@ -69,4 +69,6 @@ _globals['_DATASOURCE_PUSHOPTIONS']._serialized_end=2801 _globals['_DATASOURCE_SOURCETYPE']._serialized_start=2804 _globals['_DATASOURCE_SOURCETYPE']._serialized_end=3052 + _globals['_DATASOURCELIST']._serialized_start=3071 + _globals['_DATASOURCELIST']._serialized_end=3132 # @@protoc_insertion_point(module_scope) diff --git a/sdk/python/feast/protos/feast/core/DataSource_pb2.pyi b/sdk/python/feast/protos/feast/core/DataSource_pb2.pyi index 94336638e19..aadec3fad4c 100644 --- a/sdk/python/feast/protos/feast/core/DataSource_pb2.pyi +++ b/sdk/python/feast/protos/feast/core/DataSource_pb2.pyi @@ -557,3 +557,18 @@ class DataSource(google.protobuf.message.Message): def WhichOneof(self, oneof_group: typing_extensions.Literal["options", b"options"]) -> typing_extensions.Literal["file_options", "bigquery_options", "kafka_options", "kinesis_options", "redshift_options", "request_data_options", "custom_options", "snowflake_options", "push_options", "spark_options", "trino_options", "athena_options"] | None: ... global___DataSource = DataSource + +class DataSourceList(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + DATASOURCES_FIELD_NUMBER: builtins.int + @property + def datasources(self) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[global___DataSource]: ... + def __init__( + self, + *, + datasources: collections.abc.Iterable[global___DataSource] | None = ..., + ) -> None: ... + def ClearField(self, field_name: typing_extensions.Literal["datasources", b"datasources"]) -> None: ... + +global___DataSourceList = DataSourceList diff --git a/sdk/python/feast/protos/feast/core/Entity_pb2.py b/sdk/python/feast/protos/feast/core/Entity_pb2.py index 5a192854cab..2b3e7806736 100644 --- a/sdk/python/feast/protos/feast/core/Entity_pb2.py +++ b/sdk/python/feast/protos/feast/core/Entity_pb2.py @@ -16,7 +16,7 @@ from google.protobuf import timestamp_pb2 as google_dot_protobuf_dot_timestamp__pb2 -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x17\x66\x65\x61st/core/Entity.proto\x12\nfeast.core\x1a\x17\x66\x65\x61st/types/Value.proto\x1a\x1fgoogle/protobuf/timestamp.proto\"V\n\x06\x45ntity\x12&\n\x04spec\x18\x01 \x01(\x0b\x32\x18.feast.core.EntitySpecV2\x12$\n\x04meta\x18\x02 \x01(\x0b\x32\x16.feast.core.EntityMeta\"\xf3\x01\n\x0c\x45ntitySpecV2\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07project\x18\t \x01(\t\x12/\n\nvalue_type\x18\x02 \x01(\x0e\x32\x1b.feast.types.ValueType.Enum\x12\x13\n\x0b\x64\x65scription\x18\x03 \x01(\t\x12\x10\n\x08join_key\x18\x04 \x01(\t\x12\x30\n\x04tags\x18\x08 \x03(\x0b\x32\".feast.core.EntitySpecV2.TagsEntry\x12\r\n\x05owner\x18\n \x01(\t\x1a+\n\tTagsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"\x7f\n\nEntityMeta\x12\x35\n\x11\x63reated_timestamp\x18\x01 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12:\n\x16last_updated_timestamp\x18\x02 \x01(\x0b\x32\x1a.google.protobuf.TimestampBP\n\x10\x66\x65\x61st.proto.coreB\x0b\x45ntityProtoZ/github.com/feast-dev/feast/go/protos/feast/coreb\x06proto3') +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x17\x66\x65\x61st/core/Entity.proto\x12\nfeast.core\x1a\x17\x66\x65\x61st/types/Value.proto\x1a\x1fgoogle/protobuf/timestamp.proto\"V\n\x06\x45ntity\x12&\n\x04spec\x18\x01 \x01(\x0b\x32\x18.feast.core.EntitySpecV2\x12$\n\x04meta\x18\x02 \x01(\x0b\x32\x16.feast.core.EntityMeta\"\xf3\x01\n\x0c\x45ntitySpecV2\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07project\x18\t \x01(\t\x12/\n\nvalue_type\x18\x02 \x01(\x0e\x32\x1b.feast.types.ValueType.Enum\x12\x13\n\x0b\x64\x65scription\x18\x03 \x01(\t\x12\x10\n\x08join_key\x18\x04 \x01(\t\x12\x30\n\x04tags\x18\x08 \x03(\x0b\x32\".feast.core.EntitySpecV2.TagsEntry\x12\r\n\x05owner\x18\n \x01(\t\x1a+\n\tTagsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"\x7f\n\nEntityMeta\x12\x35\n\x11\x63reated_timestamp\x18\x01 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12:\n\x16last_updated_timestamp\x18\x02 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\"2\n\nEntityList\x12$\n\x08\x65ntities\x18\x01 \x03(\x0b\x32\x12.feast.core.EntityBP\n\x10\x66\x65\x61st.proto.coreB\x0b\x45ntityProtoZ/github.com/feast-dev/feast/go/protos/feast/coreb\x06proto3') _globals = globals() _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) @@ -34,4 +34,6 @@ _globals['_ENTITYSPECV2_TAGSENTRY']._serialized_end=429 _globals['_ENTITYMETA']._serialized_start=431 _globals['_ENTITYMETA']._serialized_end=558 + _globals['_ENTITYLIST']._serialized_start=560 + _globals['_ENTITYLIST']._serialized_end=610 # @@protoc_insertion_point(module_scope) diff --git a/sdk/python/feast/protos/feast/core/Entity_pb2.pyi b/sdk/python/feast/protos/feast/core/Entity_pb2.pyi index 732b3e10326..025817edfee 100644 --- a/sdk/python/feast/protos/feast/core/Entity_pb2.pyi +++ b/sdk/python/feast/protos/feast/core/Entity_pb2.pyi @@ -128,3 +128,18 @@ class EntityMeta(google.protobuf.message.Message): def ClearField(self, field_name: typing_extensions.Literal["created_timestamp", b"created_timestamp", "last_updated_timestamp", b"last_updated_timestamp"]) -> None: ... global___EntityMeta = EntityMeta + +class EntityList(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + ENTITIES_FIELD_NUMBER: builtins.int + @property + def entities(self) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[global___Entity]: ... + def __init__( + self, + *, + entities: collections.abc.Iterable[global___Entity] | None = ..., + ) -> None: ... + def ClearField(self, field_name: typing_extensions.Literal["entities", b"entities"]) -> None: ... + +global___EntityList = EntityList diff --git a/sdk/python/feast/protos/feast/core/FeatureService_pb2.py b/sdk/python/feast/protos/feast/core/FeatureService_pb2.py index cf6ac46ac54..642d5b010f9 100644 --- a/sdk/python/feast/protos/feast/core/FeatureService_pb2.py +++ b/sdk/python/feast/protos/feast/core/FeatureService_pb2.py @@ -16,7 +16,7 @@ from feast.protos.feast.core import FeatureViewProjection_pb2 as feast_dot_core_dot_FeatureViewProjection__pb2 -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x1f\x66\x65\x61st/core/FeatureService.proto\x12\nfeast.core\x1a\x1fgoogle/protobuf/timestamp.proto\x1a&feast/core/FeatureViewProjection.proto\"l\n\x0e\x46\x65\x61tureService\x12,\n\x04spec\x18\x01 \x01(\x0b\x32\x1e.feast.core.FeatureServiceSpec\x12,\n\x04meta\x18\x02 \x01(\x0b\x32\x1e.feast.core.FeatureServiceMeta\"\xa4\x02\n\x12\x46\x65\x61tureServiceSpec\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07project\x18\x02 \x01(\t\x12\x33\n\x08\x66\x65\x61tures\x18\x03 \x03(\x0b\x32!.feast.core.FeatureViewProjection\x12\x36\n\x04tags\x18\x04 \x03(\x0b\x32(.feast.core.FeatureServiceSpec.TagsEntry\x12\x13\n\x0b\x64\x65scription\x18\x05 \x01(\t\x12\r\n\x05owner\x18\x06 \x01(\t\x12\x31\n\x0elogging_config\x18\x07 \x01(\x0b\x32\x19.feast.core.LoggingConfig\x1a+\n\tTagsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"\x87\x01\n\x12\x46\x65\x61tureServiceMeta\x12\x35\n\x11\x63reated_timestamp\x18\x01 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12:\n\x16last_updated_timestamp\x18\x02 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\"\x9a\x07\n\rLoggingConfig\x12\x13\n\x0bsample_rate\x18\x01 \x01(\x02\x12\x45\n\x10\x66ile_destination\x18\x03 \x01(\x0b\x32).feast.core.LoggingConfig.FileDestinationH\x00\x12M\n\x14\x62igquery_destination\x18\x04 \x01(\x0b\x32-.feast.core.LoggingConfig.BigQueryDestinationH\x00\x12M\n\x14redshift_destination\x18\x05 \x01(\x0b\x32-.feast.core.LoggingConfig.RedshiftDestinationH\x00\x12O\n\x15snowflake_destination\x18\x06 \x01(\x0b\x32..feast.core.LoggingConfig.SnowflakeDestinationH\x00\x12I\n\x12\x63ustom_destination\x18\x07 \x01(\x0b\x32+.feast.core.LoggingConfig.CustomDestinationH\x00\x12I\n\x12\x61thena_destination\x18\x08 \x01(\x0b\x32+.feast.core.LoggingConfig.AthenaDestinationH\x00\x1aS\n\x0f\x46ileDestination\x12\x0c\n\x04path\x18\x01 \x01(\t\x12\x1c\n\x14s3_endpoint_override\x18\x02 \x01(\t\x12\x14\n\x0cpartition_by\x18\x03 \x03(\t\x1a(\n\x13\x42igQueryDestination\x12\x11\n\ttable_ref\x18\x01 \x01(\t\x1a)\n\x13RedshiftDestination\x12\x12\n\ntable_name\x18\x01 \x01(\t\x1a\'\n\x11\x41thenaDestination\x12\x12\n\ntable_name\x18\x01 \x01(\t\x1a*\n\x14SnowflakeDestination\x12\x12\n\ntable_name\x18\x01 \x01(\t\x1a\x99\x01\n\x11\x43ustomDestination\x12\x0c\n\x04kind\x18\x01 \x01(\t\x12G\n\x06\x63onfig\x18\x02 \x03(\x0b\x32\x37.feast.core.LoggingConfig.CustomDestination.ConfigEntry\x1a-\n\x0b\x43onfigEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\x42\r\n\x0b\x64\x65stinationBX\n\x10\x66\x65\x61st.proto.coreB\x13\x46\x65\x61tureServiceProtoZ/github.com/feast-dev/feast/go/protos/feast/coreb\x06proto3') +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x1f\x66\x65\x61st/core/FeatureService.proto\x12\nfeast.core\x1a\x1fgoogle/protobuf/timestamp.proto\x1a&feast/core/FeatureViewProjection.proto\"l\n\x0e\x46\x65\x61tureService\x12,\n\x04spec\x18\x01 \x01(\x0b\x32\x1e.feast.core.FeatureServiceSpec\x12,\n\x04meta\x18\x02 \x01(\x0b\x32\x1e.feast.core.FeatureServiceMeta\"\xa4\x02\n\x12\x46\x65\x61tureServiceSpec\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07project\x18\x02 \x01(\t\x12\x33\n\x08\x66\x65\x61tures\x18\x03 \x03(\x0b\x32!.feast.core.FeatureViewProjection\x12\x36\n\x04tags\x18\x04 \x03(\x0b\x32(.feast.core.FeatureServiceSpec.TagsEntry\x12\x13\n\x0b\x64\x65scription\x18\x05 \x01(\t\x12\r\n\x05owner\x18\x06 \x01(\t\x12\x31\n\x0elogging_config\x18\x07 \x01(\x0b\x32\x19.feast.core.LoggingConfig\x1a+\n\tTagsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"\x87\x01\n\x12\x46\x65\x61tureServiceMeta\x12\x35\n\x11\x63reated_timestamp\x18\x01 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12:\n\x16last_updated_timestamp\x18\x02 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\"\x9a\x07\n\rLoggingConfig\x12\x13\n\x0bsample_rate\x18\x01 \x01(\x02\x12\x45\n\x10\x66ile_destination\x18\x03 \x01(\x0b\x32).feast.core.LoggingConfig.FileDestinationH\x00\x12M\n\x14\x62igquery_destination\x18\x04 \x01(\x0b\x32-.feast.core.LoggingConfig.BigQueryDestinationH\x00\x12M\n\x14redshift_destination\x18\x05 \x01(\x0b\x32-.feast.core.LoggingConfig.RedshiftDestinationH\x00\x12O\n\x15snowflake_destination\x18\x06 \x01(\x0b\x32..feast.core.LoggingConfig.SnowflakeDestinationH\x00\x12I\n\x12\x63ustom_destination\x18\x07 \x01(\x0b\x32+.feast.core.LoggingConfig.CustomDestinationH\x00\x12I\n\x12\x61thena_destination\x18\x08 \x01(\x0b\x32+.feast.core.LoggingConfig.AthenaDestinationH\x00\x1aS\n\x0f\x46ileDestination\x12\x0c\n\x04path\x18\x01 \x01(\t\x12\x1c\n\x14s3_endpoint_override\x18\x02 \x01(\t\x12\x14\n\x0cpartition_by\x18\x03 \x03(\t\x1a(\n\x13\x42igQueryDestination\x12\x11\n\ttable_ref\x18\x01 \x01(\t\x1a)\n\x13RedshiftDestination\x12\x12\n\ntable_name\x18\x01 \x01(\t\x1a\'\n\x11\x41thenaDestination\x12\x12\n\ntable_name\x18\x01 \x01(\t\x1a*\n\x14SnowflakeDestination\x12\x12\n\ntable_name\x18\x01 \x01(\t\x1a\x99\x01\n\x11\x43ustomDestination\x12\x0c\n\x04kind\x18\x01 \x01(\t\x12G\n\x06\x63onfig\x18\x02 \x03(\x0b\x32\x37.feast.core.LoggingConfig.CustomDestination.ConfigEntry\x1a-\n\x0b\x43onfigEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\x42\r\n\x0b\x64\x65stination\"I\n\x12\x46\x65\x61tureServiceList\x12\x33\n\x0f\x66\x65\x61tureservices\x18\x01 \x03(\x0b\x32\x1a.feast.core.FeatureServiceBX\n\x10\x66\x65\x61st.proto.coreB\x13\x46\x65\x61tureServiceProtoZ/github.com/feast-dev/feast/go/protos/feast/coreb\x06proto3') _globals = globals() _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) @@ -52,4 +52,6 @@ _globals['_LOGGINGCONFIG_CUSTOMDESTINATION']._serialized_end=1571 _globals['_LOGGINGCONFIG_CUSTOMDESTINATION_CONFIGENTRY']._serialized_start=1526 _globals['_LOGGINGCONFIG_CUSTOMDESTINATION_CONFIGENTRY']._serialized_end=1571 + _globals['_FEATURESERVICELIST']._serialized_start=1588 + _globals['_FEATURESERVICELIST']._serialized_end=1661 # @@protoc_insertion_point(module_scope) diff --git a/sdk/python/feast/protos/feast/core/FeatureService_pb2.pyi b/sdk/python/feast/protos/feast/core/FeatureService_pb2.pyi index b3305b72df9..0b1c0baa871 100644 --- a/sdk/python/feast/protos/feast/core/FeatureService_pb2.pyi +++ b/sdk/python/feast/protos/feast/core/FeatureService_pb2.pyi @@ -264,3 +264,18 @@ class LoggingConfig(google.protobuf.message.Message): def WhichOneof(self, oneof_group: typing_extensions.Literal["destination", b"destination"]) -> typing_extensions.Literal["file_destination", "bigquery_destination", "redshift_destination", "snowflake_destination", "custom_destination", "athena_destination"] | None: ... global___LoggingConfig = LoggingConfig + +class FeatureServiceList(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + FEATURESERVICES_FIELD_NUMBER: builtins.int + @property + def featureservices(self) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[global___FeatureService]: ... + def __init__( + self, + *, + featureservices: collections.abc.Iterable[global___FeatureService] | None = ..., + ) -> None: ... + def ClearField(self, field_name: typing_extensions.Literal["featureservices", b"featureservices"]) -> None: ... + +global___FeatureServiceList = FeatureServiceList diff --git a/sdk/python/feast/protos/feast/core/FeatureView_pb2.py b/sdk/python/feast/protos/feast/core/FeatureView_pb2.py index f1480593d9a..80d04c1ec3f 100644 --- a/sdk/python/feast/protos/feast/core/FeatureView_pb2.py +++ b/sdk/python/feast/protos/feast/core/FeatureView_pb2.py @@ -18,7 +18,7 @@ from feast.protos.feast.core import Feature_pb2 as feast_dot_core_dot_Feature__pb2 -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x1c\x66\x65\x61st/core/FeatureView.proto\x12\nfeast.core\x1a\x1egoogle/protobuf/duration.proto\x1a\x1fgoogle/protobuf/timestamp.proto\x1a\x1b\x66\x65\x61st/core/DataSource.proto\x1a\x18\x66\x65\x61st/core/Feature.proto\"c\n\x0b\x46\x65\x61tureView\x12)\n\x04spec\x18\x01 \x01(\x0b\x32\x1b.feast.core.FeatureViewSpec\x12)\n\x04meta\x18\x02 \x01(\x0b\x32\x1b.feast.core.FeatureViewMeta\"\xbd\x03\n\x0f\x46\x65\x61tureViewSpec\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07project\x18\x02 \x01(\t\x12\x10\n\x08\x65ntities\x18\x03 \x03(\t\x12+\n\x08\x66\x65\x61tures\x18\x04 \x03(\x0b\x32\x19.feast.core.FeatureSpecV2\x12\x31\n\x0e\x65ntity_columns\x18\x0c \x03(\x0b\x32\x19.feast.core.FeatureSpecV2\x12\x13\n\x0b\x64\x65scription\x18\n \x01(\t\x12\x33\n\x04tags\x18\x05 \x03(\x0b\x32%.feast.core.FeatureViewSpec.TagsEntry\x12\r\n\x05owner\x18\x0b \x01(\t\x12&\n\x03ttl\x18\x06 \x01(\x0b\x32\x19.google.protobuf.Duration\x12,\n\x0c\x62\x61tch_source\x18\x07 \x01(\x0b\x32\x16.feast.core.DataSource\x12-\n\rstream_source\x18\t \x01(\x0b\x32\x16.feast.core.DataSource\x12\x0e\n\x06online\x18\x08 \x01(\x08\x1a+\n\tTagsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"\xcc\x01\n\x0f\x46\x65\x61tureViewMeta\x12\x35\n\x11\x63reated_timestamp\x18\x01 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12:\n\x16last_updated_timestamp\x18\x02 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x46\n\x19materialization_intervals\x18\x03 \x03(\x0b\x32#.feast.core.MaterializationInterval\"w\n\x17MaterializationInterval\x12.\n\nstart_time\x18\x01 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12,\n\x08\x65nd_time\x18\x02 \x01(\x0b\x32\x1a.google.protobuf.TimestampBU\n\x10\x66\x65\x61st.proto.coreB\x10\x46\x65\x61tureViewProtoZ/github.com/feast-dev/feast/go/protos/feast/coreb\x06proto3') +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x1c\x66\x65\x61st/core/FeatureView.proto\x12\nfeast.core\x1a\x1egoogle/protobuf/duration.proto\x1a\x1fgoogle/protobuf/timestamp.proto\x1a\x1b\x66\x65\x61st/core/DataSource.proto\x1a\x18\x66\x65\x61st/core/Feature.proto\"c\n\x0b\x46\x65\x61tureView\x12)\n\x04spec\x18\x01 \x01(\x0b\x32\x1b.feast.core.FeatureViewSpec\x12)\n\x04meta\x18\x02 \x01(\x0b\x32\x1b.feast.core.FeatureViewMeta\"\xbd\x03\n\x0f\x46\x65\x61tureViewSpec\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07project\x18\x02 \x01(\t\x12\x10\n\x08\x65ntities\x18\x03 \x03(\t\x12+\n\x08\x66\x65\x61tures\x18\x04 \x03(\x0b\x32\x19.feast.core.FeatureSpecV2\x12\x31\n\x0e\x65ntity_columns\x18\x0c \x03(\x0b\x32\x19.feast.core.FeatureSpecV2\x12\x13\n\x0b\x64\x65scription\x18\n \x01(\t\x12\x33\n\x04tags\x18\x05 \x03(\x0b\x32%.feast.core.FeatureViewSpec.TagsEntry\x12\r\n\x05owner\x18\x0b \x01(\t\x12&\n\x03ttl\x18\x06 \x01(\x0b\x32\x19.google.protobuf.Duration\x12,\n\x0c\x62\x61tch_source\x18\x07 \x01(\x0b\x32\x16.feast.core.DataSource\x12-\n\rstream_source\x18\t \x01(\x0b\x32\x16.feast.core.DataSource\x12\x0e\n\x06online\x18\x08 \x01(\x08\x1a+\n\tTagsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"\xcc\x01\n\x0f\x46\x65\x61tureViewMeta\x12\x35\n\x11\x63reated_timestamp\x18\x01 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12:\n\x16last_updated_timestamp\x18\x02 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x46\n\x19materialization_intervals\x18\x03 \x03(\x0b\x32#.feast.core.MaterializationInterval\"w\n\x17MaterializationInterval\x12.\n\nstart_time\x18\x01 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12,\n\x08\x65nd_time\x18\x02 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\"@\n\x0f\x46\x65\x61tureViewList\x12-\n\x0c\x66\x65\x61tureviews\x18\x01 \x03(\x0b\x32\x17.feast.core.FeatureViewBU\n\x10\x66\x65\x61st.proto.coreB\x10\x46\x65\x61tureViewProtoZ/github.com/feast-dev/feast/go/protos/feast/coreb\x06proto3') _globals = globals() _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) @@ -38,4 +38,6 @@ _globals['_FEATUREVIEWMETA']._serialized_end=918 _globals['_MATERIALIZATIONINTERVAL']._serialized_start=920 _globals['_MATERIALIZATIONINTERVAL']._serialized_end=1039 + _globals['_FEATUREVIEWLIST']._serialized_start=1041 + _globals['_FEATUREVIEWLIST']._serialized_end=1105 # @@protoc_insertion_point(module_scope) diff --git a/sdk/python/feast/protos/feast/core/FeatureView_pb2.pyi b/sdk/python/feast/protos/feast/core/FeatureView_pb2.pyi index e1d4e2dfee8..57158fc2c6c 100644 --- a/sdk/python/feast/protos/feast/core/FeatureView_pb2.pyi +++ b/sdk/python/feast/protos/feast/core/FeatureView_pb2.pyi @@ -192,3 +192,18 @@ class MaterializationInterval(google.protobuf.message.Message): def ClearField(self, field_name: typing_extensions.Literal["end_time", b"end_time", "start_time", b"start_time"]) -> None: ... global___MaterializationInterval = MaterializationInterval + +class FeatureViewList(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + FEATUREVIEWS_FIELD_NUMBER: builtins.int + @property + def featureviews(self) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[global___FeatureView]: ... + def __init__( + self, + *, + featureviews: collections.abc.Iterable[global___FeatureView] | None = ..., + ) -> None: ... + def ClearField(self, field_name: typing_extensions.Literal["featureviews", b"featureviews"]) -> None: ... + +global___FeatureViewList = FeatureViewList diff --git a/sdk/python/feast/protos/feast/core/Feature_pb2.py b/sdk/python/feast/protos/feast/core/Feature_pb2.py index dd7c6008ef1..6b1081fe811 100644 --- a/sdk/python/feast/protos/feast/core/Feature_pb2.py +++ b/sdk/python/feast/protos/feast/core/Feature_pb2.py @@ -15,7 +15,7 @@ from feast.protos.feast.types import Value_pb2 as feast_dot_types_dot_Value__pb2 -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x18\x66\x65\x61st/core/Feature.proto\x12\nfeast.core\x1a\x17\x66\x65\x61st/types/Value.proto\"\xc3\x01\n\rFeatureSpecV2\x12\x0c\n\x04name\x18\x01 \x01(\t\x12/\n\nvalue_type\x18\x02 \x01(\x0e\x32\x1b.feast.types.ValueType.Enum\x12\x31\n\x04tags\x18\x03 \x03(\x0b\x32#.feast.core.FeatureSpecV2.TagsEntry\x12\x13\n\x0b\x64\x65scription\x18\x04 \x01(\t\x1a+\n\tTagsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\x42Q\n\x10\x66\x65\x61st.proto.coreB\x0c\x46\x65\x61tureProtoZ/github.com/feast-dev/feast/go/protos/feast/coreb\x06proto3') +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x18\x66\x65\x61st/core/Feature.proto\x12\nfeast.core\x1a\x17\x66\x65\x61st/types/Value.proto\"\xf7\x01\n\rFeatureSpecV2\x12\x0c\n\x04name\x18\x01 \x01(\t\x12/\n\nvalue_type\x18\x02 \x01(\x0e\x32\x1b.feast.types.ValueType.Enum\x12\x31\n\x04tags\x18\x03 \x03(\x0b\x32#.feast.core.FeatureSpecV2.TagsEntry\x12\x13\n\x0b\x64\x65scription\x18\x04 \x01(\t\x12\x14\n\x0cvector_index\x18\x05 \x01(\x08\x12\x1c\n\x14vector_search_metric\x18\x06 \x01(\t\x1a+\n\tTagsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\x42Q\n\x10\x66\x65\x61st.proto.coreB\x0c\x46\x65\x61tureProtoZ/github.com/feast-dev/feast/go/protos/feast/coreb\x06proto3') _globals = globals() _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) @@ -26,7 +26,7 @@ _globals['_FEATURESPECV2_TAGSENTRY']._options = None _globals['_FEATURESPECV2_TAGSENTRY']._serialized_options = b'8\001' _globals['_FEATURESPECV2']._serialized_start=66 - _globals['_FEATURESPECV2']._serialized_end=261 - _globals['_FEATURESPECV2_TAGSENTRY']._serialized_start=218 - _globals['_FEATURESPECV2_TAGSENTRY']._serialized_end=261 + _globals['_FEATURESPECV2']._serialized_end=313 + _globals['_FEATURESPECV2_TAGSENTRY']._serialized_start=270 + _globals['_FEATURESPECV2_TAGSENTRY']._serialized_end=313 # @@protoc_insertion_point(module_scope) diff --git a/sdk/python/feast/protos/feast/core/Feature_pb2.pyi b/sdk/python/feast/protos/feast/core/Feature_pb2.pyi index f4235b0965b..451f1aa61ce 100644 --- a/sdk/python/feast/protos/feast/core/Feature_pb2.pyi +++ b/sdk/python/feast/protos/feast/core/Feature_pb2.pyi @@ -53,6 +53,8 @@ class FeatureSpecV2(google.protobuf.message.Message): VALUE_TYPE_FIELD_NUMBER: builtins.int TAGS_FIELD_NUMBER: builtins.int DESCRIPTION_FIELD_NUMBER: builtins.int + VECTOR_INDEX_FIELD_NUMBER: builtins.int + VECTOR_SEARCH_METRIC_FIELD_NUMBER: builtins.int name: builtins.str """Name of the feature. Not updatable.""" value_type: feast.types.Value_pb2.ValueType.Enum.ValueType @@ -62,6 +64,10 @@ class FeatureSpecV2(google.protobuf.message.Message): """Tags for user defined metadata on a feature""" description: builtins.str """Description of the feature.""" + vector_index: builtins.bool + """Field indicating the vector will be indexed for vector similarity search""" + vector_search_metric: builtins.str + """Metric used for vector similarity search.""" def __init__( self, *, @@ -69,7 +75,9 @@ class FeatureSpecV2(google.protobuf.message.Message): value_type: feast.types.Value_pb2.ValueType.Enum.ValueType = ..., tags: collections.abc.Mapping[builtins.str, builtins.str] | None = ..., description: builtins.str = ..., + vector_index: builtins.bool = ..., + vector_search_metric: builtins.str = ..., ) -> None: ... - def ClearField(self, field_name: typing_extensions.Literal["description", b"description", "name", b"name", "tags", b"tags", "value_type", b"value_type"]) -> None: ... + def ClearField(self, field_name: typing_extensions.Literal["description", b"description", "name", b"name", "tags", b"tags", "value_type", b"value_type", "vector_index", b"vector_index", "vector_search_metric", b"vector_search_metric"]) -> None: ... global___FeatureSpecV2 = FeatureSpecV2 diff --git a/sdk/python/feast/protos/feast/core/OnDemandFeatureView_pb2.py b/sdk/python/feast/protos/feast/core/OnDemandFeatureView_pb2.py index 020515a6b89..926b54df288 100644 --- a/sdk/python/feast/protos/feast/core/OnDemandFeatureView_pb2.py +++ b/sdk/python/feast/protos/feast/core/OnDemandFeatureView_pb2.py @@ -20,7 +20,7 @@ from feast.protos.feast.core import Transformation_pb2 as feast_dot_core_dot_Transformation__pb2 -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n$feast/core/OnDemandFeatureView.proto\x12\nfeast.core\x1a\x1fgoogle/protobuf/timestamp.proto\x1a\x1c\x66\x65\x61st/core/FeatureView.proto\x1a&feast/core/FeatureViewProjection.proto\x1a\x18\x66\x65\x61st/core/Feature.proto\x1a\x1b\x66\x65\x61st/core/DataSource.proto\x1a\x1f\x66\x65\x61st/core/Transformation.proto\"{\n\x13OnDemandFeatureView\x12\x31\n\x04spec\x18\x01 \x01(\x0b\x32#.feast.core.OnDemandFeatureViewSpec\x12\x31\n\x04meta\x18\x02 \x01(\x0b\x32#.feast.core.OnDemandFeatureViewMeta\"\x90\x05\n\x17OnDemandFeatureViewSpec\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07project\x18\x02 \x01(\t\x12+\n\x08\x66\x65\x61tures\x18\x03 \x03(\x0b\x32\x19.feast.core.FeatureSpecV2\x12\x41\n\x07sources\x18\x04 \x03(\x0b\x32\x30.feast.core.OnDemandFeatureViewSpec.SourcesEntry\x12\x42\n\x15user_defined_function\x18\x05 \x01(\x0b\x32\x1f.feast.core.UserDefinedFunctionB\x02\x18\x01\x12\x43\n\x16\x66\x65\x61ture_transformation\x18\n \x01(\x0b\x32#.feast.core.FeatureTransformationV2\x12\x13\n\x0b\x64\x65scription\x18\x06 \x01(\t\x12;\n\x04tags\x18\x07 \x03(\x0b\x32-.feast.core.OnDemandFeatureViewSpec.TagsEntry\x12\r\n\x05owner\x18\x08 \x01(\t\x12\x0c\n\x04mode\x18\x0b \x01(\t\x12\x1d\n\x15write_to_online_store\x18\x0c \x01(\x08\x12\x10\n\x08\x65ntities\x18\r \x03(\t\x12\x31\n\x0e\x65ntity_columns\x18\x0e \x03(\x0b\x32\x19.feast.core.FeatureSpecV2\x12\x11\n\tsingleton\x18\x0f \x01(\x08\x1aJ\n\x0cSourcesEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12)\n\x05value\x18\x02 \x01(\x0b\x32\x1a.feast.core.OnDemandSource:\x02\x38\x01\x1a+\n\tTagsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"\x8c\x01\n\x17OnDemandFeatureViewMeta\x12\x35\n\x11\x63reated_timestamp\x18\x01 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12:\n\x16last_updated_timestamp\x18\x02 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\"\xc8\x01\n\x0eOnDemandSource\x12/\n\x0c\x66\x65\x61ture_view\x18\x01 \x01(\x0b\x32\x17.feast.core.FeatureViewH\x00\x12\x44\n\x17\x66\x65\x61ture_view_projection\x18\x03 \x01(\x0b\x32!.feast.core.FeatureViewProjectionH\x00\x12\x35\n\x13request_data_source\x18\x02 \x01(\x0b\x32\x16.feast.core.DataSourceH\x00\x42\x08\n\x06source\"H\n\x13UserDefinedFunction\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0c\n\x04\x62ody\x18\x02 \x01(\x0c\x12\x11\n\tbody_text\x18\x03 \x01(\t:\x02\x18\x01\x42]\n\x10\x66\x65\x61st.proto.coreB\x18OnDemandFeatureViewProtoZ/github.com/feast-dev/feast/go/protos/feast/coreb\x06proto3') +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n$feast/core/OnDemandFeatureView.proto\x12\nfeast.core\x1a\x1fgoogle/protobuf/timestamp.proto\x1a\x1c\x66\x65\x61st/core/FeatureView.proto\x1a&feast/core/FeatureViewProjection.proto\x1a\x18\x66\x65\x61st/core/Feature.proto\x1a\x1b\x66\x65\x61st/core/DataSource.proto\x1a\x1f\x66\x65\x61st/core/Transformation.proto\"{\n\x13OnDemandFeatureView\x12\x31\n\x04spec\x18\x01 \x01(\x0b\x32#.feast.core.OnDemandFeatureViewSpec\x12\x31\n\x04meta\x18\x02 \x01(\x0b\x32#.feast.core.OnDemandFeatureViewMeta\"\x90\x05\n\x17OnDemandFeatureViewSpec\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07project\x18\x02 \x01(\t\x12+\n\x08\x66\x65\x61tures\x18\x03 \x03(\x0b\x32\x19.feast.core.FeatureSpecV2\x12\x41\n\x07sources\x18\x04 \x03(\x0b\x32\x30.feast.core.OnDemandFeatureViewSpec.SourcesEntry\x12\x42\n\x15user_defined_function\x18\x05 \x01(\x0b\x32\x1f.feast.core.UserDefinedFunctionB\x02\x18\x01\x12\x43\n\x16\x66\x65\x61ture_transformation\x18\n \x01(\x0b\x32#.feast.core.FeatureTransformationV2\x12\x13\n\x0b\x64\x65scription\x18\x06 \x01(\t\x12;\n\x04tags\x18\x07 \x03(\x0b\x32-.feast.core.OnDemandFeatureViewSpec.TagsEntry\x12\r\n\x05owner\x18\x08 \x01(\t\x12\x0c\n\x04mode\x18\x0b \x01(\t\x12\x1d\n\x15write_to_online_store\x18\x0c \x01(\x08\x12\x10\n\x08\x65ntities\x18\r \x03(\t\x12\x31\n\x0e\x65ntity_columns\x18\x0e \x03(\x0b\x32\x19.feast.core.FeatureSpecV2\x12\x11\n\tsingleton\x18\x0f \x01(\x08\x1aJ\n\x0cSourcesEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12)\n\x05value\x18\x02 \x01(\x0b\x32\x1a.feast.core.OnDemandSource:\x02\x38\x01\x1a+\n\tTagsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"\x8c\x01\n\x17OnDemandFeatureViewMeta\x12\x35\n\x11\x63reated_timestamp\x18\x01 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12:\n\x16last_updated_timestamp\x18\x02 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\"\xc8\x01\n\x0eOnDemandSource\x12/\n\x0c\x66\x65\x61ture_view\x18\x01 \x01(\x0b\x32\x17.feast.core.FeatureViewH\x00\x12\x44\n\x17\x66\x65\x61ture_view_projection\x18\x03 \x01(\x0b\x32!.feast.core.FeatureViewProjectionH\x00\x12\x35\n\x13request_data_source\x18\x02 \x01(\x0b\x32\x16.feast.core.DataSourceH\x00\x42\x08\n\x06source\"H\n\x13UserDefinedFunction\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0c\n\x04\x62ody\x18\x02 \x01(\x0c\x12\x11\n\tbody_text\x18\x03 \x01(\t:\x02\x18\x01\"X\n\x17OnDemandFeatureViewList\x12=\n\x14ondemandfeatureviews\x18\x01 \x03(\x0b\x32\x1f.feast.core.OnDemandFeatureViewB]\n\x10\x66\x65\x61st.proto.coreB\x18OnDemandFeatureViewProtoZ/github.com/feast-dev/feast/go/protos/feast/coreb\x06proto3') _globals = globals() _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) @@ -50,4 +50,6 @@ _globals['_ONDEMANDSOURCE']._serialized_end=1371 _globals['_USERDEFINEDFUNCTION']._serialized_start=1373 _globals['_USERDEFINEDFUNCTION']._serialized_end=1445 + _globals['_ONDEMANDFEATUREVIEWLIST']._serialized_start=1447 + _globals['_ONDEMANDFEATUREVIEWLIST']._serialized_end=1535 # @@protoc_insertion_point(module_scope) diff --git a/sdk/python/feast/protos/feast/core/OnDemandFeatureView_pb2.pyi b/sdk/python/feast/protos/feast/core/OnDemandFeatureView_pb2.pyi index 3380779c97e..c9fca2f550d 100644 --- a/sdk/python/feast/protos/feast/core/OnDemandFeatureView_pb2.pyi +++ b/sdk/python/feast/protos/feast/core/OnDemandFeatureView_pb2.pyi @@ -233,3 +233,18 @@ class UserDefinedFunction(google.protobuf.message.Message): def ClearField(self, field_name: typing_extensions.Literal["body", b"body", "body_text", b"body_text", "name", b"name"]) -> None: ... global___UserDefinedFunction = UserDefinedFunction + +class OnDemandFeatureViewList(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + ONDEMANDFEATUREVIEWS_FIELD_NUMBER: builtins.int + @property + def ondemandfeatureviews(self) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[global___OnDemandFeatureView]: ... + def __init__( + self, + *, + ondemandfeatureviews: collections.abc.Iterable[global___OnDemandFeatureView] | None = ..., + ) -> None: ... + def ClearField(self, field_name: typing_extensions.Literal["ondemandfeatureviews", b"ondemandfeatureviews"]) -> None: ... + +global___OnDemandFeatureViewList = OnDemandFeatureViewList diff --git a/sdk/python/feast/protos/feast/registry/RegistryServer_pb2.py b/sdk/python/feast/protos/feast/registry/RegistryServer_pb2.py index e0cae3da4b7..2d5f7b020ab 100644 --- a/sdk/python/feast/protos/feast/registry/RegistryServer_pb2.py +++ b/sdk/python/feast/protos/feast/registry/RegistryServer_pb2.py @@ -28,13 +28,14 @@ from feast.protos.feast.core import Project_pb2 as feast_dot_core_dot_Project__pb2 -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n#feast/registry/RegistryServer.proto\x12\x0e\x66\x65\x61st.registry\x1a\x1bgoogle/protobuf/empty.proto\x1a\x1fgoogle/protobuf/timestamp.proto\x1a\x19\x66\x65\x61st/core/Registry.proto\x1a\x17\x66\x65\x61st/core/Entity.proto\x1a\x1b\x66\x65\x61st/core/DataSource.proto\x1a\x1c\x66\x65\x61st/core/FeatureView.proto\x1a\"feast/core/StreamFeatureView.proto\x1a$feast/core/OnDemandFeatureView.proto\x1a\x1f\x66\x65\x61st/core/FeatureService.proto\x1a\x1d\x66\x65\x61st/core/SavedDataset.proto\x1a\"feast/core/ValidationProfile.proto\x1a\x1c\x66\x65\x61st/core/InfraObject.proto\x1a\x1b\x66\x65\x61st/core/Permission.proto\x1a\x18\x66\x65\x61st/core/Project.proto\"!\n\x0eRefreshRequest\x12\x0f\n\x07project\x18\x01 \x01(\t\"W\n\x12UpdateInfraRequest\x12 \n\x05infra\x18\x01 \x01(\x0b\x32\x11.feast.core.Infra\x12\x0f\n\x07project\x18\x02 \x01(\t\x12\x0e\n\x06\x63ommit\x18\x03 \x01(\x08\"7\n\x0fGetInfraRequest\x12\x0f\n\x07project\x18\x01 \x01(\t\x12\x13\n\x0b\x61llow_cache\x18\x02 \x01(\x08\"B\n\x1aListProjectMetadataRequest\x12\x0f\n\x07project\x18\x01 \x01(\t\x12\x13\n\x0b\x61llow_cache\x18\x02 \x01(\x08\"T\n\x1bListProjectMetadataResponse\x12\x35\n\x10project_metadata\x18\x01 \x03(\x0b\x32\x1b.feast.core.ProjectMetadata\"\xcb\x01\n\x1b\x41pplyMaterializationRequest\x12-\n\x0c\x66\x65\x61ture_view\x18\x01 \x01(\x0b\x32\x17.feast.core.FeatureView\x12\x0f\n\x07project\x18\x02 \x01(\t\x12.\n\nstart_date\x18\x03 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12,\n\x08\x65nd_date\x18\x04 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x0e\n\x06\x63ommit\x18\x05 \x01(\x08\"Y\n\x12\x41pplyEntityRequest\x12\"\n\x06\x65ntity\x18\x01 \x01(\x0b\x32\x12.feast.core.Entity\x12\x0f\n\x07project\x18\x02 \x01(\t\x12\x0e\n\x06\x63ommit\x18\x03 \x01(\x08\"F\n\x10GetEntityRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07project\x18\x02 \x01(\t\x12\x13\n\x0b\x61llow_cache\x18\x03 \x01(\x08\"\xa5\x01\n\x13ListEntitiesRequest\x12\x0f\n\x07project\x18\x01 \x01(\t\x12\x13\n\x0b\x61llow_cache\x18\x02 \x01(\x08\x12;\n\x04tags\x18\x03 \x03(\x0b\x32-.feast.registry.ListEntitiesRequest.TagsEntry\x1a+\n\tTagsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"<\n\x14ListEntitiesResponse\x12$\n\x08\x65ntities\x18\x01 \x03(\x0b\x32\x12.feast.core.Entity\"D\n\x13\x44\x65leteEntityRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07project\x18\x02 \x01(\t\x12\x0e\n\x06\x63ommit\x18\x03 \x01(\x08\"f\n\x16\x41pplyDataSourceRequest\x12+\n\x0b\x64\x61ta_source\x18\x01 \x01(\x0b\x32\x16.feast.core.DataSource\x12\x0f\n\x07project\x18\x02 \x01(\t\x12\x0e\n\x06\x63ommit\x18\x03 \x01(\x08\"J\n\x14GetDataSourceRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07project\x18\x02 \x01(\t\x12\x13\n\x0b\x61llow_cache\x18\x03 \x01(\x08\"\xab\x01\n\x16ListDataSourcesRequest\x12\x0f\n\x07project\x18\x01 \x01(\t\x12\x13\n\x0b\x61llow_cache\x18\x02 \x01(\x08\x12>\n\x04tags\x18\x03 \x03(\x0b\x32\x30.feast.registry.ListDataSourcesRequest.TagsEntry\x1a+\n\tTagsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"G\n\x17ListDataSourcesResponse\x12,\n\x0c\x64\x61ta_sources\x18\x01 \x03(\x0b\x32\x16.feast.core.DataSource\"H\n\x17\x44\x65leteDataSourceRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07project\x18\x02 \x01(\t\x12\x0e\n\x06\x63ommit\x18\x03 \x01(\x08\"\x81\x02\n\x17\x41pplyFeatureViewRequest\x12/\n\x0c\x66\x65\x61ture_view\x18\x01 \x01(\x0b\x32\x17.feast.core.FeatureViewH\x00\x12\x41\n\x16on_demand_feature_view\x18\x02 \x01(\x0b\x32\x1f.feast.core.OnDemandFeatureViewH\x00\x12<\n\x13stream_feature_view\x18\x03 \x01(\x0b\x32\x1d.feast.core.StreamFeatureViewH\x00\x12\x0f\n\x07project\x18\x04 \x01(\t\x12\x0e\n\x06\x63ommit\x18\x05 \x01(\x08\x42\x13\n\x11\x62\x61se_feature_view\"K\n\x15GetFeatureViewRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07project\x18\x02 \x01(\t\x12\x13\n\x0b\x61llow_cache\x18\x03 \x01(\x08\"\xad\x01\n\x17ListFeatureViewsRequest\x12\x0f\n\x07project\x18\x01 \x01(\t\x12\x13\n\x0b\x61llow_cache\x18\x02 \x01(\x08\x12?\n\x04tags\x18\x03 \x03(\x0b\x32\x31.feast.registry.ListFeatureViewsRequest.TagsEntry\x1a+\n\tTagsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"J\n\x18ListFeatureViewsResponse\x12.\n\rfeature_views\x18\x01 \x03(\x0b\x32\x17.feast.core.FeatureView\"I\n\x18\x44\x65leteFeatureViewRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07project\x18\x02 \x01(\t\x12\x0e\n\x06\x63ommit\x18\x03 \x01(\x08\"\xd6\x01\n\x0e\x41nyFeatureView\x12/\n\x0c\x66\x65\x61ture_view\x18\x01 \x01(\x0b\x32\x17.feast.core.FeatureViewH\x00\x12\x41\n\x16on_demand_feature_view\x18\x02 \x01(\x0b\x32\x1f.feast.core.OnDemandFeatureViewH\x00\x12<\n\x13stream_feature_view\x18\x03 \x01(\x0b\x32\x1d.feast.core.StreamFeatureViewH\x00\x42\x12\n\x10\x61ny_feature_view\"N\n\x18GetAnyFeatureViewRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07project\x18\x02 \x01(\t\x12\x13\n\x0b\x61llow_cache\x18\x03 \x01(\x08\"U\n\x19GetAnyFeatureViewResponse\x12\x38\n\x10\x61ny_feature_view\x18\x01 \x01(\x0b\x32\x1e.feast.registry.AnyFeatureView\"\xb3\x01\n\x1aListAllFeatureViewsRequest\x12\x0f\n\x07project\x18\x01 \x01(\t\x12\x13\n\x0b\x61llow_cache\x18\x02 \x01(\x08\x12\x42\n\x04tags\x18\x03 \x03(\x0b\x32\x34.feast.registry.ListAllFeatureViewsRequest.TagsEntry\x1a+\n\tTagsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"T\n\x1bListAllFeatureViewsResponse\x12\x35\n\rfeature_views\x18\x01 \x03(\x0b\x32\x1e.feast.registry.AnyFeatureView\"Q\n\x1bGetStreamFeatureViewRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07project\x18\x02 \x01(\t\x12\x13\n\x0b\x61llow_cache\x18\x03 \x01(\x08\"\xb9\x01\n\x1dListStreamFeatureViewsRequest\x12\x0f\n\x07project\x18\x01 \x01(\t\x12\x13\n\x0b\x61llow_cache\x18\x02 \x01(\x08\x12\x45\n\x04tags\x18\x03 \x03(\x0b\x32\x37.feast.registry.ListStreamFeatureViewsRequest.TagsEntry\x1a+\n\tTagsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"]\n\x1eListStreamFeatureViewsResponse\x12;\n\x14stream_feature_views\x18\x01 \x03(\x0b\x32\x1d.feast.core.StreamFeatureView\"S\n\x1dGetOnDemandFeatureViewRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07project\x18\x02 \x01(\t\x12\x13\n\x0b\x61llow_cache\x18\x03 \x01(\x08\"\xbd\x01\n\x1fListOnDemandFeatureViewsRequest\x12\x0f\n\x07project\x18\x01 \x01(\t\x12\x13\n\x0b\x61llow_cache\x18\x02 \x01(\x08\x12G\n\x04tags\x18\x03 \x03(\x0b\x32\x39.feast.registry.ListOnDemandFeatureViewsRequest.TagsEntry\x1a+\n\tTagsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"d\n ListOnDemandFeatureViewsResponse\x12@\n\x17on_demand_feature_views\x18\x01 \x03(\x0b\x32\x1f.feast.core.OnDemandFeatureView\"r\n\x1a\x41pplyFeatureServiceRequest\x12\x33\n\x0f\x66\x65\x61ture_service\x18\x01 \x01(\x0b\x32\x1a.feast.core.FeatureService\x12\x0f\n\x07project\x18\x02 \x01(\t\x12\x0e\n\x06\x63ommit\x18\x03 \x01(\x08\"N\n\x18GetFeatureServiceRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07project\x18\x02 \x01(\t\x12\x13\n\x0b\x61llow_cache\x18\x03 \x01(\x08\"\xb3\x01\n\x1aListFeatureServicesRequest\x12\x0f\n\x07project\x18\x01 \x01(\t\x12\x13\n\x0b\x61llow_cache\x18\x02 \x01(\x08\x12\x42\n\x04tags\x18\x03 \x03(\x0b\x32\x34.feast.registry.ListFeatureServicesRequest.TagsEntry\x1a+\n\tTagsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"S\n\x1bListFeatureServicesResponse\x12\x34\n\x10\x66\x65\x61ture_services\x18\x01 \x03(\x0b\x32\x1a.feast.core.FeatureService\"L\n\x1b\x44\x65leteFeatureServiceRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07project\x18\x02 \x01(\t\x12\x0e\n\x06\x63ommit\x18\x03 \x01(\x08\"l\n\x18\x41pplySavedDatasetRequest\x12/\n\rsaved_dataset\x18\x01 \x01(\x0b\x32\x18.feast.core.SavedDataset\x12\x0f\n\x07project\x18\x02 \x01(\t\x12\x0e\n\x06\x63ommit\x18\x03 \x01(\x08\"L\n\x16GetSavedDatasetRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07project\x18\x02 \x01(\t\x12\x13\n\x0b\x61llow_cache\x18\x03 \x01(\x08\"\xaf\x01\n\x18ListSavedDatasetsRequest\x12\x0f\n\x07project\x18\x01 \x01(\t\x12\x13\n\x0b\x61llow_cache\x18\x02 \x01(\x08\x12@\n\x04tags\x18\x03 \x03(\x0b\x32\x32.feast.registry.ListSavedDatasetsRequest.TagsEntry\x1a+\n\tTagsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"M\n\x19ListSavedDatasetsResponse\x12\x30\n\x0esaved_datasets\x18\x01 \x03(\x0b\x32\x18.feast.core.SavedDataset\"J\n\x19\x44\x65leteSavedDatasetRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07project\x18\x02 \x01(\t\x12\x0e\n\x06\x63ommit\x18\x03 \x01(\x08\"\x81\x01\n\x1f\x41pplyValidationReferenceRequest\x12=\n\x14validation_reference\x18\x01 \x01(\x0b\x32\x1f.feast.core.ValidationReference\x12\x0f\n\x07project\x18\x02 \x01(\t\x12\x0e\n\x06\x63ommit\x18\x03 \x01(\x08\"S\n\x1dGetValidationReferenceRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07project\x18\x02 \x01(\t\x12\x13\n\x0b\x61llow_cache\x18\x03 \x01(\x08\"\xbd\x01\n\x1fListValidationReferencesRequest\x12\x0f\n\x07project\x18\x01 \x01(\t\x12\x13\n\x0b\x61llow_cache\x18\x02 \x01(\x08\x12G\n\x04tags\x18\x03 \x03(\x0b\x32\x39.feast.registry.ListValidationReferencesRequest.TagsEntry\x1a+\n\tTagsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"b\n ListValidationReferencesResponse\x12>\n\x15validation_references\x18\x01 \x03(\x0b\x32\x1f.feast.core.ValidationReference\"Q\n DeleteValidationReferenceRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07project\x18\x02 \x01(\t\x12\x0e\n\x06\x63ommit\x18\x03 \x01(\x08\"e\n\x16\x41pplyPermissionRequest\x12*\n\npermission\x18\x01 \x01(\x0b\x32\x16.feast.core.Permission\x12\x0f\n\x07project\x18\x02 \x01(\t\x12\x0e\n\x06\x63ommit\x18\x03 \x01(\x08\"J\n\x14GetPermissionRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07project\x18\x02 \x01(\t\x12\x13\n\x0b\x61llow_cache\x18\x03 \x01(\x08\"\xab\x01\n\x16ListPermissionsRequest\x12\x0f\n\x07project\x18\x01 \x01(\t\x12\x13\n\x0b\x61llow_cache\x18\x02 \x01(\x08\x12>\n\x04tags\x18\x03 \x03(\x0b\x32\x30.feast.registry.ListPermissionsRequest.TagsEntry\x1a+\n\tTagsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"F\n\x17ListPermissionsResponse\x12+\n\x0bpermissions\x18\x01 \x03(\x0b\x32\x16.feast.core.Permission\"H\n\x17\x44\x65letePermissionRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07project\x18\x02 \x01(\t\x12\x0e\n\x06\x63ommit\x18\x03 \x01(\x08\"K\n\x13\x41pplyProjectRequest\x12$\n\x07project\x18\x01 \x01(\x0b\x32\x13.feast.core.Project\x12\x0e\n\x06\x63ommit\x18\x02 \x01(\x08\"6\n\x11GetProjectRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x13\n\x0b\x61llow_cache\x18\x02 \x01(\x08\"\x94\x01\n\x13ListProjectsRequest\x12\x13\n\x0b\x61llow_cache\x18\x01 \x01(\x08\x12;\n\x04tags\x18\x02 \x03(\x0b\x32-.feast.registry.ListProjectsRequest.TagsEntry\x1a+\n\tTagsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"=\n\x14ListProjectsResponse\x12%\n\x08projects\x18\x01 \x03(\x0b\x32\x13.feast.core.Project\"4\n\x14\x44\x65leteProjectRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0e\n\x06\x63ommit\x18\x02 \x01(\x08\x32\xcb \n\x0eRegistryServer\x12K\n\x0b\x41pplyEntity\x12\".feast.registry.ApplyEntityRequest\x1a\x16.google.protobuf.Empty\"\x00\x12\x43\n\tGetEntity\x12 .feast.registry.GetEntityRequest\x1a\x12.feast.core.Entity\"\x00\x12[\n\x0cListEntities\x12#.feast.registry.ListEntitiesRequest\x1a$.feast.registry.ListEntitiesResponse\"\x00\x12M\n\x0c\x44\x65leteEntity\x12#.feast.registry.DeleteEntityRequest\x1a\x16.google.protobuf.Empty\"\x00\x12S\n\x0f\x41pplyDataSource\x12&.feast.registry.ApplyDataSourceRequest\x1a\x16.google.protobuf.Empty\"\x00\x12O\n\rGetDataSource\x12$.feast.registry.GetDataSourceRequest\x1a\x16.feast.core.DataSource\"\x00\x12\x64\n\x0fListDataSources\x12&.feast.registry.ListDataSourcesRequest\x1a\'.feast.registry.ListDataSourcesResponse\"\x00\x12U\n\x10\x44\x65leteDataSource\x12\'.feast.registry.DeleteDataSourceRequest\x1a\x16.google.protobuf.Empty\"\x00\x12U\n\x10\x41pplyFeatureView\x12\'.feast.registry.ApplyFeatureViewRequest\x1a\x16.google.protobuf.Empty\"\x00\x12W\n\x11\x44\x65leteFeatureView\x12(.feast.registry.DeleteFeatureViewRequest\x1a\x16.google.protobuf.Empty\"\x00\x12j\n\x11GetAnyFeatureView\x12(.feast.registry.GetAnyFeatureViewRequest\x1a).feast.registry.GetAnyFeatureViewResponse\"\x00\x12p\n\x13ListAllFeatureViews\x12*.feast.registry.ListAllFeatureViewsRequest\x1a+.feast.registry.ListAllFeatureViewsResponse\"\x00\x12R\n\x0eGetFeatureView\x12%.feast.registry.GetFeatureViewRequest\x1a\x17.feast.core.FeatureView\"\x00\x12g\n\x10ListFeatureViews\x12\'.feast.registry.ListFeatureViewsRequest\x1a(.feast.registry.ListFeatureViewsResponse\"\x00\x12\x64\n\x14GetStreamFeatureView\x12+.feast.registry.GetStreamFeatureViewRequest\x1a\x1d.feast.core.StreamFeatureView\"\x00\x12y\n\x16ListStreamFeatureViews\x12-.feast.registry.ListStreamFeatureViewsRequest\x1a..feast.registry.ListStreamFeatureViewsResponse\"\x00\x12j\n\x16GetOnDemandFeatureView\x12-.feast.registry.GetOnDemandFeatureViewRequest\x1a\x1f.feast.core.OnDemandFeatureView\"\x00\x12\x7f\n\x18ListOnDemandFeatureViews\x12/.feast.registry.ListOnDemandFeatureViewsRequest\x1a\x30.feast.registry.ListOnDemandFeatureViewsResponse\"\x00\x12[\n\x13\x41pplyFeatureService\x12*.feast.registry.ApplyFeatureServiceRequest\x1a\x16.google.protobuf.Empty\"\x00\x12[\n\x11GetFeatureService\x12(.feast.registry.GetFeatureServiceRequest\x1a\x1a.feast.core.FeatureService\"\x00\x12p\n\x13ListFeatureServices\x12*.feast.registry.ListFeatureServicesRequest\x1a+.feast.registry.ListFeatureServicesResponse\"\x00\x12]\n\x14\x44\x65leteFeatureService\x12+.feast.registry.DeleteFeatureServiceRequest\x1a\x16.google.protobuf.Empty\"\x00\x12W\n\x11\x41pplySavedDataset\x12(.feast.registry.ApplySavedDatasetRequest\x1a\x16.google.protobuf.Empty\"\x00\x12U\n\x0fGetSavedDataset\x12&.feast.registry.GetSavedDatasetRequest\x1a\x18.feast.core.SavedDataset\"\x00\x12j\n\x11ListSavedDatasets\x12(.feast.registry.ListSavedDatasetsRequest\x1a).feast.registry.ListSavedDatasetsResponse\"\x00\x12Y\n\x12\x44\x65leteSavedDataset\x12).feast.registry.DeleteSavedDatasetRequest\x1a\x16.google.protobuf.Empty\"\x00\x12\x65\n\x18\x41pplyValidationReference\x12/.feast.registry.ApplyValidationReferenceRequest\x1a\x16.google.protobuf.Empty\"\x00\x12j\n\x16GetValidationReference\x12-.feast.registry.GetValidationReferenceRequest\x1a\x1f.feast.core.ValidationReference\"\x00\x12\x7f\n\x18ListValidationReferences\x12/.feast.registry.ListValidationReferencesRequest\x1a\x30.feast.registry.ListValidationReferencesResponse\"\x00\x12g\n\x19\x44\x65leteValidationReference\x12\x30.feast.registry.DeleteValidationReferenceRequest\x1a\x16.google.protobuf.Empty\"\x00\x12S\n\x0f\x41pplyPermission\x12&.feast.registry.ApplyPermissionRequest\x1a\x16.google.protobuf.Empty\"\x00\x12O\n\rGetPermission\x12$.feast.registry.GetPermissionRequest\x1a\x16.feast.core.Permission\"\x00\x12\x64\n\x0fListPermissions\x12&.feast.registry.ListPermissionsRequest\x1a\'.feast.registry.ListPermissionsResponse\"\x00\x12U\n\x10\x44\x65letePermission\x12\'.feast.registry.DeletePermissionRequest\x1a\x16.google.protobuf.Empty\"\x00\x12M\n\x0c\x41pplyProject\x12#.feast.registry.ApplyProjectRequest\x1a\x16.google.protobuf.Empty\"\x00\x12\x46\n\nGetProject\x12!.feast.registry.GetProjectRequest\x1a\x13.feast.core.Project\"\x00\x12[\n\x0cListProjects\x12#.feast.registry.ListProjectsRequest\x1a$.feast.registry.ListProjectsResponse\"\x00\x12O\n\rDeleteProject\x12$.feast.registry.DeleteProjectRequest\x1a\x16.google.protobuf.Empty\"\x00\x12]\n\x14\x41pplyMaterialization\x12+.feast.registry.ApplyMaterializationRequest\x1a\x16.google.protobuf.Empty\"\x00\x12p\n\x13ListProjectMetadata\x12*.feast.registry.ListProjectMetadataRequest\x1a+.feast.registry.ListProjectMetadataResponse\"\x00\x12K\n\x0bUpdateInfra\x12\".feast.registry.UpdateInfraRequest\x1a\x16.google.protobuf.Empty\"\x00\x12@\n\x08GetInfra\x12\x1f.feast.registry.GetInfraRequest\x1a\x11.feast.core.Infra\"\x00\x12:\n\x06\x43ommit\x12\x16.google.protobuf.Empty\x1a\x16.google.protobuf.Empty\"\x00\x12\x43\n\x07Refresh\x12\x1e.feast.registry.RefreshRequest\x1a\x16.google.protobuf.Empty\"\x00\x12\x37\n\x05Proto\x12\x16.google.protobuf.Empty\x1a\x14.feast.core.Registry\"\x00\x62\x06proto3') +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n#feast/registry/RegistryServer.proto\x12\x0e\x66\x65\x61st.registry\x1a\x1bgoogle/protobuf/empty.proto\x1a\x1fgoogle/protobuf/timestamp.proto\x1a\x19\x66\x65\x61st/core/Registry.proto\x1a\x17\x66\x65\x61st/core/Entity.proto\x1a\x1b\x66\x65\x61st/core/DataSource.proto\x1a\x1c\x66\x65\x61st/core/FeatureView.proto\x1a\"feast/core/StreamFeatureView.proto\x1a$feast/core/OnDemandFeatureView.proto\x1a\x1f\x66\x65\x61st/core/FeatureService.proto\x1a\x1d\x66\x65\x61st/core/SavedDataset.proto\x1a\"feast/core/ValidationProfile.proto\x1a\x1c\x66\x65\x61st/core/InfraObject.proto\x1a\x1b\x66\x65\x61st/core/Permission.proto\x1a\x18\x66\x65\x61st/core/Project.proto\"!\n\x0eRefreshRequest\x12\x0f\n\x07project\x18\x01 \x01(\t\"W\n\x12UpdateInfraRequest\x12 \n\x05infra\x18\x01 \x01(\x0b\x32\x11.feast.core.Infra\x12\x0f\n\x07project\x18\x02 \x01(\t\x12\x0e\n\x06\x63ommit\x18\x03 \x01(\x08\"7\n\x0fGetInfraRequest\x12\x0f\n\x07project\x18\x01 \x01(\t\x12\x13\n\x0b\x61llow_cache\x18\x02 \x01(\x08\"B\n\x1aListProjectMetadataRequest\x12\x0f\n\x07project\x18\x01 \x01(\t\x12\x13\n\x0b\x61llow_cache\x18\x02 \x01(\x08\"T\n\x1bListProjectMetadataResponse\x12\x35\n\x10project_metadata\x18\x01 \x03(\x0b\x32\x1b.feast.core.ProjectMetadata\"\xcb\x01\n\x1b\x41pplyMaterializationRequest\x12-\n\x0c\x66\x65\x61ture_view\x18\x01 \x01(\x0b\x32\x17.feast.core.FeatureView\x12\x0f\n\x07project\x18\x02 \x01(\t\x12.\n\nstart_date\x18\x03 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12,\n\x08\x65nd_date\x18\x04 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x0e\n\x06\x63ommit\x18\x05 \x01(\x08\"Y\n\x12\x41pplyEntityRequest\x12\"\n\x06\x65ntity\x18\x01 \x01(\x0b\x32\x12.feast.core.Entity\x12\x0f\n\x07project\x18\x02 \x01(\t\x12\x0e\n\x06\x63ommit\x18\x03 \x01(\x08\"F\n\x10GetEntityRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07project\x18\x02 \x01(\t\x12\x13\n\x0b\x61llow_cache\x18\x03 \x01(\x08\"\xa5\x01\n\x13ListEntitiesRequest\x12\x0f\n\x07project\x18\x01 \x01(\t\x12\x13\n\x0b\x61llow_cache\x18\x02 \x01(\x08\x12;\n\x04tags\x18\x03 \x03(\x0b\x32-.feast.registry.ListEntitiesRequest.TagsEntry\x1a+\n\tTagsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"<\n\x14ListEntitiesResponse\x12$\n\x08\x65ntities\x18\x01 \x03(\x0b\x32\x12.feast.core.Entity\"D\n\x13\x44\x65leteEntityRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07project\x18\x02 \x01(\t\x12\x0e\n\x06\x63ommit\x18\x03 \x01(\x08\"f\n\x16\x41pplyDataSourceRequest\x12+\n\x0b\x64\x61ta_source\x18\x01 \x01(\x0b\x32\x16.feast.core.DataSource\x12\x0f\n\x07project\x18\x02 \x01(\t\x12\x0e\n\x06\x63ommit\x18\x03 \x01(\x08\"J\n\x14GetDataSourceRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07project\x18\x02 \x01(\t\x12\x13\n\x0b\x61llow_cache\x18\x03 \x01(\x08\"\xab\x01\n\x16ListDataSourcesRequest\x12\x0f\n\x07project\x18\x01 \x01(\t\x12\x13\n\x0b\x61llow_cache\x18\x02 \x01(\x08\x12>\n\x04tags\x18\x03 \x03(\x0b\x32\x30.feast.registry.ListDataSourcesRequest.TagsEntry\x1a+\n\tTagsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"G\n\x17ListDataSourcesResponse\x12,\n\x0c\x64\x61ta_sources\x18\x01 \x03(\x0b\x32\x16.feast.core.DataSource\"H\n\x17\x44\x65leteDataSourceRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07project\x18\x02 \x01(\t\x12\x0e\n\x06\x63ommit\x18\x03 \x01(\x08\"\x81\x02\n\x17\x41pplyFeatureViewRequest\x12/\n\x0c\x66\x65\x61ture_view\x18\x01 \x01(\x0b\x32\x17.feast.core.FeatureViewH\x00\x12\x41\n\x16on_demand_feature_view\x18\x02 \x01(\x0b\x32\x1f.feast.core.OnDemandFeatureViewH\x00\x12<\n\x13stream_feature_view\x18\x03 \x01(\x0b\x32\x1d.feast.core.StreamFeatureViewH\x00\x12\x0f\n\x07project\x18\x04 \x01(\t\x12\x0e\n\x06\x63ommit\x18\x05 \x01(\x08\x42\x13\n\x11\x62\x61se_feature_view\"K\n\x15GetFeatureViewRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07project\x18\x02 \x01(\t\x12\x13\n\x0b\x61llow_cache\x18\x03 \x01(\x08\"\xad\x01\n\x17ListFeatureViewsRequest\x12\x0f\n\x07project\x18\x01 \x01(\t\x12\x13\n\x0b\x61llow_cache\x18\x02 \x01(\x08\x12?\n\x04tags\x18\x03 \x03(\x0b\x32\x31.feast.registry.ListFeatureViewsRequest.TagsEntry\x1a+\n\tTagsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"J\n\x18ListFeatureViewsResponse\x12.\n\rfeature_views\x18\x01 \x03(\x0b\x32\x17.feast.core.FeatureView\"I\n\x18\x44\x65leteFeatureViewRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07project\x18\x02 \x01(\t\x12\x0e\n\x06\x63ommit\x18\x03 \x01(\x08\"\xd6\x01\n\x0e\x41nyFeatureView\x12/\n\x0c\x66\x65\x61ture_view\x18\x01 \x01(\x0b\x32\x17.feast.core.FeatureViewH\x00\x12\x41\n\x16on_demand_feature_view\x18\x02 \x01(\x0b\x32\x1f.feast.core.OnDemandFeatureViewH\x00\x12<\n\x13stream_feature_view\x18\x03 \x01(\x0b\x32\x1d.feast.core.StreamFeatureViewH\x00\x42\x12\n\x10\x61ny_feature_view\"N\n\x18GetAnyFeatureViewRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07project\x18\x02 \x01(\t\x12\x13\n\x0b\x61llow_cache\x18\x03 \x01(\x08\"U\n\x19GetAnyFeatureViewResponse\x12\x38\n\x10\x61ny_feature_view\x18\x01 \x01(\x0b\x32\x1e.feast.registry.AnyFeatureView\"\xb3\x01\n\x1aListAllFeatureViewsRequest\x12\x0f\n\x07project\x18\x01 \x01(\t\x12\x13\n\x0b\x61llow_cache\x18\x02 \x01(\x08\x12\x42\n\x04tags\x18\x03 \x03(\x0b\x32\x34.feast.registry.ListAllFeatureViewsRequest.TagsEntry\x1a+\n\tTagsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"T\n\x1bListAllFeatureViewsResponse\x12\x35\n\rfeature_views\x18\x01 \x03(\x0b\x32\x1e.feast.registry.AnyFeatureView\"Q\n\x1bGetStreamFeatureViewRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07project\x18\x02 \x01(\t\x12\x13\n\x0b\x61llow_cache\x18\x03 \x01(\x08\"\xb9\x01\n\x1dListStreamFeatureViewsRequest\x12\x0f\n\x07project\x18\x01 \x01(\t\x12\x13\n\x0b\x61llow_cache\x18\x02 \x01(\x08\x12\x45\n\x04tags\x18\x03 \x03(\x0b\x32\x37.feast.registry.ListStreamFeatureViewsRequest.TagsEntry\x1a+\n\tTagsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"]\n\x1eListStreamFeatureViewsResponse\x12;\n\x14stream_feature_views\x18\x01 \x03(\x0b\x32\x1d.feast.core.StreamFeatureView\"S\n\x1dGetOnDemandFeatureViewRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07project\x18\x02 \x01(\t\x12\x13\n\x0b\x61llow_cache\x18\x03 \x01(\x08\"\xbd\x01\n\x1fListOnDemandFeatureViewsRequest\x12\x0f\n\x07project\x18\x01 \x01(\t\x12\x13\n\x0b\x61llow_cache\x18\x02 \x01(\x08\x12G\n\x04tags\x18\x03 \x03(\x0b\x32\x39.feast.registry.ListOnDemandFeatureViewsRequest.TagsEntry\x1a+\n\tTagsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"d\n ListOnDemandFeatureViewsResponse\x12@\n\x17on_demand_feature_views\x18\x01 \x03(\x0b\x32\x1f.feast.core.OnDemandFeatureView\"r\n\x1a\x41pplyFeatureServiceRequest\x12\x33\n\x0f\x66\x65\x61ture_service\x18\x01 \x01(\x0b\x32\x1a.feast.core.FeatureService\x12\x0f\n\x07project\x18\x02 \x01(\t\x12\x0e\n\x06\x63ommit\x18\x03 \x01(\x08\"N\n\x18GetFeatureServiceRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07project\x18\x02 \x01(\t\x12\x13\n\x0b\x61llow_cache\x18\x03 \x01(\x08\"\xb3\x01\n\x1aListFeatureServicesRequest\x12\x0f\n\x07project\x18\x01 \x01(\t\x12\x13\n\x0b\x61llow_cache\x18\x02 \x01(\x08\x12\x42\n\x04tags\x18\x03 \x03(\x0b\x32\x34.feast.registry.ListFeatureServicesRequest.TagsEntry\x1a+\n\tTagsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"S\n\x1bListFeatureServicesResponse\x12\x34\n\x10\x66\x65\x61ture_services\x18\x01 \x03(\x0b\x32\x1a.feast.core.FeatureService\"L\n\x1b\x44\x65leteFeatureServiceRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07project\x18\x02 \x01(\t\x12\x0e\n\x06\x63ommit\x18\x03 \x01(\x08\"l\n\x18\x41pplySavedDatasetRequest\x12/\n\rsaved_dataset\x18\x01 \x01(\x0b\x32\x18.feast.core.SavedDataset\x12\x0f\n\x07project\x18\x02 \x01(\t\x12\x0e\n\x06\x63ommit\x18\x03 \x01(\x08\"L\n\x16GetSavedDatasetRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07project\x18\x02 \x01(\t\x12\x13\n\x0b\x61llow_cache\x18\x03 \x01(\x08\"\xaf\x01\n\x18ListSavedDatasetsRequest\x12\x0f\n\x07project\x18\x01 \x01(\t\x12\x13\n\x0b\x61llow_cache\x18\x02 \x01(\x08\x12@\n\x04tags\x18\x03 \x03(\x0b\x32\x32.feast.registry.ListSavedDatasetsRequest.TagsEntry\x1a+\n\tTagsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"M\n\x19ListSavedDatasetsResponse\x12\x30\n\x0esaved_datasets\x18\x01 \x03(\x0b\x32\x18.feast.core.SavedDataset\"J\n\x19\x44\x65leteSavedDatasetRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07project\x18\x02 \x01(\t\x12\x0e\n\x06\x63ommit\x18\x03 \x01(\x08\"\x81\x01\n\x1f\x41pplyValidationReferenceRequest\x12=\n\x14validation_reference\x18\x01 \x01(\x0b\x32\x1f.feast.core.ValidationReference\x12\x0f\n\x07project\x18\x02 \x01(\t\x12\x0e\n\x06\x63ommit\x18\x03 \x01(\x08\"S\n\x1dGetValidationReferenceRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07project\x18\x02 \x01(\t\x12\x13\n\x0b\x61llow_cache\x18\x03 \x01(\x08\"\xbd\x01\n\x1fListValidationReferencesRequest\x12\x0f\n\x07project\x18\x01 \x01(\t\x12\x13\n\x0b\x61llow_cache\x18\x02 \x01(\x08\x12G\n\x04tags\x18\x03 \x03(\x0b\x32\x39.feast.registry.ListValidationReferencesRequest.TagsEntry\x1a+\n\tTagsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"b\n ListValidationReferencesResponse\x12>\n\x15validation_references\x18\x01 \x03(\x0b\x32\x1f.feast.core.ValidationReference\"Q\n DeleteValidationReferenceRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07project\x18\x02 \x01(\t\x12\x0e\n\x06\x63ommit\x18\x03 \x01(\x08\"e\n\x16\x41pplyPermissionRequest\x12*\n\npermission\x18\x01 \x01(\x0b\x32\x16.feast.core.Permission\x12\x0f\n\x07project\x18\x02 \x01(\t\x12\x0e\n\x06\x63ommit\x18\x03 \x01(\x08\"J\n\x14GetPermissionRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07project\x18\x02 \x01(\t\x12\x13\n\x0b\x61llow_cache\x18\x03 \x01(\x08\"\xab\x01\n\x16ListPermissionsRequest\x12\x0f\n\x07project\x18\x01 \x01(\t\x12\x13\n\x0b\x61llow_cache\x18\x02 \x01(\x08\x12>\n\x04tags\x18\x03 \x03(\x0b\x32\x30.feast.registry.ListPermissionsRequest.TagsEntry\x1a+\n\tTagsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"F\n\x17ListPermissionsResponse\x12+\n\x0bpermissions\x18\x01 \x03(\x0b\x32\x16.feast.core.Permission\"H\n\x17\x44\x65letePermissionRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07project\x18\x02 \x01(\t\x12\x0e\n\x06\x63ommit\x18\x03 \x01(\x08\"K\n\x13\x41pplyProjectRequest\x12$\n\x07project\x18\x01 \x01(\x0b\x32\x13.feast.core.Project\x12\x0e\n\x06\x63ommit\x18\x02 \x01(\x08\"6\n\x11GetProjectRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x13\n\x0b\x61llow_cache\x18\x02 \x01(\x08\"\x94\x01\n\x13ListProjectsRequest\x12\x13\n\x0b\x61llow_cache\x18\x01 \x01(\x08\x12;\n\x04tags\x18\x02 \x03(\x0b\x32-.feast.registry.ListProjectsRequest.TagsEntry\x1a+\n\tTagsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"=\n\x14ListProjectsResponse\x12%\n\x08projects\x18\x01 \x03(\x0b\x32\x13.feast.core.Project\"4\n\x14\x44\x65leteProjectRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0e\n\x06\x63ommit\x18\x02 \x01(\x08\x32\xcb \n\x0eRegistryServer\x12K\n\x0b\x41pplyEntity\x12\".feast.registry.ApplyEntityRequest\x1a\x16.google.protobuf.Empty\"\x00\x12\x43\n\tGetEntity\x12 .feast.registry.GetEntityRequest\x1a\x12.feast.core.Entity\"\x00\x12[\n\x0cListEntities\x12#.feast.registry.ListEntitiesRequest\x1a$.feast.registry.ListEntitiesResponse\"\x00\x12M\n\x0c\x44\x65leteEntity\x12#.feast.registry.DeleteEntityRequest\x1a\x16.google.protobuf.Empty\"\x00\x12S\n\x0f\x41pplyDataSource\x12&.feast.registry.ApplyDataSourceRequest\x1a\x16.google.protobuf.Empty\"\x00\x12O\n\rGetDataSource\x12$.feast.registry.GetDataSourceRequest\x1a\x16.feast.core.DataSource\"\x00\x12\x64\n\x0fListDataSources\x12&.feast.registry.ListDataSourcesRequest\x1a\'.feast.registry.ListDataSourcesResponse\"\x00\x12U\n\x10\x44\x65leteDataSource\x12\'.feast.registry.DeleteDataSourceRequest\x1a\x16.google.protobuf.Empty\"\x00\x12U\n\x10\x41pplyFeatureView\x12\'.feast.registry.ApplyFeatureViewRequest\x1a\x16.google.protobuf.Empty\"\x00\x12W\n\x11\x44\x65leteFeatureView\x12(.feast.registry.DeleteFeatureViewRequest\x1a\x16.google.protobuf.Empty\"\x00\x12j\n\x11GetAnyFeatureView\x12(.feast.registry.GetAnyFeatureViewRequest\x1a).feast.registry.GetAnyFeatureViewResponse\"\x00\x12p\n\x13ListAllFeatureViews\x12*.feast.registry.ListAllFeatureViewsRequest\x1a+.feast.registry.ListAllFeatureViewsResponse\"\x00\x12R\n\x0eGetFeatureView\x12%.feast.registry.GetFeatureViewRequest\x1a\x17.feast.core.FeatureView\"\x00\x12g\n\x10ListFeatureViews\x12\'.feast.registry.ListFeatureViewsRequest\x1a(.feast.registry.ListFeatureViewsResponse\"\x00\x12\x64\n\x14GetStreamFeatureView\x12+.feast.registry.GetStreamFeatureViewRequest\x1a\x1d.feast.core.StreamFeatureView\"\x00\x12y\n\x16ListStreamFeatureViews\x12-.feast.registry.ListStreamFeatureViewsRequest\x1a..feast.registry.ListStreamFeatureViewsResponse\"\x00\x12j\n\x16GetOnDemandFeatureView\x12-.feast.registry.GetOnDemandFeatureViewRequest\x1a\x1f.feast.core.OnDemandFeatureView\"\x00\x12\x7f\n\x18ListOnDemandFeatureViews\x12/.feast.registry.ListOnDemandFeatureViewsRequest\x1a\x30.feast.registry.ListOnDemandFeatureViewsResponse\"\x00\x12[\n\x13\x41pplyFeatureService\x12*.feast.registry.ApplyFeatureServiceRequest\x1a\x16.google.protobuf.Empty\"\x00\x12[\n\x11GetFeatureService\x12(.feast.registry.GetFeatureServiceRequest\x1a\x1a.feast.core.FeatureService\"\x00\x12p\n\x13ListFeatureServices\x12*.feast.registry.ListFeatureServicesRequest\x1a+.feast.registry.ListFeatureServicesResponse\"\x00\x12]\n\x14\x44\x65leteFeatureService\x12+.feast.registry.DeleteFeatureServiceRequest\x1a\x16.google.protobuf.Empty\"\x00\x12W\n\x11\x41pplySavedDataset\x12(.feast.registry.ApplySavedDatasetRequest\x1a\x16.google.protobuf.Empty\"\x00\x12U\n\x0fGetSavedDataset\x12&.feast.registry.GetSavedDatasetRequest\x1a\x18.feast.core.SavedDataset\"\x00\x12j\n\x11ListSavedDatasets\x12(.feast.registry.ListSavedDatasetsRequest\x1a).feast.registry.ListSavedDatasetsResponse\"\x00\x12Y\n\x12\x44\x65leteSavedDataset\x12).feast.registry.DeleteSavedDatasetRequest\x1a\x16.google.protobuf.Empty\"\x00\x12\x65\n\x18\x41pplyValidationReference\x12/.feast.registry.ApplyValidationReferenceRequest\x1a\x16.google.protobuf.Empty\"\x00\x12j\n\x16GetValidationReference\x12-.feast.registry.GetValidationReferenceRequest\x1a\x1f.feast.core.ValidationReference\"\x00\x12\x7f\n\x18ListValidationReferences\x12/.feast.registry.ListValidationReferencesRequest\x1a\x30.feast.registry.ListValidationReferencesResponse\"\x00\x12g\n\x19\x44\x65leteValidationReference\x12\x30.feast.registry.DeleteValidationReferenceRequest\x1a\x16.google.protobuf.Empty\"\x00\x12S\n\x0f\x41pplyPermission\x12&.feast.registry.ApplyPermissionRequest\x1a\x16.google.protobuf.Empty\"\x00\x12O\n\rGetPermission\x12$.feast.registry.GetPermissionRequest\x1a\x16.feast.core.Permission\"\x00\x12\x64\n\x0fListPermissions\x12&.feast.registry.ListPermissionsRequest\x1a\'.feast.registry.ListPermissionsResponse\"\x00\x12U\n\x10\x44\x65letePermission\x12\'.feast.registry.DeletePermissionRequest\x1a\x16.google.protobuf.Empty\"\x00\x12M\n\x0c\x41pplyProject\x12#.feast.registry.ApplyProjectRequest\x1a\x16.google.protobuf.Empty\"\x00\x12\x46\n\nGetProject\x12!.feast.registry.GetProjectRequest\x1a\x13.feast.core.Project\"\x00\x12[\n\x0cListProjects\x12#.feast.registry.ListProjectsRequest\x1a$.feast.registry.ListProjectsResponse\"\x00\x12O\n\rDeleteProject\x12$.feast.registry.DeleteProjectRequest\x1a\x16.google.protobuf.Empty\"\x00\x12]\n\x14\x41pplyMaterialization\x12+.feast.registry.ApplyMaterializationRequest\x1a\x16.google.protobuf.Empty\"\x00\x12p\n\x13ListProjectMetadata\x12*.feast.registry.ListProjectMetadataRequest\x1a+.feast.registry.ListProjectMetadataResponse\"\x00\x12K\n\x0bUpdateInfra\x12\".feast.registry.UpdateInfraRequest\x1a\x16.google.protobuf.Empty\"\x00\x12@\n\x08GetInfra\x12\x1f.feast.registry.GetInfraRequest\x1a\x11.feast.core.Infra\"\x00\x12:\n\x06\x43ommit\x12\x16.google.protobuf.Empty\x1a\x16.google.protobuf.Empty\"\x00\x12\x43\n\x07Refresh\x12\x1e.feast.registry.RefreshRequest\x1a\x16.google.protobuf.Empty\"\x00\x12\x37\n\x05Proto\x12\x16.google.protobuf.Empty\x1a\x14.feast.core.Registry\"\x00\x42\x35Z3github.com/feast-dev/feast/go/protos/feast/registryb\x06proto3') _globals = globals() _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) _builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'feast.registry.RegistryServer_pb2', _globals) if _descriptor._USE_C_DESCRIPTORS == False: - DESCRIPTOR._options = None + _globals['DESCRIPTOR']._options = None + _globals['DESCRIPTOR']._serialized_options = b'Z3github.com/feast-dev/feast/go/protos/feast/registry' _globals['_LISTENTITIESREQUEST_TAGSENTRY']._options = None _globals['_LISTENTITIESREQUEST_TAGSENTRY']._serialized_options = b'8\001' _globals['_LISTDATASOURCESREQUEST_TAGSENTRY']._options = None diff --git a/sdk/python/feast/protos/feast/serving/GrpcServer_pb2.py b/sdk/python/feast/protos/feast/serving/GrpcServer_pb2.py index 8e40630cfff..ce4db37a658 100644 --- a/sdk/python/feast/protos/feast/serving/GrpcServer_pb2.py +++ b/sdk/python/feast/protos/feast/serving/GrpcServer_pb2.py @@ -15,13 +15,14 @@ from feast.protos.feast.serving import ServingService_pb2 as feast_dot_serving_dot_ServingService__pb2 -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x1e\x66\x65\x61st/serving/GrpcServer.proto\x1a\"feast/serving/ServingService.proto\"\xb3\x01\n\x0bPushRequest\x12,\n\x08\x66\x65\x61tures\x18\x01 \x03(\x0b\x32\x1a.PushRequest.FeaturesEntry\x12\x1b\n\x13stream_feature_view\x18\x02 \x01(\t\x12\x1c\n\x14\x61llow_registry_cache\x18\x03 \x01(\x08\x12\n\n\x02to\x18\x04 \x01(\t\x1a/\n\rFeaturesEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"\x1e\n\x0cPushResponse\x12\x0e\n\x06status\x18\x01 \x01(\x08\"\xc1\x01\n\x19WriteToOnlineStoreRequest\x12:\n\x08\x66\x65\x61tures\x18\x01 \x03(\x0b\x32(.WriteToOnlineStoreRequest.FeaturesEntry\x12\x19\n\x11\x66\x65\x61ture_view_name\x18\x02 \x01(\t\x12\x1c\n\x14\x61llow_registry_cache\x18\x03 \x01(\x08\x1a/\n\rFeaturesEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\",\n\x1aWriteToOnlineStoreResponse\x12\x0e\n\x06status\x18\x01 \x01(\x08\x32\xf1\x01\n\x11GrpcFeatureServer\x12%\n\x04Push\x12\x0c.PushRequest\x1a\r.PushResponse\"\x00\x12M\n\x12WriteToOnlineStore\x12\x1a.WriteToOnlineStoreRequest\x1a\x1b.WriteToOnlineStoreResponse\x12\x66\n\x11GetOnlineFeatures\x12\'.feast.serving.GetOnlineFeaturesRequest\x1a(.feast.serving.GetOnlineFeaturesResponseb\x06proto3') +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x1e\x66\x65\x61st/serving/GrpcServer.proto\x1a\"feast/serving/ServingService.proto\"\xb3\x01\n\x0bPushRequest\x12,\n\x08\x66\x65\x61tures\x18\x01 \x03(\x0b\x32\x1a.PushRequest.FeaturesEntry\x12\x1b\n\x13stream_feature_view\x18\x02 \x01(\t\x12\x1c\n\x14\x61llow_registry_cache\x18\x03 \x01(\x08\x12\n\n\x02to\x18\x04 \x01(\t\x1a/\n\rFeaturesEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"\x1e\n\x0cPushResponse\x12\x0e\n\x06status\x18\x01 \x01(\x08\"\xc1\x01\n\x19WriteToOnlineStoreRequest\x12:\n\x08\x66\x65\x61tures\x18\x01 \x03(\x0b\x32(.WriteToOnlineStoreRequest.FeaturesEntry\x12\x19\n\x11\x66\x65\x61ture_view_name\x18\x02 \x01(\t\x12\x1c\n\x14\x61llow_registry_cache\x18\x03 \x01(\x08\x1a/\n\rFeaturesEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\",\n\x1aWriteToOnlineStoreResponse\x12\x0e\n\x06status\x18\x01 \x01(\x08\x32\xf1\x01\n\x11GrpcFeatureServer\x12%\n\x04Push\x12\x0c.PushRequest\x1a\r.PushResponse\"\x00\x12M\n\x12WriteToOnlineStore\x12\x1a.WriteToOnlineStoreRequest\x1a\x1b.WriteToOnlineStoreResponse\x12\x66\n\x11GetOnlineFeatures\x12\'.feast.serving.GetOnlineFeaturesRequest\x1a(.feast.serving.GetOnlineFeaturesResponseB4Z2github.com/feast-dev/feast/go/protos/feast/servingb\x06proto3') _globals = globals() _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) _builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'feast.serving.GrpcServer_pb2', _globals) if _descriptor._USE_C_DESCRIPTORS == False: - DESCRIPTOR._options = None + _globals['DESCRIPTOR']._options = None + _globals['DESCRIPTOR']._serialized_options = b'Z2github.com/feast-dev/feast/go/protos/feast/serving' _globals['_PUSHREQUEST_FEATURESENTRY']._options = None _globals['_PUSHREQUEST_FEATURESENTRY']._serialized_options = b'8\001' _globals['_WRITETOONLINESTOREREQUEST_FEATURESENTRY']._options = None diff --git a/sdk/python/feast/repo_config.py b/sdk/python/feast/repo_config.py index fe34a12adf8..2b8d5174e1f 100644 --- a/sdk/python/feast/repo_config.py +++ b/sdk/python/feast/repo_config.py @@ -81,6 +81,7 @@ "singlestore": "feast.infra.online_stores.singlestore_online_store.singlestore.SingleStoreOnlineStore", "qdrant": "feast.infra.online_stores.cqdrant.QdrantOnlineStore", "couchbase": "feast.infra.online_stores.couchbase_online_store.couchbase.CouchbaseOnlineStore", + "milvus": "feast.infra.online_stores.milvus_online_store.milvus.MilvusOnlineStore", **LEGACY_ONLINE_STORE_CLASS_FOR_TYPE, } diff --git a/sdk/python/feast/ssl_ca_trust_store_setup.py b/sdk/python/feast/ssl_ca_trust_store_setup.py new file mode 100644 index 00000000000..72e84132187 --- /dev/null +++ b/sdk/python/feast/ssl_ca_trust_store_setup.py @@ -0,0 +1,22 @@ +import logging +import os + +logger = logging.getLogger(__name__) +logger.setLevel(logging.INFO) + + +def configure_ca_trust_store_env_variables(): + """ + configures the environment variable so that other libraries or servers refer to the TLS ca file path. + :param ca_file_path: + :return: + """ + if ( + "FEAST_CA_CERT_FILE_PATH" in os.environ + and os.environ["FEAST_CA_CERT_FILE_PATH"] + ): + logger.info( + f"Feast CA Cert file path found in environment variable FEAST_CA_CERT_FILE_PATH={os.environ['FEAST_CA_CERT_FILE_PATH']}. Going to refer this path." + ) + os.environ["SSL_CERT_FILE"] = os.environ["FEAST_CA_CERT_FILE_PATH"] + os.environ["REQUESTS_CA_BUNDLE"] = os.environ["FEAST_CA_CERT_FILE_PATH"] diff --git a/sdk/python/feast/templates/couchbase/__init__.py b/sdk/python/feast/templates/couchbase/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/sdk/python/feast/templates/couchbase/feature_repo/__init__.py b/sdk/python/feast/templates/couchbase/feature_repo/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/sdk/python/feast/templates/couchbase/feature_repo/feature_store.yaml b/sdk/python/feast/templates/couchbase/feature_repo/feature_store.yaml new file mode 100644 index 00000000000..bc21e44defd --- /dev/null +++ b/sdk/python/feast/templates/couchbase/feature_repo/feature_store.yaml @@ -0,0 +1,11 @@ +project: my_project +registry: /path/to/registry.db +provider: local +online_store: + type: couchbase + connection_string: COUCHBASE_CONNECTION_STRING # Couchbase connection string, copied from 'Connect' page in Couchbase Capella console + user: COUCHBASE_USER # Couchbase username from database access credentials + password: COUCHBASE_PASSWORD # Couchbase password from database access credentials + bucket_name: COUCHBASE_BUCKET_NAME # Couchbase bucket name, defaults to feast + kv_port: COUCHBASE_KV_PORT # Couchbase key-value port, defaults to 11210. Required if custom ports are used. +entity_key_serialization_version: 2 diff --git a/sdk/python/feast/templates/couchbase/gitignore b/sdk/python/feast/templates/couchbase/gitignore new file mode 100644 index 00000000000..e86277f60f4 --- /dev/null +++ b/sdk/python/feast/templates/couchbase/gitignore @@ -0,0 +1,45 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*.pyo +*.pyd + +# C extensions +*.so + +# Distribution / packaging +.Python +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ +*.egg-info/ +dist/ +build/ +.venv + +# Pytest +.cache +*.cover +*.log +.coverage +nosetests.xml +coverage.xml +*.hypothesis/ +*.pytest_cache/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IDEs and Editors +.vscode/ +.idea/ +*.swp +*.swo +*.sublime-workspace +*.sublime-project + +# OS generated files +.DS_Store +Thumbs.db diff --git a/sdk/python/feast/type_map.py b/sdk/python/feast/type_map.py index 8a88c24ffc1..d47a4221f60 100644 --- a/sdk/python/feast/type_map.py +++ b/sdk/python/feast/type_map.py @@ -164,6 +164,7 @@ def python_type_to_feast_value_type( "datetime64[ns]": ValueType.UNIX_TIMESTAMP, "datetime64[ns, tz]": ValueType.UNIX_TIMESTAMP, # special dtype of pandas "datetime64[ns, utc]": ValueType.UNIX_TIMESTAMP, + "date": ValueType.UNIX_TIMESTAMP, "category": ValueType.STRING, } @@ -523,6 +524,24 @@ def python_values_to_proto_values( return proto_values +PROTO_VALUE_TO_VALUE_TYPE_MAP: Dict[str, ValueType] = { + "int32_val": ValueType.INT32, + "int64_val": ValueType.INT64, + "double_val": ValueType.DOUBLE, + "float_val": ValueType.FLOAT, + "string_val": ValueType.STRING, + "bytes_val": ValueType.BYTES, + "bool_val": ValueType.BOOL, + "int32_list_val": ValueType.INT32_LIST, + "int64_list_val": ValueType.INT64_LIST, + "double_list_val": ValueType.DOUBLE_LIST, + "float_list_val": ValueType.FLOAT_LIST, + "string_list_val": ValueType.STRING_LIST, + "bytes_list_val": ValueType.BYTES_LIST, + "bool_list_val": ValueType.BOOL_LIST, +} + + def _proto_value_to_value_type(proto_value: ProtoValue) -> ValueType: """ Returns Feast ValueType given Feast ValueType string. @@ -534,25 +553,9 @@ def _proto_value_to_value_type(proto_value: ProtoValue) -> ValueType: A variant of ValueType. """ proto_str = proto_value.WhichOneof("val") - type_map = { - "int32_val": ValueType.INT32, - "int64_val": ValueType.INT64, - "double_val": ValueType.DOUBLE, - "float_val": ValueType.FLOAT, - "string_val": ValueType.STRING, - "bytes_val": ValueType.BYTES, - "bool_val": ValueType.BOOL, - "int32_list_val": ValueType.INT32_LIST, - "int64_list_val": ValueType.INT64_LIST, - "double_list_val": ValueType.DOUBLE_LIST, - "float_list_val": ValueType.FLOAT_LIST, - "string_list_val": ValueType.STRING_LIST, - "bytes_list_val": ValueType.BYTES_LIST, - "bool_list_val": ValueType.BOOL_LIST, - None: ValueType.NULL, - } - - return type_map[proto_str] + if proto_str is None: + return ValueType.UNKNOWN + return PROTO_VALUE_TO_VALUE_TYPE_MAP[proto_str] def pa_to_feast_value_type(pa_type_as_str: str) -> ValueType: @@ -813,6 +816,7 @@ def spark_to_feast_value_type(spark_type_as_str: str) -> ValueType: "float": ValueType.FLOAT, "boolean": ValueType.BOOL, "timestamp": ValueType.UNIX_TIMESTAMP, + "date": ValueType.UNIX_TIMESTAMP, "array": ValueType.BYTES_LIST, "array": ValueType.STRING_LIST, "array": ValueType.INT32_LIST, @@ -822,6 +826,7 @@ def spark_to_feast_value_type(spark_type_as_str: str) -> ValueType: "array": ValueType.FLOAT_LIST, "array": ValueType.BOOL_LIST, "array": ValueType.UNIX_TIMESTAMP_LIST, + "array": ValueType.UNIX_TIMESTAMP_LIST, } if spark_type_as_str.startswith("decimal"): spark_type_as_str = "decimal" diff --git a/sdk/python/feast/ui/package.json b/sdk/python/feast/ui/package.json index 0382cfafee6..16e413a0002 100644 --- a/sdk/python/feast/ui/package.json +++ b/sdk/python/feast/ui/package.json @@ -4,9 +4,9 @@ "private": true, "dependencies": { "@elastic/datemath": "^5.0.3", - "@elastic/eui": "^55.0.1", + "@elastic/eui": "^72.0.0", "@emotion/react": "^11.9.0", - "@feast-dev/feast-ui": "0.42.0", + "@feast-dev/feast-ui": "0.43.0", "@testing-library/jest-dom": "^5.16.4", "@testing-library/react": "^13.2.0", "@testing-library/user-event": "^13.5.0", diff --git a/sdk/python/feast/ui/yarn.lock b/sdk/python/feast/ui/yarn.lock index 2e6af4dc7ca..b269c2e7f51 100644 --- a/sdk/python/feast/ui/yarn.lock +++ b/sdk/python/feast/ui/yarn.lock @@ -1272,10 +1272,10 @@ dependencies: tslib "^1.9.3" -"@elastic/eui@^55.0.1": - version "55.1.2" - resolved "https://registry.yarnpkg.com/@elastic/eui/-/eui-55.1.2.tgz#dd0b42f5b26c5800d6a9cb2d4c2fe1afce9d3f07" - integrity sha512-wwZz5KxMIMFlqEsoCRiQBJDc4CrluS1d0sCOmQ5lhIzKhYc91MdxnqCk2i6YkhL4sSDf2Y9KAEuMXa+uweOWUA== +"@elastic/eui@^72.0.0": + version "72.2.0" + resolved "https://registry.yarnpkg.com/@elastic/eui/-/eui-72.2.0.tgz#0d89ec4c6d8a677ba41d086abd509c5a5ea09180" + integrity sha512-3JHKLWqbU1A6qMVkw0n1VZ5PaL07sd3N44tWsRCn+DEaDv9jq68ilEmY1wdYqKXw8VyFwcPbd8ZYZpdzBD2nPA== dependencies: "@types/chroma-js" "^2.0.0" "@types/lodash" "^4.14.160" @@ -1296,7 +1296,7 @@ react-beautiful-dnd "^13.1.0" react-dropzone "^11.5.3" react-element-to-jsx-string "^14.3.4" - react-focus-on "^3.5.4" + react-focus-on "^3.7.0" react-input-autosize "^3.0.0" react-is "^17.0.2" react-virtualized-auto-sizer "^1.0.6" @@ -1307,7 +1307,7 @@ rehype-stringify "^8.0.0" remark-breaks "^2.0.2" remark-emoji "^2.1.0" - remark-parse "^8.0.3" + remark-parse-no-trim "^8.0.4" remark-rehype "^8.0.0" tabbable "^5.2.1" text-diff "^1.0.1" @@ -1570,10 +1570,10 @@ minimatch "^3.1.2" strip-json-comments "^3.1.1" -"@feast-dev/feast-ui@0.42.0": - version "0.42.0" - resolved "https://registry.yarnpkg.com/@feast-dev/feast-ui/-/feast-ui-0.42.0.tgz#b186142d6b5176c8d5784c425fa22724b16dda6f" - integrity sha512-onHkZznObLCy5kpeWv+8Z6O51WMqF7xxNfnn3SNSOj6sPJn+FpfMq5DmJG2ESRvB3lw/SrBJB1aPr+pOtMYQjQ== +"@feast-dev/feast-ui@0.43.0": + version "0.43.0" + resolved "https://registry.yarnpkg.com/@feast-dev/feast-ui/-/feast-ui-0.43.0.tgz#0228a13d3899d8a30109bc07117c5bf141dcc810" + integrity sha512-+IjDLSN+zeY62EmstbtS0JwJKnKJVEJJCGBluktlMHBdCLdoC/Pxs8DDFLd/Q94zMmzM+3kUdaj5zABs2QinWA== dependencies: "@elastic/datemath" "^5.0.3" "@elastic/eui" "^95.12.0" @@ -1587,9 +1587,9 @@ react-app-polyfill "^3.0.0" react-code-blocks "^0.1.6" react-query "^3.39.3" - react-router-dom "<6.4.0" + react-router-dom "^6.28.0" tslib "^2.3.1" - use-query-params "^1.2.3" + use-query-params "^2.2.1" zod "^3.11.6" "@hello-pangea/dnd@^16.6.0": @@ -2056,6 +2056,11 @@ resolved "https://registry.yarnpkg.com/@protobufjs/utf8/-/utf8-1.1.0.tgz#a777360b5b39a1a2e5106f8e858f2fd2d060c570" integrity sha512-Vvn3zZrhQZkkBE8LSuW3em98c0FwgO4nxzv6OdSxPKJIEKY2bGbHn+mhGIPerzI4twdxaP8/0+06HBpwf345Lw== +"@remix-run/router@1.21.1": + version "1.21.1" + resolved "https://registry.yarnpkg.com/@remix-run/router/-/router-1.21.1.tgz#bf15274d3856c395402719fa6b1dc8cc5245aaf7" + integrity sha512-KeBYSwohb8g4/wCcnksvKTYlg69O62sQeLynn2YE+5z7JWEj95if27kclW9QqbrlsQ2DINI8fjbV3zyuKfwjKg== + "@rollup/plugin-babel@^5.2.0": version "5.3.1" resolved "https://registry.yarnpkg.com/@rollup/plugin-babel/-/plugin-babel-5.3.1.tgz#04bc0608f4aa4b2e4b1aebf284344d0f68fda283" @@ -3363,13 +3368,6 @@ argparse@^2.0.1: resolved "https://registry.yarnpkg.com/argparse/-/argparse-2.0.1.tgz#246f50f3ca78a3240f6c997e8a9bd1eac49e4b38" integrity sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q== -aria-hidden@^1.1.3: - version "1.1.3" - resolved "https://registry.yarnpkg.com/aria-hidden/-/aria-hidden-1.1.3.tgz#bb48de18dc84787a3c6eee113709c473c64ec254" - integrity sha512-RhVWFtKH5BiGMycI72q2RAFMLQi8JP9bLuQXgR5a8Znp7P5KOIADSJeyfI8PCVxLEp067B2HbP5JIiI/PXIZeA== - dependencies: - tslib "^1.0.0" - aria-hidden@^1.2.2: version "1.2.4" resolved "https://registry.yarnpkg.com/aria-hidden/-/aria-hidden-1.2.4.tgz#b78e383fdbc04d05762c78b4a25a501e736c4522" @@ -5724,13 +5722,6 @@ flatted@^3.1.0: resolved "https://registry.yarnpkg.com/flatted/-/flatted-3.2.5.tgz#76c8584f4fc843db64702a6bd04ab7a8bd666da3" integrity sha512-WIWGi2L3DyTUvUrwRKgGi9TwxQMUEqPOPQBVi71R96jZXJdFskXEmf54BoZaS1kknGODoIGASGEzBUYdyMCBJg== -focus-lock@^0.11.2: - version "0.11.2" - resolved "https://registry.yarnpkg.com/focus-lock/-/focus-lock-0.11.2.tgz#aeef3caf1cea757797ac8afdebaec8fd9ab243ed" - integrity sha512-pZ2bO++NWLHhiKkgP1bEXHhR1/OjVcSvlCJ98aNJDFeb7H5OOQaO+SKOZle6041O9rv2tmbrO4JzClAvDUHf0g== - dependencies: - tslib "^2.0.3" - focus-lock@^1.3.5: version "1.3.5" resolved "https://registry.yarnpkg.com/focus-lock/-/focus-lock-1.3.5.tgz#aa644576e5ec47d227b57eb14e1efb2abf33914c" @@ -7785,15 +7776,10 @@ nano-time@1.0.0: dependencies: big-integer "^1.6.16" -nanoid@^3.3.3: - version "3.3.4" - resolved "https://registry.yarnpkg.com/nanoid/-/nanoid-3.3.4.tgz#730b67e3cd09e2deacf03c027c81c9d9dbc5e8ab" - integrity sha512-MqBkQh/OHTS2egovRtLk45wEyNXwF+cokD+1YPf9u5VfJiRdAiRwB2froX5Co9Rh20xs4siNPm8naNotSD6RBw== - -nanoid@^3.3.7: - version "3.3.7" - resolved "https://registry.yarnpkg.com/nanoid/-/nanoid-3.3.7.tgz#d0c301a691bc8d54efa0a2226ccf3fe2fd656bd8" - integrity sha512-eSRppjcPIatRIMC1U6UngP8XFcz8MQWGQdt1MTBQ7NaAmvXDfvNxbvWV3x2y6CdEUciCSsDHDQZbhYaB8QEo2g== +nanoid@^3.3.3, nanoid@^3.3.7: + version "3.3.8" + resolved "https://registry.yarnpkg.com/nanoid/-/nanoid-3.3.8.tgz#b1be3030bee36aaff18bacb375e5cce521684baf" + integrity sha512-WNLf5Sd8oZxOm+TzppcYk8gVOgP+l58xNy58D0nbUnOxOWRWvlcCV4kUF7ltmI6PsrLl/BgKEyS4mqsGChFN0w== natural-compare@^1.4.0: version "1.4.0" @@ -9103,32 +9089,7 @@ react-focus-lock@^2.11.3: use-callback-ref "^1.3.2" use-sidecar "^1.1.2" -react-focus-lock@^2.9.0: - version "2.9.1" - resolved "https://registry.yarnpkg.com/react-focus-lock/-/react-focus-lock-2.9.1.tgz#094cfc19b4f334122c73bb0bff65d77a0c92dd16" - integrity sha512-pSWOQrUmiKLkffPO6BpMXN7SNKXMsuOakl652IBuALAu1esk+IcpJyM+ALcYzPTTFz1rD0R54aB9A4HuP5t1Wg== - dependencies: - "@babel/runtime" "^7.0.0" - focus-lock "^0.11.2" - prop-types "^15.6.2" - react-clientside-effect "^1.2.6" - use-callback-ref "^1.3.0" - use-sidecar "^1.1.2" - -react-focus-on@^3.5.4: - version "3.6.0" - resolved "https://registry.yarnpkg.com/react-focus-on/-/react-focus-on-3.6.0.tgz#159e13082dad4ea1f07abe11254f0e981d5a7b79" - integrity sha512-onIRjpd9trAUenXNdDcvjc8KJUSklty4X/Gr7hAm/MzM7ekSF2pg9D8KBKL7ipige22IAPxLRRf/EmJji9KD6Q== - dependencies: - aria-hidden "^1.1.3" - react-focus-lock "^2.9.0" - react-remove-scroll "^2.5.2" - react-style-singleton "^2.2.0" - tslib "^2.3.1" - use-callback-ref "^1.3.0" - use-sidecar "^1.1.2" - -react-focus-on@^3.9.1: +react-focus-on@^3.7.0, react-focus-on@^3.9.1: version "3.9.4" resolved "https://registry.yarnpkg.com/react-focus-on/-/react-focus-on-3.9.4.tgz#0b6c13273d86243c330d1aa53af39290f543da7b" integrity sha512-NFKmeH6++wu8e7LJcbwV8TTd4L5w/U5LMXTMOdUcXhCcZ7F5VOvgeTHd4XN1PD7TNmdvldDu/ENROOykUQ4yQg== @@ -9209,14 +9170,6 @@ react-refresh@^0.11.0: resolved "https://registry.yarnpkg.com/react-refresh/-/react-refresh-0.11.0.tgz#77198b944733f0f1f1a90e791de4541f9f074046" integrity sha512-F27qZr8uUqwhWZboondsPx8tnC3Ct3SxZA3V5WyEvujRyyNv0VYPhoBg1gZ8/MV5tubQp76Trw8lTv9hzRBa+A== -react-remove-scroll-bar@^2.3.1: - version "2.3.1" - resolved "https://registry.yarnpkg.com/react-remove-scroll-bar/-/react-remove-scroll-bar-2.3.1.tgz#9f13b05b249eaa57c8d646c1ebb83006b3581f5f" - integrity sha512-IvGX3mJclEF7+hga8APZczve1UyGMkMG+tjS0o/U1iLgvZRpjFAQEUBJ4JETfvbNlfNnZnoDyWJCICkA15Mghg== - dependencies: - react-style-singleton "^2.2.0" - tslib "^2.0.0" - react-remove-scroll-bar@^2.3.4, react-remove-scroll-bar@^2.3.6: version "2.3.6" resolved "https://registry.yarnpkg.com/react-remove-scroll-bar/-/react-remove-scroll-bar-2.3.6.tgz#3e585e9d163be84a010180b18721e851ac81a29c" @@ -9225,17 +9178,6 @@ react-remove-scroll-bar@^2.3.4, react-remove-scroll-bar@^2.3.6: react-style-singleton "^2.2.1" tslib "^2.0.0" -react-remove-scroll@^2.5.2: - version "2.5.3" - resolved "https://registry.yarnpkg.com/react-remove-scroll/-/react-remove-scroll-2.5.3.tgz#a152196e710e8e5811be39dc352fd8a90b05c961" - integrity sha512-NQ1bXrxKrnK5pFo/GhLkXeo3CrK5steI+5L+jynwwIemvZyfXqaL0L5BzwJd7CSwNCU723DZaccvjuyOdoy3Xw== - dependencies: - react-remove-scroll-bar "^2.3.1" - react-style-singleton "^2.2.0" - tslib "^2.0.0" - use-callback-ref "^1.3.0" - use-sidecar "^1.1.2" - react-remove-scroll@^2.6.0: version "2.6.0" resolved "https://registry.yarnpkg.com/react-remove-scroll/-/react-remove-scroll-2.6.0.tgz#fb03a0845d7768a4f1519a99fdb84983b793dc07" @@ -9247,7 +9189,7 @@ react-remove-scroll@^2.6.0: use-callback-ref "^1.3.0" use-sidecar "^1.1.2" -react-router-dom@6, react-router-dom@<6.4.0: +react-router-dom@6: version "6.3.0" resolved "https://registry.yarnpkg.com/react-router-dom/-/react-router-dom-6.3.0.tgz#a0216da813454e521905b5fa55e0e5176123f43d" integrity sha512-uaJj7LKytRxZNQV8+RbzJWnJ8K2nPsOOEuX7aQstlMZKQT0164C+X2w6bnkqU3sjtLvpd5ojrezAyfZ1+0sStw== @@ -9255,6 +9197,21 @@ react-router-dom@6, react-router-dom@<6.4.0: history "^5.2.0" react-router "6.3.0" +react-router-dom@^6.28.0: + version "6.28.2" + resolved "https://registry.yarnpkg.com/react-router-dom/-/react-router-dom-6.28.2.tgz#9bc4f58b0cfe91d39d1a6be4beb0ef051ca9b06e" + integrity sha512-O81EWqNJWqvlN/a7eTudAdQm0TbI7hw+WIi7OwwMcTn5JMyZ0ibTFNGz+t+Lju0df4LcqowCegcrK22lB1q9Kw== + dependencies: + "@remix-run/router" "1.21.1" + react-router "6.28.2" + +react-router@6.28.2: + version "6.28.2" + resolved "https://registry.yarnpkg.com/react-router/-/react-router-6.28.2.tgz#1ddea57c2de0d99e12d00af14d1499703f1378a9" + integrity sha512-BgFY7+wEGVjHCiqaj2XiUBQ1kkzfg6UoKYwEe0wv+FF+HNPCxtS/MVPvLAPH++EsuCMReZl9RYVGqcHLk5ms3A== + dependencies: + "@remix-run/router" "1.21.1" + react-router@6.3.0: version "6.3.0" resolved "https://registry.yarnpkg.com/react-router/-/react-router-6.3.0.tgz#3970cc64b4cb4eae0c1ea5203a80334fdd175557" @@ -9317,15 +9274,6 @@ react-scripts@^5.0.0: optionalDependencies: fsevents "^2.3.2" -react-style-singleton@^2.2.0: - version "2.2.0" - resolved "https://registry.yarnpkg.com/react-style-singleton/-/react-style-singleton-2.2.0.tgz#70f45f5fef97fdb9a52eed98d1839fa6b9032b22" - integrity sha512-nK7mN92DMYZEu3cQcAhfwE48NpzO5RpxjG4okbSqRRbfal9Pk+fG2RdQXTMp+f6all1hB9LIJSt+j7dCYrU11g== - dependencies: - get-nonce "^1.0.0" - invariant "^2.2.4" - tslib "^2.0.0" - react-style-singleton@^2.2.1: version "2.2.1" resolved "https://registry.yarnpkg.com/react-style-singleton/-/react-style-singleton-2.2.1.tgz#f99e420492b2d8f34d38308ff660b60d0b1205b4" @@ -9589,28 +9537,6 @@ remark-parse-no-trim@^8.0.4: vfile-location "^3.0.0" xtend "^4.0.1" -remark-parse@^8.0.3: - version "8.0.3" - resolved "https://registry.yarnpkg.com/remark-parse/-/remark-parse-8.0.3.tgz#9c62aa3b35b79a486454c690472906075f40c7e1" - integrity sha512-E1K9+QLGgggHxCQtLt++uXltxEprmWzNfg+MxpfHsZlrddKzZ/hZyWHDbK3/Ap8HJQqYJRXP+jHczdL6q6i85Q== - dependencies: - ccount "^1.0.0" - collapse-white-space "^1.0.2" - is-alphabetical "^1.0.0" - is-decimal "^1.0.0" - is-whitespace-character "^1.0.0" - is-word-character "^1.0.0" - markdown-escapes "^1.0.0" - parse-entities "^2.0.0" - repeat-string "^1.5.4" - state-toggle "^1.0.0" - trim "0.0.1" - trim-trailing-lines "^1.0.0" - unherit "^1.0.4" - unist-util-remove-position "^2.0.0" - vfile-location "^3.0.0" - xtend "^4.0.1" - remark-rehype@^8.0.0, remark-rehype@^8.1.0: version "8.1.0" resolved "https://registry.yarnpkg.com/remark-rehype/-/remark-rehype-8.1.0.tgz#610509a043484c1e697437fa5eb3fd992617c945" @@ -9923,6 +9849,11 @@ serialize-query-params@^1.3.5: resolved "https://registry.yarnpkg.com/serialize-query-params/-/serialize-query-params-1.3.6.tgz#5dd5225db85ce747fe6fbc4897628504faafec6d" integrity sha512-VlH7sfWNyPVZClPkRacopn6sn5uQMXBsjPVz1+pBHX895VpcYVznfJtZ49e6jymcrz+l/vowkepCZn/7xEAEdw== +serialize-query-params@^2.0.2: + version "2.0.2" + resolved "https://registry.yarnpkg.com/serialize-query-params/-/serialize-query-params-2.0.2.tgz#598a3fb9e13f4ea1c1992fbd20231aa16b31db81" + integrity sha512-1chMo1dST4pFA9RDXAtF0Rbjaut4is7bzFbI1Z26IuMub68pNCILku85aYmeFhvnY//BXUPUhoRMjYcsT93J/Q== + serve-index@^1.9.1: version "1.9.1" resolved "https://registry.yarnpkg.com/serve-index/-/serve-index-1.9.1.tgz#d3768d69b1e7d82e5ce050fff5b453bea12a9239" @@ -10643,11 +10574,6 @@ trim-trailing-lines@^1.0.0: resolved "https://registry.yarnpkg.com/trim-trailing-lines/-/trim-trailing-lines-1.1.4.tgz#bd4abbec7cc880462f10b2c8b5ce1d8d1ec7c2c0" integrity sha512-rjUWSqnfTNrjbB9NQWfPMH/xRK1deHeGsHoVfpxJ++XeYXE0d6B1En37AHfw3jtfTU7dzMzZL2jjpe8Qb5gLIQ== -trim@0.0.1: - version "0.0.1" - resolved "https://registry.yarnpkg.com/trim/-/trim-0.0.1.tgz#5858547f6b290757ee95cccc666fb50084c460dd" - integrity sha1-WFhUf2spB1fulczMZm+1AITEYN0= - trough@^1.0.0: version "1.0.5" resolved "https://registry.yarnpkg.com/trough/-/trough-1.0.5.tgz#b8b639cefad7d0bb2abd37d433ff8293efa5f406" @@ -10673,7 +10599,7 @@ tslib@2.6.2: resolved "https://registry.yarnpkg.com/tslib/-/tslib-2.6.2.tgz#703ac29425e7b37cd6fd456e92404d46d1f3e4ae" integrity sha512-AEYxH93jGFPn/a2iVAwW87VuUIkR1FVUKB77NwMF7nBTDkDrrT/Hpt/IrCJ0QXhW27jTBDcf5ZY7w6RiqTMw2Q== -tslib@^1.0.0, tslib@^1.8.1, tslib@^1.9.3: +tslib@^1.8.1, tslib@^1.9.3: version "1.14.1" resolved "https://registry.yarnpkg.com/tslib/-/tslib-1.14.1.tgz#cf2d38bdc34a134bcaf1091c41f6619e2f672d00" integrity sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg== @@ -10973,6 +10899,13 @@ use-query-params@^1.2.3: dependencies: serialize-query-params "^1.3.5" +use-query-params@^2.2.1: + version "2.2.1" + resolved "https://registry.yarnpkg.com/use-query-params/-/use-query-params-2.2.1.tgz#c558ab70706f319112fbccabf6867b9f904e947d" + integrity sha512-i6alcyLB8w9i3ZK3caNftdb+UnbfBRNPDnc89CNQWkGRmDrm/gfydHvMBfVsQJRq3NoHOM2dt/ceBWG2397v1Q== + dependencies: + serialize-query-params "^2.0.2" + use-sidecar@^1.1.2: version "1.1.2" resolved "https://registry.yarnpkg.com/use-sidecar/-/use-sidecar-1.1.2.tgz#2f43126ba2d7d7e117aa5855e5d8f0276dfe73c2" diff --git a/sdk/python/feast/utils.py b/sdk/python/feast/utils.py index 51d4bf4f2cc..cfc19e37ca4 100644 --- a/sdk/python/feast/utils.py +++ b/sdk/python/feast/utils.py @@ -1192,6 +1192,10 @@ def _utc_now() -> datetime: return datetime.now(tz=timezone.utc) +def _serialize_vector_to_float_list(vector: List[float]) -> ValueProto: + return ValueProto(float_list_val=FloatListProto(val=vector)) + + def _build_retrieve_online_document_record( entity_key: Union[str, bytes], feature_value: Union[str, bytes], diff --git a/sdk/python/pytest.ini b/sdk/python/pytest.ini index a0736767601..d79459c0d0e 100644 --- a/sdk/python/pytest.ini +++ b/sdk/python/pytest.ini @@ -4,6 +4,7 @@ asyncio_mode = auto markers = universal_offline_stores: mark a test as using all offline stores. universal_online_stores: mark a test as using all online stores. + rbac_remote_integration_test: mark a integration test related to rbac and remote functionality. env = IS_TEST=True diff --git a/sdk/python/requirements/py3.10-ci-requirements.txt b/sdk/python/requirements/py3.10-ci-requirements.txt index 54a64f5b1ce..af41030a6de 100644 --- a/sdk/python/requirements/py3.10-ci-requirements.txt +++ b/sdk/python/requirements/py3.10-ci-requirements.txt @@ -1,14 +1,14 @@ # This file was autogenerated by uv via the following command: # uv pip compile -p 3.10 --system --no-strip-extras setup.py --extra ci --output-file sdk/python/requirements/py3.10-ci-requirements.txt -aiobotocore==2.15.2 +aiobotocore==2.16.0 # via feast (setup.py) -aiohappyeyeballs==2.4.3 +aiohappyeyeballs==2.4.4 # via aiohttp -aiohttp==3.11.7 +aiohttp==3.11.11 # via aiobotocore aioitertools==0.12.0 # via aiobotocore -aiosignal==1.3.1 +aiosignal==1.3.2 # via aiohttp alabaster==0.7.16 # via sphinx @@ -16,7 +16,7 @@ altair==4.2.2 # via great-expectations annotated-types==0.7.0 # via pydantic -anyio==4.6.2.post1 +anyio==4.7.0 # via # httpx # jupyter-server @@ -25,7 +25,9 @@ anyio==4.6.2.post1 appnope==0.1.4 # via ipykernel argon2-cffi==23.1.0 - # via jupyter-server + # via + # jupyter-server + # minio argon2-cffi-bindings==21.2.0 # via argon2-cffi arrow==1.3.0 @@ -34,7 +36,7 @@ asn1crypto==1.5.1 # via snowflake-connector-python assertpy==1.1 # via feast (setup.py) -asttokens==2.4.1 +asttokens==3.0.0 # via stack-data async-lru==2.0.4 # via jupyterlab @@ -46,7 +48,7 @@ async-timeout==5.0.1 # redis atpublic==5.0 # via ibis-framework -attrs==24.2.0 +attrs==24.3.0 # via # aiohttp # jsonschema @@ -69,11 +71,11 @@ bigtree==0.22.3 # via feast (setup.py) bleach==6.2.0 # via nbconvert -boto3==1.35.36 +boto3==1.35.81 # via # feast (setup.py) # moto -botocore==1.35.36 +botocore==1.35.81 # via # aiobotocore # boto3 @@ -88,7 +90,7 @@ cachetools==5.5.0 # via google-auth cassandra-driver==3.29.2 # via feast (setup.py) -certifi==2024.8.30 +certifi==2024.12.14 # via # elastic-transport # httpcore @@ -128,9 +130,9 @@ comm==0.2.2 # ipywidgets couchbase==4.3.2 # via feast (setup.py) -coverage[toml]==7.6.8 +coverage[toml]==7.6.9 # via pytest-cov -cryptography==42.0.8 +cryptography==43.0.3 # via # feast (setup.py) # azure-identity @@ -146,21 +148,21 @@ cryptography==42.0.8 # types-redis cython==3.0.11 # via thriftpy2 -dask[dataframe]==2024.11.2 +dask[dataframe]==2024.12.1 # via # feast (setup.py) # dask-expr -dask-expr==1.1.19 +dask-expr==1.1.21 # via dask db-dtypes==1.3.1 # via google-cloud-bigquery -debugpy==1.8.9 +debugpy==1.8.11 # via ipykernel decorator==5.1.1 # via ipython defusedxml==0.7.1 # via nbconvert -deltalake==0.22.0 +deltalake==0.22.3 # via feast (setup.py) deprecation==2.1.0 # via python-keycloak @@ -176,10 +178,12 @@ duckdb==1.1.3 # via ibis-framework elastic-transport==8.15.1 # via elasticsearch -elasticsearch==8.16.0 +elasticsearch==8.17.0 # via feast (setup.py) entrypoints==0.4 # via altair +environs==9.5.0 + # via pymilvus exceptiongroup==1.2.2 # via # anyio @@ -191,9 +195,9 @@ executing==2.1.0 # via stack-data faiss-cpu==1.9.0.post1 # via feast (setup.py) -fastapi==0.115.5 +fastapi==0.115.6 # via feast (setup.py) -fastjsonschema==2.20.0 +fastjsonschema==2.21.1 # via nbformat filelock==3.16.1 # via @@ -211,7 +215,7 @@ fsspec==2024.9.0 # dask geomet==0.2.1.post1 # via cassandra-driver -google-api-core[grpc]==2.23.0 +google-api-core[grpc]==2.24.0 # via # feast (setup.py) # google-cloud-bigquery @@ -220,7 +224,7 @@ google-api-core[grpc]==2.23.0 # google-cloud-core # google-cloud-datastore # google-cloud-storage -google-auth==2.36.0 +google-auth==2.37.0 # via # google-api-core # google-cloud-bigquery @@ -242,9 +246,9 @@ google-cloud-core==2.4.1 # google-cloud-bigtable # google-cloud-datastore # google-cloud-storage -google-cloud-datastore==2.20.1 +google-cloud-datastore==2.20.2 # via feast (setup.py) -google-cloud-storage==2.18.2 +google-cloud-storage==2.19.0 # via feast (setup.py) google-crc32c==1.6.0 # via @@ -264,7 +268,7 @@ great-expectations==0.18.22 # via feast (setup.py) grpc-google-iam-v1==0.13.1 # via google-cloud-bigtable -grpcio==1.68.0 +grpcio==1.68.1 # via # feast (setup.py) # google-api-core @@ -275,6 +279,7 @@ grpcio==1.68.0 # grpcio-status # grpcio-testing # grpcio-tools + # pymilvus # qdrant-client grpcio-health-checking==1.62.3 # via feast (setup.py) @@ -344,7 +349,7 @@ iniconfig==2.0.0 # via pytest ipykernel==6.29.5 # via jupyterlab -ipython==8.29.0 +ipython==8.30.0 # via # great-expectations # ipykernel @@ -357,7 +362,7 @@ isoduration==20.11.0 # via jsonschema jedi==0.19.2 # via ipython -jinja2==3.1.4 +jinja2==3.1.5 # via # feast (setup.py) # altair @@ -372,7 +377,7 @@ jmespath==1.0.1 # via # boto3 # botocore -json5==0.9.28 +json5==0.10.0 # via jupyterlab-server jsonpatch==1.33 # via great-expectations @@ -404,7 +409,7 @@ jupyter-core==5.7.2 # nbclient # nbconvert # nbformat -jupyter-events==0.10.0 +jupyter-events==0.11.0 # via jupyter-server jupyter-lsp==2.2.5 # via jupyterlab @@ -417,7 +422,7 @@ jupyter-server==2.14.2 # notebook-shim jupyter-server-terminals==0.5.3 # via jupyter-server -jupyterlab==4.2.6 +jupyterlab==4.3.4 # via notebook jupyterlab-pygments==0.3.0 # via nbconvert @@ -442,15 +447,19 @@ markupsafe==3.0.2 # jinja2 # nbconvert # werkzeug -marshmallow==3.23.1 - # via great-expectations +marshmallow==3.23.2 + # via + # environs + # great-expectations matplotlib-inline==0.1.7 # via # ipykernel # ipython mdurl==0.1.2 # via markdown-it-py -minio==7.1.0 +milvus-lite==2.4.10 + # via pymilvus +minio==7.2.11 # via feast (setup.py) mistune==3.0.2 # via @@ -480,7 +489,7 @@ mypy-extensions==1.0.0 # via mypy mypy-protobuf==3.3.0 # via feast (setup.py) -nbclient==0.10.0 +nbclient==0.10.2 # via nbconvert nbconvert==7.16.4 # via jupyter-server @@ -494,7 +503,7 @@ nest-asyncio==1.6.0 # via ipykernel nodeenv==1.9.1 # via pre-commit -notebook==7.2.2 +notebook==7.3.1 # via great-expectations notebook-shim==0.2.4 # via @@ -548,6 +557,7 @@ pandas==2.2.3 # google-cloud-bigquery # great-expectations # ibis-framework + # pymilvus # snowflake-connector-python pandocfilters==1.5.1 # via nbconvert @@ -582,13 +592,13 @@ portalocker==2.10.1 # qdrant-client pre-commit==3.3.1 # via feast (setup.py) -prometheus-client==0.21.0 +prometheus-client==0.21.1 # via # feast (setup.py) # jupyter-server prompt-toolkit==3.0.48 # via ipython -propcache==0.2.0 +propcache==0.2.1 # via # aiohttp # yarl @@ -614,6 +624,7 @@ protobuf==4.25.5 # grpcio-tools # mypy-protobuf # proto-plus + # pymilvus # substrait psutil==5.9.0 # via @@ -658,13 +669,15 @@ pybindgen==0.22.1 # via feast (setup.py) pycparser==2.22 # via cffi -pydantic==2.10.1 +pycryptodome==3.21.0 + # via minio +pydantic==2.10.4 # via # feast (setup.py) # fastapi # great-expectations # qdrant-client -pydantic-core==2.27.1 +pydantic-core==2.27.2 # via pydantic pygments==2.18.0 # via @@ -673,19 +686,21 @@ pygments==2.18.0 # nbconvert # rich # sphinx -pyjwt[crypto]==2.10.0 +pyjwt[crypto]==2.10.1 # via # feast (setup.py) # msal # singlestoredb # snowflake-connector-python +pymilvus==2.4.9 + # via feast (setup.py) pymssql==2.3.2 # via feast (setup.py) pymysql==1.1.1 # via feast (setup.py) pyodbc==5.2.0 # via feast (setup.py) -pyopenssl==24.2.1 +pyopenssl==24.3.0 # via snowflake-connector-python pyparsing==3.2.0 # via great-expectations @@ -738,8 +753,10 @@ python-dateutil==2.9.0.post0 # pandas # trino python-dotenv==1.0.1 - # via uvicorn -python-json-logger==2.0.7 + # via + # environs + # uvicorn +python-json-logger==3.2.1 # via jupyter-events python-keycloak==4.2.2 # via feast (setup.py) @@ -815,7 +832,7 @@ rfc3986-validator==0.1.1 # jupyter-events rich==13.9.4 # via ibis-framework -rpds-py==0.21.0 +rpds-py==0.22.3 # via # jsonschema # referencing @@ -825,7 +842,7 @@ ruamel-yaml==0.17.40 # via great-expectations ruamel-yaml-clib==0.2.12 # via ruamel-yaml -ruff==0.8.0 +ruff==0.8.4 # via feast (setup.py) s3transfer==0.10.4 # via boto3 @@ -839,12 +856,12 @@ setuptools==75.6.0 # jupyterlab # kubernetes # pip-tools + # pymilvus # singlestoredb singlestoredb==1.7.2 # via feast (setup.py) -six==1.16.0 +six==1.17.0 # via - # asttokens # azure-core # geomet # happybase @@ -859,7 +876,7 @@ sniffio==1.3.1 # httpx snowballstemmer==2.2.0 # via sphinx -snowflake-connector-python[pandas]==3.12.3 +snowflake-connector-python[pandas]==3.12.4 # via feast (setup.py) sortedcontainers==2.4.0 # via snowflake-connector-python @@ -883,7 +900,7 @@ sqlalchemy[mypy]==2.0.36 # via feast (setup.py) sqlglot==25.20.2 # via ibis-framework -sqlite-vec==0.1.1 +sqlite-vec==0.1.3 # via feast (setup.py) sqlparams==6.1.0 # via singlestoredb @@ -909,7 +926,7 @@ tinycss2==1.4.0 # via nbconvert toml==0.10.2 # via feast (setup.py) -tomli==2.1.0 +tomli==2.2.1 # via # build # coverage @@ -939,6 +956,7 @@ tqdm==4.67.1 # via # feast (setup.py) # great-expectations + # milvus-lite traitlets==5.14.3 # via # comm @@ -954,7 +972,7 @@ traitlets==5.14.3 # nbclient # nbconvert # nbformat -trino==0.330.0 +trino==0.331.0 # via feast (setup.py) typeguard==4.4.1 # via feast (setup.py) @@ -968,7 +986,7 @@ types-pymysql==1.1.0.20241103 # via feast (setup.py) types-pyopenssl==24.1.0.20240722 # via types-redis -types-python-dateutil==2.9.0.20241003 +types-python-dateutil==2.9.0.20241206 # via # feast (setup.py) # arrow @@ -984,7 +1002,7 @@ types-setuptools==75.6.0.20241126 # via # feast (setup.py) # types-cffi -types-tabulate==0.9.0.20240106 +types-tabulate==0.9.0.20241207 # via feast (setup.py) types-urllib3==1.26.25.14 # via types-requests @@ -1000,6 +1018,7 @@ typing-extensions==4.12.2 # ibis-framework # ipython # jwcrypto + # minio # multidict # mypy # psycopg @@ -1018,6 +1037,8 @@ tzlocal==5.2 # via # great-expectations # trino +ujson==5.10.0 + # via pymilvus uri-template==1.3.0 # via jsonschema urllib3==2.2.3 @@ -1033,7 +1054,7 @@ urllib3==2.2.3 # requests # responses # testcontainers -uvicorn[standard]==0.32.1 +uvicorn[standard]==0.34.0 # via # feast (setup.py) # uvicorn-worker @@ -1045,7 +1066,7 @@ virtualenv==20.23.0 # via # feast (setup.py) # pre-commit -watchfiles==1.0.0 +watchfiles==1.0.3 # via uvicorn wcwidth==0.2.13 # via prompt-toolkit @@ -1075,7 +1096,7 @@ wrapt==1.17.0 # testcontainers xmltodict==0.14.2 # via moto -yarl==1.18.0 +yarl==1.18.3 # via aiohttp zipp==3.21.0 # via importlib-metadata diff --git a/sdk/python/requirements/py3.10-requirements.txt b/sdk/python/requirements/py3.10-requirements.txt index 9a087b4a8eb..2488e12304b 100644 --- a/sdk/python/requirements/py3.10-requirements.txt +++ b/sdk/python/requirements/py3.10-requirements.txt @@ -2,17 +2,17 @@ # uv pip compile -p 3.10 --system --no-strip-extras setup.py --output-file sdk/python/requirements/py3.10-requirements.txt annotated-types==0.7.0 # via pydantic -anyio==4.6.2.post1 +anyio==4.7.0 # via # starlette # watchfiles -attrs==24.2.0 +attrs==24.3.0 # via # jsonschema # referencing bigtree==0.22.3 # via feast (setup.py) -certifi==2024.8.30 +certifi==2024.12.14 # via requests charset-normalizer==3.4.0 # via requests @@ -25,19 +25,19 @@ cloudpickle==3.1.0 # via dask colorama==0.4.6 # via feast (setup.py) -dask[dataframe]==2024.11.2 +dask[dataframe]==2024.12.1 # via # feast (setup.py) # dask-expr -dask-expr==1.1.19 +dask-expr==1.1.21 # via dask dill==0.3.9 # via feast (setup.py) exceptiongroup==1.2.2 # via anyio -fastapi==0.115.5 +fastapi==0.115.6 # via feast (setup.py) -fsspec==2024.10.0 +fsspec==2024.12.0 # via dask gunicorn==23.0.0 # via @@ -53,7 +53,7 @@ idna==3.10 # requests importlib-metadata==8.5.0 # via dask -jinja2==3.1.4 +jinja2==3.1.5 # via feast (setup.py) jsonschema==4.23.0 # via feast (setup.py) @@ -85,25 +85,25 @@ pandas==2.2.3 # dask-expr partd==1.4.2 # via dask -prometheus-client==0.21.0 +prometheus-client==0.21.1 # via feast (setup.py) protobuf==4.25.5 # via feast (setup.py) -psutil==6.1.0 +psutil==6.1.1 # via feast (setup.py) pyarrow==18.0.0 # via # feast (setup.py) # dask-expr -pydantic==2.10.1 +pydantic==2.10.4 # via # feast (setup.py) # fastapi -pydantic-core==2.27.1 +pydantic-core==2.27.2 # via pydantic pygments==2.18.0 # via feast (setup.py) -pyjwt==2.10.0 +pyjwt==2.10.1 # via feast (setup.py) python-dateutil==2.9.0.post0 # via pandas @@ -122,11 +122,11 @@ referencing==0.35.1 # jsonschema-specifications requests==2.32.3 # via feast (setup.py) -rpds-py==0.21.0 +rpds-py==0.22.3 # via # jsonschema # referencing -six==1.16.0 +six==1.17.0 # via python-dateutil sniffio==1.3.1 # via anyio @@ -140,7 +140,7 @@ tenacity==8.5.0 # via feast (setup.py) toml==0.10.2 # via feast (setup.py) -tomli==2.1.0 +tomli==2.2.1 # via mypy toolz==1.0.0 # via @@ -164,7 +164,7 @@ tzdata==2024.2 # via pandas urllib3==2.2.3 # via requests -uvicorn[standard]==0.32.1 +uvicorn[standard]==0.34.0 # via # feast (setup.py) # uvicorn-worker @@ -172,7 +172,7 @@ uvicorn-worker==0.2.0 # via feast (setup.py) uvloop==0.21.0 # via uvicorn -watchfiles==1.0.0 +watchfiles==1.0.3 # via uvicorn websockets==14.1 # via uvicorn diff --git a/sdk/python/requirements/py3.11-ci-requirements.txt b/sdk/python/requirements/py3.11-ci-requirements.txt index 43637fd2067..10c31f30953 100644 --- a/sdk/python/requirements/py3.11-ci-requirements.txt +++ b/sdk/python/requirements/py3.11-ci-requirements.txt @@ -1,14 +1,14 @@ # This file was autogenerated by uv via the following command: # uv pip compile -p 3.11 --system --no-strip-extras setup.py --extra ci --output-file sdk/python/requirements/py3.11-ci-requirements.txt -aiobotocore==2.15.2 +aiobotocore==2.16.0 # via feast (setup.py) -aiohappyeyeballs==2.4.3 +aiohappyeyeballs==2.4.4 # via aiohttp -aiohttp==3.11.7 +aiohttp==3.11.11 # via aiobotocore aioitertools==0.12.0 # via aiobotocore -aiosignal==1.3.1 +aiosignal==1.3.2 # via aiohttp alabaster==0.7.16 # via sphinx @@ -16,7 +16,7 @@ altair==4.2.2 # via great-expectations annotated-types==0.7.0 # via pydantic -anyio==4.6.2.post1 +anyio==4.7.0 # via # httpx # jupyter-server @@ -25,7 +25,9 @@ anyio==4.6.2.post1 appnope==0.1.4 # via ipykernel argon2-cffi==23.1.0 - # via jupyter-server + # via + # jupyter-server + # minio argon2-cffi-bindings==21.2.0 # via argon2-cffi arrow==1.3.0 @@ -34,7 +36,7 @@ asn1crypto==1.5.1 # via snowflake-connector-python assertpy==1.1 # via feast (setup.py) -asttokens==2.4.1 +asttokens==3.0.0 # via stack-data async-lru==2.0.4 # via jupyterlab @@ -44,7 +46,7 @@ async-timeout==5.0.1 # via redis atpublic==5.0 # via ibis-framework -attrs==24.2.0 +attrs==24.3.0 # via # aiohttp # jsonschema @@ -67,11 +69,11 @@ bigtree==0.22.3 # via feast (setup.py) bleach==6.2.0 # via nbconvert -boto3==1.35.36 +boto3==1.35.81 # via # feast (setup.py) # moto -botocore==1.35.36 +botocore==1.35.81 # via # aiobotocore # boto3 @@ -86,7 +88,7 @@ cachetools==5.5.0 # via google-auth cassandra-driver==3.29.2 # via feast (setup.py) -certifi==2024.8.30 +certifi==2024.12.14 # via # elastic-transport # httpcore @@ -126,9 +128,9 @@ comm==0.2.2 # ipywidgets couchbase==4.3.2 # via feast (setup.py) -coverage[toml]==7.6.8 +coverage[toml]==7.6.9 # via pytest-cov -cryptography==42.0.8 +cryptography==43.0.3 # via # feast (setup.py) # azure-identity @@ -144,21 +146,21 @@ cryptography==42.0.8 # types-redis cython==3.0.11 # via thriftpy2 -dask[dataframe]==2024.11.2 +dask[dataframe]==2024.12.1 # via # feast (setup.py) # dask-expr -dask-expr==1.1.19 +dask-expr==1.1.21 # via dask db-dtypes==1.3.1 # via google-cloud-bigquery -debugpy==1.8.9 +debugpy==1.8.11 # via ipykernel decorator==5.1.1 # via ipython defusedxml==0.7.1 # via nbconvert -deltalake==0.22.0 +deltalake==0.22.3 # via feast (setup.py) deprecation==2.1.0 # via python-keycloak @@ -174,19 +176,21 @@ duckdb==1.1.3 # via ibis-framework elastic-transport==8.15.1 # via elasticsearch -elasticsearch==8.16.0 +elasticsearch==8.17.0 # via feast (setup.py) entrypoints==0.4 # via altair +environs==9.5.0 + # via pymilvus execnet==2.1.1 # via pytest-xdist executing==2.1.0 # via stack-data faiss-cpu==1.9.0.post1 # via feast (setup.py) -fastapi==0.115.5 +fastapi==0.115.6 # via feast (setup.py) -fastjsonschema==2.20.0 +fastjsonschema==2.21.1 # via nbformat filelock==3.16.1 # via @@ -204,7 +208,7 @@ fsspec==2024.9.0 # dask geomet==0.2.1.post1 # via cassandra-driver -google-api-core[grpc]==2.23.0 +google-api-core[grpc]==2.24.0 # via # feast (setup.py) # google-cloud-bigquery @@ -213,7 +217,7 @@ google-api-core[grpc]==2.23.0 # google-cloud-core # google-cloud-datastore # google-cloud-storage -google-auth==2.36.0 +google-auth==2.37.0 # via # google-api-core # google-cloud-bigquery @@ -235,9 +239,9 @@ google-cloud-core==2.4.1 # google-cloud-bigtable # google-cloud-datastore # google-cloud-storage -google-cloud-datastore==2.20.1 +google-cloud-datastore==2.20.2 # via feast (setup.py) -google-cloud-storage==2.18.2 +google-cloud-storage==2.19.0 # via feast (setup.py) google-crc32c==1.6.0 # via @@ -257,7 +261,7 @@ great-expectations==0.18.22 # via feast (setup.py) grpc-google-iam-v1==0.13.1 # via google-cloud-bigtable -grpcio==1.68.0 +grpcio==1.68.1 # via # feast (setup.py) # google-api-core @@ -268,6 +272,7 @@ grpcio==1.68.0 # grpcio-status # grpcio-testing # grpcio-tools + # pymilvus # qdrant-client grpcio-health-checking==1.62.3 # via feast (setup.py) @@ -335,7 +340,7 @@ iniconfig==2.0.0 # via pytest ipykernel==6.29.5 # via jupyterlab -ipython==8.29.0 +ipython==8.30.0 # via # great-expectations # ipykernel @@ -348,7 +353,7 @@ isoduration==20.11.0 # via jsonschema jedi==0.19.2 # via ipython -jinja2==3.1.4 +jinja2==3.1.5 # via # feast (setup.py) # altair @@ -363,7 +368,7 @@ jmespath==1.0.1 # via # boto3 # botocore -json5==0.9.28 +json5==0.10.0 # via jupyterlab-server jsonpatch==1.33 # via great-expectations @@ -395,7 +400,7 @@ jupyter-core==5.7.2 # nbclient # nbconvert # nbformat -jupyter-events==0.10.0 +jupyter-events==0.11.0 # via jupyter-server jupyter-lsp==2.2.5 # via jupyterlab @@ -408,7 +413,7 @@ jupyter-server==2.14.2 # notebook-shim jupyter-server-terminals==0.5.3 # via jupyter-server -jupyterlab==4.2.6 +jupyterlab==4.3.4 # via notebook jupyterlab-pygments==0.3.0 # via nbconvert @@ -433,15 +438,19 @@ markupsafe==3.0.2 # jinja2 # nbconvert # werkzeug -marshmallow==3.23.1 - # via great-expectations +marshmallow==3.23.2 + # via + # environs + # great-expectations matplotlib-inline==0.1.7 # via # ipykernel # ipython mdurl==0.1.2 # via markdown-it-py -minio==7.1.0 +milvus-lite==2.4.10 + # via pymilvus +minio==7.2.11 # via feast (setup.py) mistune==3.0.2 # via @@ -471,7 +480,7 @@ mypy-extensions==1.0.0 # via mypy mypy-protobuf==3.3.0 # via feast (setup.py) -nbclient==0.10.0 +nbclient==0.10.2 # via nbconvert nbconvert==7.16.4 # via jupyter-server @@ -485,7 +494,7 @@ nest-asyncio==1.6.0 # via ipykernel nodeenv==1.9.1 # via pre-commit -notebook==7.2.2 +notebook==7.3.1 # via great-expectations notebook-shim==0.2.4 # via @@ -539,6 +548,7 @@ pandas==2.2.3 # google-cloud-bigquery # great-expectations # ibis-framework + # pymilvus # snowflake-connector-python pandocfilters==1.5.1 # via nbconvert @@ -573,13 +583,13 @@ portalocker==2.10.1 # qdrant-client pre-commit==3.3.1 # via feast (setup.py) -prometheus-client==0.21.0 +prometheus-client==0.21.1 # via # feast (setup.py) # jupyter-server prompt-toolkit==3.0.48 # via ipython -propcache==0.2.0 +propcache==0.2.1 # via # aiohttp # yarl @@ -605,6 +615,7 @@ protobuf==4.25.5 # grpcio-tools # mypy-protobuf # proto-plus + # pymilvus # substrait psutil==5.9.0 # via @@ -649,13 +660,15 @@ pybindgen==0.22.1 # via feast (setup.py) pycparser==2.22 # via cffi -pydantic==2.10.1 +pycryptodome==3.21.0 + # via minio +pydantic==2.10.4 # via # feast (setup.py) # fastapi # great-expectations # qdrant-client -pydantic-core==2.27.1 +pydantic-core==2.27.2 # via pydantic pygments==2.18.0 # via @@ -664,19 +677,21 @@ pygments==2.18.0 # nbconvert # rich # sphinx -pyjwt[crypto]==2.10.0 +pyjwt[crypto]==2.10.1 # via # feast (setup.py) # msal # singlestoredb # snowflake-connector-python +pymilvus==2.4.9 + # via feast (setup.py) pymssql==2.3.2 # via feast (setup.py) pymysql==1.1.1 # via feast (setup.py) pyodbc==5.2.0 # via feast (setup.py) -pyopenssl==24.2.1 +pyopenssl==24.3.0 # via snowflake-connector-python pyparsing==3.2.0 # via great-expectations @@ -729,8 +744,10 @@ python-dateutil==2.9.0.post0 # pandas # trino python-dotenv==1.0.1 - # via uvicorn -python-json-logger==2.0.7 + # via + # environs + # uvicorn +python-json-logger==3.2.1 # via jupyter-events python-keycloak==4.2.2 # via feast (setup.py) @@ -806,7 +823,7 @@ rfc3986-validator==0.1.1 # jupyter-events rich==13.9.4 # via ibis-framework -rpds-py==0.21.0 +rpds-py==0.22.3 # via # jsonschema # referencing @@ -816,7 +833,7 @@ ruamel-yaml==0.17.40 # via great-expectations ruamel-yaml-clib==0.2.12 # via ruamel-yaml -ruff==0.8.0 +ruff==0.8.4 # via feast (setup.py) s3transfer==0.10.4 # via boto3 @@ -830,12 +847,12 @@ setuptools==75.6.0 # jupyterlab # kubernetes # pip-tools + # pymilvus # singlestoredb singlestoredb==1.7.2 # via feast (setup.py) -six==1.16.0 +six==1.17.0 # via - # asttokens # azure-core # geomet # happybase @@ -850,7 +867,7 @@ sniffio==1.3.1 # httpx snowballstemmer==2.2.0 # via sphinx -snowflake-connector-python[pandas]==3.12.3 +snowflake-connector-python[pandas]==3.12.4 # via feast (setup.py) sortedcontainers==2.4.0 # via snowflake-connector-python @@ -874,7 +891,7 @@ sqlalchemy[mypy]==2.0.36 # via feast (setup.py) sqlglot==25.20.2 # via ibis-framework -sqlite-vec==0.1.1 +sqlite-vec==0.1.3 # via feast (setup.py) sqlparams==6.1.0 # via singlestoredb @@ -920,6 +937,7 @@ tqdm==4.67.1 # via # feast (setup.py) # great-expectations + # milvus-lite traitlets==5.14.3 # via # comm @@ -935,7 +953,7 @@ traitlets==5.14.3 # nbclient # nbconvert # nbformat -trino==0.330.0 +trino==0.331.0 # via feast (setup.py) typeguard==4.4.1 # via feast (setup.py) @@ -949,7 +967,7 @@ types-pymysql==1.1.0.20241103 # via feast (setup.py) types-pyopenssl==24.1.0.20240722 # via types-redis -types-python-dateutil==2.9.0.20241003 +types-python-dateutil==2.9.0.20241206 # via # feast (setup.py) # arrow @@ -965,12 +983,13 @@ types-setuptools==75.6.0.20241126 # via # feast (setup.py) # types-cffi -types-tabulate==0.9.0.20240106 +types-tabulate==0.9.0.20241207 # via feast (setup.py) types-urllib3==1.26.25.14 # via types-requests typing-extensions==4.12.2 # via + # anyio # azure-core # azure-identity # azure-storage-blob @@ -979,6 +998,7 @@ typing-extensions==4.12.2 # ibis-framework # ipython # jwcrypto + # minio # mypy # psycopg # psycopg-pool @@ -994,6 +1014,8 @@ tzlocal==5.2 # via # great-expectations # trino +ujson==5.10.0 + # via pymilvus uri-template==1.3.0 # via jsonschema urllib3==2.2.3 @@ -1009,7 +1031,7 @@ urllib3==2.2.3 # requests # responses # testcontainers -uvicorn[standard]==0.32.1 +uvicorn[standard]==0.34.0 # via # feast (setup.py) # uvicorn-worker @@ -1021,7 +1043,7 @@ virtualenv==20.23.0 # via # feast (setup.py) # pre-commit -watchfiles==1.0.0 +watchfiles==1.0.3 # via uvicorn wcwidth==0.2.13 # via prompt-toolkit @@ -1051,7 +1073,7 @@ wrapt==1.17.0 # testcontainers xmltodict==0.14.2 # via moto -yarl==1.18.0 +yarl==1.18.3 # via aiohttp zipp==3.21.0 # via importlib-metadata diff --git a/sdk/python/requirements/py3.11-requirements.txt b/sdk/python/requirements/py3.11-requirements.txt index 8f776fdc457..6919925f32b 100644 --- a/sdk/python/requirements/py3.11-requirements.txt +++ b/sdk/python/requirements/py3.11-requirements.txt @@ -2,17 +2,17 @@ # uv pip compile -p 3.11 --system --no-strip-extras setup.py --output-file sdk/python/requirements/py3.11-requirements.txt annotated-types==0.7.0 # via pydantic -anyio==4.6.2.post1 +anyio==4.7.0 # via # starlette # watchfiles -attrs==24.2.0 +attrs==24.3.0 # via # jsonschema # referencing bigtree==0.22.3 # via feast (setup.py) -certifi==2024.8.30 +certifi==2024.12.14 # via requests charset-normalizer==3.4.0 # via requests @@ -25,17 +25,17 @@ cloudpickle==3.1.0 # via dask colorama==0.4.6 # via feast (setup.py) -dask[dataframe]==2024.11.2 +dask[dataframe]==2024.12.1 # via # feast (setup.py) # dask-expr -dask-expr==1.1.19 +dask-expr==1.1.21 # via dask dill==0.3.9 # via feast (setup.py) -fastapi==0.115.5 +fastapi==0.115.6 # via feast (setup.py) -fsspec==2024.10.0 +fsspec==2024.12.0 # via dask gunicorn==23.0.0 # via @@ -51,7 +51,7 @@ idna==3.10 # requests importlib-metadata==8.5.0 # via dask -jinja2==3.1.4 +jinja2==3.1.5 # via feast (setup.py) jsonschema==4.23.0 # via feast (setup.py) @@ -83,25 +83,25 @@ pandas==2.2.3 # dask-expr partd==1.4.2 # via dask -prometheus-client==0.21.0 +prometheus-client==0.21.1 # via feast (setup.py) protobuf==4.25.5 # via feast (setup.py) -psutil==6.1.0 +psutil==6.1.1 # via feast (setup.py) pyarrow==18.0.0 # via # feast (setup.py) # dask-expr -pydantic==2.10.1 +pydantic==2.10.4 # via # feast (setup.py) # fastapi -pydantic-core==2.27.1 +pydantic-core==2.27.2 # via pydantic pygments==2.18.0 # via feast (setup.py) -pyjwt==2.10.0 +pyjwt==2.10.1 # via feast (setup.py) python-dateutil==2.9.0.post0 # via pandas @@ -120,11 +120,11 @@ referencing==0.35.1 # jsonschema-specifications requests==2.32.3 # via feast (setup.py) -rpds-py==0.21.0 +rpds-py==0.22.3 # via # jsonschema # referencing -six==1.16.0 +six==1.17.0 # via python-dateutil sniffio==1.3.1 # via anyio @@ -148,6 +148,7 @@ typeguard==4.4.1 # via feast (setup.py) typing-extensions==4.12.2 # via + # anyio # fastapi # mypy # pydantic @@ -158,7 +159,7 @@ tzdata==2024.2 # via pandas urllib3==2.2.3 # via requests -uvicorn[standard]==0.32.1 +uvicorn[standard]==0.34.0 # via # feast (setup.py) # uvicorn-worker @@ -166,7 +167,7 @@ uvicorn-worker==0.2.0 # via feast (setup.py) uvloop==0.21.0 # via uvicorn -watchfiles==1.0.0 +watchfiles==1.0.3 # via uvicorn websockets==14.1 # via uvicorn diff --git a/sdk/python/requirements/py3.9-ci-requirements.txt b/sdk/python/requirements/py3.9-ci-requirements.txt index 3deb441827c..b0e00409f18 100644 --- a/sdk/python/requirements/py3.9-ci-requirements.txt +++ b/sdk/python/requirements/py3.9-ci-requirements.txt @@ -1,14 +1,14 @@ # This file was autogenerated by uv via the following command: # uv pip compile -p 3.9 --system --no-strip-extras setup.py --extra ci --output-file sdk/python/requirements/py3.9-ci-requirements.txt -aiobotocore==2.15.2 +aiobotocore==2.16.0 # via feast (setup.py) -aiohappyeyeballs==2.4.3 +aiohappyeyeballs==2.4.4 # via aiohttp -aiohttp==3.11.7 +aiohttp==3.11.11 # via aiobotocore aioitertools==0.12.0 # via aiobotocore -aiosignal==1.3.1 +aiosignal==1.3.2 # via aiohttp alabaster==0.7.16 # via sphinx @@ -16,7 +16,7 @@ altair==4.2.2 # via great-expectations annotated-types==0.7.0 # via pydantic -anyio==4.6.2.post1 +anyio==4.7.0 # via # httpx # jupyter-server @@ -25,7 +25,9 @@ anyio==4.6.2.post1 appnope==0.1.4 # via ipykernel argon2-cffi==23.1.0 - # via jupyter-server + # via + # jupyter-server + # minio argon2-cffi-bindings==21.2.0 # via argon2-cffi arrow==1.3.0 @@ -34,7 +36,7 @@ asn1crypto==1.5.1 # via snowflake-connector-python assertpy==1.1 # via feast (setup.py) -asttokens==2.4.1 +asttokens==3.0.0 # via stack-data async-lru==2.0.4 # via jupyterlab @@ -46,7 +48,7 @@ async-timeout==5.0.1 # redis atpublic==4.1.0 # via ibis-framework -attrs==24.2.0 +attrs==24.3.0 # via # aiohttp # jsonschema @@ -71,11 +73,11 @@ bigtree==0.22.3 # via feast (setup.py) bleach==6.2.0 # via nbconvert -boto3==1.35.36 +boto3==1.35.81 # via # feast (setup.py) # moto -botocore==1.35.36 +botocore==1.35.81 # via # aiobotocore # boto3 @@ -90,7 +92,7 @@ cachetools==5.5.0 # via google-auth cassandra-driver==3.29.2 # via feast (setup.py) -certifi==2024.8.30 +certifi==2024.12.14 # via # elastic-transport # httpcore @@ -130,9 +132,9 @@ comm==0.2.2 # ipywidgets couchbase==4.3.2 # via feast (setup.py) -coverage[toml]==7.6.8 +coverage[toml]==7.6.9 # via pytest-cov -cryptography==42.0.8 +cryptography==43.0.3 # via # feast (setup.py) # azure-identity @@ -156,13 +158,13 @@ dask-expr==1.1.10 # via dask db-dtypes==1.3.1 # via google-cloud-bigquery -debugpy==1.8.9 +debugpy==1.8.11 # via ipykernel decorator==5.1.1 # via ipython defusedxml==0.7.1 # via nbconvert -deltalake==0.22.0 +deltalake==0.22.3 # via feast (setup.py) deprecation==2.1.0 # via python-keycloak @@ -178,10 +180,12 @@ duckdb==0.10.3 # via ibis-framework elastic-transport==8.15.1 # via elasticsearch -elasticsearch==8.16.0 +elasticsearch==8.17.0 # via feast (setup.py) entrypoints==0.4 # via altair +environs==9.5.0 + # via pymilvus exceptiongroup==1.2.2 # via # anyio @@ -193,9 +197,9 @@ executing==2.1.0 # via stack-data faiss-cpu==1.9.0.post1 # via feast (setup.py) -fastapi==0.115.5 +fastapi==0.115.6 # via feast (setup.py) -fastjsonschema==2.20.0 +fastjsonschema==2.21.1 # via nbformat filelock==3.16.1 # via @@ -213,7 +217,7 @@ fsspec==2024.9.0 # dask geomet==0.2.1.post1 # via cassandra-driver -google-api-core[grpc]==2.23.0 +google-api-core[grpc]==2.24.0 # via # feast (setup.py) # google-cloud-bigquery @@ -222,7 +226,7 @@ google-api-core[grpc]==2.23.0 # google-cloud-core # google-cloud-datastore # google-cloud-storage -google-auth==2.36.0 +google-auth==2.37.0 # via # google-api-core # google-cloud-bigquery @@ -244,9 +248,9 @@ google-cloud-core==2.4.1 # google-cloud-bigtable # google-cloud-datastore # google-cloud-storage -google-cloud-datastore==2.20.1 +google-cloud-datastore==2.20.2 # via feast (setup.py) -google-cloud-storage==2.18.2 +google-cloud-storage==2.19.0 # via feast (setup.py) google-crc32c==1.6.0 # via @@ -266,7 +270,7 @@ great-expectations==0.18.22 # via feast (setup.py) grpc-google-iam-v1==0.13.1 # via google-cloud-bigtable -grpcio==1.68.0 +grpcio==1.68.1 # via # feast (setup.py) # google-api-core @@ -277,6 +281,7 @@ grpcio==1.68.0 # grpcio-status # grpcio-testing # grpcio-tools + # pymilvus # qdrant-client grpcio-health-checking==1.62.3 # via feast (setup.py) @@ -366,7 +371,7 @@ isoduration==20.11.0 # via jsonschema jedi==0.19.2 # via ipython -jinja2==3.1.4 +jinja2==3.1.5 # via # feast (setup.py) # altair @@ -381,7 +386,7 @@ jmespath==1.0.1 # via # boto3 # botocore -json5==0.9.28 +json5==0.10.0 # via jupyterlab-server jsonpatch==1.33 # via great-expectations @@ -413,7 +418,7 @@ jupyter-core==5.7.2 # nbclient # nbconvert # nbformat -jupyter-events==0.10.0 +jupyter-events==0.11.0 # via jupyter-server jupyter-lsp==2.2.5 # via jupyterlab @@ -426,7 +431,7 @@ jupyter-server==2.14.2 # notebook-shim jupyter-server-terminals==0.5.3 # via jupyter-server -jupyterlab==4.2.6 +jupyterlab==4.3.4 # via notebook jupyterlab-pygments==0.3.0 # via nbconvert @@ -451,15 +456,19 @@ markupsafe==3.0.2 # jinja2 # nbconvert # werkzeug -marshmallow==3.23.1 - # via great-expectations +marshmallow==3.23.2 + # via + # environs + # great-expectations matplotlib-inline==0.1.7 # via # ipykernel # ipython mdurl==0.1.2 # via markdown-it-py -minio==7.1.0 +milvus-lite==2.4.10 + # via pymilvus +minio==7.2.11 # via feast (setup.py) mistune==3.0.2 # via @@ -489,7 +498,7 @@ mypy-extensions==1.0.0 # via mypy mypy-protobuf==3.3.0 # via feast (setup.py) -nbclient==0.10.0 +nbclient==0.10.2 # via nbconvert nbconvert==7.16.4 # via jupyter-server @@ -503,7 +512,7 @@ nest-asyncio==1.6.0 # via ipykernel nodeenv==1.9.1 # via pre-commit -notebook==7.2.2 +notebook==7.3.1 # via great-expectations notebook-shim==0.2.4 # via @@ -556,6 +565,7 @@ pandas==2.2.3 # google-cloud-bigquery # great-expectations # ibis-framework + # pymilvus # snowflake-connector-python pandocfilters==1.5.1 # via nbconvert @@ -590,13 +600,13 @@ portalocker==2.10.1 # qdrant-client pre-commit==3.3.1 # via feast (setup.py) -prometheus-client==0.21.0 +prometheus-client==0.21.1 # via # feast (setup.py) # jupyter-server prompt-toolkit==3.0.48 # via ipython -propcache==0.2.0 +propcache==0.2.1 # via # aiohttp # yarl @@ -622,6 +632,7 @@ protobuf==4.25.5 # grpcio-tools # mypy-protobuf # proto-plus + # pymilvus # substrait psutil==5.9.0 # via @@ -666,13 +677,15 @@ pybindgen==0.22.1 # via feast (setup.py) pycparser==2.22 # via cffi -pydantic==2.10.1 +pycryptodome==3.21.0 + # via minio +pydantic==2.10.4 # via # feast (setup.py) # fastapi # great-expectations # qdrant-client -pydantic-core==2.27.1 +pydantic-core==2.27.2 # via pydantic pygments==2.18.0 # via @@ -681,19 +694,21 @@ pygments==2.18.0 # nbconvert # rich # sphinx -pyjwt[crypto]==2.10.0 +pyjwt[crypto]==2.10.1 # via # feast (setup.py) # msal # singlestoredb # snowflake-connector-python +pymilvus==2.4.9 + # via feast (setup.py) pymssql==2.3.2 # via feast (setup.py) pymysql==1.1.1 # via feast (setup.py) pyodbc==5.2.0 # via feast (setup.py) -pyopenssl==24.2.1 +pyopenssl==24.3.0 # via snowflake-connector-python pyparsing==3.2.0 # via great-expectations @@ -746,8 +761,10 @@ python-dateutil==2.9.0.post0 # pandas # trino python-dotenv==1.0.1 - # via uvicorn -python-json-logger==2.0.7 + # via + # environs + # uvicorn +python-json-logger==3.2.1 # via jupyter-events python-keycloak==4.2.2 # via feast (setup.py) @@ -823,7 +840,7 @@ rfc3986-validator==0.1.1 # jupyter-events rich==13.9.4 # via ibis-framework -rpds-py==0.21.0 +rpds-py==0.22.3 # via # jsonschema # referencing @@ -833,7 +850,7 @@ ruamel-yaml==0.17.40 # via great-expectations ruamel-yaml-clib==0.2.12 # via ruamel-yaml -ruff==0.8.0 +ruff==0.8.4 # via feast (setup.py) s3transfer==0.10.4 # via boto3 @@ -847,12 +864,12 @@ setuptools==75.6.0 # jupyterlab # kubernetes # pip-tools + # pymilvus # singlestoredb singlestoredb==1.7.2 # via feast (setup.py) -six==1.16.0 +six==1.17.0 # via - # asttokens # azure-core # geomet # happybase @@ -867,7 +884,7 @@ sniffio==1.3.1 # httpx snowballstemmer==2.2.0 # via sphinx -snowflake-connector-python[pandas]==3.12.3 +snowflake-connector-python[pandas]==3.12.4 # via feast (setup.py) sortedcontainers==2.4.0 # via snowflake-connector-python @@ -891,7 +908,7 @@ sqlalchemy[mypy]==2.0.36 # via feast (setup.py) sqlglot==23.12.2 # via ibis-framework -sqlite-vec==0.1.1 +sqlite-vec==0.1.3 # via feast (setup.py) sqlparams==6.1.0 # via singlestoredb @@ -917,7 +934,7 @@ tinycss2==1.4.0 # via nbconvert toml==0.10.2 # via feast (setup.py) -tomli==2.1.0 +tomli==2.2.1 # via # build # coverage @@ -947,6 +964,7 @@ tqdm==4.67.1 # via # feast (setup.py) # great-expectations + # milvus-lite traitlets==5.14.3 # via # comm @@ -962,7 +980,7 @@ traitlets==5.14.3 # nbclient # nbconvert # nbformat -trino==0.330.0 +trino==0.331.0 # via feast (setup.py) typeguard==4.4.1 # via feast (setup.py) @@ -976,7 +994,7 @@ types-pymysql==1.1.0.20241103 # via feast (setup.py) types-pyopenssl==24.1.0.20240722 # via types-redis -types-python-dateutil==2.9.0.20241003 +types-python-dateutil==2.9.0.20241206 # via # feast (setup.py) # arrow @@ -992,7 +1010,7 @@ types-setuptools==75.6.0.20241126 # via # feast (setup.py) # types-cffi -types-tabulate==0.9.0.20240106 +types-tabulate==0.9.0.20241207 # via feast (setup.py) types-urllib3==1.26.25.14 # via types-requests @@ -1009,12 +1027,14 @@ typing-extensions==4.12.2 # ibis-framework # ipython # jwcrypto + # minio # multidict # mypy # psycopg # psycopg-pool # pydantic # pydantic-core + # python-json-logger # rich # snowflake-connector-python # sqlalchemy @@ -1028,6 +1048,8 @@ tzlocal==5.2 # via # great-expectations # trino +ujson==5.10.0 + # via pymilvus uri-template==1.3.0 # via jsonschema urllib3==1.26.20 @@ -1044,7 +1066,7 @@ urllib3==1.26.20 # responses # snowflake-connector-python # testcontainers -uvicorn[standard]==0.32.1 +uvicorn[standard]==0.34.0 # via # feast (setup.py) # uvicorn-worker @@ -1056,7 +1078,7 @@ virtualenv==20.23.0 # via # feast (setup.py) # pre-commit -watchfiles==1.0.0 +watchfiles==1.0.3 # via uvicorn wcwidth==0.2.13 # via prompt-toolkit @@ -1086,7 +1108,7 @@ wrapt==1.17.0 # testcontainers xmltodict==0.14.2 # via moto -yarl==1.18.0 +yarl==1.18.3 # via aiohttp zipp==3.21.0 # via importlib-metadata diff --git a/sdk/python/requirements/py3.9-requirements.txt b/sdk/python/requirements/py3.9-requirements.txt index 8c9fc036433..aab78c8ceef 100644 --- a/sdk/python/requirements/py3.9-requirements.txt +++ b/sdk/python/requirements/py3.9-requirements.txt @@ -2,17 +2,17 @@ # uv pip compile -p 3.9 --system --no-strip-extras setup.py --output-file sdk/python/requirements/py3.9-requirements.txt annotated-types==0.7.0 # via pydantic -anyio==4.6.2.post1 +anyio==4.7.0 # via # starlette # watchfiles -attrs==24.2.0 +attrs==24.3.0 # via # jsonschema # referencing bigtree==0.22.3 # via feast (setup.py) -certifi==2024.8.30 +certifi==2024.12.14 # via requests charset-normalizer==3.4.0 # via requests @@ -35,9 +35,9 @@ dill==0.3.9 # via feast (setup.py) exceptiongroup==1.2.2 # via anyio -fastapi==0.115.5 +fastapi==0.115.6 # via feast (setup.py) -fsspec==2024.10.0 +fsspec==2024.12.0 # via dask gunicorn==23.0.0 # via @@ -55,7 +55,7 @@ importlib-metadata==8.5.0 # via # dask # typeguard -jinja2==3.1.4 +jinja2==3.1.5 # via feast (setup.py) jsonschema==4.23.0 # via feast (setup.py) @@ -87,25 +87,25 @@ pandas==2.2.3 # dask-expr partd==1.4.2 # via dask -prometheus-client==0.21.0 +prometheus-client==0.21.1 # via feast (setup.py) protobuf==4.25.5 # via feast (setup.py) -psutil==6.1.0 +psutil==6.1.1 # via feast (setup.py) pyarrow==18.0.0 # via # feast (setup.py) # dask-expr -pydantic==2.10.1 +pydantic==2.10.4 # via # feast (setup.py) # fastapi -pydantic-core==2.27.1 +pydantic-core==2.27.2 # via pydantic pygments==2.18.0 # via feast (setup.py) -pyjwt==2.10.0 +pyjwt==2.10.1 # via feast (setup.py) python-dateutil==2.9.0.post0 # via pandas @@ -124,11 +124,11 @@ referencing==0.35.1 # jsonschema-specifications requests==2.32.3 # via feast (setup.py) -rpds-py==0.21.0 +rpds-py==0.22.3 # via # jsonschema # referencing -six==1.16.0 +six==1.17.0 # via python-dateutil sniffio==1.3.1 # via anyio @@ -142,7 +142,7 @@ tenacity==8.5.0 # via feast (setup.py) toml==0.10.2 # via feast (setup.py) -tomli==2.1.0 +tomli==2.2.1 # via mypy toolz==1.0.0 # via @@ -167,7 +167,7 @@ tzdata==2024.2 # via pandas urllib3==2.2.3 # via requests -uvicorn[standard]==0.32.1 +uvicorn[standard]==0.34.0 # via # feast (setup.py) # uvicorn-worker @@ -175,7 +175,7 @@ uvicorn-worker==0.2.0 # via feast (setup.py) uvloop==0.21.0 # via uvicorn -watchfiles==1.0.0 +watchfiles==1.0.3 # via uvicorn websockets==14.1 # via uvicorn diff --git a/sdk/python/tests/conftest.py b/sdk/python/tests/conftest.py index 24c8f40f742..c029648aeeb 100644 --- a/sdk/python/tests/conftest.py +++ b/sdk/python/tests/conftest.py @@ -57,8 +57,12 @@ location, ) from tests.utils.auth_permissions_util import default_store -from tests.utils.generate_self_signed_certifcate_util import generate_self_signed_cert from tests.utils.http_server import check_port_open, free_port # noqa: E402 +from tests.utils.ssl_certifcates_util import ( + combine_trust_stores, + create_ca_trust_store, + generate_self_signed_cert, +) logger = logging.getLogger(__name__) @@ -306,6 +310,10 @@ def pytest_generate_tests(metafunc: pytest.Metafunc): pytest.mark.xdist_group(name=m) for m in c.offline_store_creator.xdist_groups() ] + # Check if there are any test markers associated with the creator and add them. + if c.offline_store_creator.test_markers(): + marks.extend(c.offline_store_creator.test_markers()) + _config_cache[c] = pytest.param(c, marks=marks) configs.append(_config_cache[c]) @@ -514,17 +522,36 @@ def auth_config(request, is_integration_test): return auth_configuration -@pytest.fixture(params=[True, False], scope="module") +@pytest.fixture(scope="module") def tls_mode(request): - is_tls_mode = request.param + is_tls_mode = request.param[0] + output_combined_truststore_path = "" if is_tls_mode: certificates_path = tempfile.mkdtemp() tls_key_path = os.path.join(certificates_path, "key.pem") tls_cert_path = os.path.join(certificates_path, "cert.pem") + generate_self_signed_cert(cert_path=tls_cert_path, key_path=tls_key_path) + is_ca_trust_store_set = request.param[1] + if is_ca_trust_store_set: + # Paths + feast_ca_trust_store_path = os.path.join( + certificates_path, "feast_trust_store.pem" + ) + create_ca_trust_store( + public_key_path=tls_cert_path, + private_key_path=tls_key_path, + output_trust_store_path=feast_ca_trust_store_path, + ) + + # Combine trust stores + output_combined_path = os.path.join( + certificates_path, "combined_trust_store.pem" + ) + combine_trust_stores(feast_ca_trust_store_path, output_combined_path) else: tls_key_path = "" tls_cert_path = "" - return is_tls_mode, tls_key_path, tls_cert_path + return is_tls_mode, tls_key_path, tls_cert_path, output_combined_truststore_path diff --git a/sdk/python/tests/data/data_creator.py b/sdk/python/tests/data/data_creator.py index 6b0984f799d..dfe94913e97 100644 --- a/sdk/python/tests/data/data_creator.py +++ b/sdk/python/tests/data/data_creator.py @@ -84,6 +84,8 @@ def get_feature_values_for_dtype( def create_document_dataset() -> pd.DataFrame: data = { "item_id": [1, 2, 3], + "string_feature": ["a", "b", "c"], + "float_feature": [1.0, 2.0, 3.0], "embedding_float": [[4.0, 5.0], [1.0, 2.0], [3.0, 4.0]], "embedding_double": [[4.0, 5.0], [1.0, 2.0], [3.0, 4.0]], "ts": [ diff --git a/sdk/python/tests/example_repos/example_rag_feature_repo.py b/sdk/python/tests/example_repos/example_rag_feature_repo.py new file mode 100644 index 00000000000..2f55095bc69 --- /dev/null +++ b/sdk/python/tests/example_repos/example_rag_feature_repo.py @@ -0,0 +1,38 @@ +from datetime import timedelta + +from feast import Entity, FeatureView, Field, FileSource +from feast.types import Array, Float32, Int64, UnixTimestamp + +# This is for Milvus +# Note that file source paths are not validated, so there doesn't actually need to be any data +# at the paths for these file sources. Since these paths are effectively fake, this example +# feature repo should not be used for historical retrieval. + +rag_documents_source = FileSource( + path="data/embedded_documents.parquet", + timestamp_field="event_timestamp", + created_timestamp_column="created_timestamp", +) + +item = Entity( + name="item_id", # The name is derived from this argument, not object name. + join_keys=["item_id"], +) + +document_embeddings = FeatureView( + name="embedded_documents", + entities=[item], + schema=[ + Field( + name="vector", + dtype=Array(Float32), + vector_index=True, + vector_search_metric="L2", + ), + Field(name="item_id", dtype=Int64), + Field(name="created_timestamp", dtype=UnixTimestamp), + Field(name="event_timestamp", dtype=UnixTimestamp), + ], + source=rag_documents_source, + ttl=timedelta(hours=24), +) diff --git a/sdk/python/tests/foo_provider.py b/sdk/python/tests/foo_provider.py index 570a6d4f8d5..3d1f9219991 100644 --- a/sdk/python/tests/foo_provider.py +++ b/sdk/python/tests/foo_provider.py @@ -150,6 +150,7 @@ def retrieve_online_documents( config: RepoConfig, table: FeatureView, requested_feature: str, + requested_features: Optional[List[str]], query: List[float], top_k: int, distance_metric: Optional[str] = None, diff --git a/sdk/python/tests/integration/conftest.py b/sdk/python/tests/integration/conftest.py index 82f80b89927..21c9051d0d7 100644 --- a/sdk/python/tests/integration/conftest.py +++ b/sdk/python/tests/integration/conftest.py @@ -1,4 +1,7 @@ import logging +import random +import time +from multiprocessing import Manager import pytest from testcontainers.keycloak import KeycloakContainer @@ -9,14 +12,30 @@ from tests.utils.auth_permissions_util import setup_permissions_on_keycloak logger = logging.getLogger(__name__) +logger.setLevel(logging.INFO) + +shared_state = Manager().dict() @pytest.fixture(scope="session") def start_keycloak_server(): + # Add random sleep between 0 and 2 before checking the state to avoid concurrency issues. + random_sleep_time = random.uniform(0, 2) + time.sleep(random_sleep_time) + + # If the Keycloak instance is already started (in any worker), reuse it + if shared_state.get("keycloak_started", False): + return shared_state["keycloak_url"] logger.info("Starting keycloak instance") with KeycloakContainer("quay.io/keycloak/keycloak:24.0.1") as keycloak_container: setup_permissions_on_keycloak(keycloak_container.get_client()) - yield keycloak_container.get_url() + shared_state["keycloak_started"] = True + shared_state["keycloak_url"] = keycloak_container.get_url() + yield shared_state["keycloak_url"] + + # After the fixture is done, cleanup the shared state + del shared_state["keycloak_started"] + del shared_state["keycloak_url"] @pytest.fixture(scope="session") diff --git a/sdk/python/tests/integration/feature_repos/repo_configuration.py b/sdk/python/tests/integration/feature_repos/repo_configuration.py index bf464681600..54129f23c6e 100644 --- a/sdk/python/tests/integration/feature_repos/repo_configuration.py +++ b/sdk/python/tests/integration/feature_repos/repo_configuration.py @@ -86,6 +86,7 @@ ) DYNAMO_CONFIG = {"type": "dynamodb", "region": "us-west-2"} +MILVUS_CONFIG = {"type": "milvus"} REDIS_CONFIG = {"type": "redis", "connection_string": "localhost:6379,db=0"} REDIS_CLUSTER_CONFIG = { "type": "redis", @@ -162,6 +163,7 @@ AVAILABLE_ONLINE_STORES["datastore"] = ("datastore", None) AVAILABLE_ONLINE_STORES["snowflake"] = (SNOWFLAKE_CONFIG, None) AVAILABLE_ONLINE_STORES["bigtable"] = (BIGTABLE_CONFIG, None) + # AVAILABLE_ONLINE_STORES["milvus"] = (MILVUS_CONFIG, None) # Uncomment to test using private IKV account. Currently not enabled as # there is no dedicated IKV instance for CI testing and there is no @@ -559,6 +561,9 @@ def construct_test_environment( cache_ttl_seconds=1, ) + if test_repo_config.online_store in ["milvus", "pgvector", "qdrant"]: + entity_key_serialization_version = 3 + environment_params = { "name": project, "provider": test_repo_config.provider, diff --git a/sdk/python/tests/integration/feature_repos/universal/data_source_creator.py b/sdk/python/tests/integration/feature_repos/universal/data_source_creator.py index 513a94ee210..467db4dddce 100644 --- a/sdk/python/tests/integration/feature_repos/universal/data_source_creator.py +++ b/sdk/python/tests/integration/feature_repos/universal/data_source_creator.py @@ -2,6 +2,7 @@ from typing import Dict, Optional import pandas as pd +from _pytest.mark import MarkDecorator from feast.data_source import DataSource from feast.feature_logging import LoggingDestination @@ -64,3 +65,11 @@ def teardown(self): @staticmethod def xdist_groups() -> list[str]: return [] + + @staticmethod + def test_markers() -> list[MarkDecorator]: + """ + return the array of test markers to add dynamically to the tests created by this creator method. override this method in your implementations. By default, it will not add any markers. + :return: + """ + return [] diff --git a/sdk/python/tests/integration/feature_repos/universal/data_sources/file.py b/sdk/python/tests/integration/feature_repos/universal/data_sources/file.py index dc716f45e1e..6f6e5d68133 100644 --- a/sdk/python/tests/integration/feature_repos/universal/data_sources/file.py +++ b/sdk/python/tests/integration/feature_repos/universal/data_sources/file.py @@ -11,7 +11,9 @@ import pandas as pd import pyarrow as pa import pyarrow.parquet as pq +import pytest import yaml +from _pytest.mark import MarkDecorator from minio import Minio from testcontainers.core.generic import DockerContainer from testcontainers.core.waiting_utils import wait_for_logs @@ -34,8 +36,8 @@ DataSourceCreator, ) from tests.utils.auth_permissions_util import include_auth_config -from tests.utils.generate_self_signed_certifcate_util import generate_self_signed_cert from tests.utils.http_server import check_port_open, free_port # noqa: E402 +from tests.utils.ssl_certifcates_util import generate_self_signed_cert logger = logging.getLogger(__name__) @@ -372,6 +374,10 @@ def __init__(self, project_name: str, *args, **kwargs): self.server_port: int = 0 self.proc: Optional[Popen[bytes]] = None + @staticmethod + def test_markers() -> list[MarkDecorator]: + return [pytest.mark.rbac_remote_integration_test] + def setup(self, registry: RegistryConfig): parent_offline_config = super().create_offline_store_config() config = RepoConfig( @@ -418,6 +424,10 @@ def __init__(self, project_name: str, *args, **kwargs): self.server_port: int = 0 self.proc: Optional[Popen[bytes]] = None + @staticmethod + def test_markers() -> list[MarkDecorator]: + return [pytest.mark.rbac_remote_integration_test] + def setup(self, registry: RegistryConfig): parent_offline_config = super().create_offline_store_config() config = RepoConfig( @@ -452,9 +462,6 @@ def setup(self, registry: RegistryConfig): str(tls_key_path), "--cert", str(self.tls_cert_path), - # This is needed for the self-signed certificate, disabled verify_client for integration tests. - "--verify_client", - str(False), ] self.proc = subprocess.Popen( cmd, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL @@ -518,6 +525,10 @@ def __init__(self, project_name: str, *args, **kwargs): def xdist_groups() -> list[str]: return ["keycloak"] + @staticmethod + def test_markers() -> list[MarkDecorator]: + return [pytest.mark.rbac_remote_integration_test] + def setup(self, registry: RegistryConfig): parent_offline_config = super().create_offline_store_config() config = RepoConfig( diff --git a/sdk/python/tests/integration/feature_repos/universal/feature_views.py b/sdk/python/tests/integration/feature_repos/universal/feature_views.py index 11ddcb0ecc6..47e5746e61a 100644 --- a/sdk/python/tests/integration/feature_repos/universal/feature_views.py +++ b/sdk/python/tests/integration/feature_repos/universal/feature_views.py @@ -17,7 +17,7 @@ from feast.data_source import DataSource, RequestSource from feast.feature_view_projection import FeatureViewProjection from feast.on_demand_feature_view import PandasTransformation, SubstraitTransformation -from feast.types import Array, FeastType, Float32, Float64, Int32, Int64 +from feast.types import Array, FeastType, Float32, Float64, Int32, Int64, String from tests.integration.feature_repos.universal.entities import ( customer, driver, @@ -160,8 +160,20 @@ def create_item_embeddings_feature_view(source, infer_features: bool = False): schema=None if infer_features else [ - Field(name="embedding_double", dtype=Array(Float64)), - Field(name="embedding_float", dtype=Array(Float32)), + Field( + name="embedding_double", + dtype=Array(Float64), + vector_index=True, + vector_search_metric="L2", + ), + Field( + name="embedding_float", + dtype=Array(Float32), + vector_index=True, + vector_search_metric="L2", + ), + Field(name="string_feature", dtype=String), + Field(name="float_feature", dtype=Float32), ], source=source, ttl=timedelta(hours=2), diff --git a/sdk/python/tests/integration/feature_repos/universal/online_store/milvus.py b/sdk/python/tests/integration/feature_repos/universal/online_store/milvus.py new file mode 100644 index 00000000000..c02bd144016 --- /dev/null +++ b/sdk/python/tests/integration/feature_repos/universal/online_store/milvus.py @@ -0,0 +1,43 @@ +from typing import Any, Dict + +import docker +from testcontainers.core.container import DockerContainer +from testcontainers.core.waiting_utils import wait_for_logs + +from tests.integration.feature_repos.universal.online_store_creator import ( + OnlineStoreCreator, +) + + +class MilvusOnlineStoreCreator(OnlineStoreCreator): + def __init__(self, project_name: str, **kwargs): + super().__init__(project_name) + self.fixed_port = 19530 + self.container = DockerContainer("milvusdb/milvus:v2.4.4").with_exposed_ports( + self.fixed_port + ) + self.client = docker.from_env() + + def create_online_store(self) -> Dict[str, Any]: + self.container.start() + # Wait for Milvus server to be ready + # log_string_to_wait_for = "Ready to accept connections" + log_string_to_wait_for = "" + wait_for_logs( + container=self.container, predicate=log_string_to_wait_for, timeout=30 + ) + host = "localhost" + port = self.container.get_exposed_port(self.fixed_port) + return { + "type": "milvus", + "host": host, + "port": int(port), + "index_type": "IVF_FLAT", + "metric_type": "L2", + "embedding_dim": 2, + "vector_enabled": True, + "nlist": 1, + } + + def teardown(self): + self.container.stop() diff --git a/sdk/python/tests/integration/offline_store/test_feature_logging.py b/sdk/python/tests/integration/offline_store/test_feature_logging.py index 32f506f90b2..53147d242ef 100644 --- a/sdk/python/tests/integration/offline_store/test_feature_logging.py +++ b/sdk/python/tests/integration/offline_store/test_feature_logging.py @@ -106,7 +106,15 @@ def retrieve(): ) persisted_logs = persisted_logs[expected_columns] + logs_df = logs_df[expected_columns] + + # Convert timezone-aware datetime values to naive datetime values + logs_df[LOG_TIMESTAMP_FIELD] = logs_df[LOG_TIMESTAMP_FIELD].dt.tz_localize(None) + persisted_logs[LOG_TIMESTAMP_FIELD] = persisted_logs[ + LOG_TIMESTAMP_FIELD + ].dt.tz_localize(None) + pd.testing.assert_frame_equal( logs_df.sort_values(REQUEST_ID_FIELD).reset_index(drop=True), persisted_logs.sort_values(REQUEST_ID_FIELD).reset_index(drop=True), diff --git a/sdk/python/tests/integration/online_store/test_remote_online_store.py b/sdk/python/tests/integration/online_store/test_remote_online_store.py index 10f1180d8e6..eb03fd0c3c5 100644 --- a/sdk/python/tests/integration/online_store/test_remote_online_store.py +++ b/sdk/python/tests/integration/online_store/test_remote_online_store.py @@ -22,6 +22,10 @@ @pytest.mark.integration +@pytest.mark.rbac_remote_integration_test +@pytest.mark.parametrize( + "tls_mode", [("True", "True"), ("True", "False"), ("False", "")], indirect=True +) def test_remote_online_store_read(auth_config, tls_mode): with ( tempfile.TemporaryDirectory() as remote_server_tmp_dir, @@ -56,13 +60,13 @@ def test_remote_online_store_read(auth_config, tls_mode): ) ) assert None not in (server_store, server_url, registry_path) - _, _, tls_cert_path = tls_mode + client_store = _create_remote_client_feature_store( temp_dir=remote_client_tmp_dir, server_registry_path=str(registry_path), feature_server_url=server_url, auth_config=auth_config, - tls_cert_path=tls_cert_path, + tls_mode=tls_mode, ) assert client_store is not None _assert_non_existing_entity_feature_views_entity( @@ -172,7 +176,7 @@ def _create_server_store_spin_feature_server( ): store = default_store(str(temp_dir), auth_config, permissions_list) feast_server_port = free_port() - is_tls_mode, tls_key_path, tls_cert_path = tls_mode + is_tls_mode, tls_key_path, tls_cert_path, ca_trust_store_path = tls_mode server_url = next( start_feature_server( @@ -180,6 +184,7 @@ def _create_server_store_spin_feature_server( server_port=feast_server_port, tls_key_path=tls_key_path, tls_cert_path=tls_cert_path, + ca_trust_store_path=ca_trust_store_path, ) ) if is_tls_mode: @@ -203,20 +208,33 @@ def _create_remote_client_feature_store( server_registry_path: str, feature_server_url: str, auth_config: str, - tls_cert_path: str = "", + tls_mode, ) -> FeatureStore: project_name = "REMOTE_ONLINE_CLIENT_PROJECT" runner = CliRunner() result = runner.run(["init", project_name], cwd=temp_dir) assert result.returncode == 0 repo_path = os.path.join(temp_dir, project_name, "feature_repo") - _overwrite_remote_client_feature_store_yaml( - repo_path=str(repo_path), - registry_path=server_registry_path, - feature_server_url=feature_server_url, - auth_config=auth_config, - tls_cert_path=tls_cert_path, - ) + is_tls_mode, _, tls_cert_path, ca_trust_store_path = tls_mode + if is_tls_mode and not ca_trust_store_path: + _overwrite_remote_client_feature_store_yaml( + repo_path=str(repo_path), + registry_path=server_registry_path, + feature_server_url=feature_server_url, + auth_config=auth_config, + tls_cert_path=tls_cert_path, + ) + else: + _overwrite_remote_client_feature_store_yaml( + repo_path=str(repo_path), + registry_path=server_registry_path, + feature_server_url=feature_server_url, + auth_config=auth_config, + ) + + if is_tls_mode and ca_trust_store_path: + # configure trust store path only when is_tls_mode and ca_trust_store_path exists. + os.environ["FEAST_CA_CERT_FILE_PATH"] = ca_trust_store_path return FeatureStore(repo_path=repo_path) diff --git a/sdk/python/tests/integration/online_store/test_universal_online.py b/sdk/python/tests/integration/online_store/test_universal_online.py index 4074dcb194e..64122d2c861 100644 --- a/sdk/python/tests/integration/online_store/test_universal_online.py +++ b/sdk/python/tests/integration/online_store/test_universal_online.py @@ -614,6 +614,10 @@ def eventually_apply() -> Tuple[None, bool]: online_features = fs.get_online_features( features=features, entity_rows=entity_rows ).to_dict() + + # Debugging print statement + print("Online features values:", online_features["value"]) + assert all(v is None for v in online_features["value"]) @@ -858,8 +862,8 @@ def assert_feature_service_entity_mapping_correctness( @pytest.mark.integration @pytest.mark.universal_online_stores(only=["pgvector", "elasticsearch", "qdrant"]) -def test_retrieve_online_documents(vectordb_environment, fake_document_data): - fs = vectordb_environment.feature_store +def test_retrieve_online_documents(environment, fake_document_data): + fs = environment.feature_store df, data_source = fake_document_data item_embeddings_feature_view = create_item_embeddings_feature_view(data_source) fs.apply([item_embeddings_feature_view, item()]) @@ -891,3 +895,28 @@ def test_retrieve_online_documents(vectordb_environment, fake_document_data): top_k=2, distance_metric="wrong", ).to_dict() + + +@pytest.mark.integration +@pytest.mark.universal_online_stores(only=["milvus"]) +def test_retrieve_online_milvus_documents(environment, fake_document_data): + fs = environment.feature_store + df, data_source = fake_document_data + item_embeddings_feature_view = create_item_embeddings_feature_view(data_source) + fs.apply([item_embeddings_feature_view, item()]) + fs.write_to_online_store("item_embeddings", df) + documents = fs.retrieve_online_documents( + feature=None, + features=[ + "item_embeddings:embedding_float", + "item_embeddings:item_id", + "item_embeddings:string_feature", + ], + query=[1.0, 2.0], + top_k=2, + distance_metric="L2", + ).to_dict() + assert len(documents["embedding_float"]) == 2 + + assert len(documents["item_id"]) == 2 + assert documents["item_id"] == [2, 3] diff --git a/sdk/python/tests/integration/registration/test_universal_registry.py b/sdk/python/tests/integration/registration/test_universal_registry.py index 5e06247ebbb..3819d168d78 100644 --- a/sdk/python/tests/integration/registration/test_universal_registry.py +++ b/sdk/python/tests/integration/registration/test_universal_registry.py @@ -344,7 +344,10 @@ def mock_remote_registry(): marks=pytest.mark.xdist_group(name="mysql_registry"), ), lazy_fixture("sqlite_registry"), - lazy_fixture("mock_remote_registry"), + pytest.param( + lazy_fixture("mock_remote_registry"), + marks=pytest.mark.rbac_remote_integration_test, + ), ] sql_fixtures = [ diff --git a/sdk/python/tests/unit/infra/offline_stores/contrib/spark_offline_store/test_spark.py b/sdk/python/tests/unit/infra/offline_stores/contrib/spark_offline_store/test_spark.py index b8f8cc42474..307ba4058c1 100644 --- a/sdk/python/tests/unit/infra/offline_stores/contrib/spark_offline_store/test_spark.py +++ b/sdk/python/tests/unit/infra/offline_stores/contrib/spark_offline_store/test_spark.py @@ -71,6 +71,68 @@ def test_pull_latest_from_table_with_nested_timestamp_or_query(mock_get_spark_se assert retrieval_job.query.strip() == expected_query.strip() +@patch( + "feast.infra.offline_stores.contrib.spark_offline_store.spark.get_spark_session_or_start_new_with_repoconfig" +) +def test_pull_latest_from_table_with_nested_timestamp_or_query_and_date_partition_column_set( + mock_get_spark_session, +): + mock_spark_session = MagicMock() + mock_get_spark_session.return_value = mock_spark_session + + test_repo_config = RepoConfig( + project="test_project", + registry="test_registry", + provider="local", + offline_store=SparkOfflineStoreConfig(type="spark"), + ) + + test_data_source = SparkSource( + name="test_nested_batch_source", + description="test_nested_batch_source", + table="offline_store_database_name.offline_store_table_name", + timestamp_field="nested_timestamp", + field_mapping={ + "event_header.event_published_datetime_utc": "nested_timestamp", + }, + date_partition_column="effective_date", + ) + + # Define the parameters for the method + join_key_columns = ["key1", "key2"] + feature_name_columns = ["feature1", "feature2"] + timestamp_field = "event_header.event_published_datetime_utc" + created_timestamp_column = "created_timestamp" + start_date = datetime(2021, 1, 1) + end_date = datetime(2021, 1, 2) + + # Call the method + retrieval_job = SparkOfflineStore.pull_latest_from_table_or_query( + config=test_repo_config, + data_source=test_data_source, + join_key_columns=join_key_columns, + feature_name_columns=feature_name_columns, + timestamp_field=timestamp_field, + created_timestamp_column=created_timestamp_column, + start_date=start_date, + end_date=end_date, + ) + + expected_query = """SELECT + key1, key2, feature1, feature2, nested_timestamp, created_timestamp + + FROM ( + SELECT key1, key2, feature1, feature2, event_header.event_published_datetime_utc AS nested_timestamp, created_timestamp, + ROW_NUMBER() OVER(PARTITION BY key1, key2 ORDER BY event_header.event_published_datetime_utc DESC, created_timestamp DESC) AS feast_row_ + FROM `offline_store_database_name`.`offline_store_table_name` t1 + WHERE event_header.event_published_datetime_utc BETWEEN TIMESTAMP('2021-01-01 00:00:00.000000') AND TIMESTAMP('2021-01-02 00:00:00.000000') AND effective_date >= '2021-01-01' AND effective_date <= '2021-01-02' + ) t2 + WHERE feast_row_ = 1""" # noqa: W293, W291 + + assert isinstance(retrieval_job, RetrievalJob) + assert retrieval_job.query.strip() == expected_query.strip() + + @patch( "feast.infra.offline_stores.contrib.spark_offline_store.spark.get_spark_session_or_start_new_with_repoconfig" ) @@ -127,3 +189,62 @@ def test_pull_latest_from_table_without_nested_timestamp_or_query( assert isinstance(retrieval_job, RetrievalJob) assert retrieval_job.query.strip() == expected_query.strip() + + +@patch( + "feast.infra.offline_stores.contrib.spark_offline_store.spark.get_spark_session_or_start_new_with_repoconfig" +) +def test_pull_latest_from_table_without_nested_timestamp_or_query_and_date_partition_column_set( + mock_get_spark_session, +): + mock_spark_session = MagicMock() + mock_get_spark_session.return_value = mock_spark_session + + test_repo_config = RepoConfig( + project="test_project", + registry="test_registry", + provider="local", + offline_store=SparkOfflineStoreConfig(type="spark"), + ) + + test_data_source = SparkSource( + name="test_batch_source", + description="test_nested_batch_source", + table="offline_store_database_name.offline_store_table_name", + timestamp_field="event_published_datetime_utc", + date_partition_column="effective_date", + ) + + # Define the parameters for the method + join_key_columns = ["key1", "key2"] + feature_name_columns = ["feature1", "feature2"] + timestamp_field = "event_published_datetime_utc" + created_timestamp_column = "created_timestamp" + start_date = datetime(2021, 1, 1) + end_date = datetime(2021, 1, 2) + + # Call the method + retrieval_job = SparkOfflineStore.pull_latest_from_table_or_query( + config=test_repo_config, + data_source=test_data_source, + join_key_columns=join_key_columns, + feature_name_columns=feature_name_columns, + timestamp_field=timestamp_field, + created_timestamp_column=created_timestamp_column, + start_date=start_date, + end_date=end_date, + ) + + expected_query = """SELECT + key1, key2, feature1, feature2, event_published_datetime_utc, created_timestamp + + FROM ( + SELECT key1, key2, feature1, feature2, event_published_datetime_utc, created_timestamp, + ROW_NUMBER() OVER(PARTITION BY key1, key2 ORDER BY event_published_datetime_utc DESC, created_timestamp DESC) AS feast_row_ + FROM `offline_store_database_name`.`offline_store_table_name` t1 + WHERE event_published_datetime_utc BETWEEN TIMESTAMP('2021-01-01 00:00:00.000000') AND TIMESTAMP('2021-01-02 00:00:00.000000') AND effective_date >= '2021-01-01' AND effective_date <= '2021-01-02' + ) t2 + WHERE feast_row_ = 1""" # noqa: W293, W291 + + assert isinstance(retrieval_job, RetrievalJob) + assert retrieval_job.query.strip() == expected_query.strip() diff --git a/sdk/python/tests/unit/online_store/test_online_retrieval.py b/sdk/python/tests/unit/online_store/test_online_retrieval.py index 83184643f35..5f0796f4eed 100644 --- a/sdk/python/tests/unit/online_store/test_online_retrieval.py +++ b/sdk/python/tests/unit/online_store/test_online_retrieval.py @@ -1,5 +1,6 @@ import os import platform +import random import sqlite3 import sys import time @@ -561,3 +562,182 @@ def test_sqlite_vec_import() -> None: """).fetchall() result = [(rowid, round(distance, 2)) for rowid, distance in result] assert result == [(2, 2.39), (1, 2.39)] + + +def test_local_milvus() -> None: + import random + + from pymilvus import MilvusClient + + random.seed(42) + VECTOR_LENGTH: int = 768 + COLLECTION_NAME: str = "test_demo_collection" + + client = MilvusClient("./milvus_demo.db") + + for collection in client.list_collections(): + client.drop_collection(collection_name=collection) + client.create_collection( + collection_name=COLLECTION_NAME, + dimension=VECTOR_LENGTH, + ) + assert client.list_collections() == [COLLECTION_NAME] + + docs = [ + "Artificial intelligence was founded as an academic discipline in 1956.", + "Alan Turing was the first person to conduct substantial research in AI.", + "Born in Maida Vale, London, Turing was raised in southern England.", + ] + # Use fake representation with random vectors (vector_length dimension). + vectors = [[random.uniform(-1, 1) for _ in range(VECTOR_LENGTH)] for _ in docs] + data = [ + {"id": i, "vector": vectors[i], "text": docs[i], "subject": "history"} + for i in range(len(vectors)) + ] + + print("Data has", len(data), "entities, each with fields: ", data[0].keys()) + print("Vector dim:", len(data[0]["vector"])) + + insert_res = client.insert(collection_name=COLLECTION_NAME, data=data) + assert insert_res == {"insert_count": 3, "ids": [0, 1, 2], "cost": 0} + + query_vectors = [[random.uniform(-1, 1) for _ in range(VECTOR_LENGTH)]] + + search_res = client.search( + collection_name=COLLECTION_NAME, # target collection + data=query_vectors, # query vectors + limit=2, # number of returned entities + output_fields=["text", "subject"], # specifies fields to be returned + ) + assert [j["id"] for j in search_res[0]] == [0, 1] + query_result = client.query( + collection_name=COLLECTION_NAME, + filter="id == 0", + ) + assert list(query_result[0].keys()) == ["id", "text", "subject", "vector"] + + client.drop_collection(collection_name=COLLECTION_NAME) + + +def test_milvus_lite_get_online_documents() -> None: + """ + Test retrieving documents from the online store in local mode. + """ + + random.seed(42) + n = 10 # number of samples - note: we'll actually double it + vector_length = 10 + runner = CliRunner() + with runner.local_repo( + example_repo_py=get_example_repo("example_rag_feature_repo.py"), + offline_store="file", + online_store="milvus", + apply=False, + teardown=False, + ) as store: + from datetime import timedelta + + from feast import Entity, FeatureView, Field, FileSource + from feast.types import Array, Float32, Int64, UnixTimestamp + + # This is for Milvus + # Note that file source paths are not validated, so there doesn't actually need to be any data + # at the paths for these file sources. Since these paths are effectively fake, this example + # feature repo should not be used for historical retrieval. + + rag_documents_source = FileSource( + path="data/embedded_documents.parquet", + timestamp_field="event_timestamp", + created_timestamp_column="created_timestamp", + ) + + item = Entity( + name="item_id", # The name is derived from this argument, not object name. + join_keys=["item_id"], + ) + + document_embeddings = FeatureView( + name="embedded_documents", + entities=[item], + schema=[ + Field( + name="vector", + dtype=Array(Float32), + vector_index=True, + vector_search_metric="L2", + ), + Field(name="item_id", dtype=Int64), + Field(name="created_timestamp", dtype=UnixTimestamp), + Field(name="event_timestamp", dtype=UnixTimestamp), + ], + source=rag_documents_source, + ttl=timedelta(hours=24), + ) + + store.apply([rag_documents_source, item, document_embeddings]) + + # Write some data to two tables + document_embeddings_fv = store.get_feature_view(name="embedded_documents") + + provider = store._get_provider() + + item_keys = [ + EntityKeyProto( + join_keys=["item_id"], entity_values=[ValueProto(int64_val=i)] + ) + for i in range(n) + ] + data = [] + for item_key in item_keys: + data.append( + ( + item_key, + { + "vector": ValueProto( + float_list_val=FloatListProto( + val=np.random.random( + vector_length, + ) + ) + ) + }, + _utc_now(), + _utc_now(), + ) + ) + + provider.online_write_batch( + config=store.config, + table=document_embeddings_fv, + data=data, + progress=None, + ) + documents_df = pd.DataFrame( + { + "item_id": [str(i) for i in range(n)], + "vector": [ + np.random.random( + vector_length, + ) + for i in range(n) + ], + "event_timestamp": [_utc_now() for _ in range(n)], + "created_timestamp": [_utc_now() for _ in range(n)], + } + ) + + store.write_to_online_store( + feature_view_name="embedded_documents", + df=documents_df, + ) + + query_embedding = np.random.random( + vector_length, + ) + result = store.retrieve_online_documents( + feature="embedded_documents:vector", query=query_embedding, top_k=3 + ).to_dict() + + assert "vector" in result + assert "distance" in result + assert len(result["distance"]) == 3 diff --git a/sdk/python/tests/unit/permissions/auth/server/test_auth_registry_server.py b/sdk/python/tests/unit/permissions/auth/server/test_auth_registry_server.py index 25c5fe3eb8c..0395f995410 100644 --- a/sdk/python/tests/unit/permissions/auth/server/test_auth_registry_server.py +++ b/sdk/python/tests/unit/permissions/auth/server/test_auth_registry_server.py @@ -44,7 +44,7 @@ def start_registry_server( assertpy.assert_that(server_port).is_not_equal_to(0) - is_tls_mode, tls_key_path, tls_cert_path = tls_mode + is_tls_mode, tls_key_path, tls_cert_path, tls_ca_file_path = tls_mode if is_tls_mode: print(f"Starting Registry in TLS mode at {server_port}") server = start_server( @@ -74,6 +74,9 @@ def start_registry_server( server.stop(grace=None) # Teardown server +@pytest.mark.parametrize( + "tls_mode", [("True", "True"), ("True", "False"), ("False", "")], indirect=True +) def test_registry_apis( auth_config, tls_mode, diff --git a/sdk/python/tests/unit/test_entity.py b/sdk/python/tests/unit/test_entity.py index 78f71231049..b36f363a6ff 100644 --- a/sdk/python/tests/unit/test_entity.py +++ b/sdk/python/tests/unit/test_entity.py @@ -11,6 +11,8 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +import warnings + import assertpy import pytest @@ -73,3 +75,16 @@ def test_hash(): s4 = {entity1, entity2, entity3, entity4} assert len(s4) == 3 + + +def test_entity_without_value_type_warns(): + with pytest.warns(DeprecationWarning, match="Entity value_type will be mandatory"): + entity = Entity(name="my-entity") + assert entity.value_type == ValueType.UNKNOWN + + +def test_entity_with_value_type_no_warning(): + with warnings.catch_warnings(): + warnings.simplefilter("error") + entity = Entity(name="my-entity", value_type=ValueType.STRING) + assert entity.value_type == ValueType.STRING diff --git a/sdk/python/tests/utils/auth_permissions_util.py b/sdk/python/tests/utils/auth_permissions_util.py index 6f0a3c8eeac..8a1e7b7c4d7 100644 --- a/sdk/python/tests/utils/auth_permissions_util.py +++ b/sdk/python/tests/utils/auth_permissions_util.py @@ -60,6 +60,7 @@ def start_feature_server( metrics: bool = False, tls_key_path: str = "", tls_cert_path: str = "", + ca_trust_store_path: str = "", ): host = "0.0.0.0" cmd = [ @@ -127,18 +128,30 @@ def start_feature_server( def get_remote_registry_store(server_port, feature_store, tls_mode): - is_tls_mode, _, tls_cert_path = tls_mode + is_tls_mode, _, tls_cert_path, ca_trust_store_path = tls_mode if is_tls_mode: - registry_config = RemoteRegistryConfig( - registry_type="remote", - path=f"localhost:{server_port}", - cert=tls_cert_path, - ) + if ca_trust_store_path: + registry_config = RemoteRegistryConfig( + registry_type="remote", + path=f"localhost:{server_port}", + is_tls=True, + ) + else: + registry_config = RemoteRegistryConfig( + registry_type="remote", + path=f"localhost:{server_port}", + is_tls=True, + cert=tls_cert_path, + ) else: registry_config = RemoteRegistryConfig( registry_type="remote", path=f"localhost:{server_port}" ) + if is_tls_mode and ca_trust_store_path: + # configure trust store path only when is_tls_mode and ca_trust_store_path exists. + os.environ["FEAST_CA_CERT_FILE_PATH"] = ca_trust_store_path + store = FeatureStore( config=RepoConfig( project=PROJECT_NAME, diff --git a/sdk/python/tests/utils/cli_repo_creator.py b/sdk/python/tests/utils/cli_repo_creator.py index e00104081a2..8bb696f7d4e 100644 --- a/sdk/python/tests/utils/cli_repo_creator.py +++ b/sdk/python/tests/utils/cli_repo_creator.py @@ -51,7 +51,14 @@ def run_with_output(self, args: List[str], cwd: Path) -> Tuple[int, bytes]: return e.returncode, e.output @contextmanager - def local_repo(self, example_repo_py: str, offline_store: str): + def local_repo( + self, + example_repo_py: str, + offline_store: str, + online_store: str = "sqlite", + apply=True, + teardown=True, + ): """ Convenience method to set up all the boilerplate for a local feature repo. """ @@ -67,41 +74,61 @@ def local_repo(self, example_repo_py: str, offline_store: str): data_path = Path(data_dir_name) repo_config = repo_path / "feature_store.yaml" - - repo_config.write_text( - dedent( + if online_store == "sqlite": + yaml_config = dedent( f""" - project: {project_id} - registry: {data_path / "registry.db"} - provider: local - online_store: - path: {data_path / "online_store.db"} - offline_store: - type: {offline_store} - entity_key_serialization_version: 2 - """ + project: {project_id} + registry: {data_path / "registry.db"} + provider: local + online_store: + path: {data_path / "online_store.db"} + offline_store: + type: {offline_store} + entity_key_serialization_version: 2 + """ ) - ) + elif online_store == "milvus": + yaml_config = dedent( + f""" + project: {project_id} + registry: {data_path / "registry.db"} + provider: local + online_store: + path: {data_path / "online_store.db"} + type: milvus + vector_enabled: true + embedding_dim: 10 + offline_store: + type: {offline_store} + entity_key_serialization_version: 3 + """ + ) + else: + pass + + repo_config.write_text(yaml_config) repo_example = repo_path / "example.py" repo_example.write_text(example_repo_py) - result = self.run(["apply"], cwd=repo_path) - stdout = result.stdout.decode("utf-8") - stderr = result.stderr.decode("utf-8") - print(f"Apply stdout:\n{stdout}") - print(f"Apply stderr:\n{stderr}") - assert ( - result.returncode == 0 - ), f"stdout: {result.stdout}\nstderr: {result.stderr}" + if apply: + result = self.run(["apply"], cwd=repo_path) + stdout = result.stdout.decode("utf-8") + stderr = result.stderr.decode("utf-8") + print(f"Apply stdout:\n{stdout}") + print(f"Apply stderr:\n{stderr}") + assert ( + result.returncode == 0 + ), f"stdout: {result.stdout}\nstderr: {result.stderr}" yield FeatureStore(repo_path=str(repo_path), config=None) - result = self.run(["teardown"], cwd=repo_path) - stdout = result.stdout.decode("utf-8") - stderr = result.stderr.decode("utf-8") - print(f"Apply stdout:\n{stdout}") - print(f"Apply stderr:\n{stderr}") - assert ( - result.returncode == 0 - ), f"stdout: {result.stdout}\nstderr: {result.stderr}" + if teardown: + result = self.run(["teardown"], cwd=repo_path) + stdout = result.stdout.decode("utf-8") + stderr = result.stderr.decode("utf-8") + print(f"Apply stdout:\n{stdout}") + print(f"Apply stderr:\n{stderr}") + assert ( + result.returncode == 0 + ), f"stdout: {result.stdout}\nstderr: {result.stderr}" diff --git a/sdk/python/tests/utils/generate_self_signed_certifcate_util.py b/sdk/python/tests/utils/generate_self_signed_certifcate_util.py deleted file mode 100644 index 559ee18cde7..00000000000 --- a/sdk/python/tests/utils/generate_self_signed_certifcate_util.py +++ /dev/null @@ -1,79 +0,0 @@ -import ipaddress -import logging -from datetime import datetime, timedelta - -from cryptography import x509 -from cryptography.hazmat.backends import default_backend -from cryptography.hazmat.primitives import hashes, serialization -from cryptography.hazmat.primitives.asymmetric import rsa -from cryptography.x509.oid import NameOID - -logger = logging.getLogger(__name__) - - -def generate_self_signed_cert( - cert_path="cert.pem", key_path="key.pem", common_name="localhost" -): - """ - Generate a self-signed certificate and save it to the specified paths. - - :param cert_path: Path to save the certificate (PEM format) - :param key_path: Path to save the private key (PEM format) - :param common_name: Common name (CN) for the certificate, defaults to 'localhost' - """ - # Generate private key - key = rsa.generate_private_key( - public_exponent=65537, key_size=2048, backend=default_backend() - ) - - # Create a self-signed certificate - subject = issuer = x509.Name( - [ - x509.NameAttribute(NameOID.COUNTRY_NAME, "US"), - x509.NameAttribute(NameOID.STATE_OR_PROVINCE_NAME, "California"), - x509.NameAttribute(NameOID.LOCALITY_NAME, "San Francisco"), - x509.NameAttribute(NameOID.ORGANIZATION_NAME, "Feast"), - x509.NameAttribute(NameOID.COMMON_NAME, common_name), - ] - ) - - # Define the certificate's Subject Alternative Names (SANs) - alt_names = [ - x509.DNSName("localhost"), # Hostname - x509.IPAddress(ipaddress.IPv4Address("127.0.0.1")), # Localhost IP - x509.IPAddress(ipaddress.IPv4Address("0.0.0.0")), # Bind-all IP (optional) - ] - san = x509.SubjectAlternativeName(alt_names) - - certificate = ( - x509.CertificateBuilder() - .subject_name(subject) - .issuer_name(issuer) - .public_key(key.public_key()) - .serial_number(x509.random_serial_number()) - .not_valid_before(datetime.utcnow()) - .not_valid_after( - # Certificate valid for 1 year - datetime.utcnow() + timedelta(days=365) - ) - .add_extension(san, critical=False) - .sign(key, hashes.SHA256(), default_backend()) - ) - - # Write the private key to a file - with open(key_path, "wb") as f: - f.write( - key.private_bytes( - encoding=serialization.Encoding.PEM, - format=serialization.PrivateFormat.TraditionalOpenSSL, - encryption_algorithm=serialization.NoEncryption(), - ) - ) - - # Write the certificate to a file - with open(cert_path, "wb") as f: - f.write(certificate.public_bytes(serialization.Encoding.PEM)) - - logger.info( - f"Self-signed certificate and private key have been generated at {cert_path} and {key_path}." - ) diff --git a/sdk/python/tests/utils/ssl_certifcates_util.py b/sdk/python/tests/utils/ssl_certifcates_util.py new file mode 100644 index 00000000000..53a56e04f3d --- /dev/null +++ b/sdk/python/tests/utils/ssl_certifcates_util.py @@ -0,0 +1,174 @@ +import ipaddress +import logging +import os +import shutil +from datetime import datetime, timedelta + +import certifi +from cryptography import x509 +from cryptography.hazmat.backends import default_backend +from cryptography.hazmat.primitives import hashes, serialization +from cryptography.hazmat.primitives.asymmetric import rsa +from cryptography.x509 import load_pem_x509_certificate +from cryptography.x509.oid import NameOID + +logger = logging.getLogger(__name__) + + +def generate_self_signed_cert( + cert_path="cert.pem", key_path="key.pem", common_name="localhost" +): + """ + Generate a self-signed certificate and save it to the specified paths. + + :param cert_path: Path to save the certificate (PEM format) + :param key_path: Path to save the private key (PEM format) + :param common_name: Common name (CN) for the certificate, defaults to 'localhost' + """ + # Generate private key + key = rsa.generate_private_key( + public_exponent=65537, key_size=2048, backend=default_backend() + ) + + # Create a self-signed certificate + subject = issuer = x509.Name( + [ + x509.NameAttribute(NameOID.COUNTRY_NAME, "US"), + x509.NameAttribute(NameOID.STATE_OR_PROVINCE_NAME, "California"), + x509.NameAttribute(NameOID.LOCALITY_NAME, "San Francisco"), + x509.NameAttribute(NameOID.ORGANIZATION_NAME, "Feast"), + x509.NameAttribute(NameOID.COMMON_NAME, common_name), + ] + ) + + # Define the certificate's Subject Alternative Names (SANs) + alt_names = [ + x509.DNSName("localhost"), # Hostname + x509.IPAddress(ipaddress.IPv4Address("127.0.0.1")), # Localhost IP + x509.IPAddress(ipaddress.IPv4Address("0.0.0.0")), # Bind-all IP (optional) + ] + san = x509.SubjectAlternativeName(alt_names) + + certificate = ( + x509.CertificateBuilder() + .subject_name(subject) + .issuer_name(issuer) + .public_key(key.public_key()) + .serial_number(x509.random_serial_number()) + .not_valid_before(datetime.utcnow()) + .not_valid_after( + # Certificate valid for 1 year + datetime.utcnow() + timedelta(days=365) + ) + .add_extension(san, critical=False) + .sign(key, hashes.SHA256(), default_backend()) + ) + + # Write the private key to a file + with open(key_path, "wb") as f: + f.write( + key.private_bytes( + encoding=serialization.Encoding.PEM, + format=serialization.PrivateFormat.TraditionalOpenSSL, + encryption_algorithm=serialization.NoEncryption(), + ) + ) + + # Write the certificate to a file + with open(cert_path, "wb") as f: + f.write(certificate.public_bytes(serialization.Encoding.PEM)) + + logger.info( + f"Self-signed certificate and private key have been generated at {cert_path} and {key_path}." + ) + + +def create_ca_trust_store( + public_key_path: str, private_key_path: str, output_trust_store_path: str +): + """ + Create a new CA trust store as a copy of the existing one (if available), + and add the provided public certificate to it. + + :param public_key_path: Path to the public certificate (e.g., PEM file). + :param private_key_path: Path to the private key (optional, to verify signing authority). + :param output_trust_store_path: Path to save the new trust store. + """ + try: + # Step 1: Identify the existing trust store (if available via environment variables) + existing_trust_store = os.environ.get("SSL_CERT_FILE") or os.environ.get( + "REQUESTS_CA_BUNDLE" + ) + + # Step 2: Copy the existing trust store to the new location (if it exists) + if existing_trust_store and os.path.exists(existing_trust_store): + shutil.copy(existing_trust_store, output_trust_store_path) + logger.info( + f"Copied existing trust store from {existing_trust_store} to {output_trust_store_path}" + ) + else: + # Log the creation of a new trust store (without opening a file unnecessarily) + logger.info( + f"No existing trust store found. Creating a new trust store at {output_trust_store_path}" + ) + + # Step 3: Load and validate the public certificate + with open(public_key_path, "rb") as pub_file: + public_cert_data = pub_file.read() + public_cert = load_pem_x509_certificate( + public_cert_data, backend=default_backend() + ) + + # Verify the private key matches (optional, adds validation) + if private_key_path: + with open(private_key_path, "rb") as priv_file: + private_key_data = priv_file.read() + private_key = serialization.load_pem_private_key( + private_key_data, password=None, backend=default_backend() + ) + # Check the public/private key match + if ( + private_key.public_key().public_numbers() + != public_cert.public_key().public_numbers() + ): + raise ValueError( + "Public certificate does not match the private key." + ) + + # Step 4: Add the public certificate to the new trust store + with open(output_trust_store_path, "ab") as trust_store_file: + trust_store_file.write(public_cert.public_bytes(serialization.Encoding.PEM)) + + logger.info( + f"Trust store created/updated successfully at: {output_trust_store_path}" + ) + + except Exception as e: + logger.error(f"Error creating CA trust store: {e}") + + +def combine_trust_stores(custom_cert_path: str, output_combined_path: str): + """ + Combine the default certifi CA bundle with a custom certificate file. + + :param custom_cert_path: Path to the custom certificate PEM file. + :param output_combined_path: Path where the combined CA bundle will be saved. + """ + try: + # Get the default certifi CA bundle + certifi_ca_bundle = certifi.where() + + with open(output_combined_path, "wb") as combined_file: + # Write the default CA bundle + with open(certifi_ca_bundle, "rb") as default_file: + combined_file.write(default_file.read()) + + # Append the custom certificates + with open(custom_cert_path, "rb") as custom_file: + combined_file.write(custom_file.read()) + + logger.info(f"Combined trust store created at: {output_combined_path}") + + except Exception as e: + logger.error(f"Error combining trust stores: {e}") + raise e diff --git a/setup.py b/setup.py index 815d1b23229..034f6699105 100644 --- a/setup.py +++ b/setup.py @@ -156,18 +156,20 @@ GO_REQUIRED = ["cffi~=1.15.0"] +MILVUS_REQUIRED = ["pymilvus"] + CI_REQUIRED = ( [ "build", "virtualenv==20.23.0", - "cryptography>=35.0,<43", + "cryptography>=43.0,<44", "ruff>=0.8.0", "mypy-protobuf>=3.1", "grpcio-tools>=1.56.2,<2", "grpcio-testing>=1.56.2,<2", # FastAPI does not correctly pull starlette dependency on httpx see thread(https://github.com/tiangolo/fastapi/issues/5656). - "httpx>=0.23.3", - "minio==7.1.0", + "httpx==0.27.2", + "minio==7.2.11", "mock==2.0.0", "moto<5", "mypy>=1.4.1,<1.11.3", @@ -226,6 +228,7 @@ + OPENTELEMETRY + FAISS_REQUIRED + QDRANT_REQUIRED + + MILVUS_REQUIRED ) DOCS_REQUIRED = CI_REQUIRED @@ -254,61 +257,6 @@ PYTHON_CODE_PREFIX = "sdk/python" -def _generate_path_with_gopath(): - go_path = subprocess.check_output(["go", "env", "GOPATH"]).decode("utf-8") - go_path = go_path.strip() - path_val = os.getenv("PATH") - path_val = f"{path_val}:{go_path}/bin" - - return path_val - -class BuildGoProtosCommand(Command): - description = "Builds the proto files into Go files." - user_options = [] - - def initialize_options(self): - self.go_protoc = [ - sys.executable, - "-m", - "grpc_tools.protoc", - ] # find_executable("protoc") - self.proto_folder = os.path.join(repo_root, "protos") - self.go_folder = os.path.join(repo_root, "go/protos") - self.sub_folders = ["core", "registry", "serving", "types", "storage"] - self.path_val = _generate_path_with_gopath() - - def finalize_options(self): - pass - - def _generate_go_protos(self, path: str): - proto_files = glob.glob(os.path.join(self.proto_folder, path)) - - try: - subprocess.check_call( - self.go_protoc - + [ - "-I", - self.proto_folder, - "--go_out", - self.go_folder, - "--go_opt=module=github.com/feast-dev/feast/go/protos", - "--go-grpc_out", - self.go_folder, - "--go-grpc_opt=module=github.com/feast-dev/feast/go/protos", - ] - + proto_files, - env={"PATH": self.path_val}, - ) - except CalledProcessError as e: - print(f"Stderr: {e.stderr}") - print(f"Stdout: {e.stdout}") - - def run(self): - go_dir = Path(repo_root) / "go" / "protos" - go_dir.mkdir(exist_ok=True) - for sub_folder in self.sub_folders: - self._generate_go_protos(f"feast/{sub_folder}/*.proto") - setup( name=NAME, @@ -355,6 +303,7 @@ def run(self): "faiss": FAISS_REQUIRED, "qdrant": QDRANT_REQUIRED, "go": GO_REQUIRED, + "milvus": MILVUS_REQUIRED, }, include_package_data=True, license="Apache", @@ -372,7 +321,4 @@ def run(self): "pybindgen==0.22.0", # TODO do we need this? "setuptools_scm>=6.2", # TODO do we need this? ], - cmdclass={ - "build_go_protos": BuildGoProtosCommand - }, ) diff --git a/ui/package.json b/ui/package.json index ea7e953edcf..e057b0e5e42 100644 --- a/ui/package.json +++ b/ui/package.json @@ -1,6 +1,6 @@ { "name": "@feast-dev/feast-ui", - "version": "0.42.0", + "version": "0.43.0", "private": false, "files": [ "dist" @@ -35,9 +35,9 @@ "react-app-polyfill": "^3.0.0", "react-code-blocks": "^0.1.6", "react-query": "^3.39.3", - "react-router-dom": "<6.4.0", + "react-router-dom": "^6.28.0", "tslib": "^2.3.1", - "use-query-params": "^1.2.3", + "use-query-params": "^2.2.1", "zod": "^3.11.6" }, "scripts": { diff --git a/ui/src/FeastUI.tsx b/ui/src/FeastUI.tsx index 628b916f2d8..210ff7a4aa7 100644 --- a/ui/src/FeastUI.tsx +++ b/ui/src/FeastUI.tsx @@ -3,7 +3,7 @@ import React from "react"; import { BrowserRouter } from "react-router-dom"; import { QueryClient, QueryClientProvider } from "react-query"; import { QueryParamProvider } from "use-query-params"; -import RouteAdapter from "./hacks/RouteAdapter"; +import { ReactRouter6Adapter } from 'use-query-params/adapters/react-router-6'; import FeastUISansProviders, { FeastUIConfigs } from "./FeastUISansProviders"; interface FeastUIProps { @@ -17,11 +17,10 @@ const FeastUI = ({ reactQueryClient, feastUIConfigs }: FeastUIProps) => { const queryClient = reactQueryClient || defaultQueryClient; return ( - + // Disable v7_relativeSplatPath: custom tab routes don't currently work with it + - + diff --git a/ui/src/components/EuiCustomLink.jsx b/ui/src/components/EuiCustomLink.jsx index cf646d43f73..6872dae4275 100644 --- a/ui/src/components/EuiCustomLink.jsx +++ b/ui/src/components/EuiCustomLink.jsx @@ -39,7 +39,7 @@ export default function EuiCustomLink({ to, ...rest }) { } // Generate the correct link href (with basename accounted for) - const href = useHref({ pathname: to }); + const href = useHref(to); const props = { ...rest, href, onClick }; return ; diff --git a/ui/src/components/ProjectSelector.test.tsx b/ui/src/components/ProjectSelector.test.tsx index fc5b3c68400..dfaaab7f626 100644 --- a/ui/src/components/ProjectSelector.test.tsx +++ b/ui/src/components/ProjectSelector.test.tsx @@ -40,7 +40,7 @@ test("in a full App render, it shows the right initial project", async () => { name: "Top Level", }); - within(topLevelNavigation).getByDisplayValue("Credit Score Project"); + await within(topLevelNavigation).findByDisplayValue("Credit Score Project"); expect(options.length).toBe(1); diff --git a/ui/src/hacks/RouteAdapter.ts b/ui/src/hacks/RouteAdapter.ts deleted file mode 100644 index e7743c9d90b..00000000000 --- a/ui/src/hacks/RouteAdapter.ts +++ /dev/null @@ -1,39 +0,0 @@ -import React from "react"; -import { Location } from "history"; -import { - useLocation, - useNavigate, - Location as RouterLocation, -} from "react-router-dom"; - -// via: https://github.com/pbeshai/use-query-params/issues/196#issuecomment-996893750 -interface RouteAdapterProps { - children: React.FunctionComponent<{ - history: { - replace(location: Location): void; - push(location: Location): void; - }; - location: RouterLocation; - }>; -} - -// Via: https://github.com/pbeshai/use-query-params/blob/cd44e7fb3394620f757bfb09ff57b7f296d9a5e6/examples/react-router-6/src/index.js#L36 -const RouteAdapter = ({ children }: RouteAdapterProps) => { - const navigate = useNavigate(); - const location = useLocation(); - - const adaptedHistory = React.useMemo( - () => ({ - replace(location: Location) { - navigate(location, { replace: true, state: location.state }); - }, - push(location: Location) { - navigate(location, { replace: false, state: location.state }); - }, - }), - [navigate] - ); - return children && children({ history: adaptedHistory, location }); -}; - -export default RouteAdapter; diff --git a/ui/src/test-utils.tsx b/ui/src/test-utils.tsx index c180b01872c..0130686252d 100644 --- a/ui/src/test-utils.tsx +++ b/ui/src/test-utils.tsx @@ -2,8 +2,8 @@ import React from "react"; import { render, RenderOptions } from "@testing-library/react"; import { QueryClient, QueryClientProvider } from "react-query"; import { QueryParamProvider } from "use-query-params"; +import { ReactRouter6Adapter } from 'use-query-params/adapters/react-router-6'; import { MemoryRouter as Router } from "react-router-dom"; -import RouteAdapter from "./hacks/RouteAdapter"; interface ProvidersProps { children: React.ReactNode; @@ -14,10 +14,12 @@ const queryClient = new QueryClient(); const AllTheProviders = ({ children }: ProvidersProps) => { return ( - - + + {children} diff --git a/ui/yarn.lock b/ui/yarn.lock index 0057595f391..773260ad861 100644 --- a/ui/yarn.lock +++ b/ui/yarn.lock @@ -1869,7 +1869,7 @@ "@babel/plugin-transform-modules-commonjs" "^7.25.9" "@babel/plugin-transform-typescript" "^7.25.9" -"@babel/runtime@^7.0.0", "@babel/runtime@^7.12.13", "@babel/runtime@^7.7.6", "@babel/runtime@^7.9.2": +"@babel/runtime@^7.0.0", "@babel/runtime@^7.12.13", "@babel/runtime@^7.9.2": version "7.16.7" resolved "https://registry.yarnpkg.com/@babel/runtime/-/runtime-7.16.7.tgz#03ff99f64106588c9c403c6ecb8c3bafbbdff1fa" integrity sha512-9E9FJowqAsytyOY6LG+1KuueckRL+aQW+mKvXRXnuFGyRAyepJPmEo9vgMfXUA6O9u3IeEdv9MAkppFcaQwogQ== @@ -2709,6 +2709,11 @@ resolved "https://registry.yarnpkg.com/@protobufjs/utf8/-/utf8-1.1.0.tgz#a777360b5b39a1a2e5106f8e858f2fd2d060c570" integrity sha512-Vvn3zZrhQZkkBE8LSuW3em98c0FwgO4nxzv6OdSxPKJIEKY2bGbHn+mhGIPerzI4twdxaP8/0+06HBpwf345Lw== +"@remix-run/router@1.21.0": + version "1.21.0" + resolved "https://registry.yarnpkg.com/@remix-run/router/-/router-1.21.0.tgz#c65ae4262bdcfe415dbd4f64ec87676e4a56e2b5" + integrity sha512-xfSkCAchbdG5PnbrKqFWwia4Bi61nH+wm8wLEqfHDyp7Y3dZzgqS2itV8i4gAq9pC2HsTpwyBC6Ds8VHZ96JlA== + "@rollup/plugin-babel@^5.2.0", "@rollup/plugin-babel@^5.3.1": version "5.3.1" resolved "https://registry.yarnpkg.com/@rollup/plugin-babel/-/plugin-babel-5.3.1.tgz#04bc0608f4aa4b2e4b1aebf284344d0f68fda283" @@ -6715,13 +6720,6 @@ highlightjs-vue@^1.0.0: resolved "https://registry.yarnpkg.com/highlightjs-vue/-/highlightjs-vue-1.0.0.tgz#fdfe97fbea6354e70ee44e3a955875e114db086d" integrity sha512-PDEfEF102G23vHmPhLyPboFCD+BkMGu+GuJe2d9/eH4FsCwvgBpnc9n0pGE+ffKdph38s6foEZiEjdgHdzp+IA== -history@^5.2.0: - version "5.2.0" - resolved "https://registry.yarnpkg.com/history/-/history-5.2.0.tgz#7cdd31cf9bac3c5d31f09c231c9928fad0007b7c" - integrity sha512-uPSF6lAJb3nSePJ43hN3eKj1dTWpN9gMod0ZssbFTIsen+WehTmEadgL+kg78xLJFdRfrrC//SavDzmRVdE+Ig== - dependencies: - "@babel/runtime" "^7.7.6" - hoist-non-react-statics@^3.3.0, hoist-non-react-statics@^3.3.1, hoist-non-react-statics@^3.3.2: version "3.3.2" resolved "https://registry.yarnpkg.com/hoist-non-react-statics/-/hoist-non-react-statics-3.3.2.tgz#ece0acaf71d62c2969c2ec59feff42a4b1a85b45" @@ -9962,20 +9960,20 @@ react-remove-scroll@^2.6.0: use-callback-ref "^1.3.0" use-sidecar "^1.1.2" -react-router-dom@<6.4.0: - version "6.3.0" - resolved "https://registry.yarnpkg.com/react-router-dom/-/react-router-dom-6.3.0.tgz#a0216da813454e521905b5fa55e0e5176123f43d" - integrity sha512-uaJj7LKytRxZNQV8+RbzJWnJ8K2nPsOOEuX7aQstlMZKQT0164C+X2w6bnkqU3sjtLvpd5ojrezAyfZ1+0sStw== +react-router-dom@^6.28.0: + version "6.28.0" + resolved "https://registry.yarnpkg.com/react-router-dom/-/react-router-dom-6.28.0.tgz#f73ebb3490e59ac9f299377062ad1d10a9f579e6" + integrity sha512-kQ7Unsl5YdyOltsPGl31zOjLrDv+m2VcIEcIHqYYD3Lp0UppLjrzcfJqDJwXxFw3TH/yvapbnUvPlAj7Kx5nbg== dependencies: - history "^5.2.0" - react-router "6.3.0" + "@remix-run/router" "1.21.0" + react-router "6.28.0" -react-router@6.3.0: - version "6.3.0" - resolved "https://registry.yarnpkg.com/react-router/-/react-router-6.3.0.tgz#3970cc64b4cb4eae0c1ea5203a80334fdd175557" - integrity sha512-7Wh1DzVQ+tlFjkeo+ujvjSqSJmkt1+8JO+T5xklPlgrh70y7ogx75ODRW0ThWhY7S+6yEDks8TYrtQe/aoboBQ== +react-router@6.28.0: + version "6.28.0" + resolved "https://registry.yarnpkg.com/react-router/-/react-router-6.28.0.tgz#29247c86d7ba901d7e5a13aa79a96723c3e59d0d" + integrity sha512-HrYdIFqdrnhDw0PqG/AKjAqEqM7AvxCz0DQ4h2W8k6nqmc5uRBYDag0SBxx9iYz5G8gnuNVLzUe13wl9eAsXXg== dependencies: - history "^5.2.0" + "@remix-run/router" "1.21.0" react-style-singleton@^2.2.1: version "2.2.1" @@ -10588,10 +10586,10 @@ serialize-javascript@^6.0.0, serialize-javascript@^6.0.1: dependencies: randombytes "^2.1.0" -serialize-query-params@^1.3.5: - version "1.3.6" - resolved "https://registry.yarnpkg.com/serialize-query-params/-/serialize-query-params-1.3.6.tgz#5dd5225db85ce747fe6fbc4897628504faafec6d" - integrity sha512-VlH7sfWNyPVZClPkRacopn6sn5uQMXBsjPVz1+pBHX895VpcYVznfJtZ49e6jymcrz+l/vowkepCZn/7xEAEdw== +serialize-query-params@^2.0.2: + version "2.0.2" + resolved "https://registry.yarnpkg.com/serialize-query-params/-/serialize-query-params-2.0.2.tgz#598a3fb9e13f4ea1c1992fbd20231aa16b31db81" + integrity sha512-1chMo1dST4pFA9RDXAtF0Rbjaut4is7bzFbI1Z26IuMub68pNCILku85aYmeFhvnY//BXUPUhoRMjYcsT93J/Q== serve-index@^1.9.1: version "1.9.1" @@ -11709,12 +11707,12 @@ use-memo-one@^1.1.3: resolved "https://registry.yarnpkg.com/use-memo-one/-/use-memo-one-1.1.3.tgz#2fd2e43a2169eabc7496960ace8c79efef975e99" integrity sha512-g66/K7ZQGYrI6dy8GLpVcMsBp4s17xNkYJVSMvTEevGy3nDxHOfE6z8BVE22+5G5x7t3+bhzrlTDB7ObrEE0cQ== -use-query-params@^1.2.3: - version "1.2.3" - resolved "https://registry.yarnpkg.com/use-query-params/-/use-query-params-1.2.3.tgz#306c31a0cbc714e8a3b4bd7e91a6a9aaccaa5e22" - integrity sha512-cdG0tgbzK+FzsV6DAt2CN8Saa3WpRnze7uC4Rdh7l15epSFq7egmcB/zuREvPNwO5Yk80nUpDZpiyHsoq50d8w== +use-query-params@^2.2.1: + version "2.2.1" + resolved "https://registry.yarnpkg.com/use-query-params/-/use-query-params-2.2.1.tgz#c558ab70706f319112fbccabf6867b9f904e947d" + integrity sha512-i6alcyLB8w9i3ZK3caNftdb+UnbfBRNPDnc89CNQWkGRmDrm/gfydHvMBfVsQJRq3NoHOM2dt/ceBWG2397v1Q== dependencies: - serialize-query-params "^1.3.5" + serialize-query-params "^2.0.2" use-sidecar@^1.1.2: version "1.1.2"