diff --git a/.claude/rules/feast-components.md b/.claude/rules/feast-components.md index 5c03cb0bd3d..02b1c6f4dd4 100644 --- a/.claude/rules/feast-components.md +++ b/.claude/rules/feast-components.md @@ -24,6 +24,7 @@ For testing patterns and debugging, also read `skills/feast-testing/SKILL.md`. - **Unit tests**: add or update tests in `sdk/python/tests/unit/infra//` - **Integration tests**: run `make test-python-integration-local`; add a universal test case in `sdk/python/tests/integration/` if the change affects retrieval or materialization behavior +- **SQL registry binary columns**: in `infra/registry/sql.py`, a new column that stores a serialized proto or blob metadata must use `ProtoBytes`, not `LargeBinary` directly — `LargeBinary` maps to MySQL `BLOB` (64 KB cap) and silently truncates large protos - **Protos**: if you add a field to a proto message, recompile with `make protos` and update serialization helpers in `proto_registry_utils.py` - **Both SDKs**: if the change affects online serving, check whether the Go server (`go/`) also needs updating - **Skills/Rules**: if the change introduces new patterns, interfaces, or conventions that agents should follow, update the relevant section in `skills/feast-architecture/SKILL.md` (and `skills/feast-testing/SKILL.md` if testing patterns changed) diff --git a/.codecov.yaml b/.codecov.yaml new file mode 100644 index 00000000000..2fa599642ec --- /dev/null +++ b/.codecov.yaml @@ -0,0 +1,48 @@ +codecov: + require_ci_to_pass: true + +coverage: + precision: 2 + round: down + range: "50...70" + + status: + project: + default: + informational: true + target: auto + threshold: 1% + patch: + default: + informational: true + target: 70% + +comment: + layout: "reach,diff,flags,files,footer" + behavior: default + require_changes: false + require_base: false + require_head: true + show_carryforward_flags: true + +flags: + python-unit: + paths: + - sdk/python/feast/ + carryforward: true + go-feature-server: + paths: + - go/ + carryforward: true + +ignore: + - "sdk/python/tests/**" + - "**/*_pb2.py" + - "**/*_pb2_grpc.py" + - "sdk/python/feast/protos/**" + - "sdk/python/feast/embedded_go/**" + - "protos/**" + - "docs/**" + - "ui/**" + - "java/**" + - "infra/feast-operator/test/**" diff --git a/.cursor/rules/feast-components.mdc b/.cursor/rules/feast-components.mdc index a474f00fc47..f015619020d 100644 --- a/.cursor/rules/feast-components.mdc +++ b/.cursor/rules/feast-components.mdc @@ -20,6 +20,7 @@ For testing patterns and debugging, also read `skills/feast-testing/SKILL.md`. - **Unit tests**: add or update tests in `sdk/python/tests/unit/infra//` - **Integration tests**: run `make test-python-integration-local`; add a universal test case in `sdk/python/tests/integration/` if the change affects retrieval or materialization behavior +- **SQL registry binary columns**: in `infra/registry/sql.py`, a new column that stores a serialized proto or blob metadata must use `ProtoBytes`, not `LargeBinary` directly — `LargeBinary` maps to MySQL `BLOB` (64 KB cap) and silently truncates large protos - **Protos**: if you add a field to a proto message, recompile with `make protos` and update serialization helpers in `proto_registry_utils.py` - **Both SDKs**: if the change affects online serving, check whether the Go server (`go/`) also needs updating - **Skills/Rules**: if the change introduces new patterns, interfaces, or conventions that agents should follow, update the relevant section in `skills/feast-architecture/SKILL.md` (and `skills/feast-testing/SKILL.md` if testing patterns changed) diff --git a/.cursor/rules/feast-ui.mdc b/.cursor/rules/feast-ui.mdc new file mode 100644 index 00000000000..9072cbe335f --- /dev/null +++ b/.cursor/rules/feast-ui.mdc @@ -0,0 +1,19 @@ +--- +description: Formatting and lint rules for the Feast UI (React/TypeScript) +globs: ui/src/** +alwaysApply: false +--- + +## After editing any file under `ui/src/` + +1. **Run Prettier** before considering the task complete: + ```bash + cd ui && yarn prettier --write + ``` +2. **Verify** formatting passes: + ```bash + cd ui && yarn format:check + ``` + CI runs `yarn format:check` and will reject PRs with style violations. + +3. Prettier config lives in `ui/package.json` (no separate `.prettierrc`). Do not override it. diff --git a/.gitbook.yaml b/.gitbook.yaml index bbdd0c57e3b..8441cf23dd7 100644 --- a/.gitbook.yaml +++ b/.gitbook.yaml @@ -6,3 +6,6 @@ structure: redirects: reference/telemetry: ./reference/usage.md quickstart: ./getting-started/quickstart.md + reference/feature-store-yaml: ./reference/feature-repository/feature-store-yaml.md + reference/feast-ignore: ./reference/feature-repository/feast-ignore.md + reference/feature-repository: ./reference/feature-repository/README.md diff --git a/.github/actions/get-semantic-release-version/action.yml b/.github/actions/get-semantic-release-version/action.yml index 89f6a8f81c1..a53bc337b44 100644 --- a/.github/actions/get-semantic-release-version/action.yml +++ b/.github/actions/get-semantic-release-version/action.yml @@ -2,7 +2,7 @@ name: Get semantic release version description: "" inputs: custom_version: # Optional input for a custom version - description: "Custom version to publish (e.g., v1.2.3) -- only edit if you know what you are doing" + description: "Custom version to publish (e.g., v1.2.3 or v1.2.3.dev4) -- only edit if you know what you are doing" required: false token: description: "Personal Access Token" @@ -10,10 +10,10 @@ inputs: default: "" outputs: release_version: - description: "The release version to use (e.g., v1.2.3)" + description: "The release version to use (e.g., v1.2.3 or v1.2.3.dev4)" value: ${{ steps.get_release_version.outputs.release_version }} version_without_prefix: - description: "The release version to use without 'v' (e.g., 1.2.3)" + description: "The release version to use without 'v' (e.g., 1.2.3 or 1.2.3.dev4)" value: ${{ steps.get_release_version_without_prefix.outputs.version_without_prefix }} highest_semver_tag: description: "The highest semantic version tag without the 'v' prefix (e.g., 1.2.3)" @@ -32,10 +32,10 @@ runs: GIT_COMMITTER_EMAIL: feast-ci-bot@willem.co run: | if [[ -n "${{ inputs.custom_version }}" ]]; then - VERSION_REGEX="^v[0-9]+\.[0-9]+\.[0-9]+$" + VERSION_REGEX="^v[0-9]+\.[0-9]+\.[0-9]+(\.dev[0-9]+)?$" echo "Using custom version: ${{ inputs.custom_version }}" if [[ ! "${{ inputs.custom_version }}" =~ $VERSION_REGEX ]]; then - echo "Error: custom_version must match semantic versioning (e.g., v1.2.3)." + echo "Error: custom_version must match semantic versioning (e.g., v1.2.3 or v1.2.3.dev4)." exit 1 fi echo "::set-output name=release_version::${{ inputs.custom_version }}" @@ -84,4 +84,4 @@ runs: run: | echo $RELEASE_VERSION echo $VERSION_WITHOUT_PREFIX - echo $HIGHEST_SEMVER_TAG \ No newline at end of file + echo $HIGHEST_SEMVER_TAG diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml index de34d7b8004..c69b5d8e699 100644 --- a/.github/workflows/build_wheels.yml +++ b/.github/workflows/build_wheels.yml @@ -7,9 +7,18 @@ on: workflow_dispatch: # Allows manual trigger of the workflow inputs: custom_version: # Optional input for a custom version - description: 'Custom version to publish (e.g., v1.2.3) -- only edit if you know what you are doing' + description: 'Custom version to publish (e.g., v1.2.3 or v1.2.3.dev4) -- only edit if you know what you are doing' required: false type: string + checkout_ref: + description: 'Git ref to checkout before building wheels. Defaults to the release version.' + required: false + type: string + build_docker_images: + description: 'Build Docker images as part of release verification.' + required: true + default: true + type: boolean token: description: 'Personal Access Token' required: true @@ -18,9 +27,18 @@ on: workflow_call: inputs: custom_version: # Optional input for a custom version - description: 'Custom version to publish (e.g., v1.2.3) -- only edit if you know what you are doing' + description: 'Custom version to publish (e.g., v1.2.3 or v1.2.3.dev4) -- only edit if you know what you are doing' + required: false + type: string + checkout_ref: + description: 'Git ref to checkout before building wheels. Defaults to the release version.' required: false type: string + build_docker_images: + description: 'Build Docker images as part of release verification.' + required: false + default: true + type: boolean token: description: 'Personal Access Token' required: true @@ -48,17 +66,20 @@ jobs: - id: get-version uses: ./.github/actions/get-semantic-release-version with: - custom_version: ${{ github.event.inputs.custom_version }} - token: ${{ github.event.inputs.token }} + custom_version: ${{ inputs.custom_version }} + token: ${{ inputs.token }} - name: Checkout version and install dependencies env: VERSION: ${{ steps.get-version.outputs.release_version }} + CHECKOUT_REF: ${{ inputs.checkout_ref }} PYPI_PASSWORD: ${{ secrets.PYPI_PASSWORD }} run: | git fetch --tags - git checkout ${VERSION} + git checkout "${CHECKOUT_REF:-$VERSION}" python -m pip install build - name: Build feast + env: + SETUPTOOLS_SCM_PRETEND_VERSION: ${{ steps.get-version.outputs.version_without_prefix }} run: python -m build - uses: actions/upload-artifact@v4 with: @@ -68,6 +89,7 @@ jobs: # We add this step so the docker images can be built as part of the pre-release verification steps. build-docker-images: name: Build Docker images + if: ${{ inputs.build_docker_images }} runs-on: ubuntu-latest needs: [ build-python-wheel ] strategy: @@ -96,8 +118,8 @@ jobs: - id: get-version uses: ./.github/actions/get-semantic-release-version with: - custom_version: ${{ github.event.inputs.custom_version }} - token: ${{ github.event.inputs.token }} + custom_version: ${{ inputs.custom_version }} + token: ${{ inputs.token }} - name: Build image env: VERSION_WITHOUT_PREFIX: ${{ steps.get-version.outputs.version_without_prefix }} @@ -162,21 +184,22 @@ jobs: - id: get-version uses: ./.github/actions/get-semantic-release-version with: - custom_version: ${{ github.event.inputs.custom_version }} - token: ${{ github.event.inputs.token }} + custom_version: ${{ inputs.custom_version }} + token: ${{ inputs.token }} - name: Validate Feast Version env: VERSION_WITHOUT_PREFIX: ${{ steps.get-version.outputs.version_without_prefix }} run: | feast version - if ! VERSION_OUTPUT=$(feast version); then - echo "Error: Failed to get Feast version." - exit 1 - fi - VERSION_REGEX='[0-9]+\.[0-9]+\.[0-9]+' - OUTPUT_REGEX='^Feast SDK Version: "$VERSION_REGEX"$' - VERSION=$(echo $VERSION_OUTPUT | grep -oE "$VERSION_REGEX") - OUTPUT=$(echo $VERSION_OUTPUT | grep -E "$REGEX") + if ! VERSION_OUTPUT=$(feast version); then + echo "Error: Failed to get Feast version." + exit 1 + fi + VERSION_OUTPUT=$(printf '%s\n' "$VERSION_OUTPUT" | python -c 'import re, sys; print(re.sub(r"\x1b\[[0-9;]*[A-Za-z]", "", sys.stdin.read()).strip())') + VERSION_REGEX='[0-9]+\.[0-9]+\.[0-9]+(\.dev[0-9]+)?' + OUTPUT_REGEX="^Feast SDK Version: \"${VERSION_REGEX}\"$" + VERSION=$(echo "$VERSION_OUTPUT" | grep -oE "$VERSION_REGEX") + OUTPUT=$(echo "$VERSION_OUTPUT" | grep -E "$OUTPUT_REGEX") echo "Installed Feast Version: $VERSION and using Feast Version: $VERSION_WITHOUT_PREFIX" if [ -n "$OUTPUT" ] && [ "$VERSION" = "$VERSION_WITHOUT_PREFIX" ]; then echo "Correct Feast Version Installed" diff --git a/.github/workflows/linter.yml b/.github/workflows/linter.yml index 19e13d5f8e9..9e8a3fe3d1b 100644 --- a/.github/workflows/linter.yml +++ b/.github/workflows/linter.yml @@ -1,6 +1,16 @@ name: linter -on: [push, pull_request] +on: + pull_request: + paths-ignore: + - 'docs/**' + - 'community/**' + - 'examples/**' + push: + paths-ignore: + - 'docs/**' + - 'community/**' + - 'examples/**' concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} diff --git a/.github/workflows/master_only.yml b/.github/workflows/master_only.yml index c4a22b8c756..4900135add1 100644 --- a/.github/workflows/master_only.yml +++ b/.github/workflows/master_only.yml @@ -71,16 +71,18 @@ jobs: SNOWFLAKE_CI_ROLE: ${{ secrets.SNOWFLAKE_CI_ROLE }} SNOWFLAKE_CI_WAREHOUSE: ${{ secrets.SNOWFLAKE_CI_WAREHOUSE }} run: make test-python-integration - - name: Benchmark python - env: + - name: Benchmark python + if: matrix.python-version == '3.11' + env: SNOWFLAKE_CI_DEPLOYMENT: ${{ secrets.SNOWFLAKE_CI_DEPLOYMENT }} SNOWFLAKE_CI_USER: ${{ secrets.SNOWFLAKE_CI_USER }} SNOWFLAKE_CI_PASSWORD: ${{ secrets.SNOWFLAKE_CI_PASSWORD }} SNOWFLAKE_CI_ROLE: ${{ secrets.SNOWFLAKE_CI_ROLE }} SNOWFLAKE_CI_WAREHOUSE: ${{ secrets.SNOWFLAKE_CI_WAREHOUSE }} run: uv run pytest --verbose --color=yes sdk/python/tests --integration --benchmark --benchmark-autosave --benchmark-save-data --durations=5 - - name: Upload Benchmark Artifact to S3 - run: aws s3 cp --recursive .benchmarks s3://feast-ci-pytest-benchmark + - name: Upload Benchmark Artifact to S3 + if: matrix.python-version == '3.11' + run: aws s3 cp --recursive .benchmarks s3://feast-ci-pytest-benchmark - name: Minimize uv cache run: uv cache prune --ci @@ -92,16 +94,19 @@ jobs: include: - component: feature-server-dev target: feature-server-dev + image_name: feature-server build_args: DOCKER_PUSH=true DOCKER_PLATFORMS=linux/amd64,linux/arm64 push_mode: imagetools - component: feature-transformation-server target: feature-transformation-server + image_name: feature-transformation-server build_args: "" push_mode: all_tags - component: feast-operator target: feast-operator - build_args: "" - push_mode: all_tags + image_name: feast-operator + build_args: DOCKER_PUSH=true DOCKER_PLATFORMS=linux/amd64,linux/arm64 + push_mode: imagetools env: REGISTRY: quay.io/feastdev-ci steps: @@ -133,7 +138,7 @@ jobs: - name: Push image run: | if [[ "${{ matrix.push_mode }}" == "imagetools" ]]; then - docker buildx imagetools create -t ${REGISTRY}/feature-server:develop ${REGISTRY}/feature-server:${GITHUB_SHA} + docker buildx imagetools create -t ${REGISTRY}/${{ matrix.image_name }}:develop ${REGISTRY}/${{ matrix.image_name }}:${GITHUB_SHA} else - docker tag ${REGISTRY}/${{ matrix.target }}:${GITHUB_SHA} ${REGISTRY}/${{ matrix.target }}:develop && docker push ${REGISTRY}/${{ matrix.target }} --all-tags + docker tag ${REGISTRY}/${{ matrix.image_name }}:${GITHUB_SHA} ${REGISTRY}/${{ matrix.image_name }}:develop && docker push ${REGISTRY}/${{ matrix.image_name }} --all-tags fi diff --git a/.github/workflows/nightly_python_sdk_release.yml b/.github/workflows/nightly_python_sdk_release.yml new file mode 100644 index 00000000000..6c758360d56 --- /dev/null +++ b/.github/workflows/nightly_python_sdk_release.yml @@ -0,0 +1,95 @@ +name: nightly python sdk release + +on: + schedule: + - cron: "0 8 * * *" + workflow_dispatch: + inputs: + base_version: + description: "Optional base version without the .dev suffix (e.g., 1.2.3). Defaults to the next semantic-release version." + required: false + type: string + dev_number: + description: "Optional dev release number. Defaults to the workflow run number." + required: false + type: string + +permissions: + contents: write + +concurrency: + group: nightly-python-sdk-release + cancel-in-progress: false + +jobs: + get-nightly-version: + if: github.repository == 'feast-dev/feast' + runs-on: ubuntu-latest + outputs: + nightly_version: ${{ steps.version.outputs.nightly_version }} + env: + GITHUB_TOKEN: ${{ github.token }} + INPUT_BASE_VERSION: ${{ inputs.base_version }} + INPUT_DEV_NUMBER: ${{ inputs.dev_number }} + DEFAULT_DEV_NUMBER: ${{ github.run_number }} + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: "lts/*" + - name: Get nightly version + id: version + run: | + set -euo pipefail + + if [[ -n "$INPUT_BASE_VERSION" ]]; then + BASE_VERSION="${INPUT_BASE_VERSION#v}" + else + set +e + SEMANTIC_OUTPUT=$(npx -p @semantic-release/changelog -p @semantic-release/git -p @semantic-release/exec -p semantic-release semantic-release --dry-run 2>&1) + SEMANTIC_STATUS=$? + set -e + echo "$SEMANTIC_OUTPUT" + + BASE_VERSION=$(printf '%s\n' "$SEMANTIC_OUTPUT" | sed -nE 's/.*The next release version is ([[:digit:].]+)$/\1/p' | tail -n 1) + if [[ -z "$BASE_VERSION" ]]; then + echo "Could not determine a semantic-release next version (exit code: ${SEMANTIC_STATUS}); falling back to next patch after latest stable tag." + LATEST_TAG=$(git tag -l 'v[0-9]*.[0-9]*.[0-9]*' --sort=-v:refname | sed -nE '/^v[0-9]+\.[0-9]+\.[0-9]+$/{p;q;}') + if [[ -z "$LATEST_TAG" ]]; then + echo "Could not determine latest stable tag." + exit 1 + fi + LATEST_VERSION="${LATEST_TAG#v}" + IFS=. read -r MAJOR MINOR PATCH <<< "$LATEST_VERSION" + BASE_VERSION="${MAJOR}.${MINOR}.$((PATCH + 1))" + fi + fi + + if [[ ! "$BASE_VERSION" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + echo "Base version must match X.Y.Z, got: ${BASE_VERSION}" + exit 1 + fi + + DEV_NUMBER="${INPUT_DEV_NUMBER:-$DEFAULT_DEV_NUMBER}" + if [[ ! "$DEV_NUMBER" =~ ^[0-9]+$ ]]; then + echo "Dev number must be numeric, got: ${DEV_NUMBER}" + exit 1 + fi + + NIGHTLY_VERSION="v${BASE_VERSION}.dev${DEV_NUMBER}" + echo "Nightly version is ${NIGHTLY_VERSION}" + echo "nightly_version=${NIGHTLY_VERSION}" >> "$GITHUB_OUTPUT" + + publish-nightly-python-sdk: + needs: get-nightly-version + uses: ./.github/workflows/publish_python_sdk.yml + secrets: inherit # pragma: allowlist secret + with: + custom_version: ${{ needs.get-nightly-version.outputs.nightly_version }} + checkout_ref: ${{ github.sha }} + build_docker_images: false + token: ${{ github.token }} diff --git a/.github/workflows/pr_duckdb_integration_tests.yml b/.github/workflows/pr_duckdb_integration_tests.yml index d099d7fa582..862c5e679ea 100644 --- a/.github/workflows/pr_duckdb_integration_tests.yml +++ b/.github/workflows/pr_duckdb_integration_tests.yml @@ -26,9 +26,9 @@ jobs: token: ${{ secrets.GITHUB_TOKEN }} submodules: recursive - name: Setup pixi - uses: prefix-dev/setup-pixi@v0.8.1 + uses: prefix-dev/setup-pixi@v0.10.0 with: - pixi-version: v0.63.1 + pixi-version: v0.75.0 environments: duckdb-tests cache: true - name: Run DuckDB offline store integration tests diff --git a/.github/workflows/pr_integration_tests.yml b/.github/workflows/pr_integration_tests.yml index 936b2777f1d..60d0b13ce72 100644 --- a/.github/workflows/pr_integration_tests.yml +++ b/.github/workflows/pr_integration_tests.yml @@ -4,7 +4,6 @@ on: pull_request_target: types: - opened - - synchronize - labeled concurrency: @@ -50,6 +49,7 @@ jobs: ref: refs/pull/${{ github.event.pull_request.number }}/merge submodules: recursive persist-credentials: false + allow-unsafe-pr-checkout: true # Security: gated by label check above - name: Setup Python uses: actions/setup-python@v5 id: setup-python @@ -109,6 +109,7 @@ jobs: ref: refs/pull/${{ github.event.pull_request.number }}/merge submodules: recursive persist-credentials: false + allow-unsafe-pr-checkout: true # Security: gated by label check above - name: Setup Python uses: actions/setup-python@v5 with: diff --git a/.github/workflows/pr_ray_integration_tests.yml b/.github/workflows/pr_ray_integration_tests.yml index 4d54c8e34ed..81c82bf0c49 100644 --- a/.github/workflows/pr_ray_integration_tests.yml +++ b/.github/workflows/pr_ray_integration_tests.yml @@ -26,9 +26,9 @@ jobs: token: ${{ secrets.GITHUB_TOKEN }} submodules: recursive - name: Setup pixi - uses: prefix-dev/setup-pixi@v0.8.1 + uses: prefix-dev/setup-pixi@v0.10.0 with: - pixi-version: v0.63.1 + pixi-version: v0.75.0 environments: ray-tests cache: true - name: Run Ray integration tests (offline store + compute engine) diff --git a/.github/workflows/pr_registration_integration_tests.yml b/.github/workflows/pr_registration_integration_tests.yml index 76cbe701cf4..ab60cb22b3a 100644 --- a/.github/workflows/pr_registration_integration_tests.yml +++ b/.github/workflows/pr_registration_integration_tests.yml @@ -4,7 +4,6 @@ on: pull_request_target: types: - opened - - synchronize - labeled concurrency: @@ -28,10 +27,11 @@ jobs: ref: refs/pull/${{ github.event.pull_request.number }}/merge submodules: recursive persist-credentials: false + allow-unsafe-pr-checkout: true # Security: gated by label check above - name: Setup pixi - uses: prefix-dev/setup-pixi@v0.8.1 + uses: prefix-dev/setup-pixi@v0.10.0 with: - pixi-version: v0.63.1 + pixi-version: v0.75.0 environments: registration-tests cache: true - name: Run registration integration tests (local) @@ -59,6 +59,7 @@ jobs: ref: refs/pull/${{ github.event.pull_request.number }}/merge submodules: recursive persist-credentials: false + allow-unsafe-pr-checkout: true # Security: gated by label check above - name: Authenticate to Google Cloud uses: 'google-github-actions/auth@v2' with: @@ -81,9 +82,9 @@ jobs: - name: Install Hadoop dependencies run: make install-hadoop-dependencies-ci - name: Setup pixi - uses: prefix-dev/setup-pixi@v0.8.1 + uses: prefix-dev/setup-pixi@v0.10.0 with: - pixi-version: v0.63.1 + pixi-version: v0.75.0 environments: registration-tests cache: true - name: Run registration integration tests (CI) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index a5377982b29..ffa91034d32 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -29,14 +29,16 @@ on: token: description: 'Personal Access Token' required: true - default: "" type: string publish_ui: description: 'Publish to NPM?' required: true - default: true type: boolean +permissions: + contents: read + id-token: write + jobs: publish-python-sdk: uses: ./.github/workflows/publish_python_sdk.yml diff --git a/.github/workflows/publish_images.yml b/.github/workflows/publish_images.yml index 8b9abddcb0e..5ef0a972ae3 100644 --- a/.github/workflows/publish_images.yml +++ b/.github/workflows/publish_images.yml @@ -74,7 +74,7 @@ jobs: env: VERSION_WITHOUT_PREFIX: ${{ steps.get-version.outputs.version_without_prefix }} run: | - if [ "${{ matrix.component }}" = "feature-server" ]; then + if [[ "${{ matrix.component }}" == "feature-server" || "${{ matrix.component }}" == "feast-operator" ]]; then make build-${{ matrix.component }}-docker REGISTRY=${REGISTRY} VERSION=${VERSION_WITHOUT_PREFIX} DOCKER_PUSH=true DOCKER_PLATFORMS=linux/amd64,linux/arm64 else make build-${{ matrix.component }}-docker REGISTRY=${REGISTRY} VERSION=${VERSION_WITHOUT_PREFIX} @@ -84,8 +84,8 @@ jobs: VERSION_WITHOUT_PREFIX: ${{ steps.get-version.outputs.version_without_prefix }} HIGHEST_SEMVER_TAG: ${{ steps.get-version.outputs.highest_semver_tag }} run: | - if [ "${{ matrix.component }}" = "feature-server" ]; then - echo "feature-server image pushed via buildx during build step" + if [[ "${{ matrix.component }}" == "feature-server" || "${{ matrix.component }}" == "feast-operator" ]]; then + echo "${{ matrix.component }} image pushed via buildx during build step" else make push-${{ matrix.component }}-docker REGISTRY=${REGISTRY} VERSION=${VERSION_WITHOUT_PREFIX} fi @@ -93,8 +93,8 @@ jobs: echo "Only push to latest tag if tag is the highest semver version $HIGHEST_SEMVER_TAG" if [ "${VERSION_WITHOUT_PREFIX}" = "${HIGHEST_SEMVER_TAG:1}" ] then - if [ "${{ matrix.component }}" = "feature-server" ]; then - docker buildx imagetools create -t ${REGISTRY}/feature-server:latest ${REGISTRY}/feature-server:${VERSION_WITHOUT_PREFIX} + if [[ "${{ matrix.component }}" == "feature-server" || "${{ matrix.component }}" == "feast-operator" ]]; then + docker buildx imagetools create -t ${REGISTRY}/${{ matrix.component }}:latest ${REGISTRY}/${{ matrix.component }}:${VERSION_WITHOUT_PREFIX} else docker tag ${REGISTRY}/${{ matrix.component }}:${VERSION_WITHOUT_PREFIX} ${REGISTRY}/${{ matrix.component }}:latest docker push ${REGISTRY}/${{ matrix.component }}:latest diff --git a/.github/workflows/publish_python_sdk.yml b/.github/workflows/publish_python_sdk.yml index 03d0e989b49..85be4bc226b 100644 --- a/.github/workflows/publish_python_sdk.yml +++ b/.github/workflows/publish_python_sdk.yml @@ -4,9 +4,18 @@ on: workflow_dispatch: # Allows manual trigger of the workflow inputs: custom_version: # Optional input for a custom version - description: 'Custom version to publish (e.g., v1.2.3) -- only edit if you know what you are doing' + description: 'Custom version to publish (e.g., v1.2.3 or v1.2.3.dev4) -- only edit if you know what you are doing' required: false type: string + checkout_ref: + description: 'Git ref to checkout before building wheels. Defaults to the release version.' + required: false + type: string + build_docker_images: + description: 'Build Docker images as part of release verification.' + required: true + default: true + type: boolean token: description: 'Personal Access Token' required: true @@ -16,9 +25,18 @@ on: workflow_call: # Allows trigger of the workflow from another workflow inputs: custom_version: # Optional input for a custom version - description: 'Custom version to publish (e.g., v1.2.3) -- only edit if you know what you are doing' + description: 'Custom version to publish (e.g., v1.2.3 or v1.2.3.dev4) -- only edit if you know what you are doing' required: false type: string + checkout_ref: + description: 'Git ref to checkout before building wheels. Defaults to the release version.' + required: false + type: string + build_docker_images: + description: 'Build Docker images as part of release verification.' + required: false + default: true + type: boolean token: description: 'Personal Access Token' required: true @@ -30,8 +48,10 @@ jobs: uses: ./.github/workflows/build_wheels.yml secrets: inherit with: - custom_version: ${{ github.event.inputs.custom_version }} - token: ${{ github.event.inputs.token }} + custom_version: ${{ inputs.custom_version }} + checkout_ref: ${{ inputs.checkout_ref }} + build_docker_images: ${{ inputs.build_docker_images }} + token: ${{ inputs.token }} publish-python-sdk: if: github.repository == 'feast-dev/feast' @@ -46,4 +66,4 @@ jobs: uses: pypa/gh-action-pypi-publish@v1.4.2 with: user: __token__ - password: ${{ secrets.PYPI_PASSWORD }} \ No newline at end of file + password: ${{ secrets.PYPI_PASSWORD }} diff --git a/.github/workflows/publish_web_ui.yml b/.github/workflows/publish_web_ui.yml index f8f52f6a84c..b36165625fd 100644 --- a/.github/workflows/publish_web_ui.yml +++ b/.github/workflows/publish_web_ui.yml @@ -1,26 +1,6 @@ name: publish web ui on: - workflow_dispatch: # Allows manual trigger of the workflow - inputs: - current_version: - description: 'Current version to bump from (e.g., v1.2.3). If not provided, will auto-detect from git tags' - required: false - type: string - custom_version: # Optional input for a custom version - description: 'Custom version to publish (e.g., v1.2.3) -- only edit if you know what you are doing' - required: false - type: string - token: - description: 'Personal Access Token' - required: false - default: "" - type: string - publish_ui: - description: 'Publish to NPM?' - required: true - default: true - type: boolean workflow_call: # Allows trigger of the workflow from another workflow inputs: current_version: @@ -39,16 +19,17 @@ on: publish_ui: description: 'Publish to NPM?' required: true - default: true type: boolean +permissions: + contents: read + id-token: write + jobs: publish-web-ui-npm: if: github.repository == 'feast-dev/feast' runs-on: ubuntu-latest - env: - # This publish is working using an NPM automation token to bypass 2FA - NPM_TOKEN: ${{ secrets.NPM_TOKEN }} + environment: production steps: - uses: actions/checkout@v4 - name: Determine current version @@ -108,8 +89,10 @@ jobs: - name: Setup Node.js uses: actions/setup-node@v4 with: - node-version-file: './ui/.nvmrc' + node-version: '22.14.0' registry-url: 'https://registry.npmjs.org' + - name: Update npm for trusted publishing + run: npm install --global npm@11.5.1 - name: Bump file versions (temporarily for Web UI publish) if: github.event.inputs.custom_version != '' env: @@ -137,6 +120,3 @@ jobs: working-directory: ./ui if: github.event.inputs.publish_ui != 'false' run: npm publish - env: - # This publish is working using an NPM automation token to bypass 2FA - NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} \ No newline at end of file diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 627dfd589a1..24a76628fec 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -96,8 +96,10 @@ jobs: - name: Remove previous Helm run: sudo rm -rf $(which helm) - name: Set up Homebrew - uses: Homebrew/actions/setup-homebrew@master + uses: Homebrew/actions/setup-homebrew@main - name: Setup Helm-docs + env: + HOMEBREW_NO_SANDBOX_LINUX: 1 run: | brew install norwoodj/tap/helm-docs - name: Generate helm chart READMEs @@ -137,8 +139,10 @@ jobs: node-version-file: './ui/.nvmrc' - name: Set up Homebrew id: set-up-homebrew - uses: Homebrew/actions/setup-homebrew@master + uses: Homebrew/actions/setup-homebrew@main - name: Setup Helm-docs + env: + HOMEBREW_NO_SANDBOX_LINUX: 1 run: | brew install norwoodj/tap/helm-docs - name: Install Go @@ -191,4 +195,4 @@ jobs: - name: Reset stable branch to match release branch run: | git checkout -B stable origin/${GITHUB_REF#refs/heads/} - git push origin stable --force \ No newline at end of file + git push origin stable --force diff --git a/.github/workflows/smoke_tests.yml b/.github/workflows/smoke_tests.yml index b183f6f47e9..2a2c6615155 100644 --- a/.github/workflows/smoke_tests.yml +++ b/.github/workflows/smoke_tests.yml @@ -2,6 +2,10 @@ name: smoke-tests on: pull_request: + paths-ignore: + - 'docs/**' + - 'community/**' + - 'examples/**' concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} @@ -13,7 +17,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.10", "3.11", "3.12"] + python-version: ["3.11"] os: [ ubuntu-latest ] env: OS: ${{ matrix.os }} diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index 1311cd12635..1dee7f79963 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -2,9 +2,17 @@ name: unit-tests on: pull_request: + paths-ignore: + - 'docs/**' + - 'community/**' + - 'examples/**' push: branches: - master + paths-ignore: + - 'docs/**' + - 'community/**' + - 'examples/**' concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} @@ -71,9 +79,68 @@ jobs: fi make test-python-unit + - name: Upload Python coverage to Codecov + if: matrix.os == 'ubuntu-latest' && matrix.python-version == '3.12' + uses: codecov/codecov-action@b9fd7d16f6d7d1b5d2bec1a2887e65ceed900238 # v4.6.0 + with: + file: ./coverage.xml + flags: python-unit + fail_ci_if_error: false + env: + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} - name: Minimize uv cache run: uv cache prune --ci + unit-test-go: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Setup Go + uses: actions/setup-go@v5 + with: + go-version-file: go.mod + cache: true + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: "3.11" + architecture: x64 + - name: Install the latest version of uv + uses: astral-sh/setup-uv@v5 + with: + enable-cache: true + - name: Install system dependencies + run: | + sudo apt-get update + sudo apt-get install -y make protobuf-compiler libsqlite3-dev + - name: Install Go proto plugins + run: | + go install google.golang.org/protobuf/cmd/protoc-gen-go@latest + go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@latest + - name: Compile Go protobufs + run: make compile-protos-go + - name: Create virtual environment + run: | + uv venv + echo "${{ github.workspace }}/.venv/bin" >> $GITHUB_PATH + - name: Install feast locally + run: make install-feast-locally + - name: Run Go tests with coverage + run: | + CGO_ENABLED=1 go test \ + -coverprofile=go/coverage.out \ + -covermode=atomic \ + -skip "TestGetOnlineFeatures|TestSqliteOnlineRead" \ + ./go/... + - name: Upload Go coverage to Codecov + uses: codecov/codecov-action@b9fd7d16f6d7d1b5d2bec1a2887e65ceed900238 # v4.6.0 + with: + file: ./go/coverage.out + flags: go-feature-server + fail_ci_if_error: false + env: + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} + unit-test-ui: runs-on: ubuntu-latest env: @@ -95,6 +162,9 @@ jobs: - name: Build yarn rollup working-directory: ./ui run: yarn build:lib + - name: Build production UI + working-directory: ./ui + run: CI=true npm run build --omit=dev - name: Run yarn tests working-directory: ./ui run: yarn test --watchAll=false diff --git a/.secrets.baseline b/.secrets.baseline index 391a412320c..62d7fd33f37 100644 --- a/.secrets.baseline +++ b/.secrets.baseline @@ -142,7 +142,7 @@ "filename": ".github/workflows/publish.yml", "hashed_secret": "3e26d6750975d678acb8fa35a0f69237881576b0", "is_verified": false, - "line_number": 43 + "line_number": 45 } ], ".github/workflows/publish_python_sdk.yml": [ @@ -151,7 +151,7 @@ "filename": ".github/workflows/publish_python_sdk.yml", "hashed_secret": "3e26d6750975d678acb8fa35a0f69237881576b0", "is_verified": false, - "line_number": 31 + "line_number": 49 } ], ".prow.yaml": [ @@ -185,7 +185,7 @@ "filename": "docs/reference/online-stores/milvus.md", "hashed_secret": "5baa61e4c9b93f3f0682250b6cf8331b7ee68fd8", "is_verified": false, - "line_number": 33 + "line_number": 41 } ], "docs/reference/registries/sql.md": [ @@ -957,7 +957,7 @@ "filename": "infra/feast-operator/api/v1/featurestore_types.go", "hashed_secret": "44e17306b837162269a410204daaa5ecee4ec22c", "is_verified": false, - "line_number": 906 + "line_number": 958 } ], "infra/feast-operator/api/v1/zz_generated.deepcopy.go": [ @@ -966,21 +966,21 @@ "filename": "infra/feast-operator/api/v1/zz_generated.deepcopy.go", "hashed_secret": "f914fc9324de1bec1ad13dec94a8ea2ddb41fc87", "is_verified": false, - "line_number": 817 + "line_number": 842 }, { "type": "Secret Keyword", "filename": "infra/feast-operator/api/v1/zz_generated.deepcopy.go", "hashed_secret": "44e17306b837162269a410204daaa5ecee4ec22c", "is_verified": false, - "line_number": 878 + "line_number": 903 }, { "type": "Secret Keyword", "filename": "infra/feast-operator/api/v1/zz_generated.deepcopy.go", "hashed_secret": "c2028031c154bbe86fd69bef740855c74b927dcf", "is_verified": false, - "line_number": 1528 + "line_number": 1595 } ], "infra/feast-operator/api/v1alpha1/featurestore_types.go": [ @@ -989,7 +989,7 @@ "filename": "infra/feast-operator/api/v1alpha1/featurestore_types.go", "hashed_secret": "44e17306b837162269a410204daaa5ecee4ec22c", "is_verified": false, - "line_number": 649 + "line_number": 663 } ], "infra/feast-operator/api/v1alpha1/zz_generated.deepcopy.go": [ @@ -998,21 +998,30 @@ "filename": "infra/feast-operator/api/v1alpha1/zz_generated.deepcopy.go", "hashed_secret": "f914fc9324de1bec1ad13dec94a8ea2ddb41fc87", "is_verified": false, - "line_number": 595 + "line_number": 615 }, { "type": "Secret Keyword", "filename": "infra/feast-operator/api/v1alpha1/zz_generated.deepcopy.go", "hashed_secret": "44e17306b837162269a410204daaa5ecee4ec22c", "is_verified": false, - "line_number": 1103 + "line_number": 1123 }, { "type": "Secret Keyword", "filename": "infra/feast-operator/api/v1alpha1/zz_generated.deepcopy.go", "hashed_secret": "c2028031c154bbe86fd69bef740855c74b927dcf", "is_verified": false, - "line_number": 1108 + "line_number": 1128 + } + ], + "infra/feast-operator/bundle/manifests/openlineage-secret_v1_secret.yaml": [ + { + "type": "Secret Keyword", + "filename": "infra/feast-operator/bundle/manifests/openlineage-secret_v1_secret.yaml", + "hashed_secret": "598319ac6aa4a94e72a9d8a8d405cc7bc9e048ee", + "is_verified": false, + "line_number": 6 } ], "infra/feast-operator/config/samples/v1_featurestore_db_persistence.yaml": [ @@ -1163,7 +1172,7 @@ "filename": "infra/feast-operator/internal/controller/services/repo_config.go", "hashed_secret": "e2fb052132fd6a07a56af2013e0b62a1f510572c", "is_verified": false, - "line_number": 224 + "line_number": 235 } ], "infra/feast-operator/internal/controller/services/services.go": [ @@ -1172,7 +1181,7 @@ "filename": "infra/feast-operator/internal/controller/services/services.go", "hashed_secret": "36dc326eb15c7bdd8d91a6b87905bcea20b637d1", "is_verified": false, - "line_number": 180 + "line_number": 184 } ], "infra/feast-operator/internal/controller/services/tls_test.go": [ @@ -1555,5 +1564,5 @@ } ] }, - "generated_at": "2026-06-09T15:21:51Z" + "generated_at": "2026-07-31T05:29:18Z" } diff --git a/AGENTS.md b/AGENTS.md index e8ebb031cdc..2ade5b12f7c 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -92,7 +92,7 @@ Architecture & design intent: `docs/getting-started/architecture/` (overview, wr - Use type hints on all Python function signatures - Follow existing patterns in the module you are modifying -- PR titles must follow semantic conventions: `feat:`, `fix:`, `ci:`, `chore:`, `docs:` +- PR titles must follow conventional commit conventions with a lowercase type and a capitalized subject after the colon: `feat: Add ...`, `fix: Correct ...`, `ci: Update ...`, `chore: Refresh ...`, `docs: Add ...` - Sign off commits with `git commit -s` (DCO requirement) - Uses `ruff` for Python linting and formatting; Go uses standard `gofmt` - Recompile protos after making changes to `.proto` files (`make protos`) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7f1643e3ed0..38b88ae86df 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,217 @@ # Changelog +# [0.65.0](https://github.com/feast-dev/feast/compare/v0.64.0...v0.65.0) (2026-07-20) + + +### Bug Fixes + +* add debug logging for FIPS mode detection fallback ([6c1b24e](https://github.com/feast-dev/feast/commit/6c1b24ee6f27c469107269828b623180882de321)) +* Build embedded UI from local source ([#6525](https://github.com/feast-dev/feast/issues/6525)) ([3500349](https://github.com/feast-dev/feast/commit/35003494862f8b4af7f2d7eea321356743df074f)) +* Bump decommissioned Snowflake Python UDF runtime from 3.9 to 3.10 ([#6606](https://github.com/feast-dev/feast/issues/6606)) ([#6608](https://github.com/feast-dev/feast/issues/6608)) ([10341e4](https://github.com/feast-dev/feast/commit/10341e4d9cc05478ef863b33c3eeee3cc8da0162)) +* configure FIPS-compliant gRPC cipher suites for offline server ([6bc80a2](https://github.com/feast-dev/feast/commit/6bc80a2474e013724b1579d6de824c76e5d77f3d)) +* Correct Flink PyArrow dependency constraints ([#6604](https://github.com/feast-dev/feast/issues/6604)) ([70a9751](https://github.com/feast-dev/feast/commit/70a97515b8dc93e992d214e11c2bf9cd9ec65aa7)) +* Fix ValueError in signal handling for Trino worker threads ([#6428](https://github.com/feast-dev/feast/issues/6428)) ([506d919](https://github.com/feast-dev/feast/commit/506d919f3aaaaccdc4ac14cd23d0870302c6b13c)) +* Fixed monitoring page issues ([7946018](https://github.com/feast-dev/feast/commit/7946018c40f482bd82efb9a1c555d47dba5d4e54)) +* Make pytest config compatible with newer pytest ([#5779](https://github.com/feast-dev/feast/issues/5779)) ([a57ea33](https://github.com/feast-dev/feast/commit/a57ea331c53bf48a08e96764ff88fe2104bdb5bc)) +* Replace comma with space in DynamoDB-incompatible label tag value ([51e3a16](https://github.com/feast-dev/feast/commit/51e3a164fabf6e1cb2a1e41ae55328a4778177c4)) +* Resolve UI build warnings ([#6529](https://github.com/feast-dev/feast/issues/6529)) ([abe92af](https://github.com/feast-dev/feast/commit/abe92af5ef31283472a5d220390ed80b35baf3aa)) +* Unblock nightly UI build ([#6570](https://github.com/feast-dev/feast/issues/6570)) ([f296d4b](https://github.com/feast-dev/feast/commit/f296d4ba14c5d512429219b2b7845673e0fe524d)) +* Use LONGBLOB for SQL registry proto columns on MySQL ([#6566](https://github.com/feast-dev/feast/issues/6566)) ([7e4beb2](https://github.com/feast-dev/feast/commit/7e4beb21fdba8ee00afd7d0176989e42c10e31a1)) + + +### Features + +* Add click-to-zoom lightbox for blog post images ([#6575](https://github.com/feast-dev/feast/issues/6575)) ([1cb23fd](https://github.com/feast-dev/feast/commit/1cb23fde61862c6d53b434cd5b3ccbffea58d2a6)) +* Add dark mode support to website and blog ([#6589](https://github.com/feast-dev/feast/issues/6589)) ([7358fb8](https://github.com/feast-dev/feast/commit/7358fb8c9a9f8543f6add0e13b8b4b06ef11916b)) +* Add OnlineStore for Aerospike ([#6532](https://github.com/feast-dev/feast/issues/6532)) ([9cd35e1](https://github.com/feast-dev/feast/commit/9cd35e140a949dc44a9915300f2724dd2e702f03)) +* Add OpenLineage Consumer to Feast - receive, store, and visualize cross-producer lineage ([#6549](https://github.com/feast-dev/feast/issues/6549)) ([a834126](https://github.com/feast-dev/feast/commit/a834126b674356ea1efeafd7006f579c9148c3a1)) +* Add registry list feature views by updated since ([#6092](https://github.com/feast-dev/feast/issues/6092)) ([#6093](https://github.com/feast-dev/feast/issues/6093)) ([006c606](https://github.com/feast-dev/feast/commit/006c606183457373d8c83b1986a9f35e1d764c9a)) +* Add ScyllaDB online store with vector search ([#6508](https://github.com/feast-dev/feast/issues/6508)) ([1669661](https://github.com/feast-dev/feast/commit/1669661e15d3ba3b5ab9a9fffd19248d9c0da211)) +* Added compute and jobs UI ([ba2c05c](https://github.com/feast-dev/feast/commit/ba2c05c731be64aff8e7af0fdeefcbe8aa397308)) +* Added Iceberg REST Catalog data source support ([e0a8573](https://github.com/feast-dev/feast/commit/e0a8573eb453dd7060343c4e474e7fca7e1378f7)) +* Bring Your Own Spark - SparkApplication ([#6550](https://github.com/feast-dev/feast/issues/6550)) ([dcd496f](https://github.com/feast-dev/feast/commit/dcd496f22e109f0f77338d41e057dd71113b67d0)) +* **cassandra:** Add multi-DC support via per-datacenter execution profiles ([#6434](https://github.com/feast-dev/feast/issues/6434)) ([0de9196](https://github.com/feast-dev/feast/commit/0de9196d75a63e1ba3860de051cab40c6eba8efc)) +* Enhanced data source creation as a visual catalog with type-specific forms ([#6557](https://github.com/feast-dev/feast/issues/6557)) ([d6acbba](https://github.com/feast-dev/feast/commit/d6acbba057cde6e1c088d068e39492448c73fea1)) +* Enhanced datasets UI functionality ([de11152](https://github.com/feast-dev/feast/commit/de111525985b542b8bfa61118e1ee949254d8703)) +* Implement RegistryServer.Proto RPC with RBAC-filtered response ([#6558](https://github.com/feast-dev/feast/issues/6558)) ([#6552](https://github.com/feast-dev/feast/issues/6552)) ([0d02614](https://github.com/feast-dev/feast/commit/0d02614edcc6fb71992cbb0b539c4b0e2a50f810)) +* New zoned timestamp feature type ([#6536](https://github.com/feast-dev/feast/issues/6536)) ([#6537](https://github.com/feast-dev/feast/issues/6537)) ([eb042f0](https://github.com/feast-dev/feast/commit/eb042f04f5d9bdd7dafbaf654d5b5ec2a2572d9f)) +* **operator:** Auto-create RBAC for spark_application batch engine ([#6597](https://github.com/feast-dev/feast/issues/6597)) ([f487b37](https://github.com/feast-dev/feast/commit/f487b37fd317c63d0d0060ccf8be5d8238d484dd)) +* **operator:** integrate cluster TLS profile for OCP 5.0 compliance ([43263a6](https://github.com/feast-dev/feast/commit/43263a658abe5e2080241b5819fdd8affb4e5fef)) +* Permissions CRUD UI and OIDC auth integration in UI ([6511da1](https://github.com/feast-dev/feast/commit/6511da1323f5634595b5b2ae4e8a5055599c7885)) +* Retrieve historical features from BigQuery without entity_df ([#6569](https://github.com/feast-dev/feast/issues/6569)) ([cd5f6bb](https://github.com/feast-dev/feast/commit/cd5f6bbbd36f11f1d2e2faf8e5e773076b7a3026)), closes [#6558](https://github.com/feast-dev/feast/issues/6558) [#6552](https://github.com/feast-dev/feast/issues/6552) +* **spark:** SparkSource query+path and pre-computed offline read for BatchFeatureView ([#6440](https://github.com/feast-dev/feast/issues/6440)) ([4dc8757](https://github.com/feast-dev/feast/commit/4dc8757626c69c833a8d8174a6bd1513b1671ad7)) + + +### BREAKING CHANGES + +* total_timeout_ms is renamed to batch_total_timeout_ms. Config files using the old name must be updated. No default value change. + +Docs updated (reference + perf-tuning guide) with a short explainer on the per-attempt vs total deadline distinction. Two new unit tests pin the policy wiring: socket_timeout_ms propagates to all three scopes, and is omitted (not injected as None) when unset. + +Signed-off-by: Valentyn Kahamlyk + +* refactor(aerospike): use MAP_KEY_ORDERED, KEY_DIGEST, and instance-scoped client + +Cheap-win cleanups flagged in review, all touching the same small patch of write-path and lifecycle code. + +* Map CDTs are now created with MAP_KEY_ORDERED. map_get_by_key / map_remove_by_key on an ordered map are O(log N) in the map size instead of O(N); matters on reads of wide feature views and on the update() background scan (which walks every record in the project's set). + +* Writes drop POLICY_KEY_SEND and rely on the client default (POLICY_KEY_DIGEST). The serialized entity key is no longer stored alongside each record, saving per-record storage the read path never consumes (batch_operate preserves request order; results are paired back by zip in online_read). + +* _client moves from a class attribute to an instance attribute (set in __init__). Previously two AerospikeOnlineStore instances could share the cached client through class state until one wrote self._client. With the instance attribute the state is always per-instance from construction. + +* Drop MongoDB references from class docstrings and comments (they referred to how the storage layout was derived rather than documenting current behavior). Also rewrite the _build_batch_writes docstring to describe the policies applied on the write path. + +Unit test assertions for the write-path record are updated: bw.policy is now None (client default applies) and map ops carry map_policy={'map_order': MAP_KEY_ORDERED}. All three docker-backed integration tests still pass end-to-end (cross-FV upsert, update() background scan, full feature-store round-trip), so the read/write shape survives the ordering and policy changes against a real server. + +Signed-off-by: Valentyn Kahamlyk + +* feat(aerospike): add per-FV namespace/set overrides and prewriting hook + +Adds three configuration knobs to AerospikeOnlineStoreConfig: + +- namespace_overrides: pin individual feature views to a different + Aerospike namespace (e.g. RAM-only vs. SSD-backed) without splitting + the project across stores. +- set_overrides: place a feature view in its own set so admin ops on + it (truncate, scan-based deletes during `feast apply`) do not touch + records of other views. +- prewriting_hook: import-string-resolved callable invoked once per + online_write_batch with the rows about to be written, returning the + rows that actually go on the wire. Resolved and cached on first use; + returning [] short-circuits the wire call. + +Read, write, update and teardown paths all honour the per-FV ns/set +resolution. update() groups dropped feature views by their resolved +(ns, set) pair and issues one background scan per group. teardown() +truncates every unique (ns, set) pair the project may have written to, +including the store-level default. + +Adds 22 unit tests for the new behaviour and updates 3 existing call +sites of _build_batch_writes for the new namespace= parameter. Adds a +sample hook module under examples/online_store/aerospike_overrides_and_hooks/ +and corresponding sections in docs/reference/online-stores/aerospike.md. + +Signed-off-by: Valentyn Kahamlyk + +* test: update aerospike image tag + +Signed-off-by: Valentyn Kahamlyk + +* chore: sync README template and secrets baseline after master merge + +Signed-off-by: Valentyn Kahamlyk + +* chore: fix secrets baseline line number for v1 operator types + +Adding aerospike to the feast-operator enum shifted the allowlisted +SecretRef entry in api/v1/featurestore_types.go by one line. + +Signed-off-by: Valentyn Kahamlyk + +* docs: update aerospike docs + +Signed-off-by: Valentyn Kahamlyk + +* fix(aerospike): wire batch max_retries and fix empty projection handling + +Copilot review feedback on PR #6532: + +- Add max_retries to the batch client policy (batch_operate/batch_write path) +- Treat empty projected feature maps as present FV slots (is not None) +- Return {} from _normalize_projected_features([]) instead of None +- Fix projection unit test mock/assertions +- Correct prewriting_hook config docstring + +Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> +Signed-off-by: Valentyn Kahamlyk + +* style(aerospike): format online_read docs assignment for ruff + +Signed-off-by: Valentyn Kahamlyk + +* chore: update pixi.lock for aerospike optional extra + +Regenerate the v6 lockfile with Pixi v0.63.1 after adding the aerospike extra to pyproject.toml. + +Signed-off-by: Valentyn Kahamlyk + +* fix(aerospike): add client init lock and batch chunking + +Guard lazy client creation with a lock to avoid connection leaks under concurrent first use, and chunk batch reads/writes by batch_max_records so large materializations stay under Aerospike server batch limits. + +Signed-off-by: Valentyn Kahamlyk + +# [0.64.0](https://github.com/feast-dev/feast/compare/v0.63.0...v0.64.0) (2026-06-13) + + +### Bug Fixes + +* Add async_supported property to RedisOnlineStore ([9b088fe](https://github.com/feast-dev/feast/commit/9b088fe6144ff35926884cbda96099d0d4a0d66c)) +* Add missing feast init templates to operator CRD and enhance persistence documentation ([1941d4d](https://github.com/feast-dev/feast/commit/1941d4d184a3e13eea1d47b1b35d3305c89ecf1c)) +* Allow to publish from reference branch ([5458ec8](https://github.com/feast-dev/feast/commit/5458ec8afa0d692ed5dd908826ebdf1869098036)) +* API calls list ([4203eb7](https://github.com/feast-dev/feast/commit/4203eb749b153f55f6219c7a5d9dc1161fc5ae4e)) +* **bigquery:** Enable list inference for parquet loads in offline_write_batch ([9243497](https://github.com/feast-dev/feast/commit/92434971821b3a9486d04397af33bac94e808e24)), closes [#5845](https://github.com/feast-dev/feast/issues/5845) +* Bump grpcio dependencies ([07b4782](https://github.com/feast-dev/feast/commit/07b47826928f14751724130ea83e343f59e33049)) +* **compute-engine/local:** Honor field_mapping on join keys in dedup + join nodes ([#6395](https://github.com/feast-dev/feast/issues/6395)) ([bd01824](https://github.com/feast-dev/feast/commit/bd01824e284b44847c834ef75cb3bc6e71940a5d)) +* **dynamodb:** Avoid tag race condition by using diff-based tag updates ([#6479](https://github.com/feast-dev/feast/issues/6479)) ([bad2b7d](https://github.com/feast-dev/feast/commit/bad2b7d53d62b0b736d28beaa5d4b48d97875f15)), closes [#6418](https://github.com/feast-dev/feast/issues/6418) +* **dynamodb:** Fix mypy type for _build_projection_expression return ([217b4da](https://github.com/feast-dev/feast/commit/217b4daa49a47ae3c88e8a320569e83c1fb51b7e)) +* Fix intermittent async test failures for DynamoDB and Redis ([63c5eb1](https://github.com/feast-dev/feast/commit/63c5eb152a33bb30a75bf2d704e9aac310db2eab)) +* Fix mongodb blog title ([57d28d4](https://github.com/feast-dev/feast/commit/57d28d4c27384f7b5ebdc85f262ba24db82a879e)) +* Fix shared SQL registry crash - avoid unnecessary UDF deserialization in proto cache building ([ac588d7](https://github.com/feast-dev/feast/commit/ac588d70757288bbbcd98ec7c1e42c0993e7981b)) +* Fix SparkRetrievalJob.persist() failing for SparkSource ([209d7cd](https://github.com/feast-dev/feast/commit/209d7cd0f42b22f5a9a695fc7b3d66e85d4daa31)) +* Fixed formatting and image for mongo blog ([#6377](https://github.com/feast-dev/feast/issues/6377)) ([f8389fb](https://github.com/feast-dev/feast/commit/f8389fb4037ad0280c7b0a70fafe9ab710369409)) +* Fixes for ray source ([7f592a4](https://github.com/feast-dev/feast/commit/7f592a4fa6f230ce8a635a1ff235cbb575c254f4)) +* **go:** skip registry refresh when cache_ttl_seconds <= 0 ([97ed40c](https://github.com/feast-dev/feast/commit/97ed40ca175e29cc1df30fb8d866f4cfc3f3d62c)) +* Handle array of strings columns in Athena materialization ([#6324](https://github.com/feast-dev/feast/issues/6324)) ([4ed0278](https://github.com/feast-dev/feast/commit/4ed027807c87aad31b9062bb7ee1ddf4008d61ad)) +* make milvus VARCHAR max_length configurable, remove hardcoded 512 limit ([3b98c22](https://github.com/feast-dev/feast/commit/3b98c22426f108334222b81000acdcf215fc483b)) +* **operator:** Set appProtocol: grpc on registry gRPC Service ([#6367](https://github.com/feast-dev/feast/issues/6367)) ([c9ae2b4](https://github.com/feast-dev/feast/commit/c9ae2b41cf44fd8d17b9d55191a66c4d210b2292)) +* PyJWT 2.10+ added validation that rejects empty HMAC keys ([e756ffe](https://github.com/feast-dev/feast/commit/e756ffe26b0b4fd16e8f621269195f15f14340f4)) +* RemoteOnlineStore sends all features in a single HTTP request ([8f187dd](https://github.com/feast-dev/feast/commit/8f187dd6dd1a4923348d60c2bf53d1ef4e367a9b)) +* Remove registry proto dump to enforce RBAC and add permission checks to Commit/Refresh RPCs ([328431f](https://github.com/feast-dev/feast/commit/328431ffe083f744d5dad1ce1243ed88d921db64)) +* Remove selector migration job - no longer needed ([51c325e](https://github.com/feast-dev/feast/commit/51c325ee6e72c1f18f71a36f9fc7c8120e5d16f1)) +* replace broken .claude skill symlink with correct relative path ([4541690](https://github.com/feast-dev/feast/commit/45416901e488b657f45601edda8804d6fe82a714)) +* Replace selector label strip patch with migration Job for upgrade-safe selector uniqueness ([00dea50](https://github.com/feast-dev/feast/commit/00dea5010ae9b6cb6c88a145e16502818420d2b2)) +* Scope feature view name conflict check to current project in file-based registry ([#6369](https://github.com/feast-dev/feast/issues/6369)) ([a4fde83](https://github.com/feast-dev/feast/commit/a4fde83d125ed1ec18a353871101f07ac51b4be7)), closes [#6209](https://github.com/feast-dev/feast/issues/6209) +* **snowflake:** Stop double-quoting connection identifiers ([#6462](https://github.com/feast-dev/feast/issues/6462)) ([e914d59](https://github.com/feast-dev/feast/commit/e914d593fedae05bcab050b6d05dd45b1703b658)) +* **spark:** S3/GCS PyArrow filesystem resolution for staging paths ([#6442](https://github.com/feast-dev/feast/issues/6442)) ([ae50414](https://github.com/feast-dev/feast/commit/ae50414d258086f7968cb4ea911b4a9b49924665)) +* **trino:** Clean up temporary entity tables after retrieval ([#6381](https://github.com/feast-dev/feast/issues/6381)) ([d86b13d](https://github.com/feast-dev/feast/commit/d86b13df1d3c74fb1ba1906a7eadbc1cfc1492d8)), closes [#6306](https://github.com/feast-dev/feast/issues/6306) +* Update go-feature-server base image to Go 1.25 and fix operator Dockerfile COPY permissions ([86ef0bc](https://github.com/feast-dev/feast/commit/86ef0bcf6d66f3eb0690d7017714fd0b29c149c9)) + + +### Features + +* [Backend] Data Quality Monitoring with native compute, multi-backend support, REST API, CLI ([#6202](https://github.com/feast-dev/feast/issues/6202)) ([5458c37](https://github.com/feast-dev/feast/commit/5458c375745e32f219a15f5f62b49a1c6adaf2b0)) +* Add apache flink compute engine ([#6476](https://github.com/feast-dev/feast/issues/6476)) ([9636d6a](https://github.com/feast-dev/feast/commit/9636d6a2da52e2381b2b929a975b9f6cedaa7e0c)) +* Add demo noteboooks for users ([e362173](https://github.com/feast-dev/feast/commit/e362173c9623fd42f8bd78eb6ce1bfd9d1090345)) +* Add enabled/disabled toggle for feature views ([#6401](https://github.com/feast-dev/feast/issues/6401)) ([5f1fa0d](https://github.com/feast-dev/feast/commit/5f1fa0d98961509a0393bad0d1ef47ce03f8638a)), closes [#6395](https://github.com/feast-dev/feast/issues/6395) +* Add Label View to init template ([ec272d5](https://github.com/feast-dev/feast/commit/ec272d5206cd9ab95686621e82f50722452fe122)) +* Add mTLS support to remote registry gRPC client ([#6474](https://github.com/feast-dev/feast/issues/6474)) ([c9602d8](https://github.com/feast-dev/feast/commit/c9602d8f5d3f09010b5a15e19f4d55651b6e0737)) +* Add Prometheus gauges for FeatureStore installation telemetry ([#6354](https://github.com/feast-dev/feast/issues/6354)) ([1b681b7](https://github.com/feast-dev/feast/commit/1b681b714c56c75e75bc6f896424ebe4c3feddc2)) +* Adds registry REST API endpoints for managing entities, data sources, and feature views ([#6413](https://github.com/feast-dev/feast/issues/6413)) ([f77bd1d](https://github.com/feast-dev/feast/commit/f77bd1dc1a1d9a0920c900e0e40a37c2a33ce39e)) +* Allow CRUD on entities, data sources, and feature views from UI ([#6412](https://github.com/feast-dev/feast/issues/6412)) ([2321c07](https://github.com/feast-dev/feast/commit/2321c07938ca12c6a54d83e9ba6a0dfdb3a173eb)) +* Allow default openlineage configuration ([#6467](https://github.com/feast-dev/feast/issues/6467)) ([276b6df](https://github.com/feast-dev/feast/commit/276b6df562e16fefba7efb493736ff32046d4a76)) +* **bigquery:** Support DATE-type event timestamp columns ([#6362](https://github.com/feast-dev/feast/issues/6362)) ([753dee5](https://github.com/feast-dev/feast/commit/753dee5ea4fdde07b2ee74a9a74b0a7b855c6716)), closes [#2530](https://github.com/feast-dev/feast/issues/2530) +* **cli:** Add `feast projects delete` command (closes [#5095](https://github.com/feast-dev/feast/issues/5095)) ([#6318](https://github.com/feast-dev/feast/issues/6318)) ([1a4b96c](https://github.com/feast-dev/feast/commit/1a4b96c73ef383e8fcecf8a97eb3592be5d441e2)) +* Data Quality Monitoring added in feast UI ([#6422](https://github.com/feast-dev/feast/issues/6422)) ([fa271be](https://github.com/feast-dev/feast/commit/fa271be3cbe00fd930b7bc091e7c3010ae2f241e)) +* **dynamodb:** Use ProjectionExpression when requested_features is set ([0adc906](https://github.com/feast-dev/feast/commit/0adc9060d80a675b64422d5a1ddd5c8bec1f4996)), closes [#6058](https://github.com/feast-dev/feast/issues/6058) +* Enhance DataSource and FeatureView modals with error handling and submission states ([96d7169](https://github.com/feast-dev/feast/commit/96d7169f8f42926a7f149f0715b59b31b081a2e8)) +* Expose registry endpoints on feature server for MCP access ([f77981c](https://github.com/feast-dev/feast/commit/f77981c3a0dc4637bfc6c51178ad53e8789a07d1)) +* Feast First-Class LabelView Implementation ([#6292](https://github.com/feast-dev/feast/issues/6292)) ([c0e7e5d](https://github.com/feast-dev/feast/commit/c0e7e5d558347fd474f9c0316abc723d7d138118)) +* Feast-MLflow Integration ([#6235](https://github.com/feast-dev/feast/issues/6235)) ([7279c75](https://github.com/feast-dev/feast/commit/7279c75fb5681565cfa27914ca5ad17818e11089)) +* Operational metrics for offline store and SOX metrics for both ([#6340](https://github.com/feast-dev/feast/issues/6340)) ([65b1b80](https://github.com/feast-dev/feast/commit/65b1b801fce5b5e0ed89f4dd8ca16ada2461e006)) +* Pre-compute feature service ([8011550](https://github.com/feast-dev/feast/commit/80115507e9c20d15a772df56b9f089ad028b4046)) +* REST API-backed UI for RBAC compatibility and per-page lazy loading ([#6414](https://github.com/feast-dev/feast/issues/6414)) ([6ae80af](https://github.com/feast-dev/feast/commit/6ae80af1ba542ebe12e78c9a05ad2624ffd1a127)) +* Support non-string map key types ([#6382](https://github.com/feast-dev/feast/issues/6382)) ([#6383](https://github.com/feast-dev/feast/issues/6383)) ([728aa2e](https://github.com/feast-dev/feast/commit/728aa2e039dab8d51f2f714f544cf1afeea78acd)) +* Update FeatureStore CRD with DRA Fields ([01241e4](https://github.com/feast-dev/feast/commit/01241e4f587994d7abd5a6f40b503d101656ed3f)) + + +### Performance Improvements + +* Cache feature view resolution in get_online_features to reduce per-request overhead ([55c2f18](https://github.com/feast-dev/feast/commit/55c2f185f015e4fc4052a828c9785a79b9819104)) +* Optimize feature serving latency with batched async Redis, cached checks fix ([103809a](https://github.com/feast-dev/feast/commit/103809a24839fb40f625de7e111454f582431eee)) +* Replace MessageToDict with optimized custom dict builder ([#6015](https://github.com/feast-dev/feast/issues/6015)) ([9902064](https://github.com/feast-dev/feast/commit/99020646118f2c723ab4afb5842055863605c05a)) + # [0.63.0](https://github.com/feast-dev/feast/compare/v0.62.0...v0.63.0) (2026-05-04) diff --git a/Makefile b/Makefile index e277295e84c..d5a6aeeaba9 100644 --- a/Makefile +++ b/Makefile @@ -112,7 +112,7 @@ install-python-dependencies-ci: ## Install Python CI dependencies using uv pip s # Install CPU-only torch first to prevent CUDA dependency issues (Linux only) @if [ "$$(uname -s)" = "Linux" ]; then \ echo "Installing dependencies with torch CPU index for Linux..."; \ - uv pip sync --extra-index-url https://download.pytorch.org/whl/cpu --index-strategy unsafe-best-match sdk/python/requirements/py$(PYTHON_VERSION)-ci-requirements.txt; \ + uv pip sync --torch-backend cpu sdk/python/requirements/py$(PYTHON_VERSION)-ci-requirements.txt; \ else \ echo "Installing dependencies from PyPI for macOS..."; \ uv pip sync sdk/python/requirements/py$(PYTHON_VERSION)-ci-requirements.txt; \ @@ -150,9 +150,11 @@ lock-python-dependencies-all: ## Recompile and lock all Python dependency sets f pixi run --environment $(call get_env_name,$(ver)) --manifest-path infra/scripts/pixi/pixi.toml \ "uv pip compile -p $(ver) --no-strip-extras pyproject.toml --extra minimal-sdist-build \ --no-emit-package milvus-lite \ + --no-emit-package pymilvus \ + --no-emit-package faiss-cpu \ --generate-hashes --output-file sdk/python/requirements/py$(ver)-minimal-sdist-requirements.txt" && \ pixi run --environment $(call get_env_name,$(ver)) --manifest-path infra/scripts/pixi/pixi.toml \ - "uv pip install -p $(ver) pybuild-deps==0.5.0 pip==25.0.1 && \ + "uv pip install -p $(ver) pybuild-deps==0.5.0 pip==25.0.1 typing_extensions && \ pybuild-deps compile --generate-hashes \ -o sdk/python/requirements/py$(ver)-minimal-sdist-requirements-build.txt \ sdk/python/requirements/py$(ver)-minimal-sdist-requirements.txt" && \ @@ -171,6 +173,9 @@ benchmark-python-local: ## Run integration + benchmark tests for Python (local d test-python-unit: ## Run Python unit tests (use pattern= to filter tests, e.g., pattern=milvus, pattern=test_online_retrieval.py, pattern=test_online_retrieval.py::test_get_online_features_milvus) uv run python -m pytest -n 8 --color=yes $(if $(pattern),-k "$(pattern)") \ + --cov=feast \ + --cov-report=xml \ + --cov-report=term-missing \ sdk/python/tests/unit # Fast unit tests only @@ -726,10 +731,16 @@ push-feast-operator-docker: ## Push Feast Operator Docker image $(MAKE) docker-push build-feast-operator-docker: ## Build Feast Operator Docker image - cd infra/feast-operator && \ - IMAGE_TAG_BASE=$(REGISTRY)/feast-operator \ - VERSION=$(VERSION) \ - $(MAKE) docker-build + @if [ -n "$(DOCKER_PLATFORMS)" ]; then \ + cd infra/feast-operator && \ + docker buildx build --push --platform=$(DOCKER_PLATFORMS) \ + --tag $(REGISTRY)/feast-operator:$(VERSION) -f Dockerfile .; \ + else \ + cd infra/feast-operator && \ + IMAGE_TAG_BASE=$(REGISTRY)/feast-operator \ + VERSION=$(VERSION) \ + $(MAKE) docker-build; \ + fi build-feast-operator-docker-on-mac: ## Build Feast Operator Docker image on Mac cd infra/feast-operator && \ @@ -813,13 +824,12 @@ build-helm-docs: ## Build helm docs # Note: these require node and yarn to be installed build-ui: ## Build Feast UI - cd $(ROOT_DIR)/sdk/python/feast/ui && yarn upgrade @feast-dev/feast-ui --latest && yarn install && npm run build --omit=dev - -build-ui-local: ## Build Feast UI locally cd $(ROOT_DIR)/ui && yarn install && npm run build --omit=dev rm -rf $(ROOT_DIR)/sdk/python/feast/ui/build cp -r $(ROOT_DIR)/ui/build $(ROOT_DIR)/sdk/python/feast/ui/ +build-ui-local: build-ui ## Build Feast UI locally + format-ui: ## Format Feast UI cd $(ROOT_DIR)/ui && NPM_TOKEN= yarn install && NPM_TOKEN= yarn format diff --git a/README.md b/README.md index 115bd37903f..a91faccae83 100644 --- a/README.md +++ b/README.md @@ -19,7 +19,7 @@ ## Join us on Slack! -👋👋👋 [Come say hi on Slack!](https://communityinviter.com/apps/feastopensource/feast-the-open-source-feature-store) +👋👋👋 [Come say hi on Slack!](https://slack.feast.dev/) [Check out our DeepWiki!](https://deepwiki.com/feast-dev/feast) @@ -227,6 +227,7 @@ The list below contains the functionality that contributors are planning to deve * [x] [SingleStore](https://docs.feast.dev/reference/online-stores/singlestore) * [x] [Couchbase](https://docs.feast.dev/reference/online-stores/couchbase) * [x] [MongoDB](https://docs.feast.dev/reference/online-stores/mongodb) + * [x] [Aerospike](https://docs.feast.dev/reference/online-stores/aerospike) * [x] [Qdrant (vector store)](https://docs.feast.dev/reference/online-stores/qdrant) * [x] [Milvus (vector store)](https://docs.feast.dev/reference/online-stores/milvus) * [x] [Faiss (vector store)](https://docs.feast.dev/reference/online-stores/faiss) @@ -254,7 +255,7 @@ The list below contains the functionality that contributors are planning to deve * [x] [Offline Feature Server (alpha)](https://docs.feast.dev/reference/feature-servers/offline-feature-server) * [x] [Registry server (alpha)](https://github.com/feast-dev/feast/blob/master/docs/reference/feature-servers/registry-server.md) * **Data Quality Management (See [RFC](https://docs.google.com/document/d/110F72d4NTv80p35wDSONxhhPBqWRwbZXG4f9mNEMd98/edit))** - * [x] Data profiling and validation (Great Expectations) + * [x] [Feature Quality Monitoring](https://docs.feast.dev/how-to-guides/feature-monitoring) — built-in metrics, drift detection, serving log monitoring, and UI dashboard * **Feature Discovery and Governance** * [x] Python SDK for browsing feature registry * [x] CLI for browsing feature registry diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 00000000000..a7a6d645642 --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,30 @@ +# Security Policy + +The Feast community takes security bugs seriously, and we appreciate the effort it takes to find and report them. We follow [GitHub's coordinated disclosure process](https://docs.github.com/en/code-security/security-advisories/guidance-on-reporting-and-writing-information-about-vulnerabilities/about-coordinated-disclosure-of-security-vulnerabilities) so that a fix can be prepared before details become public. + +## Reporting a vulnerability + +Report vulnerabilities privately through GitHub, using **[Report a vulnerability](https://github.com/feast-dev/feast/security/advisories/new)** on this repository's Security tab. Only the maintainers can see the report, and you will be credited on the published advisory if you would like to be. + +Before reporting, please check the [published advisories](https://github.com/feast-dev/feast/security/advisories) to confirm the issue has not already been addressed. + +A report needs to show a clear, reproducible security impact. Please include: + +- the affected version or commit, and the configuration involved +- a proof of concept, or steps that reproduce the issue +- the actual impact, rather than a theoretical concern + +Raw scanner or dependency-audit output does not meet that bar on its own, since it does not establish that the issue is reachable in Feast. Reports that have not been manually verified against Feast, including bulk, automated, or AI-generated submissions, may be closed without further response. + +> [!WARNING] +> Do not open a public GitHub issue, pull request, or Slack message for a security vulnerability. Those are visible to everyone and disclose the problem before a fix exists. + +For anything that is not a vulnerability, including hardening suggestions and questions about how Feast's authentication and authorization work, a normal [GitHub issue](https://github.com/feast-dev/feast/issues) is the right place. + +## Supported versions + +Security fixes are applied to the latest release. Feast releases roughly monthly and offers best-effort community support, as described in the [versioning policy](docs/project/versioning-policy.md); there is no long-term support branch, so upgrading to the current release is the supported way to receive a fix. + +## Published advisories + +Past advisories for this project are listed under [Security advisories](https://github.com/feast-dev/feast/security/advisories). diff --git a/docs/README.md b/docs/README.md index 8229ac10587..e8588be340f 100644 --- a/docs/README.md +++ b/docs/README.md @@ -71,7 +71,7 @@ Feast helps ML platform/MLOps teams with DevOps experience productionize real-ti * **batch feature engineering**: Feast supports on-demand and streaming transformations. Feast is also investing in supporting batch transformations. * **native streaming feature integration:** Feast enables users to push streaming features, but does not pull from streaming sources or manage streaming pipelines. * **lineage:** Feast helps tie feature values to model versions, but is not a complete solution for capturing end-to-end lineage from raw data sources to model versions. Feast also has community contributed plugins with [DataHub](https://datahubproject.io/docs/generated/ingestion/sources/feast/) and [Amundsen](https://github.com/amundsen-io/amundsen/blob/4a9d60176767c4d68d1cad5b093320ea22e26a49/databuilder/databuilder/extractor/feast\_extractor.py). -* **data quality / drift detection**: Feast has experimental integrations with [Great Expectations](https://greatexpectations.io/), but is not purpose built to solve data drift / data quality issues. This requires more sophisticated monitoring across data pipelines, served feature values, labels, and model versions. +* **data quality / drift detection**: Feast includes built-in [Feature Quality Monitoring](how-to-guides/feature-monitoring.md) that computes statistical metrics (null rates, distributions, percentiles), detects drift across batch data and serving logs, and provides a monitoring UI dashboard. ## Example use cases diff --git a/docs/SUMMARY.md b/docs/SUMMARY.md index c1dc8ee8e45..091014d3ead 100644 --- a/docs/SUMMARY.md +++ b/docs/SUMMARY.md @@ -23,7 +23,12 @@ * [Project](getting-started/concepts/project.md) * [Data ingestion](getting-started/concepts/data-ingestion.md) * [Entity](getting-started/concepts/entity.md) + * [Data types](getting-started/concepts/feast-types.md) + * [Feature repository](getting-started/concepts/feature-repo.md) * [Feature view](getting-started/concepts/feature-view.md) + * [Batch feature view](getting-started/concepts/batch-feature-view.md) + * [Stream feature view](getting-started/concepts/stream-feature-view.md) + * [Tiling with Intermediate Representations](getting-started/concepts/tiling.md) * [Label view](getting-started/concepts/label-view.md) * [Feature retrieval](getting-started/concepts/feature-retrieval.md) * [Point-in-time joins](getting-started/concepts/point-in-time-joins.md) @@ -38,6 +43,7 @@ * [Online store](getting-started/components/online-store.md) * [Feature server](getting-started/components/feature-server.md) * [Compute Engine](getting-started/components/compute-engine.md) + * [Stream Processor](getting-started/components/stream-processor.md) * [Provider](getting-started/components/provider.md) * [Authorization Manager](getting-started/components/authz_manager.md) * [OpenTelemetry Integration](getting-started/components/open-telemetry.md) @@ -51,7 +57,6 @@ * [Fraud detection on GCP](tutorials/tutorials-overview/fraud-detection.md) * [Real-time credit scoring on AWS](tutorials/tutorials-overview/real-time-credit-scoring-on-aws.md) * [Driver stats on Snowflake](tutorials/tutorials-overview/driver-stats-on-snowflake.md) -* [Validating historical features with Great Expectations](tutorials/validating-historical-features.md) * [Building streaming features](tutorials/building-streaming-features.md) * [Retrieval Augmented Generation (RAG) with Feast](tutorials/rag-with-docling.md) * [RAG Fine Tuning with Feast and Milvus](../examples/rag-retriever/README.md) @@ -85,13 +90,14 @@ * [Feast Production Deployment Topologies](how-to-guides/production-deployment-topologies.md) * [Online Server Performance Tuning](how-to-guides/online-server-performance-tuning.md) * [Customizing Feast](how-to-guides/customizing-feast/README.md) - * [Adding a custom batch materialization engine](how-to-guides/customizing-feast/creating-a-custom-materialization-engine.md) + * [Adding a custom compute engine](how-to-guides/customizing-feast/creating-a-custom-compute-engine.md) * [Adding a new offline store](how-to-guides/customizing-feast/adding-a-new-offline-store.md) * [Adding a new online store](how-to-guides/customizing-feast/adding-support-for-a-new-online-store.md) * [Adding a custom provider](how-to-guides/customizing-feast/creating-a-custom-provider.md) * [Adding or reusing tests](how-to-guides/adding-or-reusing-tests.md) * [Starting Feast servers in TLS(SSL) Mode](how-to-guides/starting-feast-servers-tls-mode.md) * [Importing Features from dbt](how-to-guides/dbt-integration.md) +* [Entity Key Serialization (v2 to v3)](how-to-guides/entity-reserialization-of-from-v2-to-v3.md) * [Feature Quality Monitoring](how-to-guides/feature-monitoring.md) ## Reference @@ -143,6 +149,7 @@ * [Snowflake](reference/online-stores/snowflake.md) * [Redis](reference/online-stores/redis.md) * [Dragonfly](reference/online-stores/dragonfly.md) + * [Valkey](reference/online-stores/valkey.md) * [Datastore](reference/online-stores/datastore.md) * [DynamoDB](reference/online-stores/dynamodb.md) * [Bigtable](reference/online-stores/bigtable.md) @@ -157,17 +164,21 @@ * [SingleStore](reference/online-stores/singlestore.md) * [Milvus](reference/online-stores/milvus.md) * [MongoDB](reference/online-stores/mongodb.md) + * [Aerospike](reference/online-stores/aerospike.md) * [Elasticsearch](reference/online-stores/elasticsearch.md) * [Qdrant](reference/online-stores/qdrant.md) * [Faiss](reference/online-stores/faiss.md) * [Hybrid](reference/online-stores/hybrid.md) * [Registries](reference/registries/README.md) + * [Metadata](reference/registries/metadata.md) * [Local](reference/registries/local.md) * [S3](reference/registries/s3.md) * [GCS](reference/registries/gcs.md) * [SQL](reference/registries/sql.md) * [Snowflake](reference/registries/snowflake.md) + * [HDFS](reference/registries/hdfs.md) * [Remote](reference/registries/remote.md) + * [Registry permissions](reference/registry/registry-permissions.md) * [Providers](reference/providers/README.md) * [Local](reference/providers/local.md) * [Google Cloud Platform](reference/providers/google-cloud-platform.md) @@ -177,10 +188,15 @@ * [Snowflake](reference/compute-engine/snowflake.md) * [AWS Lambda (alpha)](reference/compute-engine/lambda.md) * [Spark (contrib)](reference/compute-engine/spark.md) + * [SparkApplication](reference/compute-engine/spark_application.md) + * [Apache Flink](reference/compute-engine/flink.md) * [Ray (contrib)](reference/compute-engine/ray.md) * [Feature repository](reference/feature-repository/README.md) * [feature\_store.yaml](reference/feature-repository/feature-store-yaml.md) * [.feastignore](reference/feature-repository/feast-ignore.md) + * [Registration inferencing](reference/feature-repository/registration-inferencing.md) +* [Kubernetes auth setup](reference/auth/kubernetes_auth_setup.md) +* [User token provisioning](reference/auth/user_token_provisioning.md) * [Feature servers](reference/feature-servers/README.md) * [Python feature server](reference/feature-servers/python-feature-server.md) * [\[Alpha\] Go feature server](reference/feature-servers/go-feature-server.md) @@ -191,7 +207,10 @@ * [\[Beta\] On demand feature view](reference/beta-on-demand-feature-view.md) * [\[Alpha\] Static Artifacts Loading](reference/alpha-static-artifacts.md) * [\[Alpha\] Vector Database](reference/alpha-vector-database.md) -* [\[Alpha\] Data quality monitoring](reference/dqm.md) +* [\[Alpha\] OpenAI-Compatible Vector Store API](reference/alpha-vector-database.md#alpha-openai-compatible-vector-store-api) + +* [Data Quality Monitoring](reference/dqm.md) +* [\[Deprecated\] Data quality monitoring (Great Expectations)](reference/dqm.md) * [\[Alpha\] Streaming feature computation with Denormalized](reference/denormalized.md) * [\[Alpha\] Feature View Versioning](reference/alpha-feature-view-versioning.md) * [OpenLineage Integration](reference/openlineage.md) @@ -221,3 +240,4 @@ * [ADR-0009: Contribution and Extensibility](adr/ADR-0009-contribution-extensibility.md) * [ADR-0010: Vector Database Integration](adr/ADR-0010-vector-database-integration.md) * [ADR-0011: Data Quality Monitoring](adr/ADR-0011-data-quality-monitoring.md) + * [ADR-0012: LabelView](adr/ADR-0012-label-view.md) diff --git a/docs/adr/ADR-0011-data-quality-monitoring.md b/docs/adr/ADR-0011-data-quality-monitoring.md index 55df3aa1ddd..657d219c48c 100644 --- a/docs/adr/ADR-0011-data-quality-monitoring.md +++ b/docs/adr/ADR-0011-data-quality-monitoring.md @@ -2,7 +2,7 @@ ## Status -Accepted +Superseded — The original external-library-based validation has been replaced by Feast's native [Feature Quality Monitoring](../how-to-guides/feature-monitoring.md) system (`feast monitor run`). ## Context @@ -12,79 +12,51 @@ Data quality issues can significantly impact ML model performance. Several compl - **Upstream pipeline bugs**: Bugs in upstream pipelines can cause invalid values to overwrite existing valid values in an online store. - **Training/serving skew**: Distribution shift between training and serving data can decrease model performance. -Feast needed a mechanism to validate data at retrieval time to catch these issues before they affect model training or serving. +Feast needed a mechanism to validate data to catch these issues before they affect model training or serving. ## Decision -Introduce a Data Quality Monitoring (DQM) module that validates datasets against user-curated rules, initially targeting historical retrieval (training dataset generation). +Introduce a Data Quality Monitoring (DQM) module that validates datasets against user-curated rules. -### Design +### Original Design (now replaced) -The validation process uses a **reference dataset** and a **profiler** pattern: +The original validation process used a **reference dataset** and a **profiler** pattern: 1. User prepares a reference dataset (saved from a known-good historical retrieval). 2. User defines a profiler function that produces a profile (set of expectations) from a dataset. 3. Validation is performed by comparing the tested dataset against the reference profile. -### Integration with Great Expectations +This approach was limited to historical retrieval only, required additional dependencies, and offered no built-in UI or automation. -The initial implementation uses [Great Expectations](https://greatexpectations.io/) as the validation engine: +### Current Design -```python -from feast.dqm.profilers.ge_profiler import ge_profiler -from great_expectations.dataset import Dataset -from great_expectations.core.expectation_suite import ExpectationSuite +The current system (`feast monitor run`) provides: -@ge_profiler -def my_profiler(dataset: Dataset) -> ExpectationSuite: - dataset.expect_column_max_to_be_between("column", 1, 2) - dataset.expect_column_values_to_not_be_null("important_feature") - return dataset.get_expectation_suite() -``` +- Automatic metric computation (null rates, percentiles, histograms) with no external dependencies +- Monitoring across batch data and serving logs +- CLI and REST API for automation +- Built-in UI monitoring dashboard +- Support for all offline store backends via SQL push-down -### Usage - -Validation is triggered during historical feature retrieval via a `validation_reference` parameter: - -```python -from feast import FeatureStore - -store = FeatureStore(".") - -job = store.get_historical_features(...) -df = job.to_df( - validation_reference=store - .get_saved_dataset("my_reference_dataset") - .as_reference(profiler=my_profiler) -) -``` - -If validation fails, a `ValidationFailed` exception is raised with details for all expectations that didn't pass. If validation succeeds, the materialized dataset is returned normally. - -### Key Decisions - -- **Profiler-based approach**: Users define their own validation rules via profiler functions rather than Feast prescribing fixed validation rules. -- **Great Expectations integration**: Leverages an established data validation framework rather than building custom validation logic. -- **Validation at retrieval time**: Validation is performed when datasets are materialized (`.to_df()` or `.to_arrow()`), not during ingestion. -- **ValidationReference as a registry object**: Saved datasets and their validation references are stored in the Feast registry for reuse. +See [Feature Quality Monitoring](../how-to-guides/feature-monitoring.md) for full documentation. ## Consequences ### Positive - Users can detect data quality issues before they affect model training. -- Flexible profiler pattern allows custom validation rules per use case. -- Integration with Great Expectations provides a rich set of built-in expectations. -- Reference datasets provide a baseline for detecting data drift. +- Native integration requires no extra dependencies. +- Covers both batch data and serving logs. +- Built-in UI provides immediate visibility into feature health. +- Baselines computed automatically on `feast apply`. ### Negative -- Currently limited to historical retrieval; online store write/read validation is planned but not yet implemented. -- Dependency on Great Expectations adds to the install footprint (optional via `feast[ge]`). -- Automatic profiling capabilities are limited; manual expectation crafting is recommended. +- Migration required from the original profiler-based approach. ## References -- Original RFC: Feast RFC-027: Data Quality Monitoring -- Implementation: `sdk/python/feast/dqm/`, `sdk/python/feast/saved_dataset.py` +- Original RFC: Feast RFC-027: Data Quality Monitoring +- Implementation: `sdk/python/feast/monitoring/` - Documentation: [Data Quality Monitoring](../reference/dqm.md) +- [Feature Quality Monitoring guide](../how-to-guides/feature-monitoring.md) diff --git a/docs/blog/feast-0-18-adds-snowflake-support-and-data-quality-monitoring.md b/docs/blog/feast-0-18-adds-snowflake-support-and-data-quality-monitoring.md index 4b4321e3259..8c587bdde9f 100644 --- a/docs/blog/feast-0-18-adds-snowflake-support-and-data-quality-monitoring.md +++ b/docs/blog/feast-0-18-adds-snowflake-support-and-data-quality-monitoring.md @@ -6,7 +6,7 @@ We are delighted to announce the release of Feast [0.18](https://github.com/feas * Snowflake offline store, which allows you to define and use features stored in Snowflake. * [Experimental] Saved Datasets, which allow training datasets to be persisted in an offline store. -* [Experimental] Data quality monitoring, which allows you to validate your training data with Great Expectations. Future work will allow you to detect issues with upstream data pipelines and check for training-serving skew. +* [Experimental] Data quality monitoring, which allows you to validate your training data. This has since been superseded by Feast's native [Feature Quality Monitoring](../how-to-guides/feature-monitoring.md) system. * Python feature server graduation from alpha status. * Performance improvements to on demand feature views, protobuf serialization and deserialization, and the Python feature server. @@ -22,7 +22,7 @@ Training datasets generated via `get_historical_features` can now be persisted i ### [Experimental] Data quality monitoring -Feast 0.18 includes the first milestone of our data quality monitoring work. Many users have requested ways to validate their training and serving data, as well as monitor for training-serving skew. Feast 0.18 allows users to validate their training data through an integration with [Great Expectations](https://greatexpectations.io/). Users can declare one of the previously generated training datasets as a reference for this validation by persisting it as a "saved dataset" (see previous section). More details about future milestones of data quality monitoring can be found [here](https://docs.feastsite.wpenginepowered.com/v/master/reference/data-quality). There's also a [tutorial on validating historical features](https://docs.feastsite.wpenginepowered.com/v/master/how-to-guides/validation/validating-historical-features) that demonstrates all new concepts in action. +Feast 0.18 includes the first milestone of our data quality monitoring work. Many users have requested ways to validate their training and serving data, as well as monitor for training-serving skew. Feast 0.18 allows users to validate their training data by declaring previously generated training datasets as a reference for validation, persisted as "saved datasets" (see previous section). This initial integration has since been superseded by Feast's native [Feature Quality Monitoring](../how-to-guides/feature-monitoring.md) system, which provides built-in metrics computation, drift detection, serving log monitoring, and a UI dashboard. ### Performance improvements diff --git a/docs/community.md b/docs/community.md index 640b5238b8b..c22c4c41dd4 100644 --- a/docs/community.md +++ b/docs/community.md @@ -2,7 +2,7 @@ ## Links & Resources -* [Come say hi on Slack!](https://communityinviter.com/apps/feastopensource/feast-the-open-source-feature-store) +* [Come say hi on Slack!](https://slack.feast.dev/) * As a part of the Linux Foundation, we ask community members to adhere to the [Linux Foundation Code of Conduct](https://events.linuxfoundation.org/about/code-of-conduct/) * [GitHub Repository](https://github.com/feast-dev/feast/): Find the complete Feast codebase on GitHub. * [Community Governance Doc](https://github.com/feast-dev/feast/blob/master/community): See the governance model of Feast, including who the maintainers are and how decisions are made. diff --git a/docs/getting-started/architecture/model-inference.md b/docs/getting-started/architecture/model-inference.md index 582657dbc43..5983cb63451 100644 --- a/docs/getting-started/architecture/model-inference.md +++ b/docs/getting-started/architecture/model-inference.md @@ -17,7 +17,7 @@ of model inference): *Note: online features can be sourced from batch, streaming, or request data sources.* -These three approaches have different tradeoffs but, in general, have significant implementation differences. +These four approaches have different tradeoffs but, in general, have significant implementation differences. ## 1. Online Model Inference with Online Features Online model inference with online features is a powerful approach to serving data-driven machine learning applications. @@ -78,7 +78,7 @@ if features.to_dict().get('user_data:model_predictions') is None: model_predictions = model_server.predict(features) store.write_to_online_store(feature_view_name="user_data", df=pd.DataFrame(model_predictions)) ``` -Note that in this case a seperate call to `write_to_online_store` is required when the underlying data changes and +Note that in this case a separate call to `write_to_online_store` is required when the underlying data changes and predictions change along with it. ```python diff --git a/docs/getting-started/components/README.md b/docs/getting-started/components/README.md index b07b5f8389e..bf49563d6f6 100644 --- a/docs/getting-started/components/README.md +++ b/docs/getting-started/components/README.md @@ -20,6 +20,10 @@ [compute-engine.md](compute-engine.md) {% endcontent-ref %} +{% content-ref url="stream-processor.md" %} +[stream-processor.md](stream-processor.md) +{% endcontent-ref %} + {% content-ref url="provider.md" %} [provider.md](provider.md) {% endcontent-ref %} diff --git a/docs/getting-started/components/authz_manager.md b/docs/getting-started/components/authz_manager.md index eae3fece50b..12fdf2f39e5 100644 --- a/docs/getting-started/components/authz_manager.md +++ b/docs/getting-started/components/authz_manager.md @@ -44,10 +44,11 @@ The server, in turn, uses the same OIDC server to validate the token and extract Some assumptions are made in the OIDC server configuration: * The OIDC token refers to a client with roles matching the RBAC roles of the configured `Permission`s (*) -* The roles are exposed in the access token under `resource_access..roles` -* The JWT token is expected to have a verified signature and not be expired. The Feast OIDC token parser logic validates for `verify_signature` and `verify_exp` so make sure that the given OIDC provider is configured to meet these requirements. -* The `preferred_username` should be part of the JWT token claim. +* The roles are exposed in the access token under `resource_access..roles` (Keycloak) or in the top-level `roles` claim (Entra ID app roles). Roles found in both are merged. +* The JWT token is expected to have a verified signature and not be expired. The Feast OIDC token parser logic validates for `verify_signature` and `verify_exp` so make sure that the given OIDC provider is configured to meet these requirements. The token's audience and issuer claims are **not** verified by default; both checks can be enabled with the `audience` and `issuer` options (see [Server-Side Configuration](#server-side-configuration)). +* The username is read from the first of `preferred_username`, `upn`, `azp`, `appid`, `sub` present in the token. Entra ID client-credentials (app-only) tokens carry no user claim, so they authenticate as the calling application. * For `GroupBasedPolicy` support, the `groups` claim should be present in the access token (requires a "Group Membership" protocol mapper in Keycloak). +* **Entra ID limitation**: Group claims use object IDs (GUIDs) instead of names, and are omitted entirely when a user exceeds the group overage threshold. GroupBasedPolicy must reference GUIDs and cannot be used for principals with large group memberships. (*) Please note that **the role match is case-sensitive**, e.g. the name of the role in the OIDC server and in the `Permission` configuration must be exactly the same. @@ -69,6 +70,16 @@ For example, the access token for a client `app` of a user with `reader` role an } ``` +A Microsoft Entra ID (Azure AD) client-credentials (app-only) token has no user claim; the application authenticates as itself, and its app roles arrive in the top-level `roles` claim: +```json +{ + "azp": "11111111-2222-3333-4444-555555555555", + "roles": [ + "reader" + ] +} +``` + #### Server-Side Configuration The server requires `auth_discovery_url` and `client_id` to validate incoming JWT tokens via JWKS: @@ -94,6 +105,36 @@ auth: Setting `verify_ssl: false` disables TLS certificate verification for all OIDC provider communication (discovery, JWKS, token endpoint). Only use this in development or internal environments where you accept the security risk. {% endhint %} +By default the server verifies only the token's signature and expiry: any validly-signed, unexpired token from the configured provider is accepted regardless of the audience it was minted for, and authorization (role matching) is the only remaining gate. For defense in depth, set `audience` and/or `issuer` to additionally require a matching `aud` / `iss` claim: + +```yaml +auth: + type: oidc + client_id: _CLIENT_ID_ + auth_discovery_url: https://login.example.com/.well-known/openid-configuration + audience: api://feast-feature-server + issuer: https://login.example.com/realms/master +``` + +A token whose `aud` (or `iss`) claim does not match is rejected at authentication. The two options are independent; leave one unset to skip that check. + +{% hint style="warning" %} +Set these to the values your IdP puts **in the token itself**, which are not always the ones in the discovery document. For example, Microsoft Entra ID commonly issues v1.0 tokens (`iss: https://sts.windows.net//`, `aud: api://`) even when `auth_discovery_url` points at the v2.0 endpoint. That setup keeps working with these options unset, or set to the v1.0 values — but copying the v2.0 issuer from the discovery document would reject every v1.0 token. +{% endhint %} + +To validate token signatures the server fetches the provider's JWKS document and caches it, refetching when the cache expires or when a token presents an unknown key id. Two options tune that behavior: + +```yaml +auth: + type: oidc + client_id: _CLIENT_ID_ + auth_discovery_url: https://login.example.com/.well-known/openid-configuration + jwks_cache_lifespan_seconds: 300 # default; how long the fetched key set is reused + jwks_request_timeout_seconds: 10 # default; network timeout for the JWKS fetch +``` + +`jwks_cache_lifespan_seconds` also bounds how long a key the provider has **revoked** continues to validate tokens, so lower it if your provider rotates or revokes aggressively; each reduction costs proportionally more JWKS fetches. Key rotations that introduce a new key id are picked up immediately regardless of this setting, because an unknown key id triggers a refetch. `jwks_request_timeout_seconds` bounds how long an unresponsive provider can block request serving. Both must be greater than zero. + #### Client-Side Configuration The client supports multiple token source modes. The SDK resolves tokens in the following priority order: diff --git a/docs/getting-started/components/compute-engine.md b/docs/getting-started/components/compute-engine.md index 60da1575932..8b69b9d1a8b 100644 --- a/docs/getting-started/components/compute-engine.md +++ b/docs/getting-started/components/compute-engine.md @@ -8,7 +8,7 @@ functions (UDFs). A materialization task abstracts over specific technologies or frameworks that are used to materialize data. It allows users to use a pure local serialized approach (which is the default LocalComputeEngine), or delegates the -materialization to seperate components (e.g. AWS Lambda, as implemented by the the LambdaComputeEngine). +materialization to separate components (e.g. AWS Lambda, as implemented by the LambdaComputeEngine). If the built-in engines are not sufficient, you can create your own custom materialization engine. Please see [this guide](../../how-to-guides/customizing-feast/creating-a-custom-compute-engine.md) for more details. @@ -24,7 +24,7 @@ engines. | SparkComputeEngine | Runs on Apache Spark, designed for large-scale distributed feature generation. | ✅ | | | SnowflakeComputeEngine | Runs on Snowflake, designed for scalable feature generation using Snowflake SQL. | ✅ | | | LambdaComputeEngine | Runs on AWS Lambda, designed for serverless feature generation. | ✅ | | -| FlinkComputeEngine | Runs on Apache Flink, designed for stream processing and real-time feature generation. | ❌ | | +| FlinkComputeEngine | Runs on Apache Flink, designed for distributed feature generation through PyFlink Table API. | ✅ | | | RayComputeEngine | Runs on Ray, designed for distributed feature generation and machine learning workloads. | ✅ | | ``` @@ -156,4 +156,4 @@ DAG nodes are defined as follows: +----------------+ +----------------+ | OnlineStoreWrite| OfflineStoreWrite| +----------------+ +----------------+ -``` \ No newline at end of file +``` diff --git a/docs/getting-started/components/feature-server.md b/docs/getting-started/components/feature-server.md index 4d961054ecb..1b5521a2b86 100644 --- a/docs/getting-started/components/feature-server.md +++ b/docs/getting-started/components/feature-server.md @@ -37,6 +37,8 @@ The Feature Server operates as a stateless service backed by two key components: | `/push` | Pushes feature data to the online and/or offline store. | | `/materialize` | Materializes features within a specific time range to the online store. | | `/materialize-incremental` | Incrementally materializes features up to the current timestamp. | -| `/retrieve-online-documents` | Supports Vector Similarity Search for RAG (Alpha end-ponit) | +| `/search` | Vector similarity search for RAG (Alpha endpoint) | +| `/v1/vector_stores/{id}/search` | OpenAI-compatible vector search with server-side embedding | +| `/retrieve-online-documents` | **Deprecated.** Use `/search` instead. | | `/docs` | API Contract for available endpoints | diff --git a/docs/getting-started/concepts/README.md b/docs/getting-started/concepts/README.md index 47ef8553781..06dbc5f9897 100644 --- a/docs/getting-started/concepts/README.md +++ b/docs/getting-started/concepts/README.md @@ -16,6 +16,14 @@ [entity.md](entity.md) {% endcontent-ref %} +{% content-ref url="feast-types.md" %} +[feast-types.md](feast-types.md) +{% endcontent-ref %} + +{% content-ref url="feature-repo.md" %} +[feature-repo.md](feature-repo.md) +{% endcontent-ref %} + {% content-ref url="feature-view.md" %} [feature-view.md](feature-view.md) {% endcontent-ref %} @@ -44,6 +52,10 @@ [dataset.md](dataset.md) {% endcontent-ref %} +{% content-ref url="label-view.md" %} +[label-view.md](label-view.md) +{% endcontent-ref %} + {% content-ref url="permission.md" %} [permission.md](permission.md) {% endcontent-ref %} diff --git a/docs/getting-started/concepts/dataset.md b/docs/getting-started/concepts/dataset.md index 3fabc48a140..c86c13503ed 100644 --- a/docs/getting-started/concepts/dataset.md +++ b/docs/getting-started/concepts/dataset.md @@ -1,6 +1,6 @@ # \[Alpha] Saved dataset -Feast datasets allow for conveniently saving dataframes that include both features and entities to be subsequently used for data analysis and model training. [Data Quality Monitoring](https://docs.google.com/document/d/110F72d4NTv80p35wDSONxhhPBqWRwbZXG4f9mNEMd98) was the primary motivation for creating dataset concept. +Feast datasets allow for conveniently saving dataframes that include both features and entities to be subsequently used for data analysis and model training. Data Quality Monitoring was the original motivation for creating the dataset concept. Dataset's metadata is stored in the Feast registry and raw data (features, entities, additional input keys and timestamp) is stored in the [offline store](../components/offline-store.md). diff --git a/docs/getting-started/concepts/feast-types.md b/docs/getting-started/concepts/feast-types.md index 7d864b6a18f..62cabcd3940 100644 --- a/docs/getting-started/concepts/feast-types.md +++ b/docs/getting-started/concepts/feast-types.md @@ -8,6 +8,7 @@ Feast's type system is built on top of [protobuf](https://github.com/protocolbuf Feast supports the following categories of data types: - **Primitive types**: numerical values (`Int32`, `Int64`, `Float32`, `Float64`), `String`, `Bytes`, `Bool`, and `UnixTimestamp`. +- **Zoned timestamp type**: `ZonedTimestamp` stores a timezone-aware datetime as both the UTC instant and its originating zone, so the original wall-clock zone round-trips losslessly. This differs from `UnixTimestamp`, which is always decoded as UTC and discards the source zone. Use `ZonedTimestamp` when local time-of-day or the offset/zone itself is meaningful. It must be explicitly declared in schema (it is not inferred by any backend), and is not supported as an entity key. - **Domain-specific primitives**: `PdfBytes` (PDF binary data for RAG/document pipelines) and `ImageBytes` (image binary data for multimodal pipelines). These are semantic aliases over `Bytes` and must be explicitly declared in schema — no backend infers them. - **UUID types**: `Uuid` and `TimeUuid` for universally unique identifiers. Stored as strings at the proto level but deserialized to `uuid.UUID` objects in Python. - **Array types**: ordered lists of any primitive type, e.g. `Array(Int64)`, `Array(String)`, `Array(Uuid)`. diff --git a/docs/getting-started/concepts/feature-view.md b/docs/getting-started/concepts/feature-view.md index 5be9b287305..27ded82cb84 100644 --- a/docs/getting-started/concepts/feature-view.md +++ b/docs/getting-started/concepts/feature-view.md @@ -91,7 +91,7 @@ If the `schema` parameter is not specified in the creation of the feature view, "Entity aliases" can be specified to join `entity_dataframe` columns that do not match the column names in the source table of a FeatureView. -This could be used if a user has no control over these column names or if there are multiple entities are a subclass of a more general entity. For example, "spammer" and "reporter" could be aliases of a "user" entity, and "origin" and "destination" could be aliases of a "location" entity as shown below. +This could be used if a user has no control over these column names or if multiple entities are subclasses of a more general entity. For example, "spammer" and "reporter" could be aliases of a "user" entity, and "origin" and "destination" could be aliases of a "location" entity as shown below. It is suggested that you dynamically specify the new FeatureView name using `.with_name` and `join_key_map` override using `.with_join_key_map` instead of needing to register each new copy. @@ -322,4 +322,4 @@ def driver_hourly_stats_stream(df: DataFrame): ) ``` -See [here](https://github.com/feast-dev/streaming-tutorial) for a example of how to use stream feature views to register your own streaming data pipelines in Feast. +See [here](https://github.com/feast-dev/streaming-tutorial) for an example of how to use stream feature views to register your own streaming data pipelines in Feast. diff --git a/docs/getting-started/concepts/label-view.md b/docs/getting-started/concepts/label-view.md index b2f378d759f..8990cedb85c 100644 --- a/docs/getting-started/concepts/label-view.md +++ b/docs/getting-started/concepts/label-view.md @@ -30,7 +30,7 @@ Before using label views, you need: **Generate training datasets.** Compose label views with feature views in a `FeatureService` and retrieve features + labels together with point-in-time correctness. -**Annotate in the UI.** Configure annotation profiles so data scientists can label data directly in the Feast UI — entity forms, document spans, bulk review, or active learning. +**Data label in the UI.** Configure data labeling profiles (annotation profiles) so data scientists can label data directly in the Feast UI — entity forms, document spans, bulk review, or active learning. ## Feedback vs Expectations @@ -85,10 +85,10 @@ store.push("agent_feedback_labels_push_source", labels_df) | Use a **FeatureView** when… | Use a **LabelView** when… | |---|---| -| Data is observational and append-only | Data is a judgment or annotation | +| Data is observational and append-only | Data is a judgment or data label (annotation) | | One source writes the data | Multiple labelers may disagree | | No conflict resolution needed | You need governed conflict resolution | -| No labeling UI needed | You want structured annotation workflows | +| No labeling UI needed | You want structured data labeling workflows (annotation) | ## How Label Views Work @@ -164,14 +164,14 @@ store.push("agent_feedback_labels_push_source", labels_df) Each push appends to the offline store (full history retained) and updates the online store (latest value per key). -### Step 3: Annotate in the Feast UI +### Step 3: Data label in the Feast UI -Open the label view in the Feast UI **Annotate** tab. The UI reads annotation tags and shows the right workflow: +Open the label view in the Feast UI **Data Labeling** tab (Annotate tab). The UI reads data labeling tags (annotation tags) and shows the right workflow: 1. Open **Label Views** in the sidebar -2. Select a label view (check the **Annotation** badge on the list page) -3. Go to the **Annotate** tab -4. Choose a method (Entity Form, Document Span, Review & Edit, or Active Learning) +2. Select a label view (check the **Data Labeling** badge on the list page) +3. Go to the **Data Labeling** tab (Annotate tab) +4. Choose a data labeling method (annotation method): Entity Form, Document Span, Review & Edit, or Active Learning 5. Submit labels — they are pushed to the label view's `PushSource` ### Step 4: Join labels with features for training @@ -193,16 +193,16 @@ training_df = store.get_historical_features( Training pipelines get features and resolved labels in one retrieval call. -## Annotation Profiles +## Data Labeling Profiles (Annotation Profiles) -Annotation profiles configure **how** labels are created in the UI. Set them via `tags` — no schema changes required. +Data labeling profiles (annotation profiles) configure **how** labels are created in the UI. Set them via `tags` — no schema changes required. ### Supported profiles | Profile | Best for | UI experience | |---|---|---| | `entity-form` | RLHF, safety review, per-entity feedback | Form — one entity at a time | -| `document-span` | RAG chunk labeling, span annotation | Load document, label chunks | +| `document-span` | RAG chunk labeling, span labeling (annotation) | Load document, label chunks | | `table` | Bulk review, correcting existing labels | Editable table with dropdowns | | `active-learning` | Label high-value unlabeled entities | Queue from a reference feature view | @@ -215,7 +215,7 @@ Answer one question: 3. **"I need to correct labels in bulk"** → `table` 4. **"I want to label only the most valuable unlabeled items"** → `active-learning` (requires `reference_feature_view`) -The **Annotate** tab shows only relevant methods per profile: +The **Data Labeling** tab (Annotate tab) shows only relevant data labeling methods (annotation methods) per profile: | Profile | Methods shown | |---|---| @@ -228,7 +228,7 @@ The **Annotate** tab shows only relevant methods per profile: | Tag | Purpose | Example values | |---|---|---| -| `feast.io/labeling-method` | Primary UI workflow | `entity-form`, `document-span`, `table` | +| `feast.io/labeling-method` | Primary data labeling method (annotation method) | `entity-form`, `document-span`, `table` | | `feast.io/field-role:` | Semantic role of a field | `feedback`, `expectation`, `label`, `metadata`, `content`, `span_start`, `span_end` | | `feast.io/label-values:` | Allowed label values | `relevant,irrelevant` | | `feast.io/label-widget:` | Input widget type | `enum`, `binary`, `text`, `number` | @@ -325,7 +325,7 @@ When multiple labelers write different values for the same entity, `ConflictPoli |---|---| | `LAST_WRITE_WINS` | Default. Most recent write wins. | | `LABELER_PRIORITY` | Trusted labelers override others (e.g. human over LLM judge). | -| `MAJORITY_VOTE` | Consensus labeling (e.g. multiple annotators on RAG chunks). | +| `MAJORITY_VOTE` | Consensus labeling (e.g. multiple labelers on RAG chunks). | {% hint style="info" %} Conflict policies apply to the **offline store** (training). The **online store** always uses last-write-wins. Full label history is always retained in the offline store. @@ -337,9 +337,9 @@ Conflict policies apply to the **offline store** (training). The **online store* **Tag field roles.** Set `feast.io/field-role:` to `feedback` or `expectation` so your team and UI know what each field means. -**Match conflict policy to label type.** Use `LABELER_PRIORITY` when humans correct automated judges. Use `MAJORITY_VOTE` for multi-annotator consensus. Use `LAST_WRITE_WINS` for simple feedback streams. +**Match conflict policy to label type.** Use `LABELER_PRIORITY` when humans correct automated judges. Use `MAJORITY_VOTE` for multi-labeler consensus. Use `LAST_WRITE_WINS` for simple feedback streams. -**Link to features.** Set `reference_feature_view` so the UI and documentation show which feature view the labels annotate. +**Link to features.** Set `reference_feature_view` so the UI and documentation show which feature view the labels apply to. **Separate noisy feedback from stable ground truth.** When possible, put expectations in dedicated fields or views with stricter writer conventions (human-only). @@ -347,10 +347,10 @@ Conflict policies apply to the **offline store** (training). The **online store* * Conflict policies are enforced on offline reads only; online store is always last-write-wins. * `LABELER_PRIORITY` requires explicit labeler ordering configuration. -* Annotation profiles are UI configuration via tags — not enforced at the SDK write path. +* Data labeling profiles (annotation profiles) are UI configuration via tags — not enforced at the SDK write path. ## Next steps -* [Feature view](feature-view.md) — immutable features that label views annotate +* [Feature view](feature-view.md) — immutable features that label views apply labels to * [Feature retrieval](feature-retrieval.md) — point-in-time joins for training * [ADR-0012: LabelView](../../adr/ADR-0012-label-view.md) — full design rationale diff --git a/docs/getting-started/concepts/point-in-time-joins.md b/docs/getting-started/concepts/point-in-time-joins.md index 55672209005..9e385774327 100644 --- a/docs/getting-started/concepts/point-in-time-joins.md +++ b/docs/getting-started/concepts/point-in-time-joins.md @@ -62,3 +62,26 @@ Below is the resulting joined training dataframe. It contains both the original Three feature rows were successfully joined to the entity dataframe rows. The first row in the entity dataframe was older than the earliest feature rows in the feature view and could not be joined. The last row in the entity dataframe was outside of the TTL window \(the event happened 11 hours after the feature row\) and also couldn't be joined. +## Retrieving features as of the event time + +By default, point-in-time joins only constrain the feature's event timestamp. If a data source also has a `created_timestamp_column`, it is used to deduplicate rows that share an event timestamp \(the row with the highest created timestamp wins\), but it is not otherwise filtered. This means a value that was backfilled or corrected *after* an entity dataframe timestamp can still be returned for it. + +To restrict retrieval to feature values that were already available at each entity row's timestamp, pass `filter_by_created_timestamp=True`: + +```python +training_df = store.get_historical_features( + entity_df=entity_df, + features = [ + 'driver_hourly_stats:trips_today', + 'driver_hourly_stats:earnings_today' + ], + filter_by_created_timestamp=True, +) +``` + +This adds a `created_timestamp <= entity_timestamp` condition to the join, so each entity dataframe row only sees feature values whose created timestamp is at or before its own timestamp. This is useful to keep backfilled values from leaking into training data, and to reproduce what the online store would have served at each event time \(assuming the created timestamp reflects when the value became available online\). + +{% hint style="info" %} +Rows with a NULL created timestamp are excluded when the flag is enabled, so the column should be non-null. Not all offline stores support this flag yet; unsupported stores raise an error rather than silently ignoring it. +{% endhint %} + diff --git a/docs/getting-started/genai.md b/docs/getting-started/genai.md index f65aeac85e2..d9f682af1f0 100644 --- a/docs/getting-started/genai.md +++ b/docs/getting-started/genai.md @@ -15,6 +15,7 @@ Feast integrates with popular vector databases to store and retrieve embedding v * **Elasticsearch**: Scalable vector search capabilities * **Postgres with PGVector**: SQL-based vector operations * **Qdrant**: Purpose-built vector database integration +* **ScyllaDB**: Native `vector` type with HNSW ANN index, full `retrieve_online_documents_v2` support These integrations allow you to: - Store embeddings as features @@ -225,7 +226,8 @@ The MCP integration uses the `fastapi_mcp` library to automatically transform yo The fastapi_mcp integration automatically exposes your Feast feature server's FastAPI endpoints as MCP tools. This means AI assistants can: * **Call `/get-online-features`** to retrieve features from the feature store -* **Call `/retrieve-online-documents`** to perform vector similarity search +* **Call `/search`** to perform vector similarity search (`/retrieve-online-documents` is a deprecated alias) +* **Call `/v1/vector_stores/{feature_view}/search`** for OpenAI-compatible text search with server-side embedding * **Call `/write-to-online-store`** to persist agent state (memory, notes, interaction history) * **Use `/health`** to check server status diff --git a/docs/getting-started/quickstart.md b/docs/getting-started/quickstart.md index aa56d09b1d8..a05830d73e1 100644 --- a/docs/getting-started/quickstart.md +++ b/docs/getting-started/quickstart.md @@ -4,7 +4,7 @@ Feast (Feature Store) is an open-source feature store designed to facilitate the management and serving of machine learning features in a way that supports both batch and real-time applications. -* *For Data Scientists*: Feast is a a tool where you can easily define, store, and retrieve your features for both model development and model deployment. By using Feast, you can focus on what you do best: build features that power your AI/ML models and maximize the value of your data. +* *For Data Scientists*: Feast is a tool where you can easily define, store, and retrieve your features for both model development and model deployment. By using Feast, you can focus on what you do best: build features that power your AI/ML models and maximize the value of your data. * *For MLOps Engineers*: Feast is a library that allows you to connect your existing infrastructure (e.g., online database, application server, microservice, analytical database, and orchestration tooling) that enables your Data Scientists to ship features for their models to production using a friendly SDK without having to be concerned with software engineering challenges that occur from serving real-time production systems. By using Feast, you can focus on maintaining a resilient system, instead of implementing features for Data Scientists. diff --git a/docs/how-to-guides/customizing-feast/README.md b/docs/how-to-guides/customizing-feast/README.md index 91c04e2f35a..054200c67e5 100644 --- a/docs/how-to-guides/customizing-feast/README.md +++ b/docs/how-to-guides/customizing-feast/README.md @@ -15,8 +15,8 @@ Below are some guides on how to add new custom components: [adding-support-for-a-new-online-store.md](adding-support-for-a-new-online-store.md) {% endcontent-ref %} -{% content-ref url="creating-a-custom-materialization-engine.md" %} -[creating-a-custom-materialization-engine.md](creating-a-custom-materialization-engine.md) +{% content-ref url="creating-a-custom-compute-engine.md" %} +[creating-a-custom-compute-engine.md](creating-a-custom-compute-engine.md) {% endcontent-ref %} {% content-ref url="creating-a-custom-provider.md" %} diff --git a/docs/how-to-guides/customizing-feast/adding-a-new-offline-store.md b/docs/how-to-guides/customizing-feast/adding-a-new-offline-store.md index d1ca100bf74..35a46acf942 100644 --- a/docs/how-to-guides/customizing-feast/adding-a-new-offline-store.md +++ b/docs/how-to-guides/customizing-feast/adding-a-new-offline-store.md @@ -51,7 +51,7 @@ To fully implement the interface for the offline store, you will need to impleme * `pull_latest_from_table_or_query` is invoked when running materialization (using the `feast materialize` or `feast materialize-incremental` commands, or the corresponding `FeatureStore.materialize()` method. This method pull data from the offline store, and the `FeatureStore` class takes care of writing this data into the online store. * `get_historical_features` is invoked when reading values from the offline store using the `FeatureStore.get_historical_features()` method. Typically, this method is used to retrieve features when training ML models. * (optional) `offline_write_batch` is a method that supports directly pushing a pyarrow table to a feature view. Given a feature view with a specific schema, this function should write the pyarrow table to the batch source defined. More details about the push api can be found [here](../docs/reference/data-sources/push.md). This method only needs implementation if you want to support the push api in your offline store. -* (optional) `pull_all_from_table_or_query` is a method that pulls all the data from an offline store from a specified start date to a specified end date. This method is only used for **SavedDatasets** as part of data quality monitoring validation. +* (optional) `pull_all_from_table_or_query` is a method that pulls all the data from an offline store from a specified start date to a specified end date. This method is used for **SavedDatasets** and as a fallback compute path for the [Feature Quality Monitoring](../../how-to-guides/feature-monitoring.md) system (backends without native SQL push-down). * (optional) `write_logged_features` is a method that takes a pyarrow table or a path that points to a parquet file and writes the data to a defined source defined by `LoggingSource` and `LoggingConfig`. This method is only used internally for **SavedDatasets**. {% code title="feast_custom_offline_store/file.py" %} diff --git a/docs/how-to-guides/feast-operator/01-project-provisioning.md b/docs/how-to-guides/feast-operator/01-project-provisioning.md index b54ce57eeeb..787ef173403 100644 --- a/docs/how-to-guides/feast-operator/01-project-provisioning.md +++ b/docs/how-to-guides/feast-operator/01-project-provisioning.md @@ -2,7 +2,8 @@ The operator needs a Feast feature repository (a directory containing `feature_store.yaml` and Python feature-view definitions) to work from. `spec.feastProjectDir` controls how that -directory is created inside the pods. Exactly one of `git` or `init` must be set. +directory is created inside the pods. When `feastProjectDir` is specified, exactly one of +`git`, `init`, or `packaged` must be set. --- @@ -89,7 +90,7 @@ feastProjectDir: ### Full `git` field reference | Field | Type | Description | -|-------|------|-------------| +| ------- | ------ | ------------- | | `url` | string | Repository URL (HTTPS or SSH) | | `ref` | string | Branch, tag, or commit SHA. Defaults to the remote HEAD | | `featureRepoPath` | string | Relative path within the repo to the feature repository directory. Default: `feature_repo` | @@ -151,9 +152,90 @@ feastProjectDir: --- +## Option C — Use a repository packaged in an image (`feastProjectDir.packaged`) + +Use `packaged` when the feature repository is built into a feature-server image. This is +useful in air-gapped environments and in release workflows where feature definitions and +their Python dependencies are promoted together as an immutable image. + +```yaml +apiVersion: feast.dev/v1 +kind: FeatureStore +metadata: + name: packaged-feature-store +spec: + feastProject: credit_scoring + feastProjectDir: + packaged: + image: registry.example.com/feature-server@sha256:0123456789abcdef + featureRepoPath: /opt/feast/feature_repo +``` + +The repository can be added to a Feast feature-server image with a Dockerfile such as: + +```dockerfile +FROM quay.io/feastdev/feature-server:latest +COPY feature_repo/ /opt/feast/feature_repo/ +``` + +`featureRepoPath` must be a canonical absolute, non-root path: do not use `.`, `..`, +repeated separators, or a trailing separator. Put it outside operator-mounted locations +such as `/feast-data`; a volume mounted there would hide files baked into the image. When +init containers are enabled, the packaged path also must not equal, contain, or be +contained by the staged repository path. + +With init containers enabled (the default), each Pod starts in this order: + +1. `feast-init` replaces the operator-managed staged repository with a fresh copy of the + repository from `packaged.featureRepoPath`. With the default storage configuration, + for example, it copies `/opt/feast/feature_repo` from the image to + `/feast-data//feature_repo`. +2. In the staged copy only, `feast-init` replaces `feature_store.yaml` (if exists in the + baked image) with the configuration generated from the FeatureStore resource. The file + baked into the image is not modified. The Python feature definitions come from the + packaged repository, while the FeatureStore resource remains authoritative for runtime + configuration. +3. When `services.runFeastApplyOnInit` is omitted or `true` (the default), `feast-apply` + runs `feast apply` from the staged repository using the packaged image. Setting it to + `false` skips only this step; repository staging still occurs. +4. The Feast service containers start with the staged repository as their working + directory. + +The repository baked into the image is therefore the source artifact, while the staged +repository is the runtime copy used by `feast apply` and the Feast services. + +For a baked repository whose own `feature_store.yaml` must remain authoritative, disable +init containers: + +```yaml +services: + disableInitContainers: true +``` + +In that mode, Feast service containers use `featureRepoPath` directly and neither staging +nor `feast apply` runs during pod initialization. The Operator does not update the registry, +so `feast apply` must be handled separately—for example, by CI/CD or a separately managed +Kubernetes Job or CronJob—whenever the packaged feature definitions change. + +The packaged `image` is optional. When set, it is the default for repository initialization, +`feast apply`, and Feast services. `services.initImage` takes precedence for the +`feast-init` and `feast-apply` init containers, while an explicit image on an individual +service takes precedence for that service. When the packaged image is omitted, the operator +uses `RELATED_IMAGE_FEATURE_SERVER` or its built-in feature-server image fallback. + +### Full `packaged` field reference + +| Field | Type | Required | Description | +|-------|------|----------|-------------| +| `featureRepoPath` | string | yes | Canonical absolute, non-root path to the feature repository in the image; it must not overlap the staged repository path | +| `image` | string | no | Image containing the repository; defaults to the operator feature-server image | + +--- + ## `feast apply` on startup -By default, when the init container completes (git clone or `feast init`), the operator runs +By default, when repository initialization completes (git clone, `feast init`, or packaged +repository staging), the operator runs `feast apply` before starting the servers. This registers all feature definitions with the registry. @@ -175,9 +257,8 @@ services: ## When `feastProjectDir` is omitted -If neither `git` nor `init` is set, the operator mounts an empty directory. In this case -you must supply a `feature_store.yaml` through another mechanism (e.g. a ConfigMap volume -mount via `services.volumes` + `volumeMounts`). +If `feastProjectDir` is not set, the operator defaults to `feastProjectDir.init: {}` and +creates a local template repository. --- @@ -188,3 +269,4 @@ mount via `services.volumes` + `volumeMounts`). - [Sample: private git repo with token](https://github.com/feast-dev/feast/blob/stable/infra/feast-operator/config/samples/v1_featurestore_git_token.yaml) - [Sample: monorepo with featureRepoPath](https://github.com/feast-dev/feast/blob/stable/infra/feast-operator/config/samples/v1_featurestore_git_repopath.yaml) - [Sample: feast init](https://github.com/feast-dev/feast/blob/stable/infra/feast-operator/config/samples/v1_featurestore_init.yaml) +- [Sample: packaged feature repository](https://github.com/feast-dev/feast/blob/stable/infra/feast-operator/config/samples/v1_featurestore_packaged.yaml) diff --git a/docs/how-to-guides/feast-operator/05-security.md b/docs/how-to-guides/feast-operator/05-security.md index 4c28aba9805..002c3a9f846 100644 --- a/docs/how-to-guides/feast-operator/05-security.md +++ b/docs/how-to-guides/feast-operator/05-security.md @@ -39,7 +39,7 @@ to subjects using standard Kubernetes `ClusterRoleBinding` or `RoleBinding` reso > Kubernetes auth requires all services to be exposed as servers (the controller rejects > partial configurations where some services are local while RBAC is enabled). -**SDK docs**: [Feast RBAC](../reference/auth/rbac.md) +**SDK docs**: [Feast RBAC](../../getting-started/architecture/rbac.md) --- @@ -62,8 +62,20 @@ stringData: client_secret: username: # used for client-credentials flow password: + audience: # optional: reject tokens whose aud claim differs + issuer: # optional: reject tokens whose iss claim differs ``` +The optional `audience` and `issuer` keys enable audience and issuer claim verification on the standard OIDC/JWKS validation path; when omitted, the `aud` and `iss` claims are not checked. Set them to the values your IdP puts in the token itself, which are not always the ones in the discovery document (see [OIDC Authorization](../../getting-started/components/authz_manager.md#oidc-authorization)). The Secret key `issuer` is distinct from the CR's `issuerUrl`, which selects the discovery endpoint and plays no part in claim verification. Kubernetes ServiceAccount tokens (validated via TokenReview) and intra-server communication follow separate paths and are not subject to these checks. + +{% hint style="warning" %} +Before enabling these, three operational caveats: + +* **Existing Secret keys take effect on operator upgrade.** Keys named `audience` or `issuer` already present in the referenced Secret were previously ignored; after upgrading they are forwarded to every Feast pod. +* **Your IdP must mint matching tokens for Feast's own clients.** Feast's client-credentials flow requests no audience, so in multi-service topologies (e.g. a remote registry) and for the UI's browser tokens, the IdP must be configured to issue tokens carrying the expected claims (e.g. a Keycloak audience mapper), or inter-service calls will be rejected. +* **Secret edits are not watched.** Changes to these keys apply on the next reconcile or pod restart, not immediately. +{% endhint %} + Reference the Secret from the CR: ```yaml @@ -92,7 +104,7 @@ authz: caCertConfigMap: oidc-ca-cert # ConfigMap with CA cert for SSL verification ``` -**SDK docs**: [Feast OIDC Auth](../reference/auth/oidc.md) +**SDK docs**: [Feast OIDC Auth](../../getting-started/components/authz_manager.md#oidc-authorization) --- diff --git a/docs/how-to-guides/feast-operator/06-batch-and-jobs.md b/docs/how-to-guides/feast-operator/06-batch-and-jobs.md index fd513168c54..71e7b6b95dc 100644 --- a/docs/how-to-guides/feast-operator/06-batch-and-jobs.md +++ b/docs/how-to-guides/feast-operator/06-batch-and-jobs.md @@ -50,12 +50,53 @@ spec: configMapKey: config # key inside the ConfigMap (default: "config") ``` +### SparkApplication batch engine (optional) + +For Bring Your Own Spark on Kubernetes, use `spark_application` instead of in-process Spark. +The Feast Operator auto-creates RBAC for this type. See +[SparkApplication](../reference/compute-engine/spark_application.md) for the full config reference. +Build an image from the reference +[Dockerfile](https://github.com/feast-dev/feast/blob/master/sdk/python/feast/infra/compute_engines/spark_application/Dockerfile) +(or equivalent): + +```yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: feast-spark-application-engine +data: + config: | + type: spark_application + image: my-registry.example.com/feast-spark-driver:latest + namespace: feast + executor_instances: 2 + driver_memory: "2g" + executor_memory: "2g" +``` + +```yaml +apiVersion: feast.dev/v1 +kind: FeatureStore +metadata: + name: sample-spark-application +spec: + feastProject: my_project + batchEngine: + configMapRef: + name: feast-spark-application-engine + configMapKey: config + # Optional: use the Spark driver image for feast-apply / init containers + # services: + # initImage: my-registry.example.com/feast-spark-driver:latest +``` + ### Engine types | `type` | Notes | |--------|-------| | `local` | Default; in-process Python, no extra infra | | `spark` | Apache Spark; requires a Spark operator or standalone cluster | +| `spark_application` | Kubeflow Spark Operator `SparkApplication` CRs; requires Spark Operator + custom image; operator auto-creates RBAC | | `ray` | Ray cluster; requires a Ray operator | | `bytewax` | Bytewax streaming engine | | `snowflake.engine` | Snowflake Snowpark compute | diff --git a/docs/how-to-guides/feast-operator/README.md b/docs/how-to-guides/feast-operator/README.md index 26e515f309f..25d71c130c7 100644 --- a/docs/how-to-guides/feast-operator/README.md +++ b/docs/how-to-guides/feast-operator/README.md @@ -23,7 +23,7 @@ look for store-specific YAML options in the Feast SDK docs. | # | Guide | Topic | |---|-------|-------| -| 1 | [Project Provisioning](01-project-provisioning.md) | `feastProjectDir`: cloning a git repo vs `feast init` templates | +| 1 | [Project Provisioning](01-project-provisioning.md) | `feastProjectDir`: git clone, `feast init`, or a repository packaged in an image | | 2 | [Persistence](02-persistence.md) | File (path + PVC) vs DB store for offline/online/registry; Secret format | | 3 | [Serving & Observability](03-serving-and-observability.md) | Feature server workers, log level, Prometheus metrics, offline push batching, MCP | | 4 | [Registry Topology](04-registry-topology.md) | Local vs remote registry, cross-namespace `feastRef`, remote TLS | diff --git a/docs/how-to-guides/feature-monitoring.md b/docs/how-to-guides/feature-monitoring.md index 7e2ebe4199d..aca36167323 100644 --- a/docs/how-to-guides/feature-monitoring.md +++ b/docs/how-to-guides/feature-monitoring.md @@ -14,6 +14,7 @@ This guide covers: 6. [On-demand exploration (transient compute)](#6-on-demand-exploration) 7. [Integrating with orchestrators](#7-integrating-with-orchestrators) 8. [Supported backends](#8-supported-backends) +9. [Monitoring in the Feast UI](#9-monitoring-in-the-feast-ui) ## 1. Prerequisites @@ -43,16 +44,16 @@ Done! The baseline reads all available source data and stores the resulting statistics with `is_baseline=TRUE`. This serves as the reference distribution for future drift detection. Baseline computation is: -- **Non-blocking** — `feast apply` returns immediately; computation runs asynchronously +- **Threaded** — runs in a background thread but completes before `feast apply` exits - **Idempotent** — only features without existing baselines are computed; re-running `feast apply` won't recompute existing baselines -### Disabling auto-baseline +### Enabling auto-baseline -To skip automatic baseline computation on `feast apply`, set the DQM config in `feature_store.yaml`: +To enable automatic baseline computation on `feast apply`, set the DQM config in `feature_store.yaml`: ```yaml -DataQualityMonitoring: - auto_baseline: false +data_quality_monitoring: + auto_baseline: true ``` When using the Feast operator, set this in the `FeatureStore` CR: @@ -63,9 +64,11 @@ kind: FeatureStore spec: feastProject: my_project dataQualityMonitoring: - autoBaseline: false + autoBaseline: true ``` +To disable it, set `auto_baseline: false` (or `autoBaseline: false` in the CR). + ## 3. Scheduled monitoring with the CLI ### Auto mode (recommended for production) @@ -386,3 +389,84 @@ Monitoring respects Feast's existing RBAC: - **Compute operations** (`POST /monitoring/compute`, `/auto_compute`, `/compute/log`, `/auto_compute/log`) require `AuthzedAction.UPDATE` - **Transient compute** (`POST /monitoring/compute/transient`) requires `AuthzedAction.DESCRIBE` - **Read operations** (`GET /monitoring/metrics/*`) require `AuthzedAction.DESCRIBE` + +## 9. Monitoring in the Feast UI + +The Feast web UI includes a built-in monitoring dashboard accessible from the **Monitoring** item in the sidebar navigation. + +### What you see + +The monitoring page has three tabs: + +| Tab | Shows | +|-----|-------| +| **Features** | Per-feature metrics table with null rate, row count, freshness, and health status | +| **Feature Views** | Aggregated data quality per feature view | +| **Feature Services** | Aggregated metrics per feature service | + +### Filters + +At the top of the monitoring page you can filter by: + +- **Feature View** — scope to a specific feature view or view all +- **Granularity** — select Baseline, Daily, Weekly, Biweekly, Monthly, or Quarterly +- **Source** — filter by batch or log data source +- **Start/End Date** — filter metrics to a specific date range (disabled for Baseline since baseline uses all data) + +### Feature detail page + +Clicking any feature row navigates to a detail page showing: + +- **Distribution histogram** — expandable/zoomable chart of the feature's value distribution +- **Statistics panel** — null rate, mean, stddev, min/max, percentiles (p50–p99) +- **Granularity dropdown** — switch between computed granularities and baseline +- **Time Series Analysis** — trend charts for aggregate metrics drift (Mean/P50/P95) and null rate evolution over time + +### Computing metrics from the UI + +Click the **Compute Metrics** button in the page header to trigger an `auto_compute` job. This computes all granularities for all feature views (or the selected feature view if filtered). Results appear after the table refreshes. + +The **Refresh** button re-fetches already computed metrics from the backend without triggering new computation. + +### When no data is available + +If no metrics have been computed yet, the page shows a prompt: + +> No monitoring data has been computed for this project. Click "Compute Metrics" to run data quality analysis on your feature views. + +If the monitoring backend is unreachable, a warning banner appears: + +> Could not connect to the monitoring API. Make sure the Feast registry server is running with monitoring enabled. + +### Enabling monitoring for the UI + +The monitoring page is always accessible in the sidebar. To see actual data: + +1. Add `data_quality_monitoring` to your `feature_store.yaml`: + + ```yaml + data_quality_monitoring: + auto_baseline: true + ``` + + Or, when using the Feast operator, set this in the `FeatureStore` CR: + + ```yaml + apiVersion: feast.dev/v1 + kind: FeatureStore + spec: + feastProject: my_project + dataQualityMonitoring: + autoBaseline: true + ``` + +2. Run `feast apply` — this computes baseline metrics automatically +3. Schedule `feast monitor run` (or click "Compute Metrics" in the UI) to generate daily/weekly/monthly metrics + +## Related: Operational and SOX Metrics + +Feature Quality Monitoring focuses on **data-level** metrics (distributions, null rates, drift). Feast also provides **operational metrics** for infrastructure observability: + +- **Prometheus metrics** (`feast_offline_store_*`, `feast_online_store_*`) — latency, throughput, and error rates for offline/online store operations. See [Python Feature Server — Metrics](../reference/feature-servers/python-feature-server.md). +- **SOX audit logging** (`feast.audit`) — structured audit events for compliance tracking of feature store operations. +- **OpenTelemetry integration** — distributed tracing for feature serving requests. See [OpenTelemetry Integration](../getting-started/components/open-telemetry.md). diff --git a/docs/how-to-guides/online-server-performance-tuning.md b/docs/how-to-guides/online-server-performance-tuning.md index b280fe53cff..f10d7bff8c8 100644 --- a/docs/how-to-guides/online-server-performance-tuning.md +++ b/docs/how-to-guides/online-server-performance-tuning.md @@ -278,6 +278,7 @@ The online store is the single largest factor in `get_online_features()` latency | **DynamoDB** | 2–5 ms | Yes | Serverless, auto-scaling on AWS | Pay-per-request cost; batch API limits (100 items) | | **PostgreSQL** | 3–10 ms | No (threadpool) | Teams with existing Postgres infra | Connection pooling needed at scale | | **MongoDB** | 2–5 ms | Yes | Flexible schema, async-native | Requires index tuning for large datasets | +| **Aerospike** | < 1 ms | No (threadpool) | Ultra-low latency, hybrid memory (RAM + SSD), large datasets | Namespace must be pre-configured on the cluster | | **Bigtable** | 3–8 ms | No (threadpool) | Large-scale GCP workloads | Row-key design affects read performance | | **Cassandra / ScyllaDB** | 2–5 ms | No (threadpool) | Multi-region, write-heavy | Tunable consistency; requires DC-aware routing | | **Remote** | Varies | No (threadpool) | Centralized feature server architecture | Adds an HTTP hop; tune connection pool | @@ -305,6 +306,7 @@ The feature server can read from the online store using either an **async** or * | **MongoDB** | Yes | Yes | Uses `motor` (async MongoDB driver) | | **PostgreSQL** | Implemented | No | Has `online_read_async` but does not yet advertise via `async_supported`; uses sync/threadpool path | | **Redis** | Implemented | **Yes** | `online_read_async` and `online_write_batch_async` both implemented; uses sync/threadpool path for `get_online_features` (overridden with batched single pipeline) | +| **Aerospike** | Implemented | No | Async methods wrap the blocking C client via `run_in_executor`; does not yet advertise via `async_supported`, so the server still uses the threadpool path | | All others | No | No | Fall back to sync with `run_in_threadpool()` | **When async matters most:** @@ -467,6 +469,34 @@ online_store: - **`connectTimeoutMS` / `socketTimeoutMS`**: Tighter timeouts improve p99 by failing fast on slow connections. - MongoDB is one of the stores with **full async support** (read and write), so it benefits from concurrent feature view reads via `asyncio.gather()`. +### Aerospike tuning + +Aerospike offers sub-millisecond reads thanks to its hybrid-memory architecture (primary index in RAM, data on SSD or RAM). Tune the per-call policies in the Feast config and rely on the Aerospike cluster's own tuning for everything else: + +```yaml +online_store: + type: aerospike + hosts: + - ["aerospike-1.internal", 3000] + - ["aerospike-2.internal", 3000] + namespace: feast + read_timeout_ms: 150 # hard deadline for a single-record get + write_timeout_ms: 300 # hard deadline for a single-record put/operate + batch_total_timeout_ms: 500 # hard deadline for online_read / online_write_batch + socket_timeout_ms: 50 # per-attempt deadline so max_retries can actually fire + max_retries: 2 + ttl_seconds: 86400 # record-level TTL; omit to use the namespace default + client_kwargs: # escape hatch for any client-config field not surfaced above + policies: + batch: + concurrent_nodes: 0 # 0 = parallel to every node (lowest latency on multi-node clusters) +``` + +- **`*_timeout_ms` (total)** vs **`socket_timeout_ms` (per-attempt)**: `*_timeout_ms` is the hard deadline for a whole call *including* retries; `socket_timeout_ms` is the per-attempt deadline that allows `max_retries` to actually fire within that budget. Without `socket_timeout_ms`, a single slow attempt can consume the entire total deadline and retries never run. +- **`hosts`**: List every seed node. The Aerospike client discovers the rest of the cluster automatically and opens one connection pool per node. +- **`ttl_seconds: 0`** means "never expire"; omit the key to inherit the namespace's `default-ttl`. Expiry is enforced by the server's `nsup` thread — nothing to delete on the client side. +- Co-locate the feature server in the **same availability zone / rack** as the Aerospike cluster; sub-millisecond reads are bandwidth- and RTT-sensitive. + ### Remote online store tuning The Remote online store connects to a Feast feature server over HTTP. Connection pooling is critical: @@ -700,6 +730,7 @@ This applies to every connection-oriented online store: | **DynamoDB** | `max_pool_connections` (HTTP pool) | 10 | No hard limit, but AWS SDK has per-process pool caps; monitor throttling | | **Redis** | Connection per worker | 1 | `maxclients` on the Redis server (default: 10,000) | | **MongoDB** | `maxPoolSize` (in `client_kwargs`) | 100 | Server's `net.maxIncomingConnections` | +| **Aerospike** | Driver manages pool per seed node | Auto | `proto-fd-max` (default 15000) on each Aerospike node | | **Cassandra** | Driver manages pool per node | Auto | `native_transport_max_threads` on each Cassandra node | | **Remote** | `connection_pool_size` (HTTP pool) | 50 | The target feature server's worker capacity | diff --git a/docs/how-to-guides/production-deployment-topologies.md b/docs/how-to-guides/production-deployment-topologies.md index ee8bb49be54..52dc2f3873b 100644 --- a/docs/how-to-guides/production-deployment-topologies.md +++ b/docs/how-to-guides/production-deployment-topologies.md @@ -1066,12 +1066,15 @@ Production environments in regulated industries (finance, government, defense) o ### Default init container behavior -When `feastProjectDir` is set on the FeatureStore CR, the operator creates up to two init containers: +When `feastProjectDir` is set on the FeatureStore CR, the operator creates up to two init containers unless `services.disableInitContainers` is `true`: -1. **`feast-init`** — bootstraps the feature repository by running either `git clone` (if `feastProjectDir.git` is set) or `feast init` (if `feastProjectDir.init` is set), then writes the generated `feature_store.yaml` into the repo directory. +1. **`feast-init`** — bootstraps the feature repository by running `git clone`, `feast init`, or copying a repository from `feastProjectDir.packaged.featureRepoPath`. It then writes the operator-generated `feature_store.yaml` into the initialized repository. 2. **`feast-apply`** — runs `feast apply` to register feature definitions in the registry. Controlled by `runFeastApplyOnInit` (defaults to `true`). Skipped when `disableInitContainers` is `true`. -In air-gapped environments, `git clone` will fail because the cluster cannot reach external Git repositories. The solution is to **pre-bake** the feature repository into a custom container image and disable the init containers entirely. +In air-gapped environments, use `feastProjectDir.packaged` to identify a feature repository baked into an image. The operator supports two lifecycle modes: + +* Keep init containers enabled to refresh shared storage from the image, generate configuration from the FeatureStore CR, and optionally run `feast apply`. +* Set `services.disableInitContainers: true` to run directly from the baked path and treat its `feature_store.yaml` as authoritative. ### Air-gapped deployment workflow @@ -1086,12 +1089,12 @@ graph TD end subgraph InternalRegistry["Internal Container Registry"] - Mirror["registry.internal.example.com
/feast/feature-server:v0.61"] + Mirror["registry.internal.example.com
/feast/feature-server:release"] end subgraph AirGappedCluster["Air-Gapped Kubernetes Cluster"] SA["ServiceAccount
(imagePullSecrets)"] - CR["FeatureStore CR
disableInitContainers: true
image: registry.internal..."] + CR["FeatureStore CR
feastProjectDir.packaged
disableInitContainers: true"] Deploy["Feast Deployment
(no init containers)"] SA --> Deploy CR --> Deploy @@ -1105,8 +1108,8 @@ graph TD 1. **Build a custom container image** that bundles the feature repository and all Python dependencies into the Feast base image. 2. **Push** the image to your internal container registry. -3. **Set `services.disableInitContainers: true`** on the FeatureStore CR to skip `git clone` / `feast init` and `feast apply`. -4. **Override the image** on each service using the per-service `image` field. +3. **Configure `feastProjectDir.packaged`** with the image and the canonical absolute path to the bundled repository. Do not use `.`, `..`, repeated separators, or a trailing separator, and keep the path outside operator-mounted locations such as `/feast-data` so it cannot overlap the staged repository. +4. **Choose the lifecycle:** leave init containers enabled for operator-managed configuration and `feast apply`, or set `services.disableInitContainers: true` to use the baked repository and configuration directly. 5. **Set `imagePullPolicy: IfNotPresent`** (or `Never` if images are pre-loaded on nodes). 6. **Configure `imagePullSecrets`** on the namespace's ServiceAccount — the FeatureStore CRD does not expose an `imagePullSecrets` field, so use the standard Kubernetes approach of attaching secrets to the ServiceAccount that the pods run under. @@ -1119,6 +1122,10 @@ metadata: name: airgap-production spec: feastProject: my_project + feastProjectDir: + packaged: + image: registry.internal.example.com/feast/feature-server:release + featureRepoPath: /opt/feast/feature_repo services: disableInitContainers: true onlineStore: @@ -1128,7 +1135,6 @@ spec: secretRef: name: feast-online-store server: - image: registry.internal.example.com/feast/feature-server:v0.61 imagePullPolicy: IfNotPresent resources: requests: @@ -1145,10 +1151,14 @@ spec: secretRef: name: feast-registry-store server: - image: registry.internal.example.com/feast/feature-server:v0.61 imagePullPolicy: IfNotPresent ``` +The packaged image is the default for every Feast service and for the `feast-init` and +`feast-apply` init containers. A per-service `image` still takes precedence for that +service, and `services.initImage` takes precedence for both init containers. Remove +`disableInitContainers: true` to use operator-managed staging and startup apply instead. + {% hint style="info" %} **Pre-populating the registry:** With init containers disabled, `feast apply` does not run on pod startup. You can populate the registry by: diff --git a/docs/project/contributing.md b/docs/project/contributing.md index d79291b9aa8..25ccd80703f 100644 --- a/docs/project/contributing.md +++ b/docs/project/contributing.md @@ -5,6 +5,8 @@ After familiarizing yourself with the documentation, the simplest way to get sta 1. Setup your developer environment by following [development guide](development-guide.md). 2. Either create a [GitHub issue](https://github.com/feast-dev/feast/issues) or make a draft PR (following [development guide](development-guide.md)) to get the ball rolling! +> **Reporting a security vulnerability?** Do not open an issue or PR. Report it privately through [GitHub's advisory form](https://github.com/feast-dev/feast/security/advisories/new); see the [security policy](https://github.com/feast-dev/feast/blob/master/SECURITY.md). + ## Decision making process *See [governance](../../community/governance.md) for more details here* diff --git a/docs/reference/alpha-vector-database.md b/docs/reference/alpha-vector-database.md index 861c3fcb114..61da02ce6f4 100644 --- a/docs/reference/alpha-vector-database.md +++ b/docs/reference/alpha-vector-database.md @@ -15,6 +15,7 @@ Below are supported vector databases and implemented features: | Faiss | [ ] | [ ] | [] | [] | | SQLite | [x] | [ ] | [x] | [x] | | Qdrant | [x] | [x] | [] | [] | +| ScyllaDB | [x] | [x] | [x] | [x] | *Note: V2 Support means the SDK supports retrieval of features along with vector embeddings from vector similarity search. @@ -30,7 +31,241 @@ Beyond that, we will then have `retrieve_online_documents` and `retrieve_online_ backwards compatibility and the adopt industry standard naming conventions. {% endhint %} -**Note**: Milvus and SQLite implement the v2 `retrieve_online_documents_v2` method in the SDK. This will be the longer-term solution so that Data Scientists can easily enable vector similarity search by just flipping a flag. +**Note**: Milvus, SQLite, and ScyllaDB implement the v2 `retrieve_online_documents_v2` method in the SDK. This will be the longer-term solution so that Data Scientists can easily enable vector similarity search by just flipping a flag. + +## Feature server search endpoints + +| Endpoint | Use when | +|----------|----------| +| `POST /search` | You have an embedding vector (or use `api_version: 2` with `query_string`) and want Feast's native online-features response format. | +| `GET /v1/vector_stores` | You want to discover available vector stores and their `vs_{hash}` IDs (OpenAI-compatible). | +| `GET /v1/vector_stores/{id}` | You want metadata for a specific vector store (OpenAI-compatible). | +| `POST /v1/vector_stores/{id}/search` | You want plain-text queries with server-side embedding and an OpenAI-compatible response. | + +`POST /retrieve-online-documents` is deprecated; use `POST /search` instead. + +## [Alpha] OpenAI-Compatible Vector Store API + +{% hint style="warning" %} +**Alpha feature.** This API surface is functional and tested, but may change in future releases. Feedback and contributions are welcome. +{% endhint %} + +Feast exposes a set of [OpenAI-compatible vector store endpoints](https://platform.openai.com/docs/api-reference/vector-stores) that let clients discover, inspect, and search vector stores using plain text queries with server-side embedding. This enables integration with AI agents, LLM tool-calling frameworks, and any OpenAI-compatible client without requiring the caller to produce raw embedding vectors. + +### Vector store IDs + +Each feature view with at least one `vector_index=True` field is automatically assigned a deterministic identifier of the form `vs_{hash}`, where `{hash}` is the first 24 characters of `SHA-256(project + ":" + feature_view_name)`. These IDs are stable across server restarts and registry refreshes. + +For example, a feature view named `product_catalog` in project `my_project` always maps to the same `vs_...` identifier. The listing endpoints return these IDs so clients can discover stores at runtime. + +### Endpoints + +| Method | Path | Permission | Description | +|--------|------|------------|-------------| +| `GET` | `/v1/vector_stores` | `DESCRIBE` | List all vector stores the caller has access to | +| `GET` | `/v1/vector_stores/{vector_store_id}` | `DESCRIBE` | Get metadata for a single vector store | +| `POST` | `/v1/vector_stores/{vector_store_id}/search` | `READ_ONLINE` | Search a vector store with a plain text query | + +All endpoints enforce RBAC when authentication is configured. The listing endpoint filters out stores the caller cannot `DESCRIBE`. + +### Requirements + +1. **Embedding model** — an `embedding_model` section in `feature_store.yaml`. Feast uses [Sentence Transformers](https://www.sbert.net/) by default for local embedding — no external API key required (`pip install sentence-transformers`): + + ```yaml + embedding_model: + provider: sentence_transformers # default; can be omitted + model: all-MiniLM-L6-v2 + ``` + +2. **Vector-indexed feature view** — at least one feature view with `vector_index=True` on a vector field, materialized to an online store that supports vector search. + +3. **Numeric filtering (optional)** — for metadata filters that use numeric or boolean comparisons, set `enable_openai_compatible_store: true` on your online store config and run `feast apply` to add the required `value_num` column. + +### Custom embedding providers + +The built-in Sentence Transformers provider works for most use cases. To use a different embedding backend (OpenAI, Cohere, a custom model, etc.), implement the `EmbeddingProvider` protocol and pass an instance to `FeatureStore`: + +```python +from feast.embedder import EmbeddingProvider + +class MyEmbeddingProvider: + def embed(self, texts: list[str]) -> list[list[float]]: + # Call your embedding API here + return my_model.encode(texts) + + async def aembed(self, texts: list[str]) -> list[list[float]]: + return await my_model.aencode(texts) + +store = FeatureStore( + repo_path=".", + embedding_provider=MyEmbeddingProvider(), +) +``` + +### Numeric storage (`enable_openai_compatible_store`) + +By default, feature values are stored as text in the online store. This means string-ordered comparisons apply (e.g., `'9' > '100'` is `true`). When `enable_openai_compatible_store: true` is set on the online store config, Feast adds a `value_num` column that stores `int`, `float`, `double`, and `bool` values natively so that numeric filters produce correct results. + +```yaml +online_store: + type: postgres # or sqlite + # ... connection settings ... + enable_openai_compatible_store: true +``` + +After changing this setting, run `feast apply` to update the database schema. + +### List vector stores + +```bash +curl http://localhost:6566/v1/vector_stores +``` + +```json +{ + "object": "list", + "data": [ + { + "id": "vs_a1b2c3d4e5f6a1b2c3d4e5f6", + "object": "vector_store", + "name": "product_catalog", + "status": "completed", + "created_at": 1717200000 + } + ] +} +``` + +### Get a single vector store + +```bash +curl http://localhost:6566/v1/vector_stores/vs_a1b2c3d4e5f6a1b2c3d4e5f6 +``` + +Returns the same object shape as a single entry in the list response. Returns `404` if the ID does not match any vector-indexed feature view. + +### Search + +Start the feature server with `feast serve`, then send a search request: + +```bash +curl -X POST http://localhost:6566/v1/vector_stores/vs_a1b2c3d4e5f6a1b2c3d4e5f6/search \ + -H "Content-Type: application/json" \ + -d '{ + "query": "wireless noise-cancelling headphones", + "max_num_results": 5 + }' +``` + +#### Request fields + +| Field | Type | Default | Description | +|-------|------|---------|-------------| +| `query` | `string` or `list[string]` | (required) | Plain text search query. Lists are joined with spaces before embedding. | +| `max_num_results` | `int` | `10` | Maximum number of results to return. | +| `filters` | `object` | `null` | OpenAI-style filters (see below). | +| `ranking_options` | `object` | `null` | Accepted for forward compatibility, but currently ignored. Setting `score_threshold` or `ranker` inside it will return a 422 error. | +| `rewrite_query` | `bool` | `null` | `false` (the default/no-op) is accepted. `true` is not yet supported and will return a 422 error. | +| `metadata` | `object` | `null` | Optional. `metadata.features_to_retrieve` selects specific features. | + +### Filters + +The endpoint supports OpenAI-style filters for narrowing results beyond vector similarity. + +**Comparison operators:** `eq`, `ne`, `gt`, `gte`, `lt`, `lte`, `in`, `nin` + +```json +{"type": "eq", "key": "category", "value": "Electronics"} +``` + +**Compound operators:** `and`, `or` (nest to arbitrary depth) + +```json +{ + "type": "and", + "filters": [ + {"type": "eq", "key": "category", "value": "Electronics"}, + {"type": "gte", "key": "rating", "value": 4.5} + ] +} +``` + +For Postgres and SQLite backends, all filtering (including string equality) requires `enable_openai_compatible_store: true` in the online store config. After enabling, run `feast apply` to update the database schema. + +ScyllaDB supports vector retrieval via `retrieve_online_documents_v2`, but OpenAI-style metadata filtering is not implemented yet. Passing `filters` raises `NotImplementedError`. + +### Response format + +Responses follow the OpenAI `vector_store.search_results.page` schema: + +```json +{ + "object": "vector_store.search_results.page", + "search_query": ["wireless noise-cancelling headphones"], + "data": [ + { + "file_id": "vs_a1b2c3d4e5f6a1b2c3d4e5f6_42", + "filename": "vs_a1b2c3d4e5f6a1b2c3d4e5f6", + "score": 0.92, + "attributes": {"name": "...", "category": "..."}, + "content": [ + {"type": "text", "text": "..."} + ] + } + ], + "has_more": false, + "next_page": null +} +``` + +The `file_id` and `filename` fields use the `vs_{hash}` identifier, not raw feature view names. + +The `score` field is a higher-is-better relevance score derived from the raw vector distance using a metric-dependent conversion: + +| Distance metric | Conversion | Range | +|----------------|------------|-------| +| L2 (default) | `1 / (1 + distance)` | (0, 1] | +| Cosine | `1 - distance` | [0, 1] | +| Inner product / dot | `-distance` | varies | + +The metric is determined by `vector_search_metric` on the feature view's vector field, not by an API parameter. When `features_to_retrieve` is omitted, all non-vector features are returned by default (vector embedding columns are excluded). + +Pagination is not yet implemented; `has_more` is always `false`. + +### SDK usage + +The OpenAI-compatible search is also available directly via the Python SDK: + +```python +import asyncio +from feast import FeatureStore + +store = FeatureStore(repo_path=".") + +result = asyncio.run(store.openai_search( + vector_store_id="product_catalog", + query="wireless noise-cancelling headphones", + max_num_results=5, + filters={"type": "eq", "key": "category", "value": "Electronics"}, +)) + +for item in result["data"]: + print(f"{item['score']:.3f} {item['attributes']}") +``` + +### Supported online stores + +The OpenAI-compatible filtering has been implemented for the following online stores: + +| Online Store | Vector Search | Metadata Filtering | Notes | +|-------------|--------------|-------------------|-------| +| Milvus | Yes | Yes | Boolean expressions | +| Elasticsearch | Yes | Yes | Query DSL clauses | +| Postgres (pgvector) | Yes | Yes | Requires `enable_openai_compatible_store: true` | +| SQLite (sqlite-vec) | Yes | Yes | Requires `enable_openai_compatible_store: true` | +| MongoDB | Yes | Yes | Aggregation pipeline | +| ScyllaDB | Yes | No | Vector search only; metadata filters are not supported yet | ## Examples diff --git a/docs/reference/alpha-web-ui.md b/docs/reference/alpha-web-ui.md index 0556482fcf8..3fe8ce052a8 100644 --- a/docs/reference/alpha-web-ui.md +++ b/docs/reference/alpha-web-ui.md @@ -153,3 +153,12 @@ const tabsRegistry = { ``` Examples of custom tabs can be found in the `ui/custom-tabs` folder. + +## Refreshing the registry + +The Feast UI caches registry data (projects, feature views, entities, etc.) using the registry cache. After running `feast apply` to make changes, it may take up to `cache_ttl_seconds` before the updates appear in the UI. + +To see changes faster: + +- **Lower the TTL**: Set `cache_ttl_seconds: 10` (or similar) in your `feature_store.yaml` registry config. This makes all registry consumers — including the UI — pick up changes within 10 seconds. +- **Refresh on demand**: The UI has a **Refresh** button that explicitly invalidates the server-side registry cache (`POST /api/v1/registry/refresh`) and reloads the UI without a full page refresh. diff --git a/docs/reference/codebase-structure.md b/docs/reference/codebase-structure.md index 80608b5929a..4783773c270 100644 --- a/docs/reference/codebase-structure.md +++ b/docs/reference/codebase-structure.md @@ -28,7 +28,7 @@ The majority of Feast logic lives in these Python files: There are also several important submodules: * `infra/` contains all the infrastructure components, such as the provider, offline store, online store, batch materialization engine, and registry. -* `dqm/` covers data quality monitoring, such as the dataset profiler. +* `dqm/` covers data quality monitoring. See [`monitoring/`](../../sdk/python/feast/monitoring/) for the built-in monitoring system. * `diff/` covers the logic for determining how to apply infrastructure changes upon feature repo changes (e.g. the output of `feast plan` and `feast apply`). * `embedded_go/` covers the Go feature server. * `ui/` contains the embedded Web UI, to be launched on the `feast ui` command. diff --git a/docs/reference/compute-engine/README.md b/docs/reference/compute-engine/README.md index dad2ede75a6..a570e5688ed 100644 --- a/docs/reference/compute-engine/README.md +++ b/docs/reference/compute-engine/README.md @@ -57,6 +57,22 @@ An example of built output from FeatureBuilder: - Supports point-in-time joins and large-scale materialization - Integrates with `SparkOfflineStore` and `SparkMaterializationJob` +### ☸️ SparkApplicationComputeEngine + +{% page-ref page="spark_application.md" %} + +- Batch materialization via Kubeflow Spark Operator `SparkApplication` CRs +- One SparkApplication per materialize call (multi–feature-view batching) +- Requires network-accessible online/offline/registry stores (no file-based backends) + +### 🌊 FlinkComputeEngine + +{% page-ref page="flink.md" %} + +- Distributed DAG execution through Apache Flink's PyFlink Table API +- Supports materialization and historical retrieval with Feast offline stores +- Integrates with `FlinkMaterializationJob` and `FlinkDAGRetrievalJob` + ### ⚡ RayComputeEngine (contrib) - Distributed DAG execution via Ray diff --git a/docs/reference/compute-engine/flink.md b/docs/reference/compute-engine/flink.md new file mode 100644 index 00000000000..0dd5560f70e --- /dev/null +++ b/docs/reference/compute-engine/flink.md @@ -0,0 +1,124 @@ +# Apache Flink + +## Description + +The Apache Flink compute engine provides a distributed execution engine for +feature pipelines through the PyFlink Table API. It implements Feast's unified +`ComputeEngine` interface and can be used for batch materialization operations +(`materialize` and `materialize-incremental`) and historical retrieval +(`get_historical_features`). + +The engine reads data through the configured Feast offline store and executes +the Feast DAG as PyFlink tables. Offline stores that expose a native +`to_flink_table(table_env)` retrieval job hand Flink tables directly to the +engine. Retrieval jobs that only expose the standard Arrow path are also +supported and are converted into Flink tables by the engine. The engine then +uses Flink Table/SQL operations for join, filter, aggregate, dedupe, and +projection steps, and writes materialization results to the configured online +and/or offline store. + +## Configuration + +Install the Flink extra from a Feast source checkout with `uv` before using the +engine: + +```bash +uv sync --extra flink --no-dev +``` + +The `flink` extra installs PyFlink directly. PyFlink currently requires +`pyarrow<21`, while the default Feast install keeps `pyarrow>=21`; Feast's uv +lock resolves the Flink extra in a separate dependency fork so normal Feast +installs do not downgrade Arrow. + +Configure the engine in `feature_store.yaml`: + +```yaml +project: my_project +registry: data/registry.db +provider: local +offline_store: + type: file +online_store: + type: sqlite + path: data/online_store.db +batch_engine: + type: flink.engine + execution_mode: batch + parallelism: 4 + table_config: + pipeline.name: "Feast Flink Compute Engine" + pandas_split_num: 4 +``` + +## Configuration Options + +| Option | Type | Default | Description | +| --- | --- | --- | --- | +| `type` | string | `flink.engine` | Must be `flink.engine`. | +| `execution_mode` | string | `batch` | PyFlink execution mode: `batch` or `streaming`. | +| `parallelism` | integer | `null` | Default Flink parallelism for jobs created by the engine. | +| `table_config` | map | `null` | Additional PyFlink table configuration entries. | +| `pandas_split_num` | integer | `1` | Number of PyFlink Arrow source splits when converting pandas entity DataFrames into Flink tables. | + +## Flink Transformations + +Use `mode="flink"` when a `BatchFeatureView` transformation should receive and +return PyFlink table objects: + +```python +from feast import BatchFeatureView, Field +from feast.types import Float32 + + +def double_rates(table): + # In production this can use PyFlink Table API operations and return a table. + return table + + +driver_stats = BatchFeatureView( + name="driver_stats", + entities=[driver], + mode="flink", + udf=double_rates, + schema=[Field(name="conv_rate", dtype=Float32)], + source=driver_stats_source, + online=True, +) +``` + +Flink transformations must return PyFlink table objects. pandas-returning UDFs +are not accepted by the Flink compute engine. + +## DAG Support + +The Flink engine implements Feast's compute DAG with Flink-specific nodes: + +- Source reads from Feast offline stores, preferring native Flink tables when a + retrieval job supports `to_flink_table(table_env)` and otherwise converting + Arrow results into Flink tables. +- Transform nodes pass PyFlink tables to `mode="flink"` UDFs and preserve native + Flink table outputs. +- Join nodes use Flink SQL temporary views for feature joins and entity joins. +- Filter nodes apply point-in-time, TTL, and custom filter expressions in Flink + SQL. +- Aggregate nodes support non-windowed Feast aggregations using Flink SQL + aggregate functions. +- Dedupe nodes use `ROW_NUMBER()` over entity keys or internal entity-row ids so + historical retrieval keeps one latest feature row per entity row. +- Validation nodes check required output columns. JSON value validation must be + handled upstream in Flink SQL. +- Output nodes write only for materialization tasks; historical retrieval is + read-only. +- Historical retrieval accepts pandas entity DataFrames and SQL-string entity + DataFrames. SQL strings are interpreted as Flink SQL queries against the + configured TableEnvironment/catalog and must select an `event_timestamp` + column. + +## Current Limitations + +- Windowed aggregations are not yet implemented in the Flink compute engine. Use + non-windowed Feast aggregations or pre-window upstream in Flink. +- JSON value validation is not implemented inside the Flink compute engine + because the engine does not collect intermediate data out of Flink for + validation. diff --git a/docs/reference/compute-engine/snowflake.md b/docs/reference/compute-engine/snowflake.md index e7b0dc5bd63..f6c633a4e40 100644 --- a/docs/reference/compute-engine/snowflake.md +++ b/docs/reference/compute-engine/snowflake.md @@ -24,5 +24,10 @@ batch_engine: role: sysadmin warehouse: demo_wh database: FEAST + python_udf_runtime_version: "3.10" ``` {% endcode %} + +## Configuration + +* `python_udf_runtime_version` *(optional, default: `"3.10"`)* -- The Snowflake Python UDF `RUNTIME_VERSION` used when Feast deploys its materialization UDFs. Snowflake periodically decommissions old Python UDF runtimes (for example, the 3.9 runtime was decommissioned, requiring Feast to bump its default to 3.10 -- see [#6606](https://github.com/feast-dev/feast/issues/6606)). If Snowflake decommissions the 3.10 runtime in the future, set this field to a still-supported version (e.g. `"3.11"`) instead of waiting for a new Feast release. diff --git a/docs/reference/compute-engine/spark_application.md b/docs/reference/compute-engine/spark_application.md new file mode 100644 index 00000000000..0c071d1ed4f --- /dev/null +++ b/docs/reference/compute-engine/spark_application.md @@ -0,0 +1,182 @@ +# SparkApplication Compute Engine + +## Description + +The **SparkApplication** compute engine runs Feast **batch materialization** on Kubernetes by creating a [Kubeflow Spark Operator](https://github.com/kubeflow/spark-operator) `SparkApplication` custom resource for each materialization job. + +Unlike the in-process [`spark.engine`](spark.md) compute engine (which uses a Spark session inside the Feast process), `spark_application` submits work to the Spark Operator. The operator starts a driver pod and executors from your configured image; Feast polls the SparkApplication until it completes. + +| Capability | Supported | +|------------|-----------| +| `materialize` / `materialize-incremental` | Yes | +| Multiple feature views in one job | Yes — one SparkApplication per materialize call | +| `get_historical_features` | Not yet | +| SparkConnect | Separate approach — not this engine | + +### Design + +1. Feast creates a ConfigMap with job tasks and a driver copy of `feature_store.yaml`. +2. Feast creates a `SparkApplication` CR pointing at the driver entrypoint (`main.py` in the image). +3. Inside the pod, the batch engine type is rewritten to `spark.engine` so materialization uses the Spark session created by `spark-submit` (avoids recursive SparkApplication creation). +4. The driver writes features to your configured **online store** and updates the **registry** (same network backends as the server). + +### Requirements + +- Kubeflow Spark Operator installed and watching the target namespace. +- A container **image** that includes the Feast SDK, PySpark, and clients for your stores. See the reference [Dockerfile](https://github.com/feast-dev/feast/blob/master/sdk/python/feast/infra/compute_engines/spark_application/Dockerfile). +- **Network-accessible** online store, offline store, and registry. File-based backends are rejected because Spark pods have an ephemeral filesystem: + +| Rejected | Examples | Use instead | +|----------|----------|-------------| +| File online | `sqlite`, `faiss` | Redis, remote online, etc. | +| File offline | `dask`, `file`, `duckdb` | `spark`, Postgres, Snowflake, BigQuery, etc. | +| File registry | `file` | SQL registry, Snowflake | + +For distributed reads, configure `offline_store.type: spark` (or another store Spark can read efficiently). + +### Kubernetes / Feast Operator notes + +When using the Feast Operator: + +- Point `spec.batchEngine.configMapRef` at a ConfigMap whose `type` is `spark_application` (see [Guide 6 — Batch Engine & Scheduled Jobs](../../how-to-guides/feast-operator/06-batch-and-jobs.md)). +- The operator auto-creates RBAC for the `spark_application` batch engine (server and driver service accounts). +- Set `spec.services.initImage` if init / `feast-apply` containers need the Spark-capable image. + +--- + +## Example + +{% code title="feature_store.yaml" %} +```yaml +project: my_project +registry: + registry_type: sql + path: postgresql+psycopg://feast:****@postgres:5432/feast +online_store: + type: redis + connection_string: redis:6379 +offline_store: + type: spark + spark_conf: + spark.master: local[*] +batch_engine: + type: spark_application + image: my-registry.example.com/feast-spark-driver:latest + namespace: feast + spark_version: "4.0.1" + driver_cores: 1 + driver_memory: "2g" + executor_instances: 2 + executor_cores: 1 + executor_memory: "2g" + spark_conf: + spark.sql.shuffle.partitions: "100" +``` +{% endcode %} + +### Feast Operator ConfigMap + +```yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: feast-spark-batch-engine + namespace: feast +data: + config: | + type: spark_application + image: my-registry.example.com/feast-spark-driver:latest + namespace: feast + executor_instances: 2 + driver_memory: "2g" + executor_memory: "2g" +--- +apiVersion: feast.dev/v1 +kind: FeatureStore +metadata: + name: feast + namespace: feast +spec: + feastProject: my_project + batchEngine: + configMapRef: + name: feast-spark-batch-engine + configMapKey: config +``` + +--- + +## Remote materialization + +If the client uses a **remote** online store (`online_store.type: remote`), `FeatureStore.materialize()` delegates to the feature server HTTP API. The server runs the SparkApplication engine. + +- Default (`run_async=False`): block until the server finishes sync materialization. +- `run_async=True`: accept asynchronously (`?async=true`); poll feature-view state in the registry for completion. +- `force=True` (with `run_async=True`): override stuck `MATERIALIZING` state on the server. + +```python +from datetime import datetime, timedelta +from feast import FeatureStore + +store = FeatureStore(repo_path=".") # client feature_store.yaml with online_store.type: remote + +store.materialize( + start_date=datetime.utcnow() - timedelta(days=1), + end_date=datetime.utcnow(), +) +``` + +--- + +## Configuration reference + +| Field | Type | Default | Description | +|-------|------|---------|-------------| +| `type` | string | `spark_application` | Engine type key | +| `image` | string | **required** | Container image for the Spark driver/executors | +| `image_pull_secrets` | list[str] | `[]` | Image pull secret names | +| `namespace` | string | `default` | Namespace for SparkApplication and ConfigMap | +| `service_account` | string | `""` | Driver service account; empty uses platform/operator default | +| `spark_version` | string | `4.0.1` | Spark version for the CR | +| `driver_cores` | int | `1` | Driver cores | +| `driver_memory` | string | `1g` | Driver memory | +| `executor_instances` | int | `1` | Number of executors | +| `executor_cores` | int | `1` | Cores per executor | +| `executor_memory` | string | `1g` | Memory per executor | +| `spark_conf` | dict | `null` | Extra Spark configuration | +| `hadoop_conf` | dict | `null` | Extra Hadoop configuration | +| `env` | list[dict] | `[]` | Driver env vars (`name` + `value` or `valueFrom`) | +| `env_from` | list[dict] | `[]` | EnvFrom sources | +| `queue_name` | string | `null` | Optional queue / Kueue label | +| `job_timeout_seconds` | int | `3600` | Max wait for SparkApplication completion | +| `poll_interval_seconds` | int | `10` | Status poll interval | +| `ttl_seconds_after_finished` | int | `3600` | CR TTL after finish | +| `restart_policy` | string | `Never` | SparkApplication restart policy | +| `max_retries` | int | `3` | Retries when restart policy allows | +| `concurrency` | int | `1` | Parallel feature views inside one driver | +| `labels` | dict | `{}` | Extra labels on the CR | +| `volumes` / `volume_mounts` | list | `[]` | Extra volumes for the driver | +| `py_files` | list[str] | `[]` | Additional Python files for Spark | +| `node_selector` | dict | `null` | Pod node selector | +| `tolerations` | list | `[]` | Pod tolerations | +| `staging_location` | string | `null` | Reserved for historical retrieval (ignored for materialize) | + +--- + +## Troubleshooting + +| Symptom | What to check | +|---------|----------------| +| SparkApplication Pending / insufficient CPU | Lower resource requests via `spark_conf` (for example `spark.kubernetes.driver.request.cores`) or free cluster capacity | +| ImagePullBackOff | Image name, tag, and `image_pull_secrets` | +| 403 on ConfigMap or SparkApplication | RBAC for the Feast server and Spark driver service accounts | +| Init `ValueError` about file-based stores | Switch online/offline/registry to network backends | +| Init / feast-apply failures missing Spark deps | Use a Spark-capable image (`initImage` with the Feast Operator) | + +--- + +## Related + +- [Spark compute engine (in-process)](spark.md) +- [Feast Operator — batch engine ConfigMap](../../how-to-guides/feast-operator/06-batch-and-jobs.md) +- [Creating a custom compute engine](../../how-to-guides/customizing-feast/creating-a-custom-compute-engine.md) diff --git a/docs/reference/data-sources/README.md b/docs/reference/data-sources/README.md index 24bf18dbe86..33e47672dcc 100644 --- a/docs/reference/data-sources/README.md +++ b/docs/reference/data-sources/README.md @@ -42,6 +42,10 @@ Please see [Data Source](../../getting-started/concepts/data-ingestion.md) for a [spark.md](spark.md) {% endcontent-ref %} +{% content-ref url="iceberg.md" %} +[iceberg.md](iceberg.md) +{% endcontent-ref %} + {% content-ref url="postgres.md" %} [postgres.md](postgres.md) {% endcontent-ref %} diff --git a/docs/reference/data-sources/iceberg.md b/docs/reference/data-sources/iceberg.md new file mode 100644 index 00000000000..6402a7ec419 --- /dev/null +++ b/docs/reference/data-sources/iceberg.md @@ -0,0 +1,161 @@ +# Iceberg source (contrib) + +## Description + +Iceberg data sources are tables managed by any supported Iceberg catalog. The `IcebergSource` class provides a unified interface with a configurable `catalog_type` parameter: + +- **`"rest"`** (default): [Apache Iceberg REST Catalog specification](https://iceberg.apache.org/concepts/catalog/#decoupling-using-the-rest-catalog) — Unity Catalog, Apache Polaris, Nessie, Snowflake Open Catalog +- **`"hive"`**: Hive Metastore catalog +- **`"glue"`**: AWS Glue catalog +- **`"sql"`**: SQL-based (JDBC) catalog +- **`"dynamodb"`**: DynamoDB-based catalog + +The data source carries catalog connection details (catalog_type, endpoint, warehouse, namespace, table, authentication). When the offline store (DuckDB, Spark) encounters this source, it resolves table metadata and credentials via the configured catalog at query time. + +## Examples + +### IcebergSource (REST catalog) + +Works with any Iceberg REST Catalog: + +```python +from feast.infra.data_sources.contrib.iceberg_catalog import IcebergSource + +my_source = IcebergSource( + catalog_type="rest", # default + endpoint="http://localhost:8081/api/2.1/unity-catalog/iceberg", + warehouse="unity", + namespace="default", + table="driver_features", + timestamp_field="event_timestamp", + token_env_var="UC_TOKEN", +) +``` + +### IcebergSource (Hive Metastore) + +```python +from feast.infra.data_sources.contrib.iceberg_catalog import IcebergSource + +my_source = IcebergSource( + catalog_type="hive", + catalog_properties={"uri": "thrift://metastore:9083"}, + warehouse="my_warehouse", + namespace="default", + table="driver_features", + timestamp_field="event_timestamp", +) +``` + +### IcebergSource (AWS Glue) + +```python +from feast.infra.data_sources.contrib.iceberg_catalog import IcebergSource + +my_source = IcebergSource( + catalog_type="glue", + catalog_properties={"region_name": "us-east-1"}, + warehouse="my_account", + namespace="my_database", + table="driver_features", + timestamp_field="event_timestamp", +) +``` + +### UnityCatalogSource (with governance) {#unity-catalog-source} + +Extends `IcebergSource` with Unity Catalog governance: + +```python +from feast.infra.data_sources.contrib.iceberg_catalog import ( + UnityCatalogSource, +) + +my_uc_source = UnityCatalogSource( + warehouse="production", + namespace="ml_features", + table="driver_stats", + timestamp_field="event_timestamp", + register_as_feature_table=True, # Register in UC on feast apply + sync_lineage=True, # Record lineage in UC +) +``` + +When `endpoint` is omitted, it defaults to `{DATABRICKS_HOST}/api/2.1/unity-catalog/iceberg`. +When `token_env_var` is omitted, it defaults to `DATABRICKS_TOKEN`. + +### Full Feature View Example + +```python +from datetime import timedelta + +from feast import Entity, FeatureView, Field +from feast.types import Float64, Int64 + +from feast.infra.data_sources.contrib.iceberg_catalog import ( + UnityCatalogSource, +) + +driver = Entity(name="driver_id", join_keys=["driver_id"]) + +driver_stats_source = UnityCatalogSource( + warehouse="production", + namespace="ml_features", + table="driver_hourly_stats", + timestamp_field="event_timestamp", + created_timestamp_column="created", +) + +driver_stats_fv = FeatureView( + name="driver_hourly_stats", + entities=[driver], + source=driver_stats_source, + schema=[ + Field(name="conv_rate", dtype=Float64), + Field(name="acc_rate", dtype=Float64), + Field(name="avg_daily_trips", dtype=Int64), + ], + ttl=timedelta(days=1), + online=True, +) +``` + +## Configuration Reference + +### IcebergSource + +| Parameter | Type | Description | +| :--- | :--- | :--- | +| `catalog_type` | `str` | Catalog backend: `"rest"` (default), `"hive"`, `"glue"`, `"sql"`, `"dynamodb"` | +| `endpoint` | `str` | Catalog endpoint URL (required for `"rest"`, optional for others) | +| `warehouse` | `str` | Catalog/warehouse name | +| `namespace` | `str` | Schema/namespace within the catalog | +| `table` | `str` | Table name | +| `catalog_properties` | `dict` | Additional catalog-specific properties passed to PyIceberg | +| `timestamp_field` | `str` | Event timestamp column for point-in-time joins | +| `created_timestamp_column` | `str` | Optional column indicating row creation time | +| `token_env_var` | `str` | Environment variable name holding the auth token | +| `credential_vending` | `bool` | Whether to request scoped credentials (default: `True`) | +| `field_mapping` | `dict` | Column name mapping from source to feature names | + +### UnityCatalogSource (additional parameters) + +| Parameter | Type | Description | +| :--- | :--- | :--- | +| `register_as_feature_table` | `bool` | Register as UC feature table on `feast apply` (default: `True`) | +| `sync_lineage` | `bool` | Sync lineage metadata to Unity Catalog (default: `True`) | + +## Supported Types + +| Iceberg Type | Feast Type | +| :--- | :--- | +| `boolean` | `BOOL` | +| `int` | `INT32` | +| `long` | `INT64` | +| `float` | `FLOAT` | +| `double` | `DOUBLE` | +| `string` | `STRING` | +| `binary` | `BYTES` | +| `timestamp` / `timestamptz` | `INT64` | +| `decimal` | `DOUBLE` | +| `uuid` | `STRING` | diff --git a/docs/reference/data-sources/kafka.md b/docs/reference/data-sources/kafka.md index 8794c7a1e81..dd7203a6149 100644 --- a/docs/reference/data-sources/kafka.md +++ b/docs/reference/data-sources/kafka.md @@ -72,4 +72,4 @@ def driver_hourly_stats_stream(df: DataFrame): ``` ### Ingesting data -See [here](https://github.com/feast-dev/streaming-tutorial) for a example of how to ingest data from a Kafka source into Feast. +See [here](https://github.com/feast-dev/streaming-tutorial) for an example of how to ingest data from a Kafka source into Feast. diff --git a/docs/reference/data-sources/kinesis.md b/docs/reference/data-sources/kinesis.md index f2adadfec03..09706617da9 100644 --- a/docs/reference/data-sources/kinesis.md +++ b/docs/reference/data-sources/kinesis.md @@ -71,4 +71,4 @@ def driver_hourly_stats_stream(df: DataFrame): ``` ### Ingesting data -See [here](https://github.com/feast-dev/streaming-tutorial) for a example of how to ingest data from a Kafka source into Feast. The approach used in the tutorial can be easily adapted to work for Kinesis as well. +See [here](https://github.com/feast-dev/streaming-tutorial) for an example of how to ingest data from a Kafka source into Feast. The approach used in the tutorial can be easily adapted to work for Kinesis as well. diff --git a/docs/reference/dqm.md b/docs/reference/dqm.md index 5a02413e534..47090b5dd1c 100644 --- a/docs/reference/dqm.md +++ b/docs/reference/dqm.md @@ -1,77 +1,81 @@ # Data Quality Monitoring -Data Quality Monitoring (DQM) is a Feast module aimed to help users to validate their data with the user-curated set of rules. -Validation could be applied during: -* Historical retrieval (training dataset generation) -* [planned] Writing features into an online store -* [planned] Reading features from an online store +Feast's Data Quality Monitoring (DQM) system computes, stores, and serves statistical metrics for every registered feature. It gives you visibility into feature health — distributions, null rates, percentiles, histograms — across batch data and feature serving logs. -Its goal is to address several complex data problems, namely: -* Data consistency - new training datasets can be significantly different from previous datasets. This might require a change in model architecture. -* Issues/bugs in the upstream pipeline - bugs in upstream pipelines can cause invalid values to overwrite existing valid values in an online store. -* Training/serving skew - distribution shift could significantly decrease the performance of the model. +Its goal is to address several complex data problems: -> To monitor data quality, we check that the characteristics of the tested dataset (aka the tested dataset's profile) are "equivalent" to the characteristics of the reference dataset. -> How exactly profile equivalency should be measured is up to the user. +* **Data consistency** — new training datasets can differ significantly from previous datasets, potentially requiring changes in model architecture. +* **Upstream pipeline bugs** — bugs in upstream pipelines can cause invalid values to overwrite existing valid values in an online store. +* **Training/serving skew** — distribution shift between training and serving data can decrease model performance. ### Overview -The validation process consists of the following steps: -1. User prepares reference dataset (currently only [saved datasets](../getting-started/concepts/dataset.md) from historical retrieval are supported). -2. User defines profiler function, which should produce profile by given dataset (currently only profilers based on [Great Expectations](https://docs.greatexpectations.io) are allowed). -3. Validation of tested dataset is performed with reference dataset and profiler provided as parameters. +Feast's DQM system works natively with your configured offline store — no additional infrastructure or external dependencies are required. The workflow is: -### Preparations -Feast with Great Expectations support can be installed via -```shell -pip install 'feast[ge]' +1. **Register features** — run `feast apply` to register feature views. If `auto_baseline: true` is configured, baseline metrics are computed automatically. +2. **Schedule monitoring** — run `feast monitor run` on a schedule (daily recommended) to compute metrics across multiple time windows. +3. **Read metrics** — query metrics via the REST API or view them in the Feast UI. + +### Configuration + +Enable DQM in your `feature_store.yaml`: + +```yaml +data_quality_monitoring: + auto_baseline: true ``` -### Dataset profile -Currently, Feast supports only [Great Expectation's](https://greatexpectations.io/) [ExpectationSuite](https://legacy.docs.greatexpectations.io/en/latest/autoapi/great_expectations/core/expectation_suite/index.html#great_expectations.core.expectation_suite.ExpectationSuite) -as dataset's profile. Hence, the user needs to define a function (profiler) that would receive a dataset and return an [ExpectationSuite](https://legacy.docs.greatexpectations.io/en/latest/autoapi/great_expectations/core/expectation_suite/index.html#great_expectations.core.expectation_suite.ExpectationSuite). +### Computing Metrics -Great Expectations supports automatic profiling as well as manually specifying expectations: -```python -from great_expectations.dataset import Dataset -from great_expectations.core.expectation_suite import ExpectationSuite +**Auto mode (recommended for production):** -from feast.dqm.profilers.ge_profiler import ge_profiler +```bash +feast monitor run +``` -@ge_profiler -def automatic_profiler(dataset: Dataset) -> ExpectationSuite: - from great_expectations.profile.user_configurable_profiler import UserConfigurableProfiler +This detects the latest event timestamp in the source data and computes metrics for 5 time windows: daily, weekly, biweekly, monthly, and quarterly. - return UserConfigurableProfiler( - profile_dataset=dataset, - ignored_columns=['conv_rate'], - value_set_threshold='few' - ).build_suite() +**Target a specific feature view:** + +```bash +feast monitor run --feature-view driver_stats ``` -However, from our experience capabilities of automatic profiler are quite limited. So we would recommend crafting your own expectations: -```python -@ge_profiler -def manual_profiler(dataset: Dataset) -> ExpectationSuite: - dataset.expect_column_max_to_be_between("column", 1, 2) - return dataset.get_expectation_suite() + +**Explicit date range:** + +```bash +feast monitor run \ + --feature-view driver_stats \ + --start-date 2025-01-01 \ + --end-date 2025-01-07 \ + --granularity weekly ``` +**Set a manual baseline:** +```bash +feast monitor run \ + --feature-view driver_stats \ + --start-date 2025-01-01 \ + --end-date 2025-03-31 \ + --granularity daily \ + --set-baseline +``` + +### Monitoring Feature Serving Logs + +If your feature services have logging configured, you can compute metrics from the actual features served to models in production: -### Validating Training Dataset -During retrieval of historical features, `validation_reference` can be passed as a parameter to methods `.to_df(validation_reference=...)` or `.to_arrow(validation_reference=...)` of RetrievalJob. -If parameter is provided Feast will run validation once dataset is materialized. In case if validation successful materialized dataset is returned. -Otherwise, `feast.dqm.errors.ValidationFailed` exception would be raised. It will consist of all details for expectations that didn't pass. +```bash +feast monitor run --source-type log +``` -```python -from feast import FeatureStore +### Reading Metrics -fs = FeatureStore(".") +Metrics are accessible via the REST API: -job = fs.get_historical_features(...) -job.to_df( - validation_reference=fs - .get_saved_dataset("my_reference_dataset") - .as_reference(profiler=manual_profiler) -) ``` +GET /monitoring/metrics/features?project=my_project&feature_view_name=driver_stats&granularity=daily +``` + +See the [Feature Quality Monitoring guide](../how-to-guides/feature-monitoring.md) for full API reference, UI integration, and orchestrator examples. diff --git a/docs/reference/feast-cli-commands.md b/docs/reference/feast-cli-commands.md index 99a99ab1707..85781f6abc2 100644 --- a/docs/reference/feast-cli-commands.md +++ b/docs/reference/feast-cli-commands.md @@ -28,6 +28,7 @@ Commands: materialize Run a (non-incremental) materialization job to... materialize-incremental Run an incremental materialization job to ingest... permissions Access permissions + registry Manage the feature registry registry-dump Print contents of the metadata registry teardown Tear down deployed feature store infrastructure version Display Feast SDK version @@ -483,6 +484,18 @@ reader driver_hourly_stats_fresh FeatureView DESCRIBE ``` +## Registry + +### create-schema + +Pre-create the SQL registry schema so the application does not need DDL privileges at runtime. Use this with `schema_mode: verify` or `schema_mode: skip` in your `feature_store.yaml`. + +```text +feast registry create-schema +``` + +This command only applies to SQL-based registries (`registry_type: sql`). It is safe to run multiple times — existing tables are not modified. + ## Teardown Tear down deployed feature store infrastructure diff --git a/docs/reference/feature-repository/README.md b/docs/reference/feature-repository/README.md index 2c1b112a783..38968825c1d 100644 --- a/docs/reference/feature-repository/README.md +++ b/docs/reference/feature-repository/README.md @@ -127,4 +127,4 @@ To declare new feature definitions, just add code to the feature repository, eit ### Next steps * See [Create a feature repository](../../how-to-guides/feast-snowflake-gcp-aws/create-a-feature-repository.md) to get started with an example feature repository. -* See [feature_store.yaml](feature-store-yaml.md), [.feastignore](feast-ignore.md), or [Feature Views](../../getting-started/concepts/feature-view.md) for more information on the configuration files that live in a feature registry. +* See [feature_store.yaml](feature-store-yaml.md), [.feastignore](feast-ignore.md), [Registration inferencing](registration-inferencing.md), or [Feature Views](../../getting-started/concepts/feature-view.md) for more information on the configuration files that live in a feature registry. diff --git a/docs/reference/feature-servers/python-feature-server.md b/docs/reference/feature-servers/python-feature-server.md index 4802599866d..b1b873cc7d2 100644 --- a/docs/reference/feature-servers/python-feature-server.md +++ b/docs/reference/feature-servers/python-feature-server.md @@ -527,6 +527,42 @@ Prometheus adds an `instance` label per pod, so there is no duplication. Use `sum(rate(...))` or `histogram_quantile(...)` across instances as usual. +## Vector Search (`POST /search`) + +The feature server exposes `POST /search` for vector similarity search against online document embeddings. Pass a pre-computed embedding in `query`, or use `api_version: 2` with `query_string` for text-based search when the online store supports it. + +`POST /retrieve-online-documents` is a deprecated alias with the same request body and response; new integrations should use `/search`. + +## [Alpha] OpenAI-Compatible Vector Store API + +{% hint style="warning" %} +**Alpha feature.** This API surface is functional and tested, but may change in future releases. +{% endhint %} + +The feature server exposes OpenAI-compatible vector store endpoints. This allows clients (including LLM agents and tool-calling frameworks) to discover and search vector data with plain text queries, without computing embeddings client-side. + +Each feature view with vector-indexed fields gets a deterministic `vs_{hash}` identifier derived from `SHA-256(project + ":" + feature_view_name)`. These IDs are stable across server restarts. + +### Endpoints + +| Method | Path | RBAC | Description | +|---|---|---|---| +| `GET` | `/v1/vector_stores` | `DESCRIBE` | List all vector stores (filtered by caller permissions) | +| `GET` | `/v1/vector_stores/{vector_store_id}` | `DESCRIBE` | Get metadata for a single vector store | +| `POST` | `/v1/vector_stores/{vector_store_id}/search` | `READ_ONLINE` | Search a vector store with server-side embedding | + +### Configuration + +Add an `embedding_model` section to your `feature_store.yaml`: + +```yaml +embedding_model: + provider: sentence_transformers # default; can be omitted + model: all-MiniLM-L6-v2 +``` + +Feast uses **Sentence Transformers** (default) for local embedding inference — no external API key required. Custom embedding providers can be plugged in by implementing the `EmbeddingProvider` protocol. See [\[Alpha\] Vector Database](../alpha-vector-database.md#alpha-openai-compatible-vector-store-api) for full configuration, custom providers, filter details, and SDK usage. + ## Starting the feature server in TLS(SSL) mode Enabling TLS mode ensures that data between the Feast client and server is transmitted securely. For an ideal production environment, it is recommended to start the feature server in TLS mode. @@ -598,7 +634,11 @@ The [PyTorch NLP template](https://github.com/feast-dev/feast/tree/main/sdk/pyth | Endpoint | Resource Type | Permission | Description | |----------------------------|---------------------------------|-------------------------------------------------------|----------------------------------------------------------------| | /get-online-features | FeatureView,OnDemandFeatureView | Read Online | Get online features from the feature store | -| /retrieve-online-documents | FeatureView | Read Online | Retrieve online documents from the feature store for RAG | +| /search | FeatureView | Read Online | Vector similarity search for RAG (embedding vector or text query) | +| /retrieve-online-documents | FeatureView | Read Online | **Deprecated.** Use `/search` instead. | +| /v1/vector_stores | FeatureView | Describe | [Alpha] List all vector stores | +| /v1/vector_stores/{id} | FeatureView | Describe | [Alpha] Get a single vector store | +| /v1/vector_stores/{id}/search | FeatureView | Read Online | [Alpha] OpenAI-compatible vector search with server-side embedding | | /push | FeatureView | Write Online, Write Offline, Write Online and Offline | Push features to the feature store (online, offline, or both) | | /write-to-online-store | FeatureView | Write Online | Write features to the online store | | /materialize | FeatureView | Write Online | Materialize features within a specified time range | diff --git a/docs/reference/feature-servers/registry-server.md b/docs/reference/feature-servers/registry-server.md index 496eaa8badc..4558a10ce63 100644 --- a/docs/reference/feature-servers/registry-server.md +++ b/docs/reference/feature-servers/registry-server.md @@ -214,6 +214,7 @@ Most endpoints support these common query parameters: - `feature` (optional): Filter feature views by feature name - `feature_service` (optional): Filter feature views by feature service name - `data_source` (optional): Filter feature views by data source name + - `updated_since` (optional): Only return feature views updated at or after this ISO-8601 UTC timestamp (e.g. `2024-01-01T00:00:00Z`) - `page` (optional): Page number for pagination - `limit` (optional): Number of items per page - `sort_by` (optional): Field to sort by @@ -223,27 +224,31 @@ Most endpoints support these common query parameters: # Basic list curl -H "Authorization: Bearer " \ "http://localhost:6572/api/v1/feature_views?project=my_project" - + # With pagination and relationships curl -H "Authorization: Bearer " \ "http://localhost:6572/api/v1/feature_views?project=my_project&include_relationships=true&page=1&limit=5&sort_by=name" - + # Filter by entity curl -H "Authorization: Bearer " \ "http://localhost:6572/api/v1/feature_views?project=my_project&entity=user" - + # Filter by feature curl -H "Authorization: Bearer " \ "http://localhost:6572/api/v1/feature_views?project=my_project&feature=age" - + # Filter by data source curl -H "Authorization: Bearer " \ "http://localhost:6572/api/v1/feature_views?project=my_project&data_source=user_profile_source" - + # Filter by feature service curl -H "Authorization: Bearer " \ "http://localhost:6572/api/v1/feature_views?project=my_project&feature_service=user_service" - + + # Filter by last-updated timestamp + curl -H "Authorization: Bearer " \ + "http://localhost:6572/api/v1/feature_views?project=my_project&updated_since=2024-06-01T00:00:00Z" + # Multiple filters combined curl -H "Authorization: Bearer " \ "http://localhost:6572/api/v1/feature_views?project=my_project&entity=user&feature=age" diff --git a/docs/reference/feature-store-yaml.md b/docs/reference/feature-store-yaml.md index 9c8975eb335..1aac166bd8b 100644 --- a/docs/reference/feature-store-yaml.md +++ b/docs/reference/feature-store-yaml.md @@ -66,6 +66,7 @@ registry: |-------|------|---------|-------------| | `registry_type` | string | `file` | Registry backend (`file`, `sql`, etc.) | | `path` | string | — | Connection string or file path | +| `schema_mode` | string | `auto` | SQL registry only. `auto`: create tables on startup; `verify`: check tables exist, error if missing; `skip`: no DDL or verification. See [SQL Registry docs](registries/sql.md#schema-management-schema_mode). | | `mcp.enabled` | bool | `false` | Enable MCP (Model Context Protocol) on the REST registry server | When `registry.mcp.enabled` is `true`, the REST registry server exposes registry diff --git a/docs/reference/online-stores/README.md b/docs/reference/online-stores/README.md index 6f31993f896..257864b9b30 100644 --- a/docs/reference/online-stores/README.md +++ b/docs/reference/online-stores/README.md @@ -22,6 +22,10 @@ Please see [Online Store](../../getting-started/components/online-store.md) for [dragonfly.md](dragonfly.md) {% endcontent-ref %} +{% content-ref url="valkey.md" %} +[valkey.md](valkey.md) +{% endcontent-ref %} + {% content-ref url="datastore.md" %} [datastore.md](datastore.md) {% endcontent-ref %} @@ -31,7 +35,7 @@ Please see [Online Store](../../getting-started/components/online-store.md) for {% endcontent-ref %} {% content-ref url="bigtable.md" %} -[bigtable.md](mysql.md) +[bigtable.md](bigtable.md) {% endcontent-ref %} {% content-ref url="postgres.md" %} @@ -58,6 +62,10 @@ Please see [Online Store](../../getting-started/components/online-store.md) for [mongodb.md](mongodb.md) {% endcontent-ref %} +{% content-ref url="aerospike.md" %} +[aerospike.md](aerospike.md) +{% endcontent-ref %} + {% content-ref url="hazelcast.md" %} [hazelcast.md](hazelcast.md) {% endcontent-ref %} diff --git a/docs/reference/online-stores/aerospike.md b/docs/reference/online-stores/aerospike.md new file mode 100644 index 00000000000..e5a9754796b --- /dev/null +++ b/docs/reference/online-stores/aerospike.md @@ -0,0 +1,389 @@ +# Aerospike online store (Preview) + +## Description + +The [Aerospike](https://aerospike.com/) online store provides support for materializing feature values into an Aerospike cluster for serving online features. + +{% hint style="warning" %} +The Aerospike online store is currently in **preview**. Some functionality may be unstable, and breaking changes may occur in future releases. +{% endhint %} + +## Features + +* Supports both synchronous and asynchronous read/write paths (`online_read` / `online_read_async`, `online_write_batch` / `online_write_batch_async`). Async methods wrap the blocking client in `run_in_executor`, keeping the event loop responsive in feature-server workloads. +* Partial, server-side upserts via Aerospike Map CDT operations — writing one feature view never clobbers another feature view stored on the same entity. +* Record-level TTL controlled by a single `ttl_seconds` config option (honours the namespace default, a "never expire" sentinel, or an explicit number of seconds). +* Per-feature-view **namespace overrides** and **set overrides** — pin individual feature views to RAM-only or SSD-backed namespaces, or isolate one view in its own set, without splitting projects. +* **Prewriting hook** — a configurable, import-string-resolved callable applied to every write batch for cross-cutting concerns like PII masking, application-side encryption, or value coercion. +* Authentication and TLS options for Aerospike Enterprise Edition passed straight through to the Aerospike Python client. +* `client_kwargs` escape hatch for any advanced client-config field not surfaced on `AerospikeOnlineStoreConfig`. +* Baseline: Aerospike Server **≥ 6.0** (uses batch-write / batch-operate APIs). The store has been developed against CE 8.x. + +## Getting started + +Install the Aerospike extra (alongside the dependency for the offline store of choice): + +```bash +pip install 'feast[aerospike]' +``` + +You can start from any of the standard templates (e.g. `feast init -t local` or `feast init -t aws`) and then swap in Aerospike as the online store as shown below. + +## Examples + +### Basic configuration — local Aerospike CE + +{% code title="feature_store.yaml" %} +```yaml +project: my_feature_repo +registry: data/registry.db +provider: local +online_store: + type: aerospike + hosts: + - ["127.0.0.1", 3000] + namespace: feast +``` +{% endcode %} + +### Multi-node cluster + +{% code title="feature_store.yaml" %} +```yaml +project: my_feature_repo +registry: data/registry.db +provider: local +online_store: + type: aerospike + hosts: + - ["aerospike-1.internal", 3000] + - ["aerospike-2.internal", 3000] + - ["aerospike-3.internal", 3000] + namespace: feast + ttl_seconds: 86400 # 24h record-level TTL + read_timeout_ms: 150 # hard deadline for a single-record get + write_timeout_ms: 300 # hard deadline for a single-record put/operate + batch_total_timeout_ms: 500 # hard deadline for online_read / online_write_batch + batch_max_records: 1000 # chunk size for batch_write / batch_operate + socket_timeout_ms: 50 # per-attempt deadline so max_retries can fire + max_retries: 2 +``` +{% endcode %} + +> **Timeout semantics.** The Aerospike client distinguishes per-attempt +> (`socket_timeout`) from total (`total_timeout`) deadlines. `*_timeout_ms` map +> to `total_timeout` — the overall budget for a call including retries. Set +> `socket_timeout_ms` as well so each individual attempt has its own (shorter) +> deadline; without it, `max_retries` effectively never fires because the +> first attempt is allowed to consume the entire total deadline. + +> **Batch chunking.** `online_read` and `online_write_batch` split large +> requests into chunks of at most `batch_max_records` (default `1000`). +> Aerospike enforces a per-node batch limit via the server `batch-max-requests` +> setting (historically `5000`). Lower `batch_max_records` if your cluster cap +> is tighter; raise it only when the server limit and client timeouts allow. + +### Aerospike Enterprise with authentication + +> Requires Aerospike Enterprise Edition. The Community Edition server has no built-in user/security model and will reject these config keys. + +{% code title="feature_store.yaml" %} +```yaml +project: my_feature_repo +registry: data/registry.db +provider: local +online_store: + type: aerospike + hosts: + - ["aerospike.internal", 3000] + namespace: feast + user: feast_user + password: ${AEROSPIKE_PASSWORD} # pragma: allowlist secret + auth_mode: internal # internal | external | pki +``` +{% endcode %} + +### Aerospike Enterprise with TLS + +> Requires Aerospike Enterprise Edition. The Community Edition server does not implement TLS, so `tls` config is effective only against EE clusters. + +{% code title="feature_store.yaml" %} +```yaml +project: my_feature_repo +registry: data/registry.db +provider: local +online_store: + type: aerospike + hosts: + - ["aerospike-1.internal", 4333, "aerospike-tls"] + namespace: feast + tls: + enable: true + cafile: /etc/aerospike/certs/ca.pem + certfile: /etc/aerospike/certs/client.pem + keyfile: /etc/aerospike/certs/client.key +``` +{% endcode %} + +### Per-feature-view namespace and set overrides + +Two `Dict[str, str]` config fields — `namespace_overrides` and `set_overrides` — let you place individual feature views on a different Aerospike namespace or set without splitting your project across stores. Anything not listed in either map falls back to the store-level default (`namespace` / `set_name_template`). + +Common reasons to reach for these: + +* A **hot, latency-sensitive view** belongs on a RAM-only namespace; a **wide, cold view** belongs on an SSD-backed namespace. Same project, different storage tiers. +* You want `feast apply` deletions or `truncate` on one feature view to be O(1) without scanning records of the others — give that view its own set. + +{% code title="feature_store.yaml" %} +```yaml +project: my_feature_repo +registry: data/registry.db +provider: local +online_store: + type: aerospike + hosts: + - ["aerospike.internal", 3000] + namespace: feast # default namespace + set_name_template: "{project}_{collection_suffix}" + namespace_overrides: + driver_realtime_stats: feast_ram # in-memory namespace + driver_history_lookup: feast_ssd # device-backed namespace + set_overrides: + isolated_view: my_feature_repo_isolated +``` +{% endcode %} + +> **Tradeoffs.** +> +> * Every namespace listed in `namespace_overrides` MUST already exist on the cluster — Aerospike cannot create namespaces at runtime, and a missing namespace surfaces as an opaque `AEROSPIKE_ERR_PARAM` on the first read or write. +> * Putting feature views on different sets means a multi-feature-view read for the same entity becomes one Aerospike round trip per set, not one round trip total. Only opt in when the operational isolation is worth that cost. Reads that touch a single feature view are unaffected. +> * Admin operations honour the overrides automatically: `update()` (called by `feast apply`) groups dropped feature views by their resolved `(namespace, set)` and issues one background scan per group; `teardown()` truncates every unique `(namespace, set)` pair the project may have written to (including the store-level default). + +### Prewriting hooks + +`prewriting_hook` is the import path of a callable that is invoked once per `online_write_batch` call, receives the rows about to be written, and returns the rows that actually go on the wire. Use it for cross-cutting write-side concerns that you don't want sprinkled through every materialization job — PII masking, application-side encryption, dual-write fan-out, value coercion, etc. + +Hooks are referenced by import string (rather than as a Python `Callable` value) so the config survives YAML/JSON serialisation and remote-feature-server transport. The resolved callable is cached on the store instance, so import cost is paid once per store lifetime. + +**Hook signature:** + +```python +def hook( + config: RepoConfig, + table: FeatureView, + data: list[ + tuple[ + EntityKeyProto, + dict[str, ValueProto], + datetime, + datetime | None, + ] + ], +) -> list[ + tuple[ + EntityKeyProto, + dict[str, ValueProto], + datetime, + datetime | None, + ] +]: + ... +``` + +The hook MUST return a row list with the same schema as its input. Returning `[]` short-circuits the write — same path as an empty input, no wire call is issued. Hooks that raise will fail the whole batch; there is no per-row fallback. + +**1. Drop a hook function in your project.** Any module on the `PYTHONPATH` of every process that writes through Feast will do (the materialization workers, the registry CLI host, and the feature server, if you run one). + +{% code title="my_feature_repo/hooks.py" %} +```python +"""Prewriting hooks for the Aerospike online store.""" +from __future__ import annotations + +import hashlib +import os +from datetime import datetime +from typing import Optional + +from feast import FeatureView +from feast.protos.feast.types.EntityKey_pb2 import EntityKey as EntityKeyProto +from feast.protos.feast.types.Value_pb2 import Value as ValueProto +from feast.repo_config import RepoConfig + +# Names of features that must never reach the online store as plaintext. +# Matched by exact feature name; tweak to your project's conventions. +_SENSITIVE_FEATURES = {"email", "phone_number", "ssn"} + + +def hash_pii_string_features( + config: RepoConfig, + table: FeatureView, + data: list[ + tuple[ + EntityKeyProto, + dict[str, ValueProto], + datetime, + Optional[datetime], + ] + ], +) -> list[ + tuple[ + EntityKeyProto, + dict[str, ValueProto], + datetime, + Optional[datetime], + ] +]: + """Replace any sensitive string feature with a salted SHA-256 hex digest. + + The hash is deterministic (same input → same digest) so downstream lookups + that hash the candidate value the same way still hit. ``FEAST_PII_SALT`` + must be set on every process that materialises features; an unset salt + raises rather than silently falling back to plaintext. + """ + salt = os.environ.get("FEAST_PII_SALT") + if salt is None: + raise RuntimeError( + "FEAST_PII_SALT is not set; refusing to write feature batches " + "without a configured PII salt." + ) + salt_bytes = salt.encode("utf-8") + + def _digest(plaintext: str) -> str: + h = hashlib.sha256() + h.update(salt_bytes) + h.update(plaintext.encode("utf-8")) + return h.hexdigest() + + transformed: list[ + tuple[ + EntityKeyProto, + dict[str, ValueProto], + datetime, + Optional[datetime], + ] + ] = [] + for entity_key, values, event_ts, created_ts in data: + new_values = dict(values) + for feature_name in _SENSITIVE_FEATURES.intersection(new_values): + v = new_values[feature_name] + if v.HasField("string_val") and v.string_val: + new_values[feature_name] = ValueProto(string_val=_digest(v.string_val)) + transformed.append((entity_key, new_values, event_ts, created_ts)) + return transformed +``` +{% endcode %} + +**2. Reference the hook from `feature_store.yaml`:** + +{% code title="feature_store.yaml" %} +```yaml +project: my_feature_repo +registry: data/registry.db +provider: local +online_store: + type: aerospike + hosts: + - ["aerospike.internal", 3000] + namespace: feast + prewriting_hook: my_feature_repo.hooks.hash_pii_string_features +``` +{% endcode %} + +> **Operational notes.** +> +> * The hook is **only invoked on the write path**; reads pass through the store untouched. If your hook is one-way (e.g. hashing) you have to apply the same transformation to the candidate value at read time yourself. +> * Hooks run inside the same process as the writer — they're not RPCs and not sandboxed. They can read environment variables, open files, call out to KMS, etc. Treat them as part of your trusted code base. +> * A misconfigured `prewriting_hook` (bad import path, missing function, non-callable target) raises `ValueError` / `TypeError` on the *first* `online_write_batch` call, not on store construction. Add a smoke test that writes one row at deploy time so misconfigurations surface before a real batch. + +The full set of configuration options is available in [`AerospikeOnlineStoreConfig`](https://rtd.feast.dev/en/latest/#feast.infra.online_stores.aerospike_online_store.aerospike.AerospikeOnlineStoreConfig). + +## Data Model + +The Aerospike online store uses a **single set per project** with entity-key collocation. Features from multiple feature views for the same entity are stored together on a single Aerospike record, analogous to the MongoDB online store's "one document per entity" layout. + +| Aerospike concept | Feast mapping | +| :---------------- | :---------------------------------------------------------------------------- | +| Namespace | `online_store.namespace` (must be pre-configured on the cluster); per-feature-view override via `online_store.namespace_overrides` | +| Set | `online_store.set_name_template` → `"{project}_{collection_suffix}"` by default; per-feature-view override via `online_store.set_overrides` | +| Key | `serialize_entity_key(entity_key)` as `bytearray` user key | +| Bin `features` | Map CDT keyed by feature-view name, each value a map of `feature → native` | +| Bin `event_ts` | Map CDT keyed by feature-view name, each value an int64 epoch-ms timestamp | +| Bin `created_ts` | Top-level int64 epoch-ms timestamp (last `feast materialize`) | + +### Example record + +For a single entity carrying features from two feature views (`driver_stats` and `pricing`): + +```text +key: (ns="feast", set="my_feature_repo_latest", user_key=) +bins: + features: + driver_stats: + rating: 4.91 + trips_last_7d: 132 + pricing: + surge_multiplier: 1.2 + event_ts: + driver_stats: 1737374400000 # 2025-01-20T12:00:00Z + pricing: 1737447000000 # 2025-01-21T08:30:00Z + created_ts: 1737460805000 # 2025-01-21T12:00:05Z +``` + +### Key design decisions + +* **Record per entity, bin per concept.** `features` and `event_ts` are Aerospike Map CDT bins, not dynamic bins, which keeps the store within the 15-byte Aerospike bin-name limit regardless of how many feature views a project has. +* **Partial upserts via Map CDT ops.** Writes use `batch_write` with `map_put_items("features", {: {...}})` and `map_put("event_ts", , )`. Concurrent writes to different feature views on the same entity never clobber each other — each write mutates only its own map keys. +* **Entity-key bytes as the Aerospike user key.** Feast's `serialize_entity_key` output is passed as a `bytearray` user key (not `bytes` — the Python client hashes only the first byte of `bytes` keys, which would collapse distinct entities). +* **Timestamps as int64 epoch milliseconds.** Aerospike has no native datetime type; tz-naive timestamps are treated as UTC per the `OnlineStore` contract. + +### TTL and expiry + +`ttl_seconds` is written as record-level metadata on every `online_write_batch` call: + +| `ttl_seconds` | Aerospike TTL | Effect | +| :------------ | :-------------------------------- | :---------------------------------------------------------- | +| not set / `null` | `TTL_NAMESPACE_DEFAULT` | Record inherits the namespace's configured `default-ttl`. | +| `0` | `TTL_NEVER_EXPIRE` | Record is kept until explicitly deleted. | +| `>0` | that many seconds | Record is evicted by the server's `nsup` thread. | + +There is no per-feature-view TTL override in this version — the setting is applied uniformly for every write made by the online store. + +### Indexes + +No secondary indexes are created. All access goes through the primary key, which is the serialized entity key. + +## Async support + +Async read/write are provided by running the Aerospike Python client's blocking calls on the default thread-pool executor (`loop.run_in_executor`). The underlying C client releases the GIL during network I/O, so `await store.online_read_async(...)` keeps the event loop responsive. A native asyncio Aerospike client is not currently used. + +Both sync and async methods are fully supported: + +* `online_read` / `online_read_async` +* `online_write_batch` / `online_write_batch_async` +* `initialize` / `close` — `initialize(config)` eagerly opens the connection so feature servers pay the TCP/handshake cost at startup; `close()` releases the cached client. + +## Functionality Matrix + +The set of functionality supported by online stores is described in detail [here](overview.md#functionality). +Below is a matrix indicating which functionality is supported by the Aerospike online store. + +| | Aerospike | +| :-------------------------------------------------------- | :-------- | +| write feature values to the online store | yes | +| read feature values from the online store | yes | +| update infrastructure (e.g. tables) in the online store | yes | +| teardown infrastructure (e.g. tables) in the online store | yes | +| generate a plan of infrastructure changes | no | +| support for on-demand transforms | yes | +| readable by Python SDK | yes | +| readable by Java | no | +| readable by Go | no | +| support for entityless feature views | yes | +| support for concurrent writing to the same key | yes | +| support for ttl (time to live) at retrieval | yes | +| support for deleting expired data | yes | +| collocated by feature view | no | +| collocated by feature service | no | +| collocated by entity key | yes | + +To compare this set of functionality against other online stores, please see the full [functionality matrix](overview.md#functionality-matrix). diff --git a/docs/reference/online-stores/cassandra.md b/docs/reference/online-stores/cassandra.md index 198f15ca47f..5d95e526421 100644 --- a/docs/reference/online-stores/cassandra.md +++ b/docs/reference/online-stores/cassandra.md @@ -37,6 +37,51 @@ online_store: ``` {% endcode %} +### Example (Cassandra — multi-DC) + +Use `datacenters` instead of `hosts` when your cluster spans multiple datacenters. +Each entry gets a named Cassandra **execution profile** keyed by its `name` field, +enabling per-DC routing. The default profile is determined by `load_balancing.local_dc` +(or the first datacenter entry when `load_balancing` is absent). Use the optional +`routing` block to direct reads and writes to specific datacenters. The keyspace must +already exist; Feast does not create it automatically. + +`datacenters` is mutually exclusive with `hosts` and `secure_bundle_path`. + +{% code title="feature_store.yaml" %} +```yaml +project: my_feature_repo +registry: data/registry.db +provider: local +online_store: + type: cassandra + keyspace: KeyspaceName + datacenters: + - name: dc1 + hosts: + - 192.168.1.1 + - 192.168.1.2 + replication_factor: 3 # optional, informational only + replication_strategy: NetworkTopologyStrategy # optional, informational only + - name: dc2 + hosts: + - 10.0.0.1 + replication_factor: 2 # optional, informational only + routing: # optional + read_dc: dc2 # DC to use for reads (default: load_balancing.local_dc) + write_dc: dc1 # DC to use for writes (default: load_balancing.local_dc) + port: 9042 # optional + username: user # optional + password: secret # optional + protocol_version: 5 # optional + load_balancing: # optional + local_dc: 'dc1' # sets the default execution profile + load_balancing_policy: 'TokenAwarePolicy(DCAwareRoundRobinPolicy)' # optional + read_concurrency: 100 # optional + write_concurrency: 100 # optional +``` +{% endcode %} + ### Example (Astra DB) {% code title="feature_store.yaml" %} diff --git a/docs/reference/online-stores/dynamodb.md b/docs/reference/online-stores/dynamodb.md index 68d3d29ca3b..a7f6b7392c9 100644 --- a/docs/reference/online-stores/dynamodb.md +++ b/docs/reference/online-stores/dynamodb.md @@ -24,7 +24,7 @@ The full set of configuration options is available in [DynamoDBOnlineStoreConfig ## Configuration -Below is a example with performance tuning options: +Below is an example with performance tuning options: {% code title="feature_store.yaml" %} ```yaml diff --git a/docs/reference/online-stores/milvus.md b/docs/reference/online-stores/milvus.md index 014c7bd68a5..58f7dbd167a 100644 --- a/docs/reference/online-stores/milvus.md +++ b/docs/reference/online-stores/milvus.md @@ -11,6 +11,14 @@ In order to use this online store, you'll need to install the Milvus extra (alon `pip install 'feast[milvus]'` +{% hint style="warning" %} +**Upgrading to milvus-lite 3.0.0+** + +Feast supports both milvus-lite 2.x and 3.x. However, if you upgrade from milvus-lite 2.x.x to 3.0.0+, the `.db` files created by the original storage format are **not compatible** with the milvus-lite 3.0.0+ engine. You will need to re-import your data into a new database — automatic migration is not available. + +See the [milvus-lite GitHub page](https://github.com/milvus-io/milvus-lite) for more details. +{% endhint %} + You can get started by using any of the other templates (e.g. `feast init -t gcp` or `feast init -t snowflake` or `feast init -t aws`), and then swapping in Redis as the online store as seen below in the examples. ## Examples diff --git a/docs/reference/online-stores/overview.md b/docs/reference/online-stores/overview.md index 6ee076b0669..663a48836dc 100644 --- a/docs/reference/online-stores/overview.md +++ b/docs/reference/online-stores/overview.md @@ -29,26 +29,26 @@ See this [issue](https://github.com/feast-dev/feast/issues/2254) for a discussio ## Functionality Matrix There are currently five core online store implementations: `SqliteOnlineStore`, `RedisOnlineStore`, `DynamoDBOnlineStore`, `SnowflakeOnlineStore`, and `DatastoreOnlineStore`. -There are several additional implementations contributed by the Feast community (`PostgreSQLOnlineStore`, `HbaseOnlineStore` and `CassandraOnlineStore`), which are not guaranteed to be stable or to match the functionality of the core implementations. +There are several additional implementations contributed by the Feast community (`PostgreSQLOnlineStore`, `HbaseOnlineStore`, `CassandraOnlineStore` and `ScyllaDBOnlineStore`), which are not guaranteed to be stable or to match the functionality of the core implementations. Details for each specific online store, such as how to configure it in a `feature_store.yaml`, can be found [here](README.md). Below is a matrix indicating which online stores support what functionality. -| | Sqlite | Redis | DynamoDB | Snowflake | Datastore | Postgres | Hbase | [[Cassandra](https://cassandra.apache.org/_/index.html) / [Astra DB](https://www.datastax.com/products/datastax-astra?utm_source=feast)] | Milvus | -| :-------------------------------------------------------- | :-- | :-- | :-- | :-- | :-- | :-- | :-- | :-- |:----| -| write feature values to the online store | yes | yes | yes | yes | yes | yes | yes | yes | yes | -| read feature values from the online store | yes | yes | yes | yes | yes | yes | yes | yes | yes | -| update infrastructure (e.g. tables) in the online store | yes | yes | yes | yes | yes | yes | yes | yes | yes | -| teardown infrastructure (e.g. tables) in the online store | yes | yes | yes | yes | yes | yes | yes | yes | yes | -| generate a plan of infrastructure changes | yes | no | no | no | no | no | no | yes | no | -| support for on-demand transforms | yes | yes | yes | yes | yes | yes | yes | yes | yes | -| readable by Python SDK | yes | yes | yes | yes | yes | yes | yes | yes | yes | -| readable by Java | no | yes | no | no | no | no | no | no | no | -| readable by Go | yes | yes | no | no | no | no | no | no | no | -| support for entityless feature views | yes | yes | yes | yes | yes | yes | yes | yes | yes | -| support for concurrent writing to the same key | no | yes | no | no | no | no | no | no | yes | -| support for ttl (time to live) at retrieval | no | yes | no | no | no | no | no | no | no | -| support for deleting expired data | no | yes | no | no | no | no | no | no | no | -| collocated by feature view | yes | no | yes | yes | yes | yes | yes | yes | no | -| collocated by feature service | no | no | no | no | no | no | no | no | no | -| collocated by entity key | no | yes | no | no | no | no | no | no | yes | +| | Sqlite | Redis | DynamoDB | Snowflake | Datastore | Postgres | Hbase | [[Cassandra](https://cassandra.apache.org/_/index.html) / [Astra DB](https://www.datastax.com/products/datastax-astra?utm_source=feast)] | Milvus | ScyllaDB | +| :-------------------------------------------------------- | :-- | :-- | :-- | :-- | :-- | :-- | :-- | :-- |:----| :-- | +| write feature values to the online store | yes | yes | yes | yes | yes | yes | yes | yes | yes | yes | +| read feature values from the online store | yes | yes | yes | yes | yes | yes | yes | yes | yes | yes | +| update infrastructure (e.g. tables) in the online store | yes | yes | yes | yes | yes | yes | yes | yes | yes | yes | +| teardown infrastructure (e.g. tables) in the online store | yes | yes | yes | yes | yes | yes | yes | yes | yes | yes | +| generate a plan of infrastructure changes | yes | no | no | no | no | no | no | yes | no | no | +| support for on-demand transforms | yes | yes | yes | yes | yes | yes | yes | yes | yes | yes | +| readable by Python SDK | yes | yes | yes | yes | yes | yes | yes | yes | yes | yes | +| readable by Java | no | yes | no | no | no | no | no | no | no | no | +| readable by Go | yes | yes | no | no | no | no | no | no | no | no | +| support for entityless feature views | yes | yes | yes | yes | yes | yes | yes | yes | yes | yes | +| support for concurrent writing to the same key | no | yes | no | no | no | no | no | no | yes | no | +| support for ttl (time to live) at retrieval | no | yes | no | no | no | no | no | no | no | yes | +| support for deleting expired data | no | yes | no | no | no | no | no | no | no | yes | +| collocated by feature view | yes | no | yes | yes | yes | yes | yes | yes | no | yes | +| collocated by feature service | no | no | no | no | no | no | no | no | no | no | +| collocated by entity key | no | yes | no | no | no | no | no | no | yes | no | diff --git a/docs/reference/online-stores/scylladb.md b/docs/reference/online-stores/scylladb.md index c8583ac101a..98dc03b24cb 100644 --- a/docs/reference/online-stores/scylladb.md +++ b/docs/reference/online-stores/scylladb.md @@ -2,20 +2,15 @@ ## Description -ScyllaDB is a low-latency and high-performance Cassandra-compatible (uses CQL) database. You can use the existing Cassandra connector to use ScyllaDB as an online store in Feast. - -The [ScyllaDB](https://www.scylladb.com/) online store provides support for materializing feature values into a ScyllaDB or [ScyllaDB Cloud](https://www.scylladb.com/product/scylla-cloud/) cluster for serving online features real-time. +[ScyllaDB](https://www.scylladb.com/) is a distributed real-time NoSQL database with vector search support. +This integration uses the native **`scylla-driver`** Python driver for optimised performance and supports materializing feature values into a [ScyllaDB Cloud](https://www.scylladb.com/product/scylla-cloud/) cluster for real-time online feature serving. ## Getting started -Install Feast with Cassandra support: -```bash -pip install "feast[cassandra]" -``` +Install Feast with the `scylladb` extra, which pulls in `scylla-driver` automatically: -Create a new Feast project: ```bash -feast init REPO_NAME -t cassandra +pip install feast[scylladb] ``` ### Example (ScyllaDB) @@ -26,7 +21,7 @@ project: scylla_feature_repo registry: data/registry.db provider: local online_store: - type: cassandra + type: scylladb hosts: - 172.17.0.2 keyspace: feast @@ -43,44 +38,106 @@ project: scylla_feature_repo registry: data/registry.db provider: local online_store: - type: cassandra + type: scylladb hosts: - node-0.aws_us_east_1.xxxxxxxx.clusters.scylla.cloud - node-1.aws_us_east_1.xxxxxxxx.clusters.scylla.cloud - node-2.aws_us_east_1.xxxxxxxx.clusters.scylla.cloud keyspace: feast username: scylla - password: password + password: xxxxxx + local_dc: AWS_US_EAST_1 ``` {% endcode %} - -The full set of configuration options is available in [CassandraOnlineStoreConfig](https://rtd.feast.dev/en/master/#feast.infra.online_stores.cassandra_online_store.cassandra_online_store.CassandraOnlineStoreConfig). -For a full explanation of configuration options please look at file -`sdk/python/feast/infra/online_stores/contrib/cassandra_online_store/README.md`. +## Configuration options + +| Parameter | Type | Default | Description | +| :--- | :--- | :--- | :--- | +| `hosts` | list[str] | *(required)* | Contact-point host addresses. | +| `port` | int | `9042` | CQL port. | +| `keyspace` | str | `feast_keyspace` | Target ScyllaDB keyspace. | +| `username` | str | `None` | Auth username. | +| `password` | str | `None` | Auth password. | +| `local_dc` | str | `None` | Local datacenter name for DC-aware load balancing. | +| `request_timeout` | float | `None` | Driver request timeout in seconds. | +| `read_concurrency` | int | `100` | `concurrency` argument passed to the driver's `execute_concurrent_with_args` for reads. Controls how many CQL statements are in-flight at once. | +| `write_concurrency` | int | `100` | `concurrency` argument passed to the driver's `execute_concurrent_with_args` for writes. Controls how many CQL statements are in-flight at once. | +| `vector_similarity_function` | str | `COSINE` | Default similarity function for vector indexes. Supported: `COSINE`, `DOT_PRODUCT`, `EUCLIDEAN`. Can be overridden per-feature via the `similarity_function` Field tag. | Storage specifications can be found at `docs/specs/online_store_format.md`. +## Vector Search + +ScyllaDB Cloud supports approximate nearest-neighbour (ANN) vector search. +To enable it for a feature view, tag the embedding `Field` with `vector_index=true` and specify the number of dimensions: + +{% code title="feature_definitions.py" %} +```python +from feast import FeatureView, Field +from feast.types import Array, Float32, String + +documents_fv = FeatureView( + name="documents", + entities=[item], + schema=[ + Field(name="text", dtype=String), + Field( + name="embedding", + dtype=Array(Float32), + tags={ + "vector_index": "true", + "dimensions": "768", + "similarity_function": "COSINE", # COSINE | DOT_PRODUCT | EUCLIDEAN + }, + ), + ], + online=True, + source=push_source, +) +``` +{% endcode %} + +When `feast apply` runs, the store automatically creates the necessary tables and HNSW ANN index for any feature view with vector-tagged fields. + +To query the top-k most similar documents: + +```python +result = store.retrieve_online_documents_v2( + features=["documents:text", "documents:embedding"], + query=[0.1, 0.2, ...], # your query embedding + top_k=10, + distance_metric="COSINE", +) +``` + +### Metadata filtering (OpenAI-compatible) + +ScyllaDB supports vector similarity search, but OpenAI-style metadata filtering is **not supported yet**. +Passing `filters` to `retrieve_online_documents_v2` or the OpenAI-compatible search endpoint raises `NotImplementedError`. + +For filtered vector search today, use one of the backends that implement metadata filters (for example Milvus, Elasticsearch, Postgres, SQLite, or MongoDB). See [Alpha Vector Database](../alpha-vector-database.md#supported-online-stores). + ## Functionality Matrix The set of functionality supported by online stores is described in detail [here](overview.md#functionality). -Below is a matrix indicating which functionality is supported by the Cassandra plugin. +Below is a matrix indicating which functionality is supported by the ScyllaDB online store. -| | Cassandra | +| | ScyllaDB | | :-------------------------------------------------------- | :-------- | | write feature values to the online store | yes | | read feature values from the online store | yes | | update infrastructure (e.g. tables) in the online store | yes | | teardown infrastructure (e.g. tables) in the online store | yes | -| generate a plan of infrastructure changes | yes | +| generate a plan of infrastructure changes | no | | support for on-demand transforms | yes | | readable by Python SDK | yes | | readable by Java | no | | readable by Go | no | | support for entityless feature views | yes | | support for concurrent writing to the same key | no | -| support for ttl (time to live) at retrieval | no | -| support for deleting expired data | no | +| support for ttl (time to live) at retrieval | yes | +| support for deleting expired data | yes | | collocated by feature view | yes | | collocated by feature service | no | | collocated by entity key | no | @@ -89,6 +146,6 @@ To compare this set of functionality against other online stores, please see the ## Resources -* [Sample application with ScyllaDB](https://feature-store.scylladb.com/stable/) +* [ScyllaDB Vector Search documentation](https://cloud.docs.scylladb.com/stable/vector-search/) * [ScyllaDB website](https://www.scylladb.com/) * [ScyllaDB Cloud documentation](https://cloud.docs.scylladb.com/stable/) diff --git a/docs/reference/online-stores/valkey.md b/docs/reference/online-stores/valkey.md new file mode 100644 index 00000000000..4ede3f65b5d --- /dev/null +++ b/docs/reference/online-stores/valkey.md @@ -0,0 +1,94 @@ +# Valkey online store + +## Description + +[Valkey](https://valkey.io/) is an open source (BSD-3-Clause), high-performance key/value datastore hosted by the Linux Foundation, created as a community fork of Redis. It maintains compatibility with the Redis wire protocol, so it can act as a drop-in replacement for Redis. Valkey is also offered as a managed engine by major cloud providers (for example, Amazon ElastiCache for Valkey). + +Similar to Redis and [Dragonfly](dragonfly.md), Valkey can be used as an online feature store for Feast: Feast's Redis online store only issues core commands (hash reads/writes, scans, key expiry, pipelines), all of which Valkey implements. + +Feast's standard online store operations have been verified against Valkey 8.1: `feast apply`, `feast materialize`, online retrieval via `get_online_features`, `feast teardown`, and key expiry via the `key_ttl_seconds` option. Features that depend on Redis modules (such as vector search) are outside the scope of this page. + +## Using Valkey as a drop-in Feast online store instead of Redis + +Make sure you have Python and `pip` installed. + +Install the Feast SDK and CLI + +`pip install feast` + +In order to use Valkey as the online store, you'll need to install the redis extra: + +`pip install 'feast[redis]'` + +### 1. Create a feature repository + +Bootstrap a new feature repository: + +``` +feast init feast_valkey +cd feast_valkey/feature_repo +``` + +Update `feature_repo/feature_store.yaml` with the below contents: + +``` +project: feast_valkey +registry: data/registry.db +provider: local +online_store: + type: redis + connection_string: "localhost:6379" +``` + +Note that the online store `type` remains `redis`: Feast talks to Valkey over the Redis protocol, and all options of the [Redis online store](redis.md) (such as `key_ttl_seconds`) apply unchanged. + +### 2. Start Valkey + +There are several options available to get Valkey up and running quickly. We will be using Docker for this tutorial. + +`docker run -d -p 6379:6379 valkey/valkey:8.1` + +### 3. Register feature definitions and deploy your feature store + +`feast apply` + +The `apply` command scans python files in the current directory for feature view/entity definitions, registers the objects, and deploys infrastructure. +You should see the following output: + +``` +.... +Created entity driver +Created feature view driver_hourly_stats_fresh +Created feature view driver_hourly_stats +Created on demand feature view transformed_conv_rate +Created on demand feature view transformed_conv_rate_fresh +Created feature service driver_activity_v1 +Created feature service driver_activity_v3 +Created feature service driver_activity_v2 +``` + +## Functionality Matrix + +The set of functionality supported by online stores is described in detail [here](overview.md#functionality). +Below is a matrix indicating which functionality is supported by the Redis online store, which Feast uses to communicate with Valkey. + +| | Redis | +| :-------------------------------------------------------- | :---- | +| write feature values to the online store | yes | +| read feature values from the online store | yes | +| update infrastructure (e.g. tables) in the online store | yes | +| teardown infrastructure (e.g. tables) in the online store | yes | +| generate a plan of infrastructure changes | no | +| support for on-demand transforms | yes | +| readable by Python SDK | yes | +| readable by Java | yes | +| readable by Go | yes | +| support for entityless feature views | yes | +| support for concurrent writing to the same key | yes | +| support for ttl (time to live) at retrieval | yes | +| support for deleting expired data | yes | +| collocated by feature view | no | +| collocated by feature service | no | +| collocated by entity key | yes | + +To compare this set of functionality against other online stores, please see the full [functionality matrix](overview.md#functionality-matrix). diff --git a/docs/reference/openlineage.md b/docs/reference/openlineage.md index 78438082d44..bf9e18750ed 100644 --- a/docs/reference/openlineage.md +++ b/docs/reference/openlineage.md @@ -186,6 +186,12 @@ Captures materialization run metadata: ## Lineage Visualization +### Option 1: Feast UI (Built-in) + +Feast includes a built-in OpenLineage consumer that can receive, store, and visualize lineage from **all** OpenLineage producers (Airflow, Spark, dbt, Feast itself, etc.) directly in the Feast UI. See the [OpenLineage Consumer](#openlineage-consumer) section below. + +### Option 2: Marquez + Use [Marquez](https://marquezproject.ai/) to visualize your Feast lineage: ```bash @@ -216,3 +222,257 @@ Then access the Marquez UI at http://localhost:3000 to see your feature lineage. | Entity | InputDataset | | FeatureService | OutputDataset | | Materialization | RunEvent (START/COMPLETE/FAIL) | + +--- + +## OpenLineage Consumer + +Feast can act as an **OpenLineage consumer**, receiving lineage events from any OpenLineage-compatible producer and displaying them in the Feast UI. This eliminates the need for a separate Marquez deployment when you want to visualize cross-system data lineage alongside your feature store. + +### Consumer Architecture + +``` +Producers (Airflow, Spark, dbt, Feast, Flink, …) + │ + ▼ + POST /api/v1/lineage ──→ Event Processor ──→ Lineage Store (SQL) + │ + ▼ + Feast UI + ┌──────────────────────────┐ + │ Lineage tab │ + │ ├─ OpenLineage Graph │ + │ │ (all producers) │ + │ └─ ☐ Feast Only Lineage │ + │ (registry view) │ + │ │ + │ Events tab │ + │ └─ Event browser │ + └──────────────────────────┘ +``` + +When the consumer is **not** enabled, the Feast UI shows only the original registry-based lineage view — no tabs are added. + +### Enabling the Consumer + +Add the `consumer` section under `openlineage` in your `feature_store.yaml`: + +```yaml +project: my_project +registry: + registry_type: sql + path: postgresql://user:****@host:5432/feast # pragma: allowlist secret + +openlineage: + enabled: true + namespace: my_project + consumer: + enabled: true + store_type: sql + # Optional: separate database for lineage storage. + # If omitted, the SQL registry database is reused. + # connection_string: postgresql://user:****@host:5432/feast_lineage + api_key: "change-me" # pragma: allowlist secret + namespace_mapping: + airflow_ns: my_project + spark_ns: my_project +``` + +Or via environment variables: + +```bash +export FEAST_OPENLINEAGE_CONSUMER_ENABLED=true +export FEAST_OPENLINEAGE_CONSUMER_STORE_TYPE=sql +export FEAST_OPENLINEAGE_CONSUMER_API_KEY=change-me # pragma: allowlist secret +# Optional separate DB: +# export FEAST_OPENLINEAGE_CONSUMER_CONNECTION_STRING=postgresql://... +``` + +### Consumer Configuration Options + +| Option | Default | Description | +|--------|---------|-------------| +| `consumer.enabled` | `false` | Enable the OpenLineage consumer | +| `consumer.store_type` | `sql` | Storage backend type. Currently only `sql` is supported | +| `consumer.connection_string` | - | Optional separate database connection string. If omitted, reuses the SQL registry database | +| `consumer.api_key` | - | API key that producers must provide when sending events | +| `consumer.namespace_mapping` | `{}` | Maps OpenLineage namespaces to Feast projects for RBAC scoping | + +### Consumer API Endpoints + +When the consumer is enabled, the following endpoints are available on the Feast REST registry server: + +#### Event Receiver (Producer-facing) + +| Endpoint | Method | Description | +|----------|--------|-------------| +| `/api/v1/lineage` | `POST` | Receive a single OpenLineage event (or array of events) | +| `/api/v1/lineage/batch` | `POST` | Receive a batch of OpenLineage events | + +Both endpoints require the `X-API-Key` header (or `Authorization: Bearer `) if `consumer.api_key` is configured. + +#### Admin Endpoints + +| Endpoint | Method | Description | +|----------|--------|-------------| +| `/lineage/openlineage/reset` | `DELETE` | Purge all OpenLineage data. Accepts optional `?namespace=X` to delete only a specific namespace. Requires API key. | + +#### OpenLineage Query Endpoints (UI-facing) + +| Endpoint | Method | Description | +|----------|--------|-------------| +| `/lineage/openlineage/graph` | `GET` | Full lineage graph with all nodes, edges, and symlinks | +| `/lineage/openlineage/graph/{node_type}/{namespace}/{name}` | `GET` | Lineage graph centered on a specific node | +| `/lineage/openlineage/events` | `GET` | Browse stored events with filtering | +| `/lineage/openlineage/jobs` | `GET` | List all known OpenLineage jobs | +| `/lineage/openlineage/datasets` | `GET` | List all known OpenLineage datasets | +| `/lineage/openlineage/runs` | `GET` | List runs with optional `?job_namespace=X&job_name=Y` filtering | +| `/lineage/openlineage/runs/{run_id}` | `GET` | Single run detail with input/output datasets | + +#### Registry Query Endpoints + +| Endpoint | Method | Description | +|----------|--------|-------------| +| `/lineage/registry` | `GET` | Feast registry lineage (entities, feature views, services) | +| `/lineage/registry/all` | `GET` | All registry objects with full metadata | +| `/lineage/objects/{object_type}/{object_name}` | `GET` | Detail for a specific registry object | +| `/lineage/complete` | `GET` | Complete registry lineage with relationships | +| `/lineage/complete/all` | `GET` | Complete registry lineage for all objects | + +### Configuring Producers to Send Events to Feast + +Configure any OpenLineage producer to send events to your Feast instance: + +#### Airflow + +```python +# In airflow.cfg or environment +OPENLINEAGE_URL = "http://feast-registry:8080/api" +OPENLINEAGE_API_KEY = "change-me" # pragma: allowlist secret +``` + +#### Spark + +```properties +spark.openlineage.transport.type=http +spark.openlineage.transport.url=http://feast-registry:8080/api +spark.openlineage.transport.endpoint=/v1/lineage +spark.openlineage.transport.auth.type=api_key +spark.openlineage.transport.auth.apiKey=change-me +``` + +#### dbt + +```yaml +# In profiles.yml or environment +OPENLINEAGE_URL: "http://feast-registry:8080/api" +OPENLINEAGE_API_KEY: "change-me" # pragma: allowlist secret +``` + +#### Feast (Self-reporting) + +When both the OpenLineage producer and consumer are enabled, Feast's own events (from `feast apply`, materialization, etc.) are automatically ingested into the local consumer store — no HTTP transport is needed. + +```yaml +# In feature_store.yaml +openlineage: + enabled: true + namespace: my_project + consumer: + enabled: true + api_key: change-me # pragma: allowlist secret +``` + +### Feast UI Lineage Views + +When the consumer is enabled, the lineage page in the Feast UI shows two tabs: + +**Lineage tab** + +- **OpenLineage Graph** (default) — shows lineage from all OpenLineage producers with cross-producer connectivity. Nodes are color-coded by producer (colors generated dynamically). The graph supports filtering by type, producer, and object name. Clicking a node opens a **detail panel** showing description, schema, tags, features, entities, data quality metrics, data source info, other facets, and **run history** (for job nodes — see [Per-Run Lineage](#per-run-lineage-run-history)). +- **Feast Only Lineage** (checkbox) — switches to the original Feast registry view (DataSource → FeatureView → FeatureService) powered entirely by the Feast registry. + +**Events tab** + +- Browse individual OpenLineage events with filtering by event type, job name, and run ID. Expand any event to inspect the full JSON payload. + +### Cross-Producer Lineage Connectivity + +The consumer automatically links datasets across different producers when they refer to the same physical data. Linking mechanisms: + +1. **Shared namespace + name** — If Airflow writes to `s3://bucket/path` and Spark reads from the same `s3://bucket/path`, the graph connects them automatically. +2. **SymlinksDatasetFacet** — Producers can declare aliases. For example, Feast can declare that its internal `driver_hourly_stats` is a symlink to the Spark output at `s3://bucket/features/driver_hourly_stats/`. +3. **dataSource URI matching** — Datasets with matching `dataSource.uri` facets are linked even if their namespace or name differ. + +Compatible producers include Airflow, Spark, dbt, Flink, Feast, and Dagster. + +### RBAC for Lineage + +The OpenLineage consumer integrates with Feast's existing RBAC: + +- **Write access** (producers sending events): Authenticated via API key in the `X-API-Key` header +- **Read access** (UI viewing lineage): Namespace-based filtering maps OpenLineage namespaces to Feast projects. Users see only lineage data for namespaces they have access to via the `namespace_mapping` configuration + +### Lineage Cleanup / Reset + +Over time the OpenLineage store accumulates historical data. Two mechanisms are provided for cleanup: + +#### Admin Reset Endpoint + +Use the `DELETE /lineage/openlineage/reset` endpoint to purge lineage data. The endpoint requires the same API key used for event ingestion. + +```bash +# Purge ALL OpenLineage data +curl -X DELETE -H "X-API-Key: your-key" \ + http://localhost:8080/api/v1/lineage/openlineage/reset + +# Purge only a specific namespace +curl -X DELETE -H "X-API-Key: your-key" \ + "http://localhost:8080/api/v1/lineage/openlineage/reset?namespace=airflow://prod-cluster" +``` + +A full purge deletes data from all seven `openlineage_*` tables. A namespace-scoped purge deletes jobs, datasets, runs, events, edges, and symlinks associated with that namespace, leaving other namespaces intact. + +#### Feast Teardown Hook + +When you run `feast teardown`, Feast automatically cleans up OpenLineage data for the project's namespace (if the consumer is configured). This ensures that tearing down a Feast project doesn't leave orphaned lineage data behind. + +```bash +# Tears down the Feast project AND its OpenLineage lineage +feast teardown +``` + +### Per-Run Lineage (Run History) + +The consumer tracks individual pipeline runs in the `openlineage_runs` table. When you click on a **job node** in the OpenLineage Graph, the detail panel shows a **Run History** section with: + +- A table of past runs: truncated run ID, status badge (COMPLETE, FAIL, RUNNING, ABORT), start time, and duration +- Click any run to expand its **inputs and outputs** — the specific datasets that run consumed and produced + +#### Run History API + +```bash +# List runs for a specific job +curl "http://localhost:8080/api/v1/lineage/openlineage/runs?job_namespace=spark://emr-cluster&job_name=feature_engineering" + +# Get a single run with its I/O datasets +curl "http://localhost:8080/api/v1/lineage/openlineage/runs/{run_id}" +``` + +The run detail response includes `inputs` and `outputs` arrays, each containing the dataset namespace, name, and any I/O facets recorded by the producer. + +### Database Schema + +The consumer creates the following tables (automatically on first startup): + +| Table | Purpose | +|-------|---------| +| `openlineage_events` | Raw event storage with JSON payloads | +| `openlineage_jobs` | Deduplicated job records with producer, description, and facets | +| `openlineage_datasets` | Deduplicated dataset records with schema, facets, and Feast mapping | +| `openlineage_runs` | Run lifecycle tracking (START/COMPLETE/FAIL) | +| `openlineage_run_io` | Input/output relationships between runs and datasets | +| `openlineage_lineage_edges` | Materialized lineage graph edges for efficient traversal | +| `openlineage_dataset_symlinks` | Cross-producer dataset linking via `SymlinksDatasetFacet` and `dataSource` URI matching | + +By default these tables are created in the **same database** as the SQL registry (hybrid storage). Set `consumer.connection_string` to store them in a separate database instead. diff --git a/docs/reference/registries/sql.md b/docs/reference/registries/sql.md index ef9993c8753..e8d1bcef17a 100644 --- a/docs/reference/registries/sql.md +++ b/docs/reference/registries/sql.md @@ -80,10 +80,114 @@ docker build \ If you are running Feast in Kubernetes, set the `image.repository` and `imagePullSecrets` Helm values accordingly to utilize your custom image. +## Schema management (`schema_mode`) + +By default, the SQL registry creates its tables on every startup (`schema_mode: auto`). In production environments where the application should not have DDL privileges, you can pre-create the schema and configure the registry to only verify it: + +```yaml +registry: + registry_type: sql + path: postgresql://db:5432/feast + schema_mode: verify # or "skip" +``` + +| Value | Behavior | +|---|---| +| `auto` (default) | Creates tables if they don't exist. Current behavior, no breaking change. | +| `verify` | Skips DDL. Checks that all expected tables exist on startup; raises an error listing missing tables if any are absent. When a separate `read_path` is configured, the read replica is also verified — a lagging replica (e.g. mid-migration) will block startup. Note: this is a table-level check only — it does not verify individual columns. A schema created by an older Feast version (missing newer columns) will pass verification but may fail at query time. | +| `skip` | Skips both creation and verification. Use when schema is managed entirely outside Feast (e.g. by a migration tool). | + +### Pre-creating the schema + +When using `verify` or `skip` mode, run the following CLI command with a user that has DDL privileges to create the schema before starting the application: + +```shell +feast registry create-schema +``` + +This reads `feature_store.yaml`, connects to the configured database, and creates all required tables. It is safe to run multiple times — existing tables are not modified. + There are some things to note about how the SQL registry works: -- Once instantiated, the Registry ensures the tables needed to store data exist, and creates them if they do not. -- Upon tearing down the feast project, the registry ensures that the tables are dropped from the database. -- The schema for how data is laid out in tables can be found . It is intentionally simple, storing the serialized protobuf versions of each Feast object keyed by its name. +- When `schema_mode` is `auto` (the default), the Registry ensures the tables needed to store data exist, and creates them if they do not. +- Upon tearing down the feast project, the registry deletes all rows from the registry tables (it does not drop the tables themselves). This runs regardless of `schema_mode` and requires only DML (`DELETE`) privileges, not DDL. +- The schema for how data is laid out in tables can be found in the table definitions in [`sdk/python/feast/infra/registry/sql.py`](https://github.com/feast-dev/feast/blob/master/sdk/python/feast/infra/registry/sql.py). It is intentionally simple, storing the serialized protobuf versions of each Feast object keyed by its name. + +## MySQL: serialized-proto columns use `LONGBLOB` + +The registry stores each Feast object as a serialized protobuf in a binary +column. On MySQL these columns are created as `LONGBLOB` (up to 4 GB). Earlier +versions created them as `BLOB`, which caps at 64 KB — a single `FeatureView` +proto routinely exceeds that, so MySQL would silently truncate the write and the +registry would later fail to load with a protobuf `DecodeError` (for example, +`feast serve` failing to start). Other dialects (PostgreSQL, SQLite) were never +affected. + +New deployments get the correct schema automatically — the registry creates its +tables as `LONGBLOB` on first use. When an existing MySQL/MariaDB registry still +has `BLOB` columns, the registry logs an error at startup listing the affected +columns (it does not refuse to start — a registry whose protos all fit in 64 KB +is unaffected). **Existing deployments are not migrated automatically**: the +registry only creates tables that do not already exist, and it has no +schema-migration step, so previously created `BLOB` columns remain `BLOB`. To +upgrade an existing MySQL registry, alter each serialized-proto column to +`LONGBLOB`, for example: + +> ⚠️ **Run the migration carefully on a live registry.** A `BLOB`→`LONGBLOB` +> change is a column *data-type* change, which MySQL InnoDB performs with +> `ALGORITHM=COPY` — a full table rebuild under a metadata lock that blocks +> readers and writers for the duration (potentially minutes on a large table +> such as `feature_view_version_history`). `ALGORITHM=INPLACE` is **not** +> generally supported for this change and is rejected with +> `ER_ALTER_OPERATION_NOT_SUPPORTED_REASON` on most builds — do not rely on it. +> +> **Before running any `ALTER TABLE`:** +> +> 1. **Stop all `feast apply` and materialization jobs.** This is required, not +> optional — a write of a `>64 KB` proto to a not-yet-widened `BLOB` column +> truncates silently with no error, and concurrent writes also extend the +> `ALTER`'s lock duration. +> 2. Confirm there are no active writers (e.g. `SHOW PROCESSLIST`). +> 3. Verify you have a backup of the registry database. +> +> Then, to minimize the lock window: +> +> - On large tables, or on managed MySQL (AWS RDS, Aurora) without shell access, +> use an online schema-change tool — +> [`pt-online-schema-change`](https://docs.percona.com/percona-toolkit/pt-online-schema-change.html) +> (Percona Toolkit) or [`gh-ost`](https://github.com/github/gh-ost) — which +> rebuild the table without a long-held lock. For small tables a plain +> `ALTER TABLE` in the maintenance window is fine. +> - Apply one table at a time so a failure is easy to isolate and re-run. +> - Resume jobs only after all `ALTER TABLE` statements complete successfully. +> - Rollback is safe (revert `MODIFY ... BLOB`) **only** while no stored proto +> exceeds 64 KB; otherwise a revert re-introduces truncation. + +```sql +ALTER TABLE projects MODIFY project_proto LONGBLOB NOT NULL; +ALTER TABLE entities MODIFY entity_proto LONGBLOB NOT NULL; +ALTER TABLE data_sources MODIFY data_source_proto LONGBLOB NOT NULL; +ALTER TABLE feature_views MODIFY materialized_intervals LONGBLOB, + MODIFY feature_view_proto LONGBLOB NOT NULL, + MODIFY user_metadata LONGBLOB; +ALTER TABLE stream_feature_views MODIFY feature_view_proto LONGBLOB NOT NULL, + MODIFY user_metadata LONGBLOB; +ALTER TABLE on_demand_feature_views MODIFY feature_view_proto LONGBLOB NOT NULL, + MODIFY user_metadata LONGBLOB; +ALTER TABLE label_views MODIFY feature_view_proto LONGBLOB NOT NULL, + MODIFY user_metadata LONGBLOB; +ALTER TABLE feature_services MODIFY feature_service_proto LONGBLOB NOT NULL; +ALTER TABLE saved_datasets MODIFY saved_dataset_proto LONGBLOB NOT NULL; +ALTER TABLE validation_references MODIFY validation_reference_proto LONGBLOB NOT NULL; +ALTER TABLE managed_infra MODIFY infra_proto LONGBLOB NOT NULL; +ALTER TABLE permissions MODIFY permission_proto LONGBLOB NOT NULL; +-- LARGE TABLE: one row per versioned apply — likely the slowest ALTER. Use +-- pt-online-schema-change or gh-ost if this registry has significant history. +ALTER TABLE feature_view_version_history MODIFY feature_view_proto LONGBLOB NOT NULL; +``` + +Any object whose proto already exceeded 64 KB before the upgrade may have been +stored truncated; re-run `feast apply` for those objects after altering the +columns so the full proto is rewritten. ## Example Usage: Concurrent materialization The SQL Registry should be used when materializing feature views concurrently to ensure correctness of data in the registry. This can be achieved by simply running feast materialize or feature_store.materialize multiple times using a correctly configured feature_store.yaml. This will make each materialization process talk to the registry database concurrently, and ensure the metadata updates are serialized. diff --git a/docs/reference/type-system.md b/docs/reference/type-system.md index eb483c6e769..97cc6036dc8 100644 --- a/docs/reference/type-system.md +++ b/docs/reference/type-system.md @@ -24,6 +24,7 @@ Feast supports the following data types: | `Bytes` | `bytes` | Binary data | | `Bool` | `bool` | Boolean value | | `UnixTimestamp` | `datetime` | Unix timestamp (nullable) | +| `ZonedTimestamp` | `datetime` | Timezone-aware datetime preserving its source zone (nullable) | | `Uuid` | `uuid.UUID` | UUID (any version) | | `TimeUuid` | `uuid.UUID` | Time-based UUID (version 1) | | `Decimal` | `decimal.Decimal` | Arbitrary-precision decimal number | @@ -202,7 +203,8 @@ from datetime import timedelta from feast import Entity, FeatureView, Field, FileSource from feast.types import ( Int32, Int64, Float32, Float64, String, Bytes, Bool, UnixTimestamp, - Uuid, TimeUuid, Decimal, Array, Set, Map, ScalarMap, Json, Struct + Uuid, TimeUuid, Decimal, Array, Set, Map, ScalarMap, Json, Struct, + ZonedTimestamp ) # Define a data source @@ -232,6 +234,7 @@ user_features = FeatureView( Field(name="profile_picture", dtype=Bytes), Field(name="is_active", dtype=Bool), Field(name="last_login", dtype=UnixTimestamp), + Field(name="event_time", dtype=ZonedTimestamp), Field(name="session_id", dtype=Uuid), Field(name="event_id", dtype=TimeUuid), Field(name="price", dtype=Decimal), @@ -362,6 +365,43 @@ unique_prices = {decimal.Decimal("9.99"), decimal.Decimal("19.99"), decimal.Deci `Decimal` is **not** inferred from any backend schema. You must declare it explicitly in your feature view schema. The pandas dtype for `Decimal` columns is `object` (holding `decimal.Decimal` instances), not a numeric dtype. {% endhint %} +### ZonedTimestamp Type Usage Examples + +The `ZonedTimestamp` type stores a timezone-aware `datetime` as both the UTC instant +and its originating zone, so the original wall-clock zone round-trips losslessly. +By contrast, `UnixTimestamp` always decodes to UTC and discards the source zone. + +```python +from datetime import datetime, timezone +from zoneinfo import ZoneInfo + +# A datetime in a specific zone — both the instant and "America/Los_Angeles" are kept +event_time = datetime(2026, 6, 17, 9, 0, 0, tzinfo=ZoneInfo("America/Los_Angeles")) + +# ZonedTimestamp values are returned as tz-aware datetime objects, in their own zone +response = store.get_online_features( + features=["event_features:event_time"], + entity_rows=[{"user_id": 1001}], +) +result = response.to_dict() +# result["event_time"][0] == event_time (same instant AND same zone, e.g. 09:00-07:00) + +# Two values at the same instant but different zones stay distinct +la = datetime(2026, 6, 17, 9, 0, 0, tzinfo=ZoneInfo("America/Los_Angeles")) +utc = datetime(2026, 6, 17, 16, 0, 0, tzinfo=timezone.utc) # same instant as `la` + +# A naive (tz-less) datetime is interpreted as UTC +naive = datetime(2026, 6, 17, 12, 0, 0) # stored zone is empty, decoded as UTC +``` + +{% hint style="warning" %} +`ZonedTimestamp` is **not** inferred from any backend schema — you must declare it +explicitly in your feature view schema. It is not supported as an entity key. The +zone is stored as an IANA name (e.g. `America/Los_Angeles`) when available, falling +back to a fixed-offset string; offline stores that cannot natively carry a zone may +normalize to UTC on that backend. +{% endhint %} + ### Nested Collection Type Usage Examples ```python diff --git a/docs/roadmap.md b/docs/roadmap.md index e47aa79b573..d92ffa38f24 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -62,6 +62,7 @@ The list below contains the functionality that contributors are planning to deve * [x] [SingleStore](https://docs.feast.dev/reference/online-stores/singlestore) * [x] [Couchbase](https://docs.feast.dev/reference/online-stores/couchbase) * [x] [MongoDB](https://docs.feast.dev/reference/online-stores/mongodb) + * [x] [Aerospike](https://docs.feast.dev/reference/online-stores/aerospike) * [x] [Qdrant (vector store)](https://docs.feast.dev/reference/online-stores/qdrant) * [x] [Milvus (vector store)](https://docs.feast.dev/reference/online-stores/milvus) * [x] [Faiss (vector store)](https://docs.feast.dev/reference/online-stores/faiss) @@ -89,7 +90,7 @@ The list below contains the functionality that contributors are planning to deve * [x] [Offline Feature Server (alpha)](https://docs.feast.dev/reference/feature-servers/offline-feature-server) * [x] [Registry server (alpha)](https://github.com/feast-dev/feast/blob/master/docs/reference/feature-servers/registry-server.md) * **Data Quality Management (See [RFC](https://docs.google.com/document/d/110F72d4NTv80p35wDSONxhhPBqWRwbZXG4f9mNEMd98/edit))** - * [x] Data profiling and validation (Great Expectations) + * [x] [Feature Quality Monitoring](https://docs.feast.dev/how-to-guides/feature-monitoring) — built-in metrics, drift detection, serving log monitoring, and UI dashboard * **Feature Discovery and Governance** * [x] Python SDK for browsing feature registry * [x] CLI for browsing feature registry diff --git a/docs/tutorials/validating-historical-features.md b/docs/tutorials/validating-historical-features.md deleted file mode 100644 index 1984adcdcf9..00000000000 --- a/docs/tutorials/validating-historical-features.md +++ /dev/null @@ -1,916 +0,0 @@ -# Validating historical features with Great Expectations - -In this tutorial, we will use the public dataset of Chicago taxi trips to present data validation capabilities of Feast. -- The original dataset is stored in BigQuery and consists of raw data for each taxi trip (one row per trip) since 2013. -- We will generate several training datasets (aka historical features in Feast) for different periods and evaluate expectations made on one dataset against another. - -Types of features we're ingesting and generating: -- Features that aggregate raw data with daily intervals (eg, trips per day, average fare or speed for a specific day, etc.). -- Features using SQL while pulling data from BigQuery (like total trips time or total miles travelled). -- Features calculated on the fly when requested using Feast's on-demand transformations - -Our plan: - -0. Prepare environment -1. Pull data from BigQuery (optional) -2. Declare & apply features and feature views in Feast -3. Generate reference dataset -4. Develop & test profiler function -5. Run validation on different dataset using reference dataset & profiler - - -> The original notebook and datasets for this tutorial can be found on [GitHub](https://github.com/feast-dev/dqm-tutorial). - -### 0. Setup - -Install Feast Python SDK and great expectations: - - -```python -!pip install 'feast[ge]' -``` - - -### 1. Dataset preparation (Optional) - -**You can skip this step if you don't have GCP account. Please use parquet files that are coming with this tutorial instead** - - -```python -!pip install google-cloud-bigquery -``` - - -```python -import pyarrow.parquet - -from google.cloud.bigquery import Client -``` - - -```python -bq_client = Client(project='kf-feast') -``` - -Running some basic aggregations while pulling data from BigQuery. Grouping by taxi_id and day: - - -```python -data_query = """SELECT - taxi_id, - TIMESTAMP_TRUNC(trip_start_timestamp, DAY) as day, - SUM(trip_miles) as total_miles_travelled, - SUM(trip_seconds) as total_trip_seconds, - SUM(fare) as total_earned, - COUNT(*) as trip_count -FROM `bigquery-public-data.chicago_taxi_trips.taxi_trips` -WHERE - trip_miles > 0 AND trip_seconds > 60 AND - trip_start_timestamp BETWEEN '2019-01-01' and '2020-12-31' AND - trip_total < 1000 -GROUP BY taxi_id, TIMESTAMP_TRUNC(trip_start_timestamp, DAY)""" -``` - - -```python -driver_stats_table = bq_client.query(data_query).to_arrow() - -# Storing resulting dataset into parquet file -pyarrow.parquet.write_table(driver_stats_table, "trips_stats.parquet") -``` - - -```python -def entities_query(year): - return f"""SELECT - distinct taxi_id -FROM `bigquery-public-data.chicago_taxi_trips.taxi_trips` -WHERE - trip_miles > 0 AND trip_seconds > 0 AND - trip_start_timestamp BETWEEN '{year}-01-01' and '{year}-12-31' -""" -``` - - -```python -entities_2019_table = bq_client.query(entities_query(2019)).to_arrow() - -# Storing entities (taxi ids) into parquet file -pyarrow.parquet.write_table(entities_2019_table, "entities.parquet") -``` - - -## 2. Declaring features - - -```python -import pyarrow.parquet -import pandas as pd - -from feast import FeatureView, Entity, FeatureStore, Field, BatchFeatureView -from feast.types import Float64, Int64 -from feast.value_type import ValueType -from feast.data_format import ParquetFormat -from feast.on_demand_feature_view import on_demand_feature_view -from feast.infra.offline_stores.file_source import FileSource -from feast.infra.offline_stores.file import SavedDatasetFileStorage -from datetime import timedelta - -``` - - -```python -batch_source = FileSource( - timestamp_field="day", - path="trips_stats.parquet", # using parquet file that we created on previous step - file_format=ParquetFormat() -) -``` - - -```python -taxi_entity = Entity(name='taxi', join_keys=['taxi_id']) -``` - - -```python -trips_stats_fv = BatchFeatureView( - name='trip_stats', - entities=[taxi_entity], - schema=[ - Field(name="total_miles_travelled", dtype=Float64), - Field(name="total_trip_seconds", dtype=Float64), - Field(name="total_earned", dtype=Float64), - Field(name="trip_count", dtype=Int64), - - ], - ttl=timedelta(seconds=86400), - source=batch_source, -) -``` - -*Read more about feature views in [Feast docs](https://docs.feast.dev/getting-started/concepts/feature-view)* - - -```python -@on_demand_feature_view( - sources=[ - trips_stats_fv, - ], - schema=[ - Field(name="avg_fare", dtype=Float64), - Field(name="avg_speed", dtype=Float64), - Field(name="avg_trip_seconds", dtype=Float64), - Field(name="earned_per_hour", dtype=Float64), - ] -) -def on_demand_stats(inp: pd.DataFrame) -> pd.DataFrame: - out = pd.DataFrame() - out["avg_fare"] = inp["total_earned"] / inp["trip_count"] - out["avg_speed"] = 3600 * inp["total_miles_travelled"] / inp["total_trip_seconds"] - out["avg_trip_seconds"] = inp["total_trip_seconds"] / inp["trip_count"] - out["earned_per_hour"] = 3600 * inp["total_earned"] / inp["total_trip_seconds"] - return out -``` - -*Read more about on demand feature views [here](../reference/beta-on-demand-feature-view.md)* - - -```python -store = FeatureStore(".") # using feature_store.yaml that stored in the same directory -``` - - -```python -store.apply([taxi_entity, trips_stats_fv, on_demand_stats]) # writing to the registry -``` - - -## 3. Generating training (reference) dataset - - -```python -taxi_ids = pyarrow.parquet.read_table("entities.parquet").to_pandas() -``` - -Generating range of timestamps with daily frequency: - - -```python -timestamps = pd.DataFrame() -timestamps["event_timestamp"] = pd.date_range("2019-06-01", "2019-07-01", freq='D') -``` - -Cross merge (aka relation multiplication) produces entity dataframe with each taxi_id repeated for each timestamp: - - -```python -entity_df = pd.merge(taxi_ids, timestamps, how='cross') -entity_df -``` - - - - -
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
taxi_idevent_timestamp
091d5288487e87c5917b813ba6f75ab1c3a9749af906a2d...2019-06-01
191d5288487e87c5917b813ba6f75ab1c3a9749af906a2d...2019-06-02
291d5288487e87c5917b813ba6f75ab1c3a9749af906a2d...2019-06-03
391d5288487e87c5917b813ba6f75ab1c3a9749af906a2d...2019-06-04
491d5288487e87c5917b813ba6f75ab1c3a9749af906a2d...2019-06-05
.........
1569797ebf27414a0c7b128e7925e1da56d51a8b81484f7630cf...2019-06-27
1569807ebf27414a0c7b128e7925e1da56d51a8b81484f7630cf...2019-06-28
1569817ebf27414a0c7b128e7925e1da56d51a8b81484f7630cf...2019-06-29
1569827ebf27414a0c7b128e7925e1da56d51a8b81484f7630cf...2019-06-30
1569837ebf27414a0c7b128e7925e1da56d51a8b81484f7630cf...2019-07-01
-

156984 rows × 2 columns

-
- - - -Retrieving historical features for resulting entity dataframe and persisting output as a saved dataset: - - -```python -job = store.get_historical_features( - entity_df=entity_df, - features=[ - "trip_stats:total_miles_travelled", - "trip_stats:total_trip_seconds", - "trip_stats:total_earned", - "trip_stats:trip_count", - "on_demand_stats:avg_fare", - "on_demand_stats:avg_trip_seconds", - "on_demand_stats:avg_speed", - "on_demand_stats:earned_per_hour", - ] -) - -store.create_saved_dataset( - from_=job, - name='my_training_ds', - storage=SavedDatasetFileStorage(path='my_training_ds.parquet') -) -``` - -```python -, full_feature_names = False, tags = {}, _retrieval_job = , min_event_timestamp = 2019-06-01 00:00:00, max_event_timestamp = 2019-07-01 00:00:00)> -``` - - -## 4. Developing dataset profiler - -Dataset profiler is a function that accepts dataset and generates set of its characteristics. This charasteristics will be then used to evaluate (validate) next datasets. - -**Important: datasets are not compared to each other! -Feast use a reference dataset and a profiler function to generate a reference profile. -This profile will be then used during validation of the tested dataset.** - - -```python -import numpy as np - -from feast.dqm.profilers.ge_profiler import ge_profiler - -from great_expectations.core.expectation_suite import ExpectationSuite -from great_expectations.dataset import PandasDataset -``` - - -Loading saved dataset first and exploring the data: - - -```python -ds = store.get_saved_dataset('my_training_ds') -ds.to_df() -``` - -
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
total_earnedavg_trip_secondstaxi_idtotal_miles_travelledtrip_countearned_per_hourevent_timestamptotal_trip_secondsavg_fareavg_speed
068.252270.00000091d5288487e87c5917b813ba6f75ab1c3a9749af906a2d...24.702.054.1189432019-06-01 00:00:00+00:004540.034.12500019.585903
1221.00560.5000007a4a6162eaf27805aef407d25d5cb21fe779cd962922cb...54.1824.059.1436222019-06-01 00:00:00+00:0013452.09.20833314.499554
2160.501010.769231f4c9d05b215d7cbd08eca76252dae51cdb7aca9651d4ef...41.3013.043.9726032019-06-01 00:00:00+00:0013140.012.34615411.315068
3183.75697.550000c1f533318f8480a59173a9728ea0248c0d3eb187f4b897...37.3020.047.4159562019-06-01 00:00:00+00:0013951.09.1875009.625116
4217.751054.076923455b6b5cae6ca5a17cddd251485f2266d13d6a2c92f07c...69.6913.057.2064512019-06-01 00:00:00+00:0013703.016.75000018.308692
.................................
15697938.001980.0000000cccf0ec1f46d1e0beefcfdeaf5188d67e170cdff92618...14.901.069.0909092019-07-01 00:00:00+00:001980.038.00000027.090909
156980135.00551.250000beefd3462e3f5a8e854942a2796876f6db73ebbd25b435...28.4016.055.1020412019-07-01 00:00:00+00:008820.08.43750011.591837
156981NaNNaN9a3c52aa112f46cf0d129fafbd42051b0fb9b0ff8dcb0e...NaNNaNNaN2019-07-01 00:00:00+00:00NaNNaNNaN
15698263.00815.00000008308c31cd99f495dea73ca276d19a6258d7b4c9c88e43...19.964.069.5705522019-07-01 00:00:00+00:003260.015.75000022.041718
156983NaNNaN7ebf27414a0c7b128e7925e1da56d51a8b81484f7630cf...NaNNaNNaN2019-07-01 00:00:00+00:00NaNNaNNaN
-

156984 rows × 10 columns

-
- - - -Feast uses [Great Expectations](https://docs.greatexpectations.io/docs/) as a validation engine and [ExpectationSuite](https://legacy.docs.greatexpectations.io/en/latest/autoapi/great_expectations/core/expectation_suite/index.html#great_expectations.core.expectation_suite.ExpectationSuite) as a dataset's profile. Hence, we need to develop a function that will generate ExpectationSuite. This function will receive instance of [PandasDataset](https://legacy.docs.greatexpectations.io/en/latest/autoapi/great_expectations/dataset/index.html?highlight=pandasdataset#great_expectations.dataset.PandasDataset) (wrapper around pandas.DataFrame) so we can utilize both Pandas DataFrame API and some helper functions from PandasDataset during profiling. - - -```python -DELTA = 0.1 # controlling allowed window in fraction of the value on scale [0, 1] - -@ge_profiler -def stats_profiler(ds: PandasDataset) -> ExpectationSuite: - # simple checks on data consistency - ds.expect_column_values_to_be_between( - "avg_speed", - min_value=0, - max_value=60, - mostly=0.99 # allow some outliers - ) - - ds.expect_column_values_to_be_between( - "total_miles_travelled", - min_value=0, - max_value=500, - mostly=0.99 # allow some outliers - ) - - # expectation of means based on observed values - observed_mean = ds.trip_count.mean() - ds.expect_column_mean_to_be_between("trip_count", - min_value=observed_mean * (1 - DELTA), - max_value=observed_mean * (1 + DELTA)) - - observed_mean = ds.earned_per_hour.mean() - ds.expect_column_mean_to_be_between("earned_per_hour", - min_value=observed_mean * (1 - DELTA), - max_value=observed_mean * (1 + DELTA)) - - - # expectation of quantiles - qs = [0.5, 0.75, 0.9, 0.95] - observed_quantiles = ds.avg_fare.quantile(qs) - - ds.expect_column_quantile_values_to_be_between( - "avg_fare", - quantile_ranges={ - "quantiles": qs, - "value_ranges": [[None, max_value] for max_value in observed_quantiles] - }) - - return ds.get_expectation_suite() -``` - -Testing our profiler function: - - -```python -ds.get_profile(profiler=stats_profiler) -``` - 02/02/2022 02:43:47 PM INFO: 5 expectation(s) included in expectation_suite. result_format settings filtered. - - - - -**Verify that all expectations that we coded in our profiler are present here. Otherwise (if you can't find some expectations) it means that it failed to pass on the reference dataset (do it silently is default behavior of Great Expectations).** - -Now we can create validation reference from dataset and profiler function: - - -```python -validation_reference = ds.as_reference(name="validation_reference_dataset", profiler=stats_profiler) -``` - -and test it against our existing retrieval job - - -```python -_ = job.to_df(validation_reference=validation_reference) -``` - - 02/02/2022 02:43:52 PM INFO: 5 expectation(s) included in expectation_suite. result_format settings filtered. - 02/02/2022 02:43:53 PM INFO: Validating data_asset_name None with expectation_suite_name default - - -Validation successfully passed as no exception were raised. - - -### 5. Validating new historical retrieval - -Creating new timestamps for Dec 2020: - - -```python -from feast.dqm.errors import ValidationFailed -``` - - -```python -timestamps = pd.DataFrame() -timestamps["event_timestamp"] = pd.date_range("2020-12-01", "2020-12-07", freq='D') -``` - - -```python -entity_df = pd.merge(taxi_ids, timestamps, how='cross') -entity_df -``` - -
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
taxi_idevent_timestamp
091d5288487e87c5917b813ba6f75ab1c3a9749af906a2d...2020-12-01
191d5288487e87c5917b813ba6f75ab1c3a9749af906a2d...2020-12-02
291d5288487e87c5917b813ba6f75ab1c3a9749af906a2d...2020-12-03
391d5288487e87c5917b813ba6f75ab1c3a9749af906a2d...2020-12-04
491d5288487e87c5917b813ba6f75ab1c3a9749af906a2d...2020-12-05
.........
354437ebf27414a0c7b128e7925e1da56d51a8b81484f7630cf...2020-12-03
354447ebf27414a0c7b128e7925e1da56d51a8b81484f7630cf...2020-12-04
354457ebf27414a0c7b128e7925e1da56d51a8b81484f7630cf...2020-12-05
354467ebf27414a0c7b128e7925e1da56d51a8b81484f7630cf...2020-12-06
354477ebf27414a0c7b128e7925e1da56d51a8b81484f7630cf...2020-12-07
-

35448 rows × 2 columns

-
- - -```python -job = store.get_historical_features( - entity_df=entity_df, - features=[ - "trip_stats:total_miles_travelled", - "trip_stats:total_trip_seconds", - "trip_stats:total_earned", - "trip_stats:trip_count", - "on_demand_stats:avg_fare", - "on_demand_stats:avg_trip_seconds", - "on_demand_stats:avg_speed", - "on_demand_stats:earned_per_hour", - ] -) -``` - -Execute retrieval job with validation reference: - - -```python -try: - df = job.to_df(validation_reference=validation_reference) -except ValidationFailed as exc: - print(exc.validation_report) -``` - - 02/02/2022 02:43:58 PM INFO: 5 expectation(s) included in expectation_suite. result_format settings filtered. - 02/02/2022 02:43:59 PM INFO: Validating data_asset_name None with expectation_suite_name default - - [ - { - "expectation_config": { - "expectation_type": "expect_column_mean_to_be_between", - "kwargs": { - "column": "trip_count", - "min_value": 10.387244591346153, - "max_value": 12.695521167200855, - "result_format": "COMPLETE" - }, - "meta": {} - }, - "meta": {}, - "result": { - "observed_value": 6.692920555429092, - "element_count": 35448, - "missing_count": 31055, - "missing_percent": 87.6071992778154 - }, - "exception_info": { - "raised_exception": false, - "exception_message": null, - "exception_traceback": null - }, - "success": false - }, - { - "expectation_config": { - "expectation_type": "expect_column_mean_to_be_between", - "kwargs": { - "column": "earned_per_hour", - "min_value": 52.320624975640214, - "max_value": 63.94743052578249, - "result_format": "COMPLETE" - }, - "meta": {} - }, - "meta": {}, - "result": { - "observed_value": 68.99268345164135, - "element_count": 35448, - "missing_count": 31055, - "missing_percent": 87.6071992778154 - }, - "exception_info": { - "raised_exception": false, - "exception_message": null, - "exception_traceback": null - }, - "success": false - }, - { - "expectation_config": { - "expectation_type": "expect_column_quantile_values_to_be_between", - "kwargs": { - "column": "avg_fare", - "quantile_ranges": { - "quantiles": [ - 0.5, - 0.75, - 0.9, - 0.95 - ], - "value_ranges": [ - [ - null, - 16.4 - ], - [ - null, - 26.229166666666668 - ], - [ - null, - 36.4375 - ], - [ - null, - 42.0 - ] - ] - }, - "result_format": "COMPLETE" - }, - "meta": {} - }, - "meta": {}, - "result": { - "observed_value": { - "quantiles": [ - 0.5, - 0.75, - 0.9, - 0.95 - ], - "values": [ - 19.5, - 28.1, - 38.0, - 44.125 - ] - }, - "element_count": 35448, - "missing_count": 31055, - "missing_percent": 87.6071992778154, - "details": { - "success_details": [ - false, - false, - false, - false - ] - } - }, - "exception_info": { - "raised_exception": false, - "exception_message": null, - "exception_traceback": null - }, - "success": false - } - ] - - -Validation failed since several expectations didn't pass: -* Trip count (mean) decreased more than 10% (which is expected when comparing Dec 2020 vs June 2019) -* Average Fare increased - all quantiles are higher than expected -* Earn per hour (mean) increased more than 10% (most probably due to increased fare) - diff --git a/examples/online_store/aerospike_overrides_and_hooks/README.md b/examples/online_store/aerospike_overrides_and_hooks/README.md new file mode 100644 index 00000000000..7143b6a3dff --- /dev/null +++ b/examples/online_store/aerospike_overrides_and_hooks/README.md @@ -0,0 +1,65 @@ +# Aerospike: per-feature-view overrides + prewriting hooks + +A short companion to [`docs/reference/online-stores/aerospike.md`](../../../docs/reference/online-stores/aerospike.md) +demonstrating three deployment patterns the Aerospike online store supports +without needing any Feast extension code: + +1. **Per-feature-view namespace overrides** — pin one view to a RAM-only + namespace and another to an SSD-backed one without splitting the project. +2. **Per-feature-view set overrides** — isolate one view in its own set so + `feast apply` deletions or admin truncates only touch that view. +3. **Prewriting hooks** — apply a project-wide write-side transformation + (PII masking in this example) without sprinkling it through every + materialization job. + +Nothing here is Aerospike-specific *infrastructure* — it's all configured +in `feature_store.yaml`. This directory only adds the hook-target Python +module the YAML references. + +## Files + +| file | purpose | +|---|---| +| [`hooks.py`](hooks.py) | A pure-Python prewriting-hook module containing `hash_pii_string_features`, the same example used in the docs. Drop into any module on the writer's `PYTHONPATH`. | +| [`feature_store.yaml`](feature_store.yaml) | Reference `online_store` block showing all three features wired together. Copy the `online_store` section into your own `feature_store.yaml` — the rest is project-specific scaffolding. | + +## Prerequisites + +* Feast installed with the Aerospike extra (`pip install 'feast[aerospike]'`). +* An Aerospike cluster reachable from your writer process. The + [Aerospike online-store reference](../../../docs/reference/online-stores/aerospike.md) + shows a minimal local CE config (`127.0.0.1:3000`); run Aerospike however + you normally would (Docker, Kubernetes, bare metal). +* On every process that calls `online_write_batch` through this store + (materialization workers, the registry CLI host, the feature server if + you run one), the `FEAST_PII_SALT` environment variable must be set + before the first write — `hash_pii_string_features` raises rather than + silently writing plaintext if the salt isn't configured. +* The two namespaces referenced by `namespace_overrides` (`feast_ram` and + `feast_ssd` in the sample YAML) must already exist on the Aerospike + cluster — Aerospike cannot create namespaces at runtime. + +## Trying it out + +1. Drop `hooks.py` into a module on your `PYTHONPATH` that the writer + process can import (e.g. inside your existing feature-repo package). + The example uses the qualified path + `examples.online_store.aerospike_overrides_and_hooks.hooks.hash_pii_string_features`. +2. Copy the `online_store:` block from `feature_store.yaml` into your + own feature repo, adjusting hosts / namespaces / the hook import path + for your project. +3. `export FEAST_PII_SALT=...` (anything random and stable across + processes — rotate by re-running materialization with a new salt). +4. `feast apply` — the new config is registered. +5. Materialize as usual — the hook runs once per `online_write_batch`, + and any feature named `email`, `phone_number` or `ssn` lands in + Aerospike as a salted SHA-256 hex digest instead of plaintext. + +## Read-side note + +Prewriting hooks are **only** invoked on the write path. If your hook is +a one-way transform (hashing, encryption-without-decryption-key) you +have to apply the same transform to the candidate value at read time +yourself. Two-way transforms (deterministic encryption, Base64) need a +matching post-read step in your serving code; the Aerospike store does +not currently expose a symmetric "postreading hook". diff --git a/examples/online_store/aerospike_overrides_and_hooks/feature_store.yaml b/examples/online_store/aerospike_overrides_and_hooks/feature_store.yaml new file mode 100644 index 00000000000..bd1061f30aa --- /dev/null +++ b/examples/online_store/aerospike_overrides_and_hooks/feature_store.yaml @@ -0,0 +1,59 @@ +# Reference feature_store.yaml demonstrating all three Aerospike +# extension points wired together. Copy the `online_store:` block into +# your own feature repo and adjust hosts / namespaces / hook import +# path for your project. +# +# Prerequisites: +# - The `feast_ram` and `feast_ssd` namespaces must already exist on +# the Aerospike cluster. Aerospike cannot create namespaces at +# runtime; a missing namespace surfaces as AEROSPIKE_ERR_PARAM on +# the first read or write touching that view. +# - `FEAST_PII_SALT` must be set in every process that calls +# online_write_batch through this store. + +project: my_feature_repo +registry: data/registry.db +provider: local + +online_store: + type: aerospike + + hosts: + - ["aerospike.internal", 3000] + + # Store-level defaults. Anything not listed in *_overrides below + # falls back to these. + namespace: feast + set_name_template: "{project}_{collection_suffix}" + + # Pin individual feature views to different namespaces -- typically + # one in-memory namespace for hot, latency-sensitive views and one + # device-backed namespace for cold, wide views. + namespace_overrides: + driver_realtime_stats: feast_ram + driver_history_lookup: feast_ssd + + # Isolate one feature view in its own set so that admin operations on + # it (truncate, scan-based deletion via `feast apply`) do not touch + # the records of other views. + set_overrides: + isolated_view: my_feature_repo_isolated + + # Project-wide write-side hook. The store dynamically imports the + # callable on first use and caches it. Adjust the import path to + # whatever module is on your writers' PYTHONPATH; the value below + # assumes you have the example folder on PYTHONPATH from the + # repository root. + prewriting_hook: examples.online_store.aerospike_overrides_and_hooks.hooks.hash_pii_string_features + + # Standard timing knobs (optional -- shown for completeness). + ttl_seconds: 86400 + read_timeout_ms: 150 + write_timeout_ms: 300 + batch_total_timeout_ms: 500 + socket_timeout_ms: 50 + max_retries: 2 + +# Offline store / entity_key_serialization_version / etc. are +# project-specific and intentionally omitted; this file is a snippet, +# not a runnable repo. diff --git a/examples/online_store/aerospike_overrides_and_hooks/hooks.py b/examples/online_store/aerospike_overrides_and_hooks/hooks.py new file mode 100644 index 00000000000..15e6f8a10c7 --- /dev/null +++ b/examples/online_store/aerospike_overrides_and_hooks/hooks.py @@ -0,0 +1,109 @@ +"""Sample prewriting hooks for the Feast Aerospike online store. + +Reference the callable from ``feature_store.yaml`` via its import string, +e.g.:: + + online_store: + type: aerospike + ... + prewriting_hook: examples.online_store.aerospike_overrides_and_hooks.hooks.hash_pii_string_features + +The Aerospike online store invokes the configured callable once per +``online_write_batch`` call, passing the rows about to be written. The +callable must return a row list with the same schema. Returning ``[]`` +short-circuits the write — same path as an empty input, no wire call is +issued. +""" + +from __future__ import annotations + +import hashlib +import os +from datetime import datetime +from typing import List, Optional, Tuple + +from feast import FeatureView +from feast.protos.feast.types.EntityKey_pb2 import EntityKey as EntityKeyProto +from feast.protos.feast.types.Value_pb2 import Value as ValueProto +from feast.repo_config import RepoConfig + +# Names of features that must never reach the online store as plaintext. +# Match is by exact feature name; tweak to your project's conventions +# (regex, suffix-based, FV-tag-driven, etc.). +_SENSITIVE_FEATURES = frozenset({"email", "phone_number", "ssn"}) + +# Type alias for the per-row payload Feast hands to ``online_write_batch``. +WriteRow = Tuple[ + EntityKeyProto, + dict, + datetime, + Optional[datetime], +] + + +def hash_pii_string_features( + config: RepoConfig, + table: FeatureView, + data: List[WriteRow], +) -> List[WriteRow]: + """Replace any sensitive string feature with a salted SHA-256 hex digest. + + Determinism: same plaintext + same ``FEAST_PII_SALT`` → same digest. + Downstream lookups that hash the candidate value the same way still + hit; lookups against the raw plaintext silently miss. + + Safety: an unset salt raises rather than falling back to plaintext. + Set ``FEAST_PII_SALT`` on every process that materialises features + (workers, registry CLI host, feature server). + """ + salt = os.environ.get("FEAST_PII_SALT") + if salt is None: + raise RuntimeError( + "FEAST_PII_SALT is not set; refusing to write feature batches " + "without a configured PII salt." + ) + salt_bytes = salt.encode("utf-8") + + def _digest(plaintext: str) -> str: + h = hashlib.sha256() + h.update(salt_bytes) + h.update(plaintext.encode("utf-8")) + return h.hexdigest() + + transformed: List[WriteRow] = [] + for entity_key, values, event_ts, created_ts in data: + new_values = dict(values) + for feature_name in _SENSITIVE_FEATURES.intersection(new_values): + v: ValueProto = new_values[feature_name] + if v.HasField("string_val") and v.string_val: + new_values[feature_name] = ValueProto(string_val=_digest(v.string_val)) + transformed.append((entity_key, new_values, event_ts, created_ts)) + return transformed + + +def drop_rows_with_negative_amounts( + config: RepoConfig, + table: FeatureView, + data: List[WriteRow], +) -> List[WriteRow]: + """Defensive sample hook: filter rows whose ``amount`` feature is < 0. + + Demonstrates that hooks can also *remove* rows. Returning an empty + list short-circuits the wire call entirely — useful for emergency + feature-write quarantines without a code deploy. + """ + keep: List[WriteRow] = [] + for entity_key, values, event_ts, created_ts in data: + amount: Optional[ValueProto] = values.get("amount") + if ( + amount is not None + and amount.HasField("double_val") + and amount.double_val < 0 + ): + continue + if amount is not None and amount.HasField("float_val") and amount.float_val < 0: + continue + if amount is not None and amount.HasField("int64_val") and amount.int64_val < 0: + continue + keep.append((entity_key, values, event_ts, created_ts)) + return keep diff --git a/examples/ray-llm-posttrain/.gitignore b/examples/ray-llm-posttrain/.gitignore new file mode 100644 index 00000000000..4bf1a46bf0e --- /dev/null +++ b/examples/ray-llm-posttrain/.gitignore @@ -0,0 +1,13 @@ +# Feast / Ray local artifacts +data/ +.feast/ +ray_storage/ +/tmp/ray/ +ray_results/ +.ray/ + +__pycache__/ +*.py[cod] +.pytest_cache/ +.venv/ +.env diff --git a/examples/ray-llm-posttrain/README.md b/examples/ray-llm-posttrain/README.md new file mode 100644 index 00000000000..64252a5a603 --- /dev/null +++ b/examples/ray-llm-posttrain/README.md @@ -0,0 +1,35 @@ +# How to Use Feast for SLM/LLM Post-Training (with Ray) + +| Name | Type | Fields | +|---|---|---| +| `web_documents` | FeatureView | `human`, `bot`, `human_repeat_ratio`, `bot_repeat_ratio` | +| `train_example` | OnDemandFeatureView | `cleaned_human`, `cleaned_bot`, `char_count`, `is_trainable`, `sft_text` | +| `llm_posttrain` | FeatureService | `web_documents` + `train_example` | + +Source data is **prepared parquet** (`document_id` + `event_timestamp` already present). No Feast core patches. + +## Paths + +| Flag | What happens | +|---|---| +| (default) | `to_ray_dataset()` + preprocess `sft_text` (ODFV does **not** run) | +| `--via-df` | `to_df()` so ODFV `train_example` runs | + +## Setup + +```bash +uv pip install -e "../../sdk/python[ray]" -r requirements.txt +PYTHONPATH=../../sdk/python python scripts/prepare_data.py +cd feature_repo && feast apply && cd .. +``` + +## Run (data load only) + +```bash +PYTHONPATH=../../sdk/python python scripts/train_sft.py --dry-run +PYTHONPATH=../../sdk/python python scripts/train_sft.py --dry-run --via-df +``` + +## Blog + +[How to Use Feast for SLM/LLM Post-Training with Ray](/blog/feast-ray-llm-posttrain) diff --git a/examples/ray-llm-posttrain/feature_repo/feature_definitions.py b/examples/ray-llm-posttrain/feature_repo/feature_definitions.py new file mode 100644 index 00000000000..a85470c3729 --- /dev/null +++ b/examples/ray-llm-posttrain/feature_repo/feature_definitions.py @@ -0,0 +1,108 @@ +"""Feast feature definitions for Ray + ODFV LLM post-training. + +Pipeline (supported Feast APIs only): + scripts/prepare_data.py → parquet with document_id + event_timestamp + → RaySource (parquet) + → FeatureView web_documents + → OnDemandFeatureView train_example + → FeatureService llm_posttrain +""" + +from __future__ import annotations + +from datetime import timedelta +from pathlib import Path + +from feast import Entity, FeatureService, FeatureView, Field, ValueType +from feast.infra.offline_stores.contrib.ray_offline_store.ray_source import RaySource +from feast.on_demand_feature_view import on_demand_feature_view +from feast.types import Bool, Float64, Int64, String + +_REPO_DIR = Path(__file__).resolve().parent +_PARQUET = str(_REPO_DIR / "data" / "tiny_webtext.parquet") + +document = Entity( + name="document", + join_keys=["document_id"], + value_type=ValueType.STRING, + description="Document id (added by scripts/prepare_data.py)", +) + +# Parquet already has document_id + event_timestamp (see prepare_data.py). +# Do not rely on BatchFeatureView UDFs to invent timestamps during entity-less +# retrieval — that path is not supported without Feast core changes. +tiny_web = RaySource( + name="tiny_webtext", + reader_type="parquet", + path=_PARQUET, + timestamp_field="event_timestamp", +) + +web_documents = FeatureView( + name="web_documents", + entities=[document], + ttl=timedelta(days=365), + schema=[ + Field(name="human", dtype=String), + Field(name="bot", dtype=String), + Field(name="human_repeat_ratio", dtype=Float64), + Field(name="bot_repeat_ratio", dtype=Float64), + ], + source=tiny_web, + online=False, + description="Conversation columns from prepared parquet", + tags={"use_case": "llm_posttrain", "source": "parquet"}, +) + + +@on_demand_feature_view( + sources=[web_documents], + schema=[ + Field(name="cleaned_human", dtype=String), + Field(name="cleaned_bot", dtype=String), + Field(name="char_count", dtype=Int64), + Field(name="is_trainable", dtype=Bool), + Field(name="sft_text", dtype=String), + ], + mode="pandas", +) +def train_example(inputs): + """Quality gate + human→bot SFT formatting.""" + import pandas as pd + + min_chars = 64 + max_repeat_ratio = 0.65 + + cleaned_human = inputs["human"].fillna("").astype(str).str.strip() + cleaned_bot = inputs["bot"].fillna("").astype(str).str.strip() + char_count = cleaned_bot.str.len().astype("int64") + + human_ratio = inputs["human_repeat_ratio"].fillna(1.0).astype(float) + bot_ratio = inputs["bot_repeat_ratio"].fillna(1.0).astype(float) + is_trainable = ( + (char_count >= min_chars) + & (human_ratio <= max_repeat_ratio) + & (bot_ratio <= max_repeat_ratio) + ) + + sft_text = ( + "<|im_start|>user\n" + cleaned_human + "<|im_end|>\n" + "<|im_start|>assistant\n" + cleaned_bot + "<|im_end|>" + ) + + return pd.DataFrame( + { + "cleaned_human": cleaned_human, + "cleaned_bot": cleaned_bot, + "char_count": char_count, + "is_trainable": is_trainable, + "sft_text": sft_text, + } + ) + + +llm_posttrain = FeatureService( + name="llm_posttrain", + features=[web_documents, train_example], + tags={"use_case": "llm_posttrain", "model": "gpt2"}, +) diff --git a/examples/ray-llm-posttrain/feature_repo/feature_store.yaml b/examples/ray-llm-posttrain/feature_repo/feature_store.yaml new file mode 100644 index 00000000000..226dbdfb45c --- /dev/null +++ b/examples/ray-llm-posttrain/feature_repo/feature_store.yaml @@ -0,0 +1,25 @@ +project: ray_llm_posttrain +registry: data/registry.db +provider: local + +# Laptop-friendly Ray offline store (no KubeRay) +offline_store: + type: ray + storage_path: data/ray_storage + enable_ray_logging: false + ray_conf: + num_cpus: 2 + object_store_memory: 104857600 + _memory: 524288000 + +batch_engine: + type: ray.engine + max_workers: 2 + +online_store: + type: sqlite + path: data/online_store.db + +entity_key_serialization_version: 3 +auth: + type: no_auth diff --git a/examples/ray-llm-posttrain/requirements.txt b/examples/ray-llm-posttrain/requirements.txt new file mode 100644 index 00000000000..4a82b391c07 --- /dev/null +++ b/examples/ray-llm-posttrain/requirements.txt @@ -0,0 +1,8 @@ +# Feast + Ray offline store / compute engine +feast[ray]>=0.50.0 +datasets>=2.19.0 + +# Short GPT-2 SFT +transformers>=4.40.0 +torch>=2.1.0 +accelerate>=0.30.0 diff --git a/examples/ray-llm-posttrain/scripts/prepare_data.py b/examples/ray-llm-posttrain/scripts/prepare_data.py new file mode 100644 index 00000000000..efd1a5e032b --- /dev/null +++ b/examples/ray-llm-posttrain/scripts/prepare_data.py @@ -0,0 +1,55 @@ +#!/usr/bin/env python3 +"""Prepare a small local parquet seed for the example. + +Hugging Face tiny-webtext has no document_id / event_timestamp. Feast entity-less +retrieval needs those columns on the *source* data. We synthesize them here +(outside Feast) and write parquet — no Feast core changes required. + + PYTHONPATH=../../sdk/python python scripts/prepare_data.py +""" + +from __future__ import annotations + +import hashlib +from pathlib import Path + +import pandas as pd + +REPO_ROOT = Path(__file__).resolve().parents[1] +OUT_PATH = REPO_ROOT / "feature_repo" / "data" / "tiny_webtext.parquet" +SPLIT = "train[:2000]" +DATASET = "nampdn-ai/tiny-webtext" + + +def main() -> int: + from datasets import load_dataset + + print(f"Loading {DATASET} split={SPLIT!r}...") + ds = load_dataset(DATASET, split=SPLIT) + df = ds.to_pandas() + + demo_base_ts = pd.Timestamp("2024-06-01", tz="UTC") + demo_window_seconds = 30 * 24 * 3600 + + humans = df["human"].fillna("").astype(str) + bots = df["bot"].fillna("").astype(str) + doc_ids: list[str] = [] + timestamps: list[pd.Timestamp] = [] + for human, bot in zip(humans, bots, strict=True): + digest = hashlib.sha256(f"{human}\n{bot}".encode()).hexdigest() + doc_ids.append(digest[:16]) + offset = int(digest[:8], 16) % demo_window_seconds + timestamps.append(demo_base_ts + pd.Timedelta(seconds=offset)) + + df = df.copy() + df["document_id"] = doc_ids + df["event_timestamp"] = timestamps + + OUT_PATH.parent.mkdir(parents=True, exist_ok=True) + df.to_parquet(OUT_PATH, index=False) + print(f"Wrote {len(df)} rows → {OUT_PATH}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/examples/ray-llm-posttrain/scripts/train_sft.py b/examples/ray-llm-posttrain/scripts/train_sft.py new file mode 100644 index 00000000000..a3f4939f855 --- /dev/null +++ b/examples/ray-llm-posttrain/scripts/train_sft.py @@ -0,0 +1,217 @@ +#!/usr/bin/env python3 +"""Feast conversation features → training rows (paths match the blog). + +Paths: + A) Default: get_historical_features → to_ray_dataset() → preprocess sft_text + (ODFVs do NOT run on to_ray_dataset) + B) --via-df: get_historical_features → to_df() (ODFV train_example runs) + + PYTHONPATH=../../sdk/python python scripts/train_sft.py --dry-run + PYTHONPATH=../../sdk/python python scripts/train_sft.py --dry-run --via-df +""" + +from __future__ import annotations + +import argparse +import sys +from datetime import datetime, timezone +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parents[1] +FEATURE_REPO = REPO_ROOT / "feature_repo" +DATA_DIR = REPO_ROOT / "data" + +# Matches the blog Option A snippet (length gate) +_MIN_CHARS = 64 + + +def _parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--max-steps", type=int, default=20) + parser.add_argument("--batch-size", type=int, default=2) + parser.add_argument("--max-length", type=int, default=256) + parser.add_argument( + "--output-dir", + type=Path, + default=DATA_DIR / "gpt2-sft", + ) + parser.add_argument( + "--dry-run", + action="store_true", + help="Print a few training rows; skip the optional GPT-2 smoke", + ) + parser.add_argument( + "--via-df", + action="store_true", + help="Use to_df() so OnDemandFeatureView train_example runs", + ) + return parser.parse_args() + + +def _date_window() -> tuple[datetime, datetime]: + return ( + datetime(2024, 6, 1, tzinfo=timezone.utc), + datetime(2024, 7, 1, tzinfo=timezone.utc), + ) + + +def _preprocess_sft_batch(batch): + """Build sft_text from FeatureView columns (blog Option A).""" + import pandas as pd + + if not isinstance(batch, pd.DataFrame): + batch = pd.DataFrame(batch) + + human = batch["human"].fillna("").astype(str).str.strip() + bot = batch["bot"].fillna("").astype(str).str.strip() + ok = bot.str.len() >= _MIN_CHARS + sft_text = ( + "<|im_start|>user\n" + human + "<|im_end|>\n" + "<|im_start|>assistant\n" + bot + "<|im_end|>" + ) + return pd.DataFrame({"sft_text": sft_text}).loc[ok].reset_index(drop=True) + + +def retrieve_via_ray_stream(): + """Option A: to_ray_dataset() + preprocess (ODFV does not run).""" + from feast import FeatureStore + + store = FeatureStore(repo_path=str(FEATURE_REPO)) + start_date, end_date = _date_window() + + print("get_historical_features → to_ray_dataset() (preprocess sft_text on Ray)") + job = store.get_historical_features( + features=[ + "web_documents:human", + "web_documents:bot", + "web_documents:human_repeat_ratio", + "web_documents:bot_repeat_ratio", + ], + start_date=start_date, + end_date=end_date, + ) + ds = job.to_ray_dataset() + return ds.map_batches(_preprocess_sft_batch, batch_format="pandas") + + +def retrieve_via_df(): + """Option B: to_df() so ODFV train_example runs, then Ray from pandas.""" + import ray + from feast import FeatureStore + + store = FeatureStore(repo_path=str(FEATURE_REPO)) + start_date, end_date = _date_window() + + print("get_historical_features → to_df() (ODFV train_example runs)") + df = store.get_historical_features( + features=store.get_feature_service("llm_posttrain"), + start_date=start_date, + end_date=end_date, + ).to_df() + + if "is_trainable" not in df.columns or "sft_text" not in df.columns: + raise RuntimeError("Expected ODFV columns is_trainable / sft_text from to_df()") + + mask = df["is_trainable"].fillna(False).astype(bool) + mask &= df["sft_text"].fillna("").astype(str).str.len() > 0 + slim = df.loc[mask, ["sft_text"]].reset_index(drop=True) + return ray.data.from_pandas(slim) + + +def train_gpt2_optional( + ray_ds, *, max_steps: int, batch_size: int, max_length: int, output_dir: Path +) -> None: + import torch + from transformers import ( + AutoModelForCausalLM, + AutoTokenizer, + DataCollatorForLanguageModeling, + Trainer, + TrainingArguments, + ) + + rows = ray_ds.take(min(500, max(50, max_steps * batch_size * 4))) + texts = [r["sft_text"] for r in rows if r.get("sft_text")] + if not texts: + raise RuntimeError("No trainable SFT rows") + + print(f"[optional] GPT-2 smoke on {len(texts)} rows, {max_steps} steps...") + tokenizer = AutoTokenizer.from_pretrained("gpt2") + if tokenizer.pad_token is None: + tokenizer.pad_token = tokenizer.eos_token + model = AutoModelForCausalLM.from_pretrained("gpt2") + encodings = tokenizer( + texts, + truncation=True, + max_length=max_length, + padding="max_length", + return_tensors="pt", + ) + + class _TextDataset(torch.utils.data.Dataset): + def __len__(self) -> int: + return encodings["input_ids"].shape[0] + + def __getitem__(self, idx: int) -> dict: + return { + "input_ids": encodings["input_ids"][idx], + "attention_mask": encodings["attention_mask"][idx], + "labels": encodings["input_ids"][idx].clone(), + } + + output_dir.mkdir(parents=True, exist_ok=True) + args = TrainingArguments( + output_dir=str(output_dir), + per_device_train_batch_size=batch_size, + max_steps=max_steps, + logging_steps=max(1, max_steps // 5), + save_steps=max_steps, + learning_rate=5e-5, + report_to=[], + remove_unused_columns=False, + ) + trainer = Trainer( + model=model, + args=args, + train_dataset=_TextDataset(), + data_collator=DataCollatorForLanguageModeling(tokenizer=tokenizer, mlm=False), + ) + trainer.train() + trainer.save_model(str(output_dir)) + tokenizer.save_pretrained(str(output_dir)) + print(f"Saved optional checkpoint to {output_dir}") + + +def main() -> int: + args = _parse_args() + if not (FEATURE_REPO / "feature_store.yaml").exists(): + print(f"Missing feature repo at {FEATURE_REPO}", file=sys.stderr) + return 1 + + if args.via_df: + ds = retrieve_via_df() + else: + ds = retrieve_via_ray_stream() + + sample = ds.take(3) + print(f"Sample training rows: {len(sample)}") + for i, row in enumerate(sample): + preview = str(row.get("sft_text", row))[:160].replace("\n", "\\n") + print(f" [{i}] {preview}...") + + if args.dry_run: + print("Done (trainer skipped).") + return 0 + + train_gpt2_optional( + ds, + max_steps=args.max_steps, + batch_size=args.batch_size, + max_length=args.max_length, + output_dir=args.output_dir, + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/infra/charts/feast-feature-server/Chart.yaml b/infra/charts/feast-feature-server/Chart.yaml index 711dd910e95..f77f882a9d2 100644 --- a/infra/charts/feast-feature-server/Chart.yaml +++ b/infra/charts/feast-feature-server/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: feast-feature-server description: Feast Feature Server in Go or Python type: application -version: 0.63.0 +version: 0.65.0 keywords: - machine learning - big data diff --git a/infra/charts/feast-feature-server/README.md b/infra/charts/feast-feature-server/README.md index 24013786f14..cd8cc475031 100644 --- a/infra/charts/feast-feature-server/README.md +++ b/infra/charts/feast-feature-server/README.md @@ -1,6 +1,6 @@ # Feast Python / Go Feature Server Helm Charts -Current chart version is `0.63.0` +Current chart version is `0.65.0` ## Installation @@ -42,7 +42,7 @@ See [here](https://github.com/feast-dev/feast/tree/master/examples/python-helm-d | fullnameOverride | string | `""` | | | image.pullPolicy | string | `"IfNotPresent"` | | | image.repository | string | `"quay.io/feastdev/feature-server"` | Docker image for Feature Server repository | -| image.tag | string | `"0.63.0"` | The Docker image tag (can be overwritten if custom feature server deps are needed for on demand transforms) | +| image.tag | string | `"0.65.0"` | The Docker image tag (can be overwritten if custom feature server deps are needed for on demand transforms) | | imagePullSecrets | list | `[]` | | | livenessProbe.initialDelaySeconds | int | `30` | | | livenessProbe.periodSeconds | int | `30` | | diff --git a/infra/charts/feast-feature-server/values.yaml b/infra/charts/feast-feature-server/values.yaml index 6a3cfda4419..f2bc97d7a2b 100644 --- a/infra/charts/feast-feature-server/values.yaml +++ b/infra/charts/feast-feature-server/values.yaml @@ -9,7 +9,7 @@ image: repository: quay.io/feastdev/feature-server pullPolicy: IfNotPresent # image.tag -- The Docker image tag (can be overwritten if custom feature server deps are needed for on demand transforms) - tag: 0.63.0 + tag: 0.65.0 logLevel: "WARNING" # Set log level DEBUG, INFO, WARNING, ERROR, and CRITICAL (case-insensitive) diff --git a/infra/charts/feast/Chart.yaml b/infra/charts/feast/Chart.yaml index 04f4994176e..dc49ff3fb2f 100644 --- a/infra/charts/feast/Chart.yaml +++ b/infra/charts/feast/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v1 description: Feature store for machine learning name: feast -version: 0.63.0 +version: 0.65.0 keywords: - machine learning - big data diff --git a/infra/charts/feast/README.md b/infra/charts/feast/README.md index a79e14e4196..2a288bb48aa 100644 --- a/infra/charts/feast/README.md +++ b/infra/charts/feast/README.md @@ -8,7 +8,7 @@ This repo contains Helm charts for Feast Java components that are being installe ## Chart: Feast -Feature store for machine learning Current chart version is `0.63.0` +Feature store for machine learning Current chart version is `0.65.0` ## Installation @@ -65,8 +65,8 @@ See [here](https://github.com/feast-dev/feast/tree/master/examples/java-demo) fo | Repository | Name | Version | |------------|------|---------| | https://charts.helm.sh/stable | redis | 10.5.6 | -| https://feast-helm-charts.storage.googleapis.com | feature-server(feature-server) | 0.63.0 | -| https://feast-helm-charts.storage.googleapis.com | transformation-service(transformation-service) | 0.63.0 | +| https://feast-helm-charts.storage.googleapis.com | feature-server(feature-server) | 0.65.0 | +| https://feast-helm-charts.storage.googleapis.com | transformation-service(transformation-service) | 0.65.0 | ## Values diff --git a/infra/charts/feast/charts/feature-server/Chart.yaml b/infra/charts/feast/charts/feature-server/Chart.yaml index 1c3f0314d00..b20c1778a18 100644 --- a/infra/charts/feast/charts/feature-server/Chart.yaml +++ b/infra/charts/feast/charts/feature-server/Chart.yaml @@ -1,8 +1,8 @@ apiVersion: v1 description: "Feast Feature Server: Online feature serving service for Feast" name: feature-server -version: 0.63.0 -appVersion: v0.63.0 +version: 0.65.0 +appVersion: v0.65.0 keywords: - machine learning - big data diff --git a/infra/charts/feast/charts/feature-server/README.md b/infra/charts/feast/charts/feature-server/README.md index 1899373fee2..571449d9009 100644 --- a/infra/charts/feast/charts/feature-server/README.md +++ b/infra/charts/feast/charts/feature-server/README.md @@ -1,6 +1,6 @@ # feature-server -![Version: 0.63.0](https://img.shields.io/badge/Version-0.63.0-informational?style=flat-square) ![AppVersion: v0.63.0](https://img.shields.io/badge/AppVersion-v0.63.0-informational?style=flat-square) +![Version: 0.65.0](https://img.shields.io/badge/Version-0.65.0-informational?style=flat-square) ![AppVersion: v0.65.0](https://img.shields.io/badge/AppVersion-v0.65.0-informational?style=flat-square) Feast Feature Server: Online feature serving service for Feast @@ -17,7 +17,7 @@ Feast Feature Server: Online feature serving service for Feast | envOverrides | object | `{}` | Extra environment variables to set | | image.pullPolicy | string | `"IfNotPresent"` | Image pull policy | | image.repository | string | `"quay.io/feastdev/feature-server-java"` | Docker image for Feature Server repository | -| image.tag | string | `"0.63.0"` | Image tag | +| image.tag | string | `"0.65.0"` | Image tag | | ingress.grpc.annotations | object | `{}` | Extra annotations for the ingress | | ingress.grpc.auth.enabled | bool | `false` | Flag to enable auth | | ingress.grpc.class | string | `"nginx"` | Which ingress controller to use | diff --git a/infra/charts/feast/charts/feature-server/values.yaml b/infra/charts/feast/charts/feature-server/values.yaml index 655bcec15e5..3367dd665fa 100644 --- a/infra/charts/feast/charts/feature-server/values.yaml +++ b/infra/charts/feast/charts/feature-server/values.yaml @@ -5,7 +5,7 @@ image: # image.repository -- Docker image for Feature Server repository repository: quay.io/feastdev/feature-server-java # image.tag -- Image tag - tag: 0.63.0 + tag: 0.65.0 # image.pullPolicy -- Image pull policy pullPolicy: IfNotPresent diff --git a/infra/charts/feast/charts/transformation-service/Chart.yaml b/infra/charts/feast/charts/transformation-service/Chart.yaml index 6252eb0d4bb..9dbc3f73cb4 100644 --- a/infra/charts/feast/charts/transformation-service/Chart.yaml +++ b/infra/charts/feast/charts/transformation-service/Chart.yaml @@ -1,8 +1,8 @@ apiVersion: v1 description: "Transformation service: to compute on-demand features" name: transformation-service -version: 0.63.0 -appVersion: v0.63.0 +version: 0.65.0 +appVersion: v0.65.0 keywords: - machine learning - big data diff --git a/infra/charts/feast/charts/transformation-service/README.md b/infra/charts/feast/charts/transformation-service/README.md index 5c7d6de89eb..ad1dd75cd65 100644 --- a/infra/charts/feast/charts/transformation-service/README.md +++ b/infra/charts/feast/charts/transformation-service/README.md @@ -1,6 +1,6 @@ # transformation-service -![Version: 0.63.0](https://img.shields.io/badge/Version-0.63.0-informational?style=flat-square) ![AppVersion: v0.63.0](https://img.shields.io/badge/AppVersion-v0.63.0-informational?style=flat-square) +![Version: 0.65.0](https://img.shields.io/badge/Version-0.65.0-informational?style=flat-square) ![AppVersion: v0.65.0](https://img.shields.io/badge/AppVersion-v0.65.0-informational?style=flat-square) Transformation service: to compute on-demand features @@ -13,7 +13,7 @@ Transformation service: to compute on-demand features | envOverrides | object | `{}` | Extra environment variables to set | | image.pullPolicy | string | `"IfNotPresent"` | Image pull policy | | image.repository | string | `"quay.io/feastdev/feature-transformation-server"` | Docker image for Transformation Server repository | -| image.tag | string | `"0.63.0"` | Image tag | +| image.tag | string | `"0.65.0"` | Image tag | | nodeSelector | object | `{}` | Node labels for pod assignment | | podLabels | object | `{}` | Labels to be added to Feast Serving pods | | replicaCount | int | `1` | Number of pods that will be created | diff --git a/infra/charts/feast/charts/transformation-service/values.yaml b/infra/charts/feast/charts/transformation-service/values.yaml index 03d2f8acf78..266cd4b48aa 100644 --- a/infra/charts/feast/charts/transformation-service/values.yaml +++ b/infra/charts/feast/charts/transformation-service/values.yaml @@ -5,7 +5,7 @@ image: # image.repository -- Docker image for Transformation Server repository repository: quay.io/feastdev/feature-transformation-server # image.tag -- Image tag - tag: 0.63.0 + tag: 0.65.0 # image.pullPolicy -- Image pull policy pullPolicy: IfNotPresent diff --git a/infra/charts/feast/requirements.yaml b/infra/charts/feast/requirements.yaml index 1b962cc0b89..3f29ad7dfdc 100644 --- a/infra/charts/feast/requirements.yaml +++ b/infra/charts/feast/requirements.yaml @@ -1,12 +1,12 @@ dependencies: - name: feature-server alias: feature-server - version: 0.63.0 + version: 0.65.0 condition: feature-server.enabled repository: https://feast-helm-charts.storage.googleapis.com - name: transformation-service alias: transformation-service - version: 0.63.0 + version: 0.65.0 condition: transformation-service.enabled repository: https://feast-helm-charts.storage.googleapis.com - name: redis diff --git a/infra/feast-operator/Dockerfile b/infra/feast-operator/Dockerfile index 65e2bd83cd5..a7ad3fa044d 100644 --- a/infra/feast-operator/Dockerfile +++ b/infra/feast-operator/Dockerfile @@ -12,7 +12,7 @@ COPY --chown=1001:0 go.sum go.sum RUN go mod download # Copy the go source -COPY --chown=1001:0 cmd/main.go cmd/main.go +COPY --chown=1001:0 cmd/ cmd/ COPY --chown=1001:0 api/ api/ COPY --chown=1001:0 internal/controller/ internal/controller/ @@ -21,9 +21,9 @@ COPY --chown=1001:0 internal/controller/ internal/controller/ # was called. For example, if we call make docker-build in a local env which has the Apple Silicon M1 SO # the docker BUILDPLATFORM arg will be linux/arm64 when for Apple x86 it will be linux/amd64. Therefore, # by leaving it empty we can ensure that the container and binary shipped on it will have the same platform. -RUN CGO_ENABLED=0 GOOS=${TARGETOS:-linux} GOARCH=${TARGETARCH} go build -a -o manager cmd/main.go +RUN CGO_ENABLED=0 GOOS=${TARGETOS:-linux} GOARCH=${TARGETARCH} go build -a -o manager ./cmd/ -FROM registry.access.redhat.com/ubi9/ubi-minimal:9.5 +FROM registry.access.redhat.com/ubi9/ubi-minimal:9.8 WORKDIR / COPY --from=builder /opt/app-root/src/manager . USER 65532:65532 diff --git a/infra/feast-operator/Makefile b/infra/feast-operator/Makefile index 94daf1df4e5..b70608a389e 100644 --- a/infra/feast-operator/Makefile +++ b/infra/feast-operator/Makefile @@ -3,7 +3,7 @@ # To re-generate a bundle for another specific version without changing the standard setup, you can: # - use the VERSION as arg of the bundle target (e.g make bundle VERSION=0.0.2) # - use environment variables to overwrite this value (e.g export VERSION=0.0.2) -VERSION ?= 0.63.0 +VERSION ?= 0.65.0 # CHANNELS define the bundle channels used in the bundle. # Add a new line here if you would like to change its default config. (E.g CHANNELS = "candidate,fast,stable") diff --git a/infra/feast-operator/README.md b/infra/feast-operator/README.md index c639be54fde..f879dff1cc1 100644 --- a/infra/feast-operator/README.md +++ b/infra/feast-operator/README.md @@ -7,7 +7,7 @@ This is a K8s Operator that can be used to deploy and manage **Feast**, an open | Guide | Topic | |-------|-------| -| [1 — Project Provisioning](https://docs.feast.dev/how-to-guides/feast-operator/01-project-provisioning) | `feastProjectDir`: git clone vs `feast init` templates | +| [1 — Project Provisioning](https://docs.feast.dev/how-to-guides/feast-operator/01-project-provisioning) | `feastProjectDir`: git clone, `feast init`, or a repository packaged in an image | | [2 — Persistence](https://docs.feast.dev/how-to-guides/feast-operator/02-persistence) | File (path + PVC) vs DB store for offline/online/registry; Secret format | | [3 — Serving & Observability](https://docs.feast.dev/how-to-guides/feast-operator/03-serving-and-observability) | Workers, log level, Prometheus metrics, offline push batching, MCP | | [4 — Registry Topology](https://docs.feast.dev/how-to-guides/feast-operator/04-registry-topology) | Local, remote, cross-namespace `feastRef` | diff --git a/infra/feast-operator/api/feastversion/version.go b/infra/feast-operator/api/feastversion/version.go index 29ab2b9e7c6..deabd34ac38 100644 --- a/infra/feast-operator/api/feastversion/version.go +++ b/infra/feast-operator/api/feastversion/version.go @@ -17,4 +17,4 @@ limitations under the License. package feastversion // Feast release version. Keep on line #20, this is critical to release CI -const FeastVersion = "0.63.0" +const FeastVersion = "0.65.0" diff --git a/infra/feast-operator/api/v1/featurestore_types.go b/infra/feast-operator/api/v1/featurestore_types.go index 3dd68311245..3372e74f63a 100644 --- a/infra/feast-operator/api/v1/featurestore_types.go +++ b/infra/feast-operator/api/v1/featurestore_types.go @@ -111,6 +111,35 @@ type OpenLineageConfig struct { // Keys must be valid Feast OpenLineageConfig YAML field names. // +optional ExtraConfig map[string]string `json:"extraConfig,omitempty"` + // Consumer configures the OpenLineage consumer (event receiver) that enables + // Feast to receive and display lineage from external producers (Airflow, Spark, dbt, etc.). + // +optional + Consumer *OpenLineageConsumerConfig `json:"consumer,omitempty"` +} + +// OpenLineageConsumerConfig configures the OpenLineage consumer (event receiver). +// When enabled, the Feast REST server exposes POST /api/v1/lineage to receive +// OpenLineage events from any producer, storing them for visualization in the Feast UI. +type OpenLineageConsumerConfig struct { + // Enable the OpenLineage consumer. + Enabled bool `json:"enabled"` + // StoreType is the storage backend for lineage events. Currently only "sql" is supported. + // +kubebuilder:default="sql" + // +kubebuilder:validation:Enum=sql + // +optional + StoreType *string `json:"storeType,omitempty"` + // Reference to a Secret containing the key "connection_string" for a separate + // lineage database. If omitted, the SQL registry database is reused. + // +optional + ConnectionStringSecretRef *corev1.LocalObjectReference `json:"connectionStringSecretRef,omitempty"` + // Reference to a Secret containing the key "api_key" that producers must + // provide in the X-API-Key header when sending events. + // +optional + ApiKeySecretRef *corev1.LocalObjectReference `json:"apiKeySecretRef,omitempty"` + // NamespaceMapping maps OpenLineage namespaces to Feast projects for + // RBAC-based filtering of lineage data in the UI. + // +optional + NamespaceMapping map[string]string `json:"namespaceMapping,omitempty"` } // FeatureStoreSpec defines the desired state of FeatureStore @@ -146,10 +175,22 @@ type FeatureStoreSpec struct { } // FeastProjectDir defines how to create the feast project directory. -// +kubebuilder:validation:XValidation:rule="[has(self.git), has(self.init)].exists_one(c, c)",message="One selection required between init or git." +// +kubebuilder:validation:XValidation:rule="[has(self.git), has(self.init), has(self.packaged)].exists_one(c, c)",message="One selection required between init, git, or packaged." type FeastProjectDir struct { - Git *GitCloneOptions `json:"git,omitempty"` - Init *FeastInitOptions `json:"init,omitempty"` + Git *GitCloneOptions `json:"git,omitempty"` + Init *FeastInitOptions `json:"init,omitempty"` + Packaged *FeastPackagedOptions `json:"packaged,omitempty"` +} + +// FeastPackagedOptions describes a feature repository packaged in a feature server image. +// +kubebuilder:validation:XValidation:rule="self.featureRepoPath.startsWith('/') && self.featureRepoPath != '/' && !self.featureRepoPath.contains('//') && !self.featureRepoPath.endsWith('/') && !self.featureRepoPath.contains('/./') && !self.featureRepoPath.endsWith('/.') && !self.featureRepoPath.contains('/../') && !self.featureRepoPath.endsWith('/..')",message="FeatureRepoPath must be a canonical absolute, non-root path without dot segments or repeated separators." +type FeastPackagedOptions struct { + // Image containing the packaged feature repository. When set, this image is used by the + // repository initialization and feast apply containers and as the default service image. + // When omitted, the operator's configured feature server image is used. + Image string `json:"image,omitempty"` + // FeatureRepoPath is the canonical absolute path to the feature repository in the image. + FeatureRepoPath string `json:"featureRepoPath"` } // GitCloneOptions describes how a clone should be performed. @@ -378,6 +419,10 @@ type FeatureStoreServices struct { PodAnnotations map[string]string `json:"podAnnotations,omitempty"` // Disable the 'feast repo initialization' initContainer DisableInitContainers bool `json:"disableInitContainers,omitempty"` + // InitImage overrides the image for init containers (feast-init, feast-apply). + // Resolution order: InitImage → FeastProjectDir.Packaged.Image → RELATED_IMAGE_FEATURE_SERVER → DefaultImage. + // +optional + InitImage *string `json:"initImage,omitempty"` // Runs feast apply on pod start to populate the registry. Defaults to true. Ignored when DisableInitContainers is true. RunFeastApplyOnInit *bool `json:"runFeastApplyOnInit,omitempty"` // Volumes specifies the volumes to mount in the FeatureStore deployment. A corresponding `VolumeMount` should be added to whichever feast service(s) require access to said volume(s). @@ -515,6 +560,11 @@ type OnlineStore struct { // Controls metrics granularity, offline push batching, and MCP. // +optional Serving *ServingConfig `json:"serving,omitempty"` + // Disabled skips deploying the online store service entirely, including its + // serving pod and persistence. Omitting the online store block, or setting + // this to false, deploys the online store with defaults as before. + // +optional + Disabled bool `json:"disabled,omitempty"` } // ServingConfig configures the feature_server section of the generated feature_store.yaml. @@ -597,7 +647,7 @@ type OnlineStoreFilePersistence struct { // OnlineStoreDBStorePersistence configures the DB store persistence for the online store service type OnlineStoreDBStorePersistence struct { // Type of the persistence type you want to use. - // +kubebuilder:validation:Enum=snowflake.online;redis;datastore;dynamodb;bigtable;postgres;cassandra;mysql;hazelcast;singlestore;hbase;elasticsearch;qdrant;couchbase.online;milvus;hybrid;mongodb + // +kubebuilder:validation:Enum=snowflake.online;redis;datastore;dynamodb;bigtable;postgres;cassandra;mysql;hazelcast;singlestore;hbase;elasticsearch;qdrant;couchbase.online;milvus;hybrid;mongodb;aerospike;scylladb Type string `json:"type"` // Data store parameters should be placed as-is from the "feature_store.yaml" under the secret key. "registry_type" & "type" fields should be removed. SecretRef corev1.LocalObjectReference `json:"secretRef"` @@ -623,6 +673,8 @@ var ValidOnlineStoreDBStorePersistenceTypes = []string{ "milvus", "hybrid", "mongodb", + "aerospike", + "scylladb", } // LocalRegistryConfig configures the registry service diff --git a/infra/feast-operator/api/v1/zz_generated.deepcopy.go b/infra/feast-operator/api/v1/zz_generated.deepcopy.go index 9402f95e34a..3035ed066cf 100644 --- a/infra/feast-operator/api/v1/zz_generated.deepcopy.go +++ b/infra/feast-operator/api/v1/zz_generated.deepcopy.go @@ -257,6 +257,21 @@ func (in *FeastInitOptions) DeepCopy() *FeastInitOptions { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *FeastPackagedOptions) DeepCopyInto(out *FeastPackagedOptions) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new FeastPackagedOptions. +func (in *FeastPackagedOptions) DeepCopy() *FeastPackagedOptions { + if in == nil { + return nil + } + out := new(FeastPackagedOptions) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *FeastProjectDir) DeepCopyInto(out *FeastProjectDir) { *out = *in @@ -270,6 +285,11 @@ func (in *FeastProjectDir) DeepCopyInto(out *FeastProjectDir) { *out = new(FeastInitOptions) **out = **in } + if in.Packaged != nil { + in, out := &in.Packaged, &out.Packaged + *out = new(FeastPackagedOptions) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new FeastProjectDir. @@ -396,6 +416,11 @@ func (in *FeatureStoreServices) DeepCopyInto(out *FeatureStoreServices) { (*out)[key] = val } } + if in.InitImage != nil { + in, out := &in.InitImage, &out.InitImage + *out = new(string) + **out = **in + } if in.RunFeastApplyOnInit != nil { in, out := &in.RunFeastApplyOnInit, &out.RunFeastApplyOnInit *out = new(bool) @@ -1043,6 +1068,11 @@ func (in *OpenLineageConfig) DeepCopyInto(out *OpenLineageConfig) { (*out)[key] = val } } + if in.Consumer != nil { + in, out := &in.Consumer, &out.Consumer + *out = new(OpenLineageConsumerConfig) + (*in).DeepCopyInto(*out) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new OpenLineageConfig. @@ -1055,6 +1085,43 @@ func (in *OpenLineageConfig) DeepCopy() *OpenLineageConfig { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *OpenLineageConsumerConfig) DeepCopyInto(out *OpenLineageConsumerConfig) { + *out = *in + if in.StoreType != nil { + in, out := &in.StoreType, &out.StoreType + *out = new(string) + **out = **in + } + if in.ConnectionStringSecretRef != nil { + in, out := &in.ConnectionStringSecretRef, &out.ConnectionStringSecretRef + *out = new(corev1.LocalObjectReference) + **out = **in + } + if in.ApiKeySecretRef != nil { + in, out := &in.ApiKeySecretRef, &out.ApiKeySecretRef + *out = new(corev1.LocalObjectReference) + **out = **in + } + if in.NamespaceMapping != nil { + in, out := &in.NamespaceMapping, &out.NamespaceMapping + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new OpenLineageConsumerConfig. +func (in *OpenLineageConsumerConfig) DeepCopy() *OpenLineageConsumerConfig { + if in == nil { + return nil + } + out := new(OpenLineageConsumerConfig) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *OptionalCtrConfigs) DeepCopyInto(out *OptionalCtrConfigs) { *out = *in diff --git a/infra/feast-operator/api/v1alpha1/featurestore_types.go b/infra/feast-operator/api/v1alpha1/featurestore_types.go index 87d13003805..8ccde377e77 100644 --- a/infra/feast-operator/api/v1alpha1/featurestore_types.go +++ b/infra/feast-operator/api/v1alpha1/featurestore_types.go @@ -78,10 +78,22 @@ type FeatureStoreSpec struct { } // FeastProjectDir defines how to create the feast project directory. -// +kubebuilder:validation:XValidation:rule="[has(self.git), has(self.init)].exists_one(c, c)",message="One selection required between init or git." +// +kubebuilder:validation:XValidation:rule="[has(self.git), has(self.init), has(self.packaged)].exists_one(c, c)",message="One selection required between init, git, or packaged." type FeastProjectDir struct { - Git *GitCloneOptions `json:"git,omitempty"` - Init *FeastInitOptions `json:"init,omitempty"` + Git *GitCloneOptions `json:"git,omitempty"` + Init *FeastInitOptions `json:"init,omitempty"` + Packaged *FeastPackagedOptions `json:"packaged,omitempty"` +} + +// FeastPackagedOptions describes a feature repository packaged in a feature server image. +// +kubebuilder:validation:XValidation:rule="self.featureRepoPath.startsWith('/') && self.featureRepoPath != '/' && !self.featureRepoPath.contains('//') && !self.featureRepoPath.endsWith('/') && !self.featureRepoPath.contains('/./') && !self.featureRepoPath.endsWith('/.') && !self.featureRepoPath.contains('/../') && !self.featureRepoPath.endsWith('/..')",message="FeatureRepoPath must be a canonical absolute, non-root path without dot segments or repeated separators." +type FeastPackagedOptions struct { + // Image containing the packaged feature repository. When set, this image is used by the + // repository initialization and feast apply containers and as the default service image. + // When omitted, the operator's configured feature server image is used. + Image string `json:"image,omitempty"` + // FeatureRepoPath is the canonical absolute path to the feature repository in the image. + FeatureRepoPath string `json:"featureRepoPath"` } // GitCloneOptions describes how a clone should be performed. @@ -373,7 +385,7 @@ type OnlineStoreFilePersistence struct { // OnlineStoreDBStorePersistence configures the DB store persistence for the online store service type OnlineStoreDBStorePersistence struct { // Type of the persistence type you want to use. - // +kubebuilder:validation:Enum=snowflake.online;redis;datastore;dynamodb;bigtable;postgres;cassandra;mysql;hazelcast;singlestore;hbase;elasticsearch;qdrant;couchbase.online;milvus;hybrid;mongodb + // +kubebuilder:validation:Enum=snowflake.online;redis;datastore;dynamodb;bigtable;postgres;cassandra;mysql;hazelcast;singlestore;hbase;elasticsearch;qdrant;couchbase.online;milvus;hybrid;mongodb;aerospike;scylladb Type string `json:"type"` // Data store parameters should be placed as-is from the "feature_store.yaml" under the secret key. "registry_type" & "type" fields should be removed. SecretRef corev1.LocalObjectReference `json:"secretRef"` @@ -399,6 +411,8 @@ var ValidOnlineStoreDBStorePersistenceTypes = []string{ "milvus", "hybrid", "mongodb", + "aerospike", + "scylladb", } // LocalRegistryConfig configures the registry service diff --git a/infra/feast-operator/api/v1alpha1/zz_generated.deepcopy.go b/infra/feast-operator/api/v1alpha1/zz_generated.deepcopy.go index 4033c368c8b..17ae4841966 100644 --- a/infra/feast-operator/api/v1alpha1/zz_generated.deepcopy.go +++ b/infra/feast-operator/api/v1alpha1/zz_generated.deepcopy.go @@ -183,6 +183,21 @@ func (in *FeastInitOptions) DeepCopy() *FeastInitOptions { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *FeastPackagedOptions) DeepCopyInto(out *FeastPackagedOptions) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new FeastPackagedOptions. +func (in *FeastPackagedOptions) DeepCopy() *FeastPackagedOptions { + if in == nil { + return nil + } + out := new(FeastPackagedOptions) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *FeastProjectDir) DeepCopyInto(out *FeastProjectDir) { *out = *in @@ -196,6 +211,11 @@ func (in *FeastProjectDir) DeepCopyInto(out *FeastProjectDir) { *out = new(FeastInitOptions) **out = **in } + if in.Packaged != nil { + in, out := &in.Packaged, &out.Packaged + *out = new(FeastPackagedOptions) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new FeastProjectDir. diff --git a/infra/feast-operator/bundle/manifests/feast-operator.clusterserviceversion.yaml b/infra/feast-operator/bundle/manifests/feast-operator.clusterserviceversion.yaml index faa9840ac43..19af99046e9 100644 --- a/infra/feast-operator/bundle/manifests/feast-operator.clusterserviceversion.yaml +++ b/infra/feast-operator/bundle/manifests/feast-operator.clusterserviceversion.yaml @@ -62,6 +62,16 @@ metadata: "transport": "sse" } } + }, + "registry": { + "local": { + "server": { + "mcp": { + "enabled": true + }, + "restAPI": true + } + } } } } @@ -101,8 +111,10 @@ metadata: "serving": { "metrics": { "categories": { + "audit_logging": false, "freshness": false, "materialization": true, + "offline_features": true, "online_features": true, "push": true, "request": true, @@ -135,10 +147,10 @@ metadata: } ] capabilities: Basic Install - createdAt: "2026-05-12T07:35:41Z" + createdAt: "2026-07-20T13:27:58Z" operators.operatorframework.io/builder: operator-sdk-v1.41.0 operators.operatorframework.io/project_layout: go.kubebuilder.io/v4 - name: feast-operator.v0.63.0 + name: feast-operator.v0.65.0 namespace: placeholder spec: apiservicedefinitions: {} @@ -168,11 +180,11 @@ spec: resources: - configmaps - persistentvolumeclaims - - serviceaccounts - services verbs: - create - delete + - deletecollection - get - list - update @@ -181,18 +193,45 @@ spec: - "" resources: - namespaces - - pods - secrets verbs: - get - list - watch + - apiGroups: + - "" + resources: + - pods + verbs: + - create + - delete + - deletecollection + - get + - list + - watch - apiGroups: - "" resources: - pods/exec verbs: - create + - apiGroups: + - "" + resources: + - pods/log + verbs: + - get + - apiGroups: + - "" + resources: + - serviceaccounts + verbs: + - create + - delete + - get + - list + - update + - watch - apiGroups: - apps resources: @@ -234,6 +273,14 @@ spec: - patch - update - watch + - apiGroups: + - config.openshift.io + resources: + - apiservers + verbs: + - get + - list + - watch - apiGroups: - feast.dev resources: @@ -309,6 +356,14 @@ spec: - list - update - watch + - apiGroups: + - sparkoperator.k8s.io + resources: + - sparkapplications + verbs: + - create + - delete + - get - apiGroups: - authentication.k8s.io resources: @@ -352,13 +407,13 @@ spec: - /manager env: - name: RELATED_IMAGE_FEATURE_SERVER - value: quay.io/feastdev/feature-server:0.63.0 + value: quay.io/feastdev/feature-server:0.65.0 - name: RELATED_IMAGE_CRON_JOB value: quay.io/openshift/origin-cli:4.17 - name: GOMEMLIMIT value: 230MiB - name: OIDC_ISSUER_URL - image: quay.io/feastdev/feast-operator:0.63.0 + image: quay.io/feastdev/feast-operator:0.65.0 livenessProbe: httpGet: path: /healthz @@ -448,8 +503,8 @@ spec: name: Feast Community url: https://lf-aidata.atlassian.net/wiki/spaces/FEAST/ relatedImages: - - image: quay.io/feastdev/feature-server:0.63.0 + - image: quay.io/feastdev/feature-server:0.65.0 name: feature-server - image: quay.io/openshift/origin-cli:4.17 name: cron-job - version: 0.63.0 + version: 0.65.0 diff --git a/infra/feast-operator/bundle/manifests/feast.dev_featurestores.yaml b/infra/feast-operator/bundle/manifests/feast.dev_featurestores.yaml index 41651161cc4..0ab08afef51 100644 --- a/infra/feast-operator/bundle/manifests/feast.dev_featurestores.yaml +++ b/infra/feast-operator/bundle/manifests/feast.dev_featurestores.yaml @@ -161,8 +161,9 @@ spec: in a Container. properties: name: - description: Name of the environment variable. Must - be a C_IDENTIFIER. + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. type: string value: description: |- @@ -211,6 +212,35 @@ spec: - fieldPath type: object x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: The key within the env file. An + invalid key will prevent the pod from starting. + type: string + optional: + default: false + description: Specify whether the file or its + key must be defined. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '.. + type: string + volumeName: + description: The name of the volume mount containing + the env file. + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic resourceFieldRef: description: |- Selects a resource of the container: only resources limits and requests @@ -286,7 +316,7 @@ spec: x-kubernetes-map-type: atomic prefix: description: Optional text to prepend to the name of - each environment variable. Must be a C_IDENTIFIER. + each environment variable. type: string secretRef: description: The Secret to select from @@ -471,7 +501,6 @@ spec: type. type: string required: - - status - type type: object type: array @@ -529,6 +558,16 @@ spec: description: The time zone name for the given schedule, see https://en.wikipedia.org/wiki/List_of_tz_database_time_zones. type: string type: object + dataQualityMonitoring: + description: DataQualityMonitoring configures Data Quality Monitoring + behaviour. + properties: + autoBaseline: + default: true + description: AutoBaseline controls whether baseline distribution + is computed automatically on feast apply. Defaults to true. + type: boolean + type: object feastProject: description: FeastProject is the Feast project id. pattern: ^[A-Za-z0-9][A-Za-z0-9_-]*$ @@ -554,8 +593,9 @@ spec: in a Container. properties: name: - description: Name of the environment variable. Must - be a C_IDENTIFIER. + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. type: string value: description: |- @@ -604,6 +644,35 @@ spec: - fieldPath type: object x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: The key within the env file. An + invalid key will prevent the pod from starting. + type: string + optional: + default: false + description: Specify whether the file or its + key must be defined. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '.. + type: string + volumeName: + description: The name of the volume mount containing + the env file. + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic resourceFieldRef: description: |- Selects a resource of the container: only resources limits and requests @@ -679,7 +748,7 @@ spec: x-kubernetes-map-type: atomic prefix: description: Optional text to prepend to the name of - each environment variable. Must be a C_IDENTIFIER. + each environment variable. type: string secretRef: description: The Secret to select from @@ -741,10 +810,32 @@ spec: - pytorch_nlp type: string type: object + packaged: + description: FeastPackagedOptions describes a feature repository + packaged in a feature server image. + properties: + featureRepoPath: + description: FeatureRepoPath is the canonical absolute path + to the feature repository in the image. + type: string + image: + description: Image containing the packaged feature repository. + type: string + required: + - featureRepoPath + type: object + x-kubernetes-validations: + - message: FeatureRepoPath must be a canonical absolute, non-root + path without dot segments or repeated separators. + rule: self.featureRepoPath.startsWith('/') && self.featureRepoPath + != '/' && !self.featureRepoPath.contains('//') && !self.featureRepoPath.endsWith('/') + && !self.featureRepoPath.contains('/./') && !self.featureRepoPath.endsWith('/.') + && !self.featureRepoPath.contains('/../') && !self.featureRepoPath.endsWith('/..') type: object x-kubernetes-validations: - - message: One selection required between init or git. - rule: '[has(self.git), has(self.init)].exists_one(c, c)' + - message: One selection required between init, git, or packaged. + rule: '[has(self.git), has(self.init), has(self.packaged)].exists_one(c, + c)' materialization: description: |- Materialization controls feature materialization behavior (batch size, pull strategy). @@ -783,6 +874,59 @@ spec: type: string type: object x-kubernetes-map-type: atomic + consumer: + description: |- + Consumer configures the OpenLineage consumer (event receiver) that enables + Feast to receive and display lineage from... + properties: + apiKeySecretRef: + description: |- + Reference to a Secret containing the key "api_key" that producers must + provide in the X-API-Key header when sending... + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. + type: string + type: object + x-kubernetes-map-type: atomic + connectionStringSecretRef: + description: |- + Reference to a Secret containing the key "connection_string" for a separate + lineage database. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. + type: string + type: object + x-kubernetes-map-type: atomic + enabled: + description: Enable the OpenLineage consumer. + type: boolean + namespaceMapping: + additionalProperties: + type: string + description: |- + NamespaceMapping maps OpenLineage namespaces to Feast projects for + RBAC-based filtering of lineage data in the UI. + type: object + storeType: + default: sql + description: StoreType is the storage backend for lineage + events. Currently only "sql" is supported. + enum: + - sql + type: string + required: + - enabled + type: object enabled: description: Enable OpenLineage integration. type: boolean @@ -1601,6 +1745,10 @@ spec: disableInitContainers: description: Disable the 'feast repo initialization' initContainer type: boolean + initImage: + description: InitImage overrides the image for init containers + (feast-init, feast-apply). + type: string offlineStore: description: OfflineStore configures the offline store service properties: @@ -1751,8 +1899,9 @@ spec: present in a Container. properties: name: - description: Name of the environment variable. Must - be a C_IDENTIFIER. + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. type: string value: description: |- @@ -1801,6 +1950,36 @@ spec: - fieldPath type: object x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: The key within the env file. + An invalid key will prevent the pod from + starting. + type: string + optional: + default: false + description: Specify whether the file or + its key must be defined. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '.. + type: string + volumeName: + description: The name of the volume mount + containing the env file. + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic resourceFieldRef: description: |- Selects a resource of the container: only resources limits and requests @@ -1877,7 +2056,7 @@ spec: x-kubernetes-map-type: atomic prefix: description: Optional text to prepend to the name - of each environment variable. Must be a C_IDENTIFIER. + of each environment variable. type: string secretRef: description: The Secret to select from @@ -2105,6 +2284,11 @@ spec: onlineStore: description: OnlineStore configures the online store service properties: + disabled: + description: |- + Disabled skips deploying the online store service entirely, including its + serving pod and persistence. + type: boolean persistence: description: OnlineStorePersistence configures the persistence settings for the online store service @@ -2246,6 +2430,8 @@ spec: - milvus - hybrid - mongodb + - aerospike + - scylladb type: string required: - secretRef @@ -2264,8 +2450,9 @@ spec: present in a Container. properties: name: - description: Name of the environment variable. Must - be a C_IDENTIFIER. + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. type: string value: description: |- @@ -2314,6 +2501,36 @@ spec: - fieldPath type: object x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: The key within the env file. + An invalid key will prevent the pod from + starting. + type: string + optional: + default: false + description: Specify whether the file or + its key must be defined. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '.. + type: string + volumeName: + description: The name of the volume mount + containing the env file. + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic resourceFieldRef: description: |- Selects a resource of the container: only resources limits and requests @@ -2390,7 +2607,7 @@ spec: x-kubernetes-map-type: atomic prefix: description: Optional text to prepend to the name - of each environment variable. Must be a C_IDENTIFIER. + of each environment variable. type: string secretRef: description: The Secret to select from @@ -2894,8 +3111,9 @@ spec: present in a Container. properties: name: - description: Name of the environment variable. - Must be a C_IDENTIFIER. + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. type: string value: description: |- @@ -2945,6 +3163,36 @@ spec: - fieldPath type: object x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: The key within the env + file. An invalid key will prevent + the pod from starting. + type: string + optional: + default: false + description: Specify whether the file + or its key must be defined. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '.. + type: string + volumeName: + description: The name of the volume + mount containing the env file. + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic resourceFieldRef: description: |- Selects a resource of the container: only resources limits and requests @@ -3023,8 +3271,7 @@ spec: x-kubernetes-map-type: atomic prefix: description: Optional text to prepend to the - name of each environment variable. Must be - a C_IDENTIFIER. + name of each environment variable. type: string secretRef: description: The Secret to select from @@ -3284,6 +3531,9 @@ spec: x-kubernetes-validations: - message: At least one of restAPI or grpc must be true rule: self.restAPI == true || self.grpc == true || !has(self.grpc) + - message: MCP requires restAPI to be true + rule: '!has(self.mcp) || !self.mcp.enabled || (has(self.restAPI) + && self.restAPI == true)' type: object remote: description: RemoteRegistryConfig points to a remote feast @@ -3340,6 +3590,37 @@ spec: x-kubernetes-validations: - message: One selection required. rule: '[has(self.local), has(self.remote)].exists_one(c, c)' + resourceClaims: + description: |- + ResourceClaims defines which ResourceClaims must be allocated + and reserved before the Pod is allowed to start. + items: + description: |- + PodResourceClaim references exactly one ResourceClaim, either directly + or by naming a ResourceClaimTemplate which is... + properties: + name: + description: |- + Name uniquely identifies this resource claim inside the pod. + This must be a DNS_LABEL. + type: string + resourceClaimName: + description: |- + ResourceClaimName is the name of a ResourceClaim object in the same + namespace as this pod. + type: string + resourceClaimTemplateName: + description: |- + ResourceClaimTemplateName is the name of a ResourceClaimTemplate + object in the same namespace as this pod. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map runFeastApplyOnInit: description: Runs feast apply on pod start to populate the registry. Defaults to true. Ignored when DisableInitContainers is true. @@ -4165,8 +4446,9 @@ spec: in a Container. properties: name: - description: Name of the environment variable. Must - be a C_IDENTIFIER. + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. type: string value: description: |- @@ -4215,6 +4497,35 @@ spec: - fieldPath type: object x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: The key within the env file. An + invalid key will prevent the pod from starting. + type: string + optional: + default: false + description: Specify whether the file or its + key must be defined. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '.. + type: string + volumeName: + description: The name of the volume mount containing + the env file. + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic resourceFieldRef: description: |- Selects a resource of the container: only resources limits and requests @@ -4290,7 +4601,7 @@ spec: x-kubernetes-map-type: atomic prefix: description: Optional text to prepend to the name of - each environment variable. Must be a C_IDENTIFIER. + each environment variable. type: string secretRef: description: The Secret to select from @@ -5184,9 +5495,8 @@ spec: host that shares a pod's lifetime. properties: endpoints: - description: |- - endpoints is the endpoint name that details Glusterfs topology. - More info: https://examples.k8s. + description: endpoints is the endpoint name that details + Glusterfs topology. type: string path: description: |- @@ -5599,6 +5909,52 @@ spec: type: array x-kubernetes-list-type: atomic type: object + podCertificate: + description: |- + Projects an auto-rotating credential bundle (private key and certificate + chain) that the pod can use either as a TLS... + properties: + certificateChainPath: + description: |- + Write the certificate chain at this path in the projected volume. + + Most applications should use credentialBundlePath. + type: string + credentialBundlePath: + description: Write the credential bundle at + this path in the projected volume. + type: string + keyPath: + description: |- + Write the key at this path in the projected volume. + + Most applications should use credentialBundlePath. + type: string + keyType: + description: The type of keypair Kubelet will + generate for the pod. + type: string + maxExpirationSeconds: + description: |- + maxExpirationSeconds is the maximum lifetime permitted for the + certificate. + format: int32 + type: integer + signerName: + description: Kubelet's generated CSRs will + be addressed to this signer. + type: string + userAnnotations: + additionalProperties: + type: string + description: |- + userAnnotations allow pod authors to pass additional information to + the signer implementation. + type: object + required: + - keyType + - signerName + type: object secret: description: secret information about the secret data to project @@ -6116,8 +6472,9 @@ spec: present in a Container. properties: name: - description: Name of the environment variable. Must - be a C_IDENTIFIER. + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. type: string value: description: |- @@ -6166,6 +6523,36 @@ spec: - fieldPath type: object x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: The key within the env file. + An invalid key will prevent the pod from + starting. + type: string + optional: + default: false + description: Specify whether the file or + its key must be defined. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '.. + type: string + volumeName: + description: The name of the volume mount + containing the env file. + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic resourceFieldRef: description: |- Selects a resource of the container: only resources limits and requests @@ -6242,7 +6629,7 @@ spec: x-kubernetes-map-type: atomic prefix: description: Optional text to prepend to the name - of each environment variable. Must be a C_IDENTIFIER. + of each environment variable. type: string secretRef: description: The Secret to select from @@ -6429,7 +6816,6 @@ spec: condition type. type: string required: - - status - type type: object type: array @@ -6488,6 +6874,16 @@ spec: https://en.wikipedia.org/wiki/List_of_tz_database_time_zones. type: string type: object + dataQualityMonitoring: + description: DataQualityMonitoring configures Data Quality Monitoring + behaviour. + properties: + autoBaseline: + default: true + description: AutoBaseline controls whether baseline distribution + is computed automatically on feast apply. Defaults to true. + type: boolean + type: object feastProject: description: FeastProject is the Feast project id. pattern: ^[A-Za-z0-9][A-Za-z0-9_-]*$ @@ -6514,8 +6910,9 @@ spec: present in a Container. properties: name: - description: Name of the environment variable. Must - be a C_IDENTIFIER. + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. type: string value: description: |- @@ -6564,6 +6961,36 @@ spec: - fieldPath type: object x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: The key within the env file. + An invalid key will prevent the pod from + starting. + type: string + optional: + default: false + description: Specify whether the file or + its key must be defined. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '.. + type: string + volumeName: + description: The name of the volume mount + containing the env file. + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic resourceFieldRef: description: |- Selects a resource of the container: only resources limits and requests @@ -6640,7 +7067,7 @@ spec: x-kubernetes-map-type: atomic prefix: description: Optional text to prepend to the name - of each environment variable. Must be a C_IDENTIFIER. + of each environment variable. type: string secretRef: description: The Secret to select from @@ -6703,10 +7130,32 @@ spec: - pytorch_nlp type: string type: object + packaged: + description: FeastPackagedOptions describes a feature repository + packaged in a feature server image. + properties: + featureRepoPath: + description: FeatureRepoPath is the canonical absolute + path to the feature repository in the image. + type: string + image: + description: Image containing the packaged feature repository. + type: string + required: + - featureRepoPath + type: object + x-kubernetes-validations: + - message: FeatureRepoPath must be a canonical absolute, non-root + path without dot segments or repeated separators. + rule: self.featureRepoPath.startsWith('/') && self.featureRepoPath + != '/' && !self.featureRepoPath.contains('//') && !self.featureRepoPath.endsWith('/') + && !self.featureRepoPath.contains('/./') && !self.featureRepoPath.endsWith('/.') + && !self.featureRepoPath.contains('/../') && !self.featureRepoPath.endsWith('/..') type: object x-kubernetes-validations: - - message: One selection required between init or git. - rule: '[has(self.git), has(self.init)].exists_one(c, c)' + - message: One selection required between init, git, or packaged. + rule: '[has(self.git), has(self.init), has(self.packaged)].exists_one(c, + c)' materialization: description: |- Materialization controls feature materialization behavior (batch size, pull strategy). @@ -6745,6 +7194,59 @@ spec: type: string type: object x-kubernetes-map-type: atomic + consumer: + description: |- + Consumer configures the OpenLineage consumer (event receiver) that enables + Feast to receive and display lineage from... + properties: + apiKeySecretRef: + description: |- + Reference to a Secret containing the key "api_key" that producers must + provide in the X-API-Key header when sending... + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. + type: string + type: object + x-kubernetes-map-type: atomic + connectionStringSecretRef: + description: |- + Reference to a Secret containing the key "connection_string" for a separate + lineage database. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. + type: string + type: object + x-kubernetes-map-type: atomic + enabled: + description: Enable the OpenLineage consumer. + type: boolean + namespaceMapping: + additionalProperties: + type: string + description: |- + NamespaceMapping maps OpenLineage namespaces to Feast projects for + RBAC-based filtering of lineage data in the UI. + type: object + storeType: + default: sql + description: StoreType is the storage backend for lineage + events. Currently only "sql" is supported. + enum: + - sql + type: string + required: + - enabled + type: object enabled: description: Enable OpenLineage integration. type: boolean @@ -7571,6 +8073,10 @@ spec: disableInitContainers: description: Disable the 'feast repo initialization' initContainer type: boolean + initImage: + description: InitImage overrides the image for init containers + (feast-init, feast-apply). + type: string offlineStore: description: OfflineStore configures the offline store service properties: @@ -7724,8 +8230,9 @@ spec: present in a Container. properties: name: - description: Name of the environment variable. - Must be a C_IDENTIFIER. + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. type: string value: description: |- @@ -7775,6 +8282,36 @@ spec: - fieldPath type: object x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: The key within the env + file. An invalid key will prevent + the pod from starting. + type: string + optional: + default: false + description: Specify whether the file + or its key must be defined. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '.. + type: string + volumeName: + description: The name of the volume + mount containing the env file. + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic resourceFieldRef: description: |- Selects a resource of the container: only resources limits and requests @@ -7853,8 +8390,7 @@ spec: x-kubernetes-map-type: atomic prefix: description: Optional text to prepend to the - name of each environment variable. Must be - a C_IDENTIFIER. + name of each environment variable. type: string secretRef: description: The Secret to select from @@ -8083,6 +8619,11 @@ spec: onlineStore: description: OnlineStore configures the online store service properties: + disabled: + description: |- + Disabled skips deploying the online store service entirely, including its + serving pod and persistence. + type: boolean persistence: description: OnlineStorePersistence configures the persistence settings for the online store service @@ -8227,6 +8768,8 @@ spec: - milvus - hybrid - mongodb + - aerospike + - scylladb type: string required: - secretRef @@ -8246,8 +8789,9 @@ spec: present in a Container. properties: name: - description: Name of the environment variable. - Must be a C_IDENTIFIER. + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. type: string value: description: |- @@ -8297,6 +8841,36 @@ spec: - fieldPath type: object x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: The key within the env + file. An invalid key will prevent + the pod from starting. + type: string + optional: + default: false + description: Specify whether the file + or its key must be defined. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '.. + type: string + volumeName: + description: The name of the volume + mount containing the env file. + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic resourceFieldRef: description: |- Selects a resource of the container: only resources limits and requests @@ -8375,8 +8949,7 @@ spec: x-kubernetes-map-type: atomic prefix: description: Optional text to prepend to the - name of each environment variable. Must be - a C_IDENTIFIER. + name of each environment variable. type: string secretRef: description: The Secret to select from @@ -8891,8 +9464,9 @@ spec: variable present in a Container. properties: name: - description: Name of the environment variable. - Must be a C_IDENTIFIER. + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. type: string value: description: |- @@ -8943,6 +9517,36 @@ spec: - fieldPath type: object x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: The key within the + env file. An invalid key will + prevent the pod from starting. + type: string + optional: + default: false + description: Specify whether the + file or its key must be defined. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '.. + type: string + volumeName: + description: The name of the volume + mount containing the env file. + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic resourceFieldRef: description: |- Selects a resource of the container: only resources limits and requests @@ -9023,7 +9627,6 @@ spec: prefix: description: Optional text to prepend to the name of each environment variable. - Must be a C_IDENTIFIER. type: string secretRef: description: The Secret to select from @@ -9289,6 +9892,9 @@ spec: true rule: self.restAPI == true || self.grpc == true || !has(self.grpc) + - message: MCP requires restAPI to be true + rule: '!has(self.mcp) || !self.mcp.enabled || (has(self.restAPI) + && self.restAPI == true)' type: object remote: description: RemoteRegistryConfig points to a remote feast @@ -9346,6 +9952,37 @@ spec: - message: One selection required. rule: '[has(self.local), has(self.remote)].exists_one(c, c)' + resourceClaims: + description: |- + ResourceClaims defines which ResourceClaims must be allocated + and reserved before the Pod is allowed to start. + items: + description: |- + PodResourceClaim references exactly one ResourceClaim, either directly + or by naming a ResourceClaimTemplate which is... + properties: + name: + description: |- + Name uniquely identifies this resource claim inside the pod. + This must be a DNS_LABEL. + type: string + resourceClaimName: + description: |- + ResourceClaimName is the name of a ResourceClaim object in the same + namespace as this pod. + type: string + resourceClaimTemplateName: + description: |- + ResourceClaimTemplateName is the name of a ResourceClaimTemplate + object in the same namespace as this pod. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map runFeastApplyOnInit: description: Runs feast apply on pod start to populate the registry. Defaults to true. Ignored when DisableInitContainers @@ -10180,8 +10817,9 @@ spec: present in a Container. properties: name: - description: Name of the environment variable. Must - be a C_IDENTIFIER. + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. type: string value: description: |- @@ -10230,6 +10868,36 @@ spec: - fieldPath type: object x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: The key within the env file. + An invalid key will prevent the pod from + starting. + type: string + optional: + default: false + description: Specify whether the file or + its key must be defined. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '.. + type: string + volumeName: + description: The name of the volume mount + containing the env file. + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic resourceFieldRef: description: |- Selects a resource of the container: only resources limits and requests @@ -10306,7 +10974,7 @@ spec: x-kubernetes-map-type: atomic prefix: description: Optional text to prepend to the name - of each environment variable. Must be a C_IDENTIFIER. + of each environment variable. type: string secretRef: description: The Secret to select from @@ -11208,9 +11876,8 @@ spec: on the host that shares a pod's lifetime. properties: endpoints: - description: |- - endpoints is the endpoint name that details Glusterfs topology. - More info: https://examples.k8s. + description: endpoints is the endpoint name that + details Glusterfs topology. type: string path: description: |- @@ -11627,6 +12294,52 @@ spec: type: array x-kubernetes-list-type: atomic type: object + podCertificate: + description: |- + Projects an auto-rotating credential bundle (private key and certificate + chain) that the pod can use either as a TLS... + properties: + certificateChainPath: + description: |- + Write the certificate chain at this path in the projected volume. + + Most applications should use credentialBundlePath. + type: string + credentialBundlePath: + description: Write the credential bundle + at this path in the projected volume. + type: string + keyPath: + description: |- + Write the key at this path in the projected volume. + + Most applications should use credentialBundlePath. + type: string + keyType: + description: The type of keypair Kubelet + will generate for the pod. + type: string + maxExpirationSeconds: + description: |- + maxExpirationSeconds is the maximum lifetime permitted for the + certificate. + format: int32 + type: integer + signerName: + description: Kubelet's generated CSRs + will be addressed to this signer. + type: string + userAnnotations: + additionalProperties: + type: string + description: |- + userAnnotations allow pod authors to pass additional information to + the signer implementation. + type: object + required: + - keyType + - signerName + type: object secret: description: secret information about the secret data to project @@ -12225,8 +12938,9 @@ spec: in a Container. properties: name: - description: Name of the environment variable. Must - be a C_IDENTIFIER. + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. type: string value: description: |- @@ -12275,6 +12989,35 @@ spec: - fieldPath type: object x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: The key within the env file. An + invalid key will prevent the pod from starting. + type: string + optional: + default: false + description: Specify whether the file or its + key must be defined. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '.. + type: string + volumeName: + description: The name of the volume mount containing + the env file. + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic resourceFieldRef: description: |- Selects a resource of the container: only resources limits and requests @@ -12350,7 +13093,7 @@ spec: x-kubernetes-map-type: atomic prefix: description: Optional text to prepend to the name of - each environment variable. Must be a C_IDENTIFIER. + each environment variable. type: string secretRef: description: The Secret to select from @@ -12535,7 +13278,6 @@ spec: type. type: string required: - - status - type type: object type: array @@ -12618,8 +13360,9 @@ spec: in a Container. properties: name: - description: Name of the environment variable. Must - be a C_IDENTIFIER. + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. type: string value: description: |- @@ -12668,6 +13411,35 @@ spec: - fieldPath type: object x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: The key within the env file. An + invalid key will prevent the pod from starting. + type: string + optional: + default: false + description: Specify whether the file or its + key must be defined. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '.. + type: string + volumeName: + description: The name of the volume mount containing + the env file. + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic resourceFieldRef: description: |- Selects a resource of the container: only resources limits and requests @@ -12743,7 +13515,7 @@ spec: x-kubernetes-map-type: atomic prefix: description: Optional text to prepend to the name of - each environment variable. Must be a C_IDENTIFIER. + each environment variable. type: string secretRef: description: The Secret to select from @@ -12805,10 +13577,32 @@ spec: - pytorch_nlp type: string type: object + packaged: + description: FeastPackagedOptions describes a feature repository + packaged in a feature server image. + properties: + featureRepoPath: + description: FeatureRepoPath is the canonical absolute path + to the feature repository in the image. + type: string + image: + description: Image containing the packaged feature repository. + type: string + required: + - featureRepoPath + type: object + x-kubernetes-validations: + - message: FeatureRepoPath must be a canonical absolute, non-root + path without dot segments or repeated separators. + rule: self.featureRepoPath.startsWith('/') && self.featureRepoPath + != '/' && !self.featureRepoPath.contains('//') && !self.featureRepoPath.endsWith('/') + && !self.featureRepoPath.contains('/./') && !self.featureRepoPath.endsWith('/.') + && !self.featureRepoPath.contains('/../') && !self.featureRepoPath.endsWith('/..') type: object x-kubernetes-validations: - - message: One selection required between init or git. - rule: '[has(self.git), has(self.init)].exists_one(c, c)' + - message: One selection required between init, git, or packaged. + rule: '[has(self.git), has(self.init), has(self.packaged)].exists_one(c, + c)' services: description: FeatureStoreServices defines the desired feast services. An ephemeral onlineStore feature server is deployed by default. @@ -12995,8 +13789,9 @@ spec: present in a Container. properties: name: - description: Name of the environment variable. Must - be a C_IDENTIFIER. + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. type: string value: description: |- @@ -13045,6 +13840,36 @@ spec: - fieldPath type: object x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: The key within the env file. + An invalid key will prevent the pod from + starting. + type: string + optional: + default: false + description: Specify whether the file or + its key must be defined. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '.. + type: string + volumeName: + description: The name of the volume mount + containing the env file. + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic resourceFieldRef: description: |- Selects a resource of the container: only resources limits and requests @@ -13121,7 +13946,7 @@ spec: x-kubernetes-map-type: atomic prefix: description: Optional text to prepend to the name - of each environment variable. Must be a C_IDENTIFIER. + of each environment variable. type: string secretRef: description: The Secret to select from @@ -13490,6 +14315,8 @@ spec: - milvus - hybrid - mongodb + - aerospike + - scylladb type: string required: - secretRef @@ -13508,8 +14335,9 @@ spec: present in a Container. properties: name: - description: Name of the environment variable. Must - be a C_IDENTIFIER. + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. type: string value: description: |- @@ -13558,6 +14386,36 @@ spec: - fieldPath type: object x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: The key within the env file. + An invalid key will prevent the pod from + starting. + type: string + optional: + default: false + description: Specify whether the file or + its key must be defined. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '.. + type: string + volumeName: + description: The name of the volume mount + containing the env file. + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic resourceFieldRef: description: |- Selects a resource of the container: only resources limits and requests @@ -13634,7 +14492,7 @@ spec: x-kubernetes-map-type: atomic prefix: description: Optional text to prepend to the name - of each environment variable. Must be a C_IDENTIFIER. + of each environment variable. type: string secretRef: description: The Secret to select from @@ -14039,8 +14897,9 @@ spec: present in a Container. properties: name: - description: Name of the environment variable. - Must be a C_IDENTIFIER. + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. type: string value: description: |- @@ -14090,6 +14949,36 @@ spec: - fieldPath type: object x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: The key within the env + file. An invalid key will prevent + the pod from starting. + type: string + optional: + default: false + description: Specify whether the file + or its key must be defined. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '.. + type: string + volumeName: + description: The name of the volume + mount containing the env file. + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic resourceFieldRef: description: |- Selects a resource of the container: only resources limits and requests @@ -14168,8 +15057,7 @@ spec: x-kubernetes-map-type: atomic prefix: description: Optional text to prepend to the - name of each environment variable. Must be - a C_IDENTIFIER. + name of each environment variable. type: string secretRef: description: The Secret to select from @@ -14613,8 +15501,9 @@ spec: in a Container. properties: name: - description: Name of the environment variable. Must - be a C_IDENTIFIER. + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. type: string value: description: |- @@ -14663,6 +15552,35 @@ spec: - fieldPath type: object x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: The key within the env file. An + invalid key will prevent the pod from starting. + type: string + optional: + default: false + description: Specify whether the file or its + key must be defined. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '.. + type: string + volumeName: + description: The name of the volume mount containing + the env file. + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic resourceFieldRef: description: |- Selects a resource of the container: only resources limits and requests @@ -14738,7 +15656,7 @@ spec: x-kubernetes-map-type: atomic prefix: description: Optional text to prepend to the name of - each environment variable. Must be a C_IDENTIFIER. + each environment variable. type: string secretRef: description: The Secret to select from @@ -15632,9 +16550,8 @@ spec: host that shares a pod's lifetime. properties: endpoints: - description: |- - endpoints is the endpoint name that details Glusterfs topology. - More info: https://examples.k8s. + description: endpoints is the endpoint name that details + Glusterfs topology. type: string path: description: |- @@ -16047,6 +16964,52 @@ spec: type: array x-kubernetes-list-type: atomic type: object + podCertificate: + description: |- + Projects an auto-rotating credential bundle (private key and certificate + chain) that the pod can use either as a TLS... + properties: + certificateChainPath: + description: |- + Write the certificate chain at this path in the projected volume. + + Most applications should use credentialBundlePath. + type: string + credentialBundlePath: + description: Write the credential bundle at + this path in the projected volume. + type: string + keyPath: + description: |- + Write the key at this path in the projected volume. + + Most applications should use credentialBundlePath. + type: string + keyType: + description: The type of keypair Kubelet will + generate for the pod. + type: string + maxExpirationSeconds: + description: |- + maxExpirationSeconds is the maximum lifetime permitted for the + certificate. + format: int32 + type: integer + signerName: + description: Kubelet's generated CSRs will + be addressed to this signer. + type: string + userAnnotations: + additionalProperties: + type: string + description: |- + userAnnotations allow pod authors to pass additional information to + the signer implementation. + type: object + required: + - keyType + - signerName + type: object secret: description: secret information about the secret data to project @@ -16485,8 +17448,9 @@ spec: present in a Container. properties: name: - description: Name of the environment variable. Must - be a C_IDENTIFIER. + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. type: string value: description: |- @@ -16535,6 +17499,36 @@ spec: - fieldPath type: object x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: The key within the env file. + An invalid key will prevent the pod from + starting. + type: string + optional: + default: false + description: Specify whether the file or + its key must be defined. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '.. + type: string + volumeName: + description: The name of the volume mount + containing the env file. + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic resourceFieldRef: description: |- Selects a resource of the container: only resources limits and requests @@ -16611,7 +17605,7 @@ spec: x-kubernetes-map-type: atomic prefix: description: Optional text to prepend to the name - of each environment variable. Must be a C_IDENTIFIER. + of each environment variable. type: string secretRef: description: The Secret to select from @@ -16798,7 +17792,6 @@ spec: condition type. type: string required: - - status - type type: object type: array @@ -16883,8 +17876,9 @@ spec: present in a Container. properties: name: - description: Name of the environment variable. Must - be a C_IDENTIFIER. + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. type: string value: description: |- @@ -16933,6 +17927,36 @@ spec: - fieldPath type: object x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: The key within the env file. + An invalid key will prevent the pod from + starting. + type: string + optional: + default: false + description: Specify whether the file or + its key must be defined. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '.. + type: string + volumeName: + description: The name of the volume mount + containing the env file. + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic resourceFieldRef: description: |- Selects a resource of the container: only resources limits and requests @@ -17009,7 +18033,7 @@ spec: x-kubernetes-map-type: atomic prefix: description: Optional text to prepend to the name - of each environment variable. Must be a C_IDENTIFIER. + of each environment variable. type: string secretRef: description: The Secret to select from @@ -17072,10 +18096,32 @@ spec: - pytorch_nlp type: string type: object + packaged: + description: FeastPackagedOptions describes a feature repository + packaged in a feature server image. + properties: + featureRepoPath: + description: FeatureRepoPath is the canonical absolute + path to the feature repository in the image. + type: string + image: + description: Image containing the packaged feature repository. + type: string + required: + - featureRepoPath + type: object + x-kubernetes-validations: + - message: FeatureRepoPath must be a canonical absolute, non-root + path without dot segments or repeated separators. + rule: self.featureRepoPath.startsWith('/') && self.featureRepoPath + != '/' && !self.featureRepoPath.contains('//') && !self.featureRepoPath.endsWith('/') + && !self.featureRepoPath.contains('/./') && !self.featureRepoPath.endsWith('/.') + && !self.featureRepoPath.contains('/../') && !self.featureRepoPath.endsWith('/..') type: object x-kubernetes-validations: - - message: One selection required between init or git. - rule: '[has(self.git), has(self.init)].exists_one(c, c)' + - message: One selection required between init, git, or packaged. + rule: '[has(self.git), has(self.init), has(self.packaged)].exists_one(c, + c)' services: description: FeatureStoreServices defines the desired feast services. An ephemeral onlineStore feature server is deployed by default. @@ -17265,8 +18311,9 @@ spec: present in a Container. properties: name: - description: Name of the environment variable. - Must be a C_IDENTIFIER. + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. type: string value: description: |- @@ -17316,6 +18363,36 @@ spec: - fieldPath type: object x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: The key within the env + file. An invalid key will prevent + the pod from starting. + type: string + optional: + default: false + description: Specify whether the file + or its key must be defined. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '.. + type: string + volumeName: + description: The name of the volume + mount containing the env file. + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic resourceFieldRef: description: |- Selects a resource of the container: only resources limits and requests @@ -17394,8 +18471,7 @@ spec: x-kubernetes-map-type: atomic prefix: description: Optional text to prepend to the - name of each environment variable. Must be - a C_IDENTIFIER. + name of each environment variable. type: string secretRef: description: The Secret to select from @@ -17768,6 +18844,8 @@ spec: - milvus - hybrid - mongodb + - aerospike + - scylladb type: string required: - secretRef @@ -17787,8 +18865,9 @@ spec: present in a Container. properties: name: - description: Name of the environment variable. - Must be a C_IDENTIFIER. + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. type: string value: description: |- @@ -17838,6 +18917,36 @@ spec: - fieldPath type: object x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: The key within the env + file. An invalid key will prevent + the pod from starting. + type: string + optional: + default: false + description: Specify whether the file + or its key must be defined. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '.. + type: string + volumeName: + description: The name of the volume + mount containing the env file. + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic resourceFieldRef: description: |- Selects a resource of the container: only resources limits and requests @@ -17916,8 +19025,7 @@ spec: x-kubernetes-map-type: atomic prefix: description: Optional text to prepend to the - name of each environment variable. Must be - a C_IDENTIFIER. + name of each environment variable. type: string secretRef: description: The Secret to select from @@ -18331,8 +19439,9 @@ spec: variable present in a Container. properties: name: - description: Name of the environment variable. - Must be a C_IDENTIFIER. + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. type: string value: description: |- @@ -18383,6 +19492,36 @@ spec: - fieldPath type: object x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: The key within the + env file. An invalid key will + prevent the pod from starting. + type: string + optional: + default: false + description: Specify whether the + file or its key must be defined. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '.. + type: string + volumeName: + description: The name of the volume + mount containing the env file. + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic resourceFieldRef: description: |- Selects a resource of the container: only resources limits and requests @@ -18463,7 +19602,6 @@ spec: prefix: description: Optional text to prepend to the name of each environment variable. - Must be a C_IDENTIFIER. type: string secretRef: description: The Secret to select from @@ -18918,8 +20056,9 @@ spec: present in a Container. properties: name: - description: Name of the environment variable. Must - be a C_IDENTIFIER. + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. type: string value: description: |- @@ -18968,6 +20107,36 @@ spec: - fieldPath type: object x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: The key within the env file. + An invalid key will prevent the pod from + starting. + type: string + optional: + default: false + description: Specify whether the file or + its key must be defined. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '.. + type: string + volumeName: + description: The name of the volume mount + containing the env file. + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic resourceFieldRef: description: |- Selects a resource of the container: only resources limits and requests @@ -19044,7 +20213,7 @@ spec: x-kubernetes-map-type: atomic prefix: description: Optional text to prepend to the name - of each environment variable. Must be a C_IDENTIFIER. + of each environment variable. type: string secretRef: description: The Secret to select from @@ -19946,9 +21115,8 @@ spec: on the host that shares a pod's lifetime. properties: endpoints: - description: |- - endpoints is the endpoint name that details Glusterfs topology. - More info: https://examples.k8s. + description: endpoints is the endpoint name that + details Glusterfs topology. type: string path: description: |- @@ -20365,6 +21533,52 @@ spec: type: array x-kubernetes-list-type: atomic type: object + podCertificate: + description: |- + Projects an auto-rotating credential bundle (private key and certificate + chain) that the pod can use either as a TLS... + properties: + certificateChainPath: + description: |- + Write the certificate chain at this path in the projected volume. + + Most applications should use credentialBundlePath. + type: string + credentialBundlePath: + description: Write the credential bundle + at this path in the projected volume. + type: string + keyPath: + description: |- + Write the key at this path in the projected volume. + + Most applications should use credentialBundlePath. + type: string + keyType: + description: The type of keypair Kubelet + will generate for the pod. + type: string + maxExpirationSeconds: + description: |- + maxExpirationSeconds is the maximum lifetime permitted for the + certificate. + format: int32 + type: integer + signerName: + description: Kubelet's generated CSRs + will be addressed to this signer. + type: string + userAnnotations: + additionalProperties: + type: string + description: |- + userAnnotations allow pod authors to pass additional information to + the signer implementation. + type: object + required: + - keyType + - signerName + type: object secret: description: secret information about the secret data to project diff --git a/infra/feast-operator/bundle/manifests/openlineage-secret_v1_secret.yaml b/infra/feast-operator/bundle/manifests/openlineage-secret_v1_secret.yaml index aae8a81d7f5..40483cc0c43 100644 --- a/infra/feast-operator/bundle/manifests/openlineage-secret_v1_secret.yaml +++ b/infra/feast-operator/bundle/manifests/openlineage-secret_v1_secret.yaml @@ -3,4 +3,4 @@ kind: Secret metadata: name: openlineage-secret stringData: - api_key: your-marquez-api-key #pragma: allowlist secret + api_key: your-marquez-api-key diff --git a/infra/feast-operator/cmd/main.go b/infra/feast-operator/cmd/main.go index 0e5565cce2b..5d2bbece7dc 100644 --- a/infra/feast-operator/cmd/main.go +++ b/infra/feast-operator/cmd/main.go @@ -17,6 +17,7 @@ limitations under the License. package main import ( + "context" "crypto/tls" "flag" "os" @@ -25,6 +26,8 @@ import ( // to ensure that exec-entrypoint and run can make use of them. _ "k8s.io/client-go/plugin/pkg/client/auth" + configv1 "github.com/openshift/api/config/v1" + tlspkg "github.com/openshift/controller-runtime-common/pkg/tls" appsv1 "k8s.io/api/apps/v1" autoscalingv2 "k8s.io/api/autoscaling/v2" batchv1 "k8s.io/api/batch/v1" @@ -61,6 +64,7 @@ var ( func init() { utilruntime.Must(clientgoscheme.AddToScheme(scheme)) + utilruntime.Must(configv1.Install(scheme)) utilruntime.Must(routev1.AddToScheme(scheme)) utilruntime.Must(feastdevv1alpha1.AddToScheme(scheme)) utilruntime.Must(feastdevv1.AddToScheme(scheme)) @@ -95,9 +99,8 @@ func main() { var enableLeaderElection bool var probeAddr string var secureMetrics bool - var enableHTTP2 bool var featureStoreMetrics bool - var tlsOpts []func(*tls.Config) + tlsOpts := make([]func(*tls.Config), 0, 2) flag.StringVar(&metricsAddr, "metrics-bind-address", "0", "The address the metrics endpoint binds to. "+ "Use :8443 for HTTPS or :8080 for HTTP, or leave as 0 to disable the metrics service.") flag.StringVar(&probeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.") @@ -106,8 +109,6 @@ func main() { "Enabling this will ensure there is only one active controller manager.") flag.BoolVar(&secureMetrics, "metrics-secure", true, "If set, the metrics endpoint is served securely via HTTPS. Use --metrics-secure=false to use HTTP instead.") - flag.BoolVar(&enableHTTP2, "enable-http2", false, - "If set, HTTP/2 will be enabled for the metrics and webhook servers") flag.BoolVar(&featureStoreMetrics, "feature-store-metrics", true, "Enable Prometheus gauges exposing online/offline store and registry configuration per FeatureStore. "+ "Disable with --feature-store-metrics=false.") @@ -119,20 +120,20 @@ func main() { ctrl.SetLogger(zap.New(zap.UseFlagOptions(&opts))) - // if the enable-http2 flag is false (the default), http/2 should be disabled - // due to its vulnerabilities. More specifically, disabling http/2 will - // prevent from being vulnerable to the HTTP/2 Stream Cancellation and - // Rapid Reset CVEs. For more information see: - // - https://github.com/advisories/GHSA-qppj-fm5r-hxr3 - // - https://github.com/advisories/GHSA-4374-p667-p6c8 - disableHTTP2 := func(c *tls.Config) { - setupLog.Info("disabling http/2") - c.NextProtos = []string{"http/1.1"} + // Fetch cluster TLS profile from apiservers.config.openshift.io/cluster + cfg := ctrl.GetConfigOrDie() + bootstrapClient, err := client.New(cfg, client.Options{Scheme: scheme}) + if err != nil { + setupLog.Error(err, "unable to create bootstrap client for TLS profile fetch") + os.Exit(1) } - if !enableHTTP2 { - tlsOpts = append(tlsOpts, disableHTTP2) + tlsResult, err := bootstrapTLS(context.Background(), bootstrapClient) + if err != nil { + setupLog.Error(err, "TLS bootstrap failed") + os.Exit(1) } + tlsOpts = append(tlsOpts, tlsResult.TLSOpts...) webhookServer := webhook.NewServer(webhook.Options{ TLSOpts: tlsOpts, @@ -162,7 +163,7 @@ func main() { metricsServerOptions.FilterProvider = filters.WithAuthenticationAndAuthorization } - mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{ + mgr, err := ctrl.NewManager(cfg, ctrl.Options{ Scheme: scheme, Metrics: metricsServerOptions, WebhookServer: webhookServer, @@ -230,6 +231,32 @@ func main() { } // +kubebuilder:scaffold:builder + // Register SecurityProfileWatcher to restart on TLS profile changes + ctx, cancel := context.WithCancel(ctrl.SetupSignalHandler()) + defer cancel() + + if tlsResult.ProfileFetched { + watcher := &tlspkg.SecurityProfileWatcher{ + Client: mgr.GetClient(), + InitialTLSProfileSpec: tlsResult.ProfileSpec, + OnProfileChange: func(_ context.Context, _, _ configv1.TLSProfileSpec) { + setupLog.Info("TLS profile changed, initiating shutdown to reload") + cancel() + }, + } + if tlsResult.AdherenceFetched { + watcher.InitialTLSAdherencePolicy = tlsResult.AdherencePolicy + watcher.OnAdherencePolicyChange = func(_ context.Context, _, _ configv1.TLSAdherencePolicy) { + setupLog.Info("TLS adherence policy changed, initiating shutdown to reload") + cancel() + } + } + if err := watcher.SetupWithManager(mgr); err != nil { + setupLog.Error(err, "unable to set up TLS profile watcher") + os.Exit(1) + } + } + if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil { setupLog.Error(err, "unable to set up health check") os.Exit(1) @@ -240,7 +267,7 @@ func main() { } setupLog.Info("starting manager") - if err := mgr.Start(ctrl.SetupSignalHandler()); err != nil { + if err := mgr.Start(ctx); err != nil { setupLog.Error(err, "problem running manager") os.Exit(1) } diff --git a/infra/feast-operator/cmd/tls_bootstrap.go b/infra/feast-operator/cmd/tls_bootstrap.go new file mode 100644 index 00000000000..6fe33631e02 --- /dev/null +++ b/infra/feast-operator/cmd/tls_bootstrap.go @@ -0,0 +1,134 @@ +/* +Copyright 2024 Feast Community. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "context" + "crypto/tls" + "errors" + "fmt" + "time" + + configv1 "github.com/openshift/api/config/v1" + tlspkg "github.com/openshift/controller-runtime-common/pkg/tls" + apierrors "k8s.io/apimachinery/pkg/api/errors" + apimeta "k8s.io/apimachinery/pkg/api/meta" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/log" +) + +const ( + tlsFetchTimeout = 10 * time.Second + alpnH2 = "h2" + alpnHTTP11 = "http/1.1" +) + +type tlsBootstrapResult struct { + TLSOpts []func(*tls.Config) + ProfileFetched bool + ProfileSpec configv1.TLSProfileSpec + AdherenceFetched bool + AdherencePolicy configv1.TLSAdherencePolicy + UnsupportedCiphers []string +} + +func fetchTLSProfile(ctx context.Context, k8sClient client.Client) (configv1.TLSProfileSpec, bool, error) { + fetchCtx, cancel := context.WithTimeout(ctx, tlsFetchTimeout) + defer cancel() + + profile, err := tlspkg.FetchAPIServerTLSProfile(fetchCtx, k8sClient) + if err != nil { + return classifyTLSProfileError(err) + } + return profile, true, nil +} + +func classifyTLSProfileError(err error) (configv1.TLSProfileSpec, bool, error) { + intermediate := *configv1.TLSProfiles[configv1.TLSProfileIntermediateType] + + switch { + case apimeta.IsNoMatchError(err): + return intermediate, false, nil + case apierrors.IsNotFound(err): + return intermediate, false, nil + case isTransientError(err): + return intermediate, true, nil + default: + return configv1.TLSProfileSpec{}, false, fmt.Errorf("unable to read APIServer TLS profile: %w", err) + } +} + +func fetchTLSAdherencePolicy(ctx context.Context, k8sClient client.Client) (configv1.TLSAdherencePolicy, bool, error) { + fetchCtx, cancel := context.WithTimeout(ctx, tlsFetchTimeout) + defer cancel() + + policy, err := tlspkg.FetchAPIServerTLSAdherencePolicy(fetchCtx, k8sClient) + if err == nil { + return policy, true, nil + } + + switch { + case apimeta.IsNoMatchError(err), + apierrors.IsNotFound(err), + isTransientError(err): + return "", false, nil + default: + return "", false, fmt.Errorf("unable to read APIServer TLS adherence policy: %w", err) + } +} + +func bootstrapTLS(ctx context.Context, k8sClient client.Client) (*tlsBootstrapResult, error) { + logger := log.FromContext(ctx) + result := &tlsBootstrapResult{ + TLSOpts: make([]func(*tls.Config), 0, 2), + } + + profile, profileFetched, err := fetchTLSProfile(ctx, k8sClient) + if err != nil { + return nil, err + } + result.ProfileFetched = profileFetched + result.ProfileSpec = profile + + tlsConfigFn, unsupported := tlspkg.NewTLSConfigFromProfile(profile) + result.UnsupportedCiphers = unsupported + if len(unsupported) > 0 { + logger.Info("TLS profile contains ciphers unsupported by Go", "unsupported", unsupported) + } + result.TLSOpts = append(result.TLSOpts, tlsConfigFn) + + adherence, adherenceFetched, err := fetchTLSAdherencePolicy(ctx, k8sClient) + if err != nil { + return nil, err + } + result.AdherenceFetched = adherenceFetched + result.AdherencePolicy = adherence + + result.TLSOpts = append(result.TLSOpts, func(c *tls.Config) { + c.NextProtos = []string{alpnH2, alpnHTTP11} + }) + + return result, nil +} + +func isTransientError(err error) bool { + return apierrors.IsServiceUnavailable(err) || + apierrors.IsTimeout(err) || + apierrors.IsServerTimeout(err) || + apierrors.IsTooManyRequests(err) || + errors.Is(err, context.DeadlineExceeded) +} diff --git a/infra/feast-operator/cmd/tls_bootstrap_test.go b/infra/feast-operator/cmd/tls_bootstrap_test.go new file mode 100644 index 00000000000..0bb31c14bf6 --- /dev/null +++ b/infra/feast-operator/cmd/tls_bootstrap_test.go @@ -0,0 +1,347 @@ +/* +Copyright 2024 Feast Community. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "context" + "crypto/tls" + "errors" + "testing" + + configv1 "github.com/openshift/api/config/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/meta" + "k8s.io/apimachinery/pkg/runtime/schema" +) + +func intermediateProfile() configv1.TLSProfileSpec { + return *configv1.TLSProfiles[configv1.TLSProfileIntermediateType] +} + +func TestClassifyTLSProfileError(t *testing.T) { + tests := []struct { + name string + err error + wantProfileFetched bool + wantError bool + wantIntermediate bool + }{ + { + name: "NoMatchError returns Intermediate defaults, profileFetched=false", + err: &meta.NoResourceMatchError{PartialResource: schema.GroupVersionResource{Group: "config.openshift.io"}}, + wantProfileFetched: false, + wantError: false, + wantIntermediate: true, + }, + { + name: "NotFound returns Intermediate defaults, profileFetched=false", + err: apierrors.NewNotFound(schema.GroupResource{Group: "config.openshift.io", Resource: "apiservers"}, "cluster"), + wantProfileFetched: false, + wantError: false, + wantIntermediate: true, + }, + { + name: "ServiceUnavailable is transient, profileFetched=true", + err: apierrors.NewServiceUnavailable("api server down"), + wantProfileFetched: true, + wantError: false, + wantIntermediate: true, + }, + { + name: "Timeout is transient, profileFetched=true", + err: apierrors.NewTimeoutError("timed out", 5), + wantProfileFetched: true, + wantError: false, + wantIntermediate: true, + }, + { + name: "ServerTimeout is transient, profileFetched=true", + err: apierrors.NewServerTimeout(schema.GroupResource{Group: "config.openshift.io", Resource: "apiservers"}, "GET", 5), + wantProfileFetched: true, + wantError: false, + wantIntermediate: true, + }, + { + name: "TooManyRequests is transient, profileFetched=true", + err: apierrors.NewTooManyRequests("throttled", 5), + wantProfileFetched: true, + wantError: false, + wantIntermediate: true, + }, + { + name: "DeadlineExceeded is transient, profileFetched=true", + err: context.DeadlineExceeded, + wantProfileFetched: true, + wantError: false, + wantIntermediate: true, + }, + { + name: "Forbidden is fatal, returns error", + err: apierrors.NewForbidden(schema.GroupResource{Group: "config.openshift.io", Resource: "apiservers"}, "cluster", errors.New("RBAC")), + wantProfileFetched: false, + wantError: true, + wantIntermediate: false, + }, + { + name: "Unauthorized is fatal, returns error", + err: apierrors.NewUnauthorized("no token"), + wantProfileFetched: false, + wantError: true, + wantIntermediate: false, + }, + { + name: "InternalServerError is fatal, returns error", + err: apierrors.NewInternalError(errors.New("crash")), + wantProfileFetched: false, + wantError: true, + wantIntermediate: false, + }, + { + name: "Generic error is fatal, returns error", + err: errors.New("something unexpected"), + wantProfileFetched: false, + wantError: true, + wantIntermediate: false, + }, + { + name: "Wrapped DeadlineExceeded is transient", + err: errors.Join(errors.New("fetch failed"), context.DeadlineExceeded), + wantProfileFetched: true, + wantError: false, + wantIntermediate: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + profile, fetched, err := classifyTLSProfileError(tt.err) + + if tt.wantError && err == nil { + t.Errorf("expected error, got nil") + } + if !tt.wantError && err != nil { + t.Errorf("unexpected error: %v", err) + } + if fetched != tt.wantProfileFetched { + t.Errorf("profileFetched = %v, want %v", fetched, tt.wantProfileFetched) + } + if tt.wantIntermediate { + intermediate := intermediateProfile() + if profile.MinTLSVersion != intermediate.MinTLSVersion { + t.Errorf("MinTLSVersion = %v, want %v (Intermediate)", profile.MinTLSVersion, intermediate.MinTLSVersion) + } + } + }) + } +} + +func TestIsTransientError(t *testing.T) { + tests := []struct { + name string + err error + want bool + }{ + {"ServiceUnavailable", apierrors.NewServiceUnavailable("down"), true}, + {"Timeout", apierrors.NewTimeoutError("slow", 5), true}, + {"ServerTimeout", apierrors.NewServerTimeout(schema.GroupResource{}, "GET", 5), true}, + {"TooManyRequests", apierrors.NewTooManyRequests("throttled", 5), true}, + {"DeadlineExceeded", context.DeadlineExceeded, true}, + {"Wrapped DeadlineExceeded", errors.Join(errors.New("wrapper"), context.DeadlineExceeded), true}, + {"NotFound", apierrors.NewNotFound(schema.GroupResource{}, "x"), false}, + {"Forbidden", apierrors.NewForbidden(schema.GroupResource{}, "x", errors.New("RBAC")), false}, + {"Unauthorized", apierrors.NewUnauthorized("no token"), false}, + {"InternalError", apierrors.NewInternalError(errors.New("crash")), false}, + {"Generic error", errors.New("oops"), false}, + {"Nil", nil, false}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := isTransientError(tt.err); got != tt.want { + t.Errorf("isTransientError() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestIntermediateProfileHasExpectedDefaults(t *testing.T) { + profile := intermediateProfile() + + if profile.MinTLSVersion != configv1.VersionTLS12 { + t.Errorf("Intermediate MinTLSVersion = %v, want %v", profile.MinTLSVersion, configv1.VersionTLS12) + } + if len(profile.Ciphers) == 0 { + t.Error("Intermediate profile should have non-empty cipher list") + } +} + +func TestTLSConfigFromIntermediateProfile(t *testing.T) { + profile := intermediateProfile() + tlsConfigFn := configv1ToTLSConfig(profile) + + cfg := &tls.Config{} + tlsConfigFn(cfg) + + if cfg.MinVersion != tls.VersionTLS12 { + t.Errorf("MinVersion = %v, want %v (TLS 1.2)", cfg.MinVersion, tls.VersionTLS12) + } + if len(cfg.CipherSuites) == 0 { + t.Error("CipherSuites should not be empty for Intermediate profile") + } +} + +func configv1ToTLSConfig(profile configv1.TLSProfileSpec) func(*tls.Config) { + // Thin wrapper to test the actual conversion without importing tlspkg in tests. + // tlspkg.NewTLSConfigFromProfile is what main.go uses. + var minVersion uint16 + switch profile.MinTLSVersion { + case configv1.VersionTLS10: + minVersion = tls.VersionTLS10 + case configv1.VersionTLS11: + minVersion = tls.VersionTLS11 + case configv1.VersionTLS12: + minVersion = tls.VersionTLS12 + case configv1.VersionTLS13: + minVersion = tls.VersionTLS13 + } + + return func(c *tls.Config) { + c.MinVersion = minVersion + c.CipherSuites = mapCiphers(profile.Ciphers) + } +} + +func mapCiphers(names []string) []uint16 { + cipherMap := map[string]uint16{ + "TLS_AES_128_GCM_SHA256": tls.TLS_AES_128_GCM_SHA256, + "TLS_AES_256_GCM_SHA384": tls.TLS_AES_256_GCM_SHA384, + "TLS_CHACHA20_POLY1305_SHA256": tls.TLS_CHACHA20_POLY1305_SHA256, + "TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256": tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256, + "TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256": tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, + "TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384": tls.TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384, + "TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384": tls.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384, + "TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256": tls.TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256, + "TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256": tls.TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256, + } + var ids []uint16 + for _, name := range names { + if id, ok := cipherMap[name]; ok { + ids = append(ids, id) + } + } + return ids +} + +func TestClassifyTLSProfileError_AllTransientErrorsSetProfileFetched(t *testing.T) { + transientErrors := []error{ + apierrors.NewServiceUnavailable("down"), + apierrors.NewTimeoutError("slow", 5), + apierrors.NewServerTimeout(schema.GroupResource{Group: "config.openshift.io", Resource: "apiservers"}, "GET", 5), + apierrors.NewTooManyRequests("throttled", 5), + context.DeadlineExceeded, + } + + for _, err := range transientErrors { + _, fetched, classifyErr := classifyTLSProfileError(err) + if classifyErr != nil { + t.Errorf("transient error %T should not return error, got: %v", err, classifyErr) + } + if !fetched { + t.Errorf("transient error %T should set profileFetched=true", err) + } + } +} + +func TestClassifyTLSProfileError_NonTransientErrorsDoNotSetProfileFetched(t *testing.T) { + nonTransientErrors := []error{ + &meta.NoResourceMatchError{PartialResource: schema.GroupVersionResource{Group: "config.openshift.io"}}, + apierrors.NewNotFound(schema.GroupResource{}, "cluster"), + } + + for _, err := range nonTransientErrors { + _, fetched, classifyErr := classifyTLSProfileError(err) + if classifyErr != nil { + t.Errorf("graceful error %T should not return error, got: %v", err, classifyErr) + } + if fetched { + t.Errorf("graceful error %T should set profileFetched=false", err) + } + } +} + +func TestClassifyTLSProfileError_FatalErrorsReturnError(t *testing.T) { + fatalErrors := []error{ + apierrors.NewForbidden(schema.GroupResource{}, "cluster", errors.New("RBAC")), + apierrors.NewUnauthorized("no token"), + apierrors.NewInternalError(errors.New("crash")), + errors.New("unexpected"), + } + + for _, err := range fatalErrors { + _, _, classifyErr := classifyTLSProfileError(err) + if classifyErr == nil { + t.Errorf("fatal error %T should return error", err) + } + } +} + +func TestClassifyTLSProfileError_IntermediateProfileAlwaysApplied(t *testing.T) { + allNonFatalErrors := []error{ + &meta.NoResourceMatchError{PartialResource: schema.GroupVersionResource{Group: "config.openshift.io"}}, + apierrors.NewNotFound(schema.GroupResource{}, "cluster"), + apierrors.NewServiceUnavailable("down"), + apierrors.NewTimeoutError("slow", 5), + apierrors.NewServerTimeout(schema.GroupResource{}, "GET", 5), + apierrors.NewTooManyRequests("throttled", 5), + context.DeadlineExceeded, + } + + intermediate := intermediateProfile() + for _, err := range allNonFatalErrors { + profile, _, classifyErr := classifyTLSProfileError(err) + if classifyErr != nil { + t.Fatalf("unexpected error for %T: %v", err, classifyErr) + } + if profile.MinTLSVersion != intermediate.MinTLSVersion { + t.Errorf("for error %T: MinTLSVersion = %v, want Intermediate (%v)", err, profile.MinTLSVersion, intermediate.MinTLSVersion) + } + if len(profile.Ciphers) != len(intermediate.Ciphers) { + t.Errorf("for error %T: got %d ciphers, want %d (Intermediate)", err, len(profile.Ciphers), len(intermediate.Ciphers)) + } + } +} + +func TestTLSBootstrapResult_NextProtosAlwaysSet(t *testing.T) { + // Verify that the TLSOpts from bootstrapTLS always include ALPN with h2 and http/1.1. + // We can't call bootstrapTLS without a real client, but we can verify the function + // in tls_bootstrap.go sets NextProtos. + result := &tlsBootstrapResult{ + TLSOpts: make([]func(*tls.Config), 0, 2), + } + result.TLSOpts = append(result.TLSOpts, func(c *tls.Config) { + c.NextProtos = []string{"h2", alpnHTTP11} + }) + + cfg := &tls.Config{} + for _, opt := range result.TLSOpts { + opt(cfg) + } + + if len(cfg.NextProtos) != 2 || cfg.NextProtos[0] != "h2" || cfg.NextProtos[1] != alpnHTTP11 { + t.Errorf("NextProtos = %v, want [h2, %s]", cfg.NextProtos, alpnHTTP11) + } +} diff --git a/infra/feast-operator/config/component_metadata.yaml b/infra/feast-operator/config/component_metadata.yaml index 052dcffaf32..7ee38fdb165 100644 --- a/infra/feast-operator/config/component_metadata.yaml +++ b/infra/feast-operator/config/component_metadata.yaml @@ -1,5 +1,5 @@ # This file is required to configure Feast release information for ODH/RHOAI Operator releases: - name: Feast - version: 0.63.0 + version: 0.65.0 repoUrl: https://github.com/feast-dev/feast diff --git a/infra/feast-operator/config/crd/bases/feast.dev_featurestores.yaml b/infra/feast-operator/config/crd/bases/feast.dev_featurestores.yaml index 0851b9abf96..8184906e14d 100644 --- a/infra/feast-operator/config/crd/bases/feast.dev_featurestores.yaml +++ b/infra/feast-operator/config/crd/bases/feast.dev_featurestores.yaml @@ -161,8 +161,9 @@ spec: in a Container. properties: name: - description: Name of the environment variable. Must - be a C_IDENTIFIER. + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. type: string value: description: |- @@ -211,6 +212,35 @@ spec: - fieldPath type: object x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: The key within the env file. An + invalid key will prevent the pod from starting. + type: string + optional: + default: false + description: Specify whether the file or its + key must be defined. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '.. + type: string + volumeName: + description: The name of the volume mount containing + the env file. + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic resourceFieldRef: description: |- Selects a resource of the container: only resources limits and requests @@ -286,7 +316,7 @@ spec: x-kubernetes-map-type: atomic prefix: description: Optional text to prepend to the name of - each environment variable. Must be a C_IDENTIFIER. + each environment variable. type: string secretRef: description: The Secret to select from @@ -471,7 +501,6 @@ spec: type. type: string required: - - status - type type: object type: array @@ -564,8 +593,9 @@ spec: in a Container. properties: name: - description: Name of the environment variable. Must - be a C_IDENTIFIER. + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. type: string value: description: |- @@ -614,6 +644,35 @@ spec: - fieldPath type: object x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: The key within the env file. An + invalid key will prevent the pod from starting. + type: string + optional: + default: false + description: Specify whether the file or its + key must be defined. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '.. + type: string + volumeName: + description: The name of the volume mount containing + the env file. + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic resourceFieldRef: description: |- Selects a resource of the container: only resources limits and requests @@ -689,7 +748,7 @@ spec: x-kubernetes-map-type: atomic prefix: description: Optional text to prepend to the name of - each environment variable. Must be a C_IDENTIFIER. + each environment variable. type: string secretRef: description: The Secret to select from @@ -751,10 +810,32 @@ spec: - pytorch_nlp type: string type: object + packaged: + description: FeastPackagedOptions describes a feature repository + packaged in a feature server image. + properties: + featureRepoPath: + description: FeatureRepoPath is the canonical absolute path + to the feature repository in the image. + type: string + image: + description: Image containing the packaged feature repository. + type: string + required: + - featureRepoPath + type: object + x-kubernetes-validations: + - message: FeatureRepoPath must be a canonical absolute, non-root + path without dot segments or repeated separators. + rule: self.featureRepoPath.startsWith('/') && self.featureRepoPath + != '/' && !self.featureRepoPath.contains('//') && !self.featureRepoPath.endsWith('/') + && !self.featureRepoPath.contains('/./') && !self.featureRepoPath.endsWith('/.') + && !self.featureRepoPath.contains('/../') && !self.featureRepoPath.endsWith('/..') type: object x-kubernetes-validations: - - message: One selection required between init or git. - rule: '[has(self.git), has(self.init)].exists_one(c, c)' + - message: One selection required between init, git, or packaged. + rule: '[has(self.git), has(self.init), has(self.packaged)].exists_one(c, + c)' materialization: description: |- Materialization controls feature materialization behavior (batch size, pull strategy). @@ -793,6 +874,59 @@ spec: type: string type: object x-kubernetes-map-type: atomic + consumer: + description: |- + Consumer configures the OpenLineage consumer (event receiver) that enables + Feast to receive and display lineage from... + properties: + apiKeySecretRef: + description: |- + Reference to a Secret containing the key "api_key" that producers must + provide in the X-API-Key header when sending... + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. + type: string + type: object + x-kubernetes-map-type: atomic + connectionStringSecretRef: + description: |- + Reference to a Secret containing the key "connection_string" for a separate + lineage database. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. + type: string + type: object + x-kubernetes-map-type: atomic + enabled: + description: Enable the OpenLineage consumer. + type: boolean + namespaceMapping: + additionalProperties: + type: string + description: |- + NamespaceMapping maps OpenLineage namespaces to Feast projects for + RBAC-based filtering of lineage data in the UI. + type: object + storeType: + default: sql + description: StoreType is the storage backend for lineage + events. Currently only "sql" is supported. + enum: + - sql + type: string + required: + - enabled + type: object enabled: description: Enable OpenLineage integration. type: boolean @@ -1611,6 +1745,10 @@ spec: disableInitContainers: description: Disable the 'feast repo initialization' initContainer type: boolean + initImage: + description: InitImage overrides the image for init containers + (feast-init, feast-apply). + type: string offlineStore: description: OfflineStore configures the offline store service properties: @@ -1761,8 +1899,9 @@ spec: present in a Container. properties: name: - description: Name of the environment variable. Must - be a C_IDENTIFIER. + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. type: string value: description: |- @@ -1811,6 +1950,36 @@ spec: - fieldPath type: object x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: The key within the env file. + An invalid key will prevent the pod from + starting. + type: string + optional: + default: false + description: Specify whether the file or + its key must be defined. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '.. + type: string + volumeName: + description: The name of the volume mount + containing the env file. + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic resourceFieldRef: description: |- Selects a resource of the container: only resources limits and requests @@ -1887,7 +2056,7 @@ spec: x-kubernetes-map-type: atomic prefix: description: Optional text to prepend to the name - of each environment variable. Must be a C_IDENTIFIER. + of each environment variable. type: string secretRef: description: The Secret to select from @@ -2115,6 +2284,11 @@ spec: onlineStore: description: OnlineStore configures the online store service properties: + disabled: + description: |- + Disabled skips deploying the online store service entirely, including its + serving pod and persistence. + type: boolean persistence: description: OnlineStorePersistence configures the persistence settings for the online store service @@ -2256,6 +2430,8 @@ spec: - milvus - hybrid - mongodb + - aerospike + - scylladb type: string required: - secretRef @@ -2274,8 +2450,9 @@ spec: present in a Container. properties: name: - description: Name of the environment variable. Must - be a C_IDENTIFIER. + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. type: string value: description: |- @@ -2324,6 +2501,36 @@ spec: - fieldPath type: object x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: The key within the env file. + An invalid key will prevent the pod from + starting. + type: string + optional: + default: false + description: Specify whether the file or + its key must be defined. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '.. + type: string + volumeName: + description: The name of the volume mount + containing the env file. + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic resourceFieldRef: description: |- Selects a resource of the container: only resources limits and requests @@ -2400,7 +2607,7 @@ spec: x-kubernetes-map-type: atomic prefix: description: Optional text to prepend to the name - of each environment variable. Must be a C_IDENTIFIER. + of each environment variable. type: string secretRef: description: The Secret to select from @@ -2904,8 +3111,9 @@ spec: present in a Container. properties: name: - description: Name of the environment variable. - Must be a C_IDENTIFIER. + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. type: string value: description: |- @@ -2955,6 +3163,36 @@ spec: - fieldPath type: object x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: The key within the env + file. An invalid key will prevent + the pod from starting. + type: string + optional: + default: false + description: Specify whether the file + or its key must be defined. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '.. + type: string + volumeName: + description: The name of the volume + mount containing the env file. + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic resourceFieldRef: description: |- Selects a resource of the container: only resources limits and requests @@ -3033,8 +3271,7 @@ spec: x-kubernetes-map-type: atomic prefix: description: Optional text to prepend to the - name of each environment variable. Must be - a C_IDENTIFIER. + name of each environment variable. type: string secretRef: description: The Secret to select from @@ -4209,8 +4446,9 @@ spec: in a Container. properties: name: - description: Name of the environment variable. Must - be a C_IDENTIFIER. + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. type: string value: description: |- @@ -4259,6 +4497,35 @@ spec: - fieldPath type: object x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: The key within the env file. An + invalid key will prevent the pod from starting. + type: string + optional: + default: false + description: Specify whether the file or its + key must be defined. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '.. + type: string + volumeName: + description: The name of the volume mount containing + the env file. + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic resourceFieldRef: description: |- Selects a resource of the container: only resources limits and requests @@ -4334,7 +4601,7 @@ spec: x-kubernetes-map-type: atomic prefix: description: Optional text to prepend to the name of - each environment variable. Must be a C_IDENTIFIER. + each environment variable. type: string secretRef: description: The Secret to select from @@ -5228,9 +5495,8 @@ spec: host that shares a pod's lifetime. properties: endpoints: - description: |- - endpoints is the endpoint name that details Glusterfs topology. - More info: https://examples.k8s. + description: endpoints is the endpoint name that details + Glusterfs topology. type: string path: description: |- @@ -5643,6 +5909,52 @@ spec: type: array x-kubernetes-list-type: atomic type: object + podCertificate: + description: |- + Projects an auto-rotating credential bundle (private key and certificate + chain) that the pod can use either as a TLS... + properties: + certificateChainPath: + description: |- + Write the certificate chain at this path in the projected volume. + + Most applications should use credentialBundlePath. + type: string + credentialBundlePath: + description: Write the credential bundle at + this path in the projected volume. + type: string + keyPath: + description: |- + Write the key at this path in the projected volume. + + Most applications should use credentialBundlePath. + type: string + keyType: + description: The type of keypair Kubelet will + generate for the pod. + type: string + maxExpirationSeconds: + description: |- + maxExpirationSeconds is the maximum lifetime permitted for the + certificate. + format: int32 + type: integer + signerName: + description: Kubelet's generated CSRs will + be addressed to this signer. + type: string + userAnnotations: + additionalProperties: + type: string + description: |- + userAnnotations allow pod authors to pass additional information to + the signer implementation. + type: object + required: + - keyType + - signerName + type: object secret: description: secret information about the secret data to project @@ -6160,8 +6472,9 @@ spec: present in a Container. properties: name: - description: Name of the environment variable. Must - be a C_IDENTIFIER. + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. type: string value: description: |- @@ -6210,6 +6523,36 @@ spec: - fieldPath type: object x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: The key within the env file. + An invalid key will prevent the pod from + starting. + type: string + optional: + default: false + description: Specify whether the file or + its key must be defined. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '.. + type: string + volumeName: + description: The name of the volume mount + containing the env file. + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic resourceFieldRef: description: |- Selects a resource of the container: only resources limits and requests @@ -6286,7 +6629,7 @@ spec: x-kubernetes-map-type: atomic prefix: description: Optional text to prepend to the name - of each environment variable. Must be a C_IDENTIFIER. + of each environment variable. type: string secretRef: description: The Secret to select from @@ -6473,7 +6816,6 @@ spec: condition type. type: string required: - - status - type type: object type: array @@ -6568,8 +6910,9 @@ spec: present in a Container. properties: name: - description: Name of the environment variable. Must - be a C_IDENTIFIER. + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. type: string value: description: |- @@ -6618,6 +6961,36 @@ spec: - fieldPath type: object x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: The key within the env file. + An invalid key will prevent the pod from + starting. + type: string + optional: + default: false + description: Specify whether the file or + its key must be defined. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '.. + type: string + volumeName: + description: The name of the volume mount + containing the env file. + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic resourceFieldRef: description: |- Selects a resource of the container: only resources limits and requests @@ -6694,7 +7067,7 @@ spec: x-kubernetes-map-type: atomic prefix: description: Optional text to prepend to the name - of each environment variable. Must be a C_IDENTIFIER. + of each environment variable. type: string secretRef: description: The Secret to select from @@ -6757,10 +7130,32 @@ spec: - pytorch_nlp type: string type: object + packaged: + description: FeastPackagedOptions describes a feature repository + packaged in a feature server image. + properties: + featureRepoPath: + description: FeatureRepoPath is the canonical absolute + path to the feature repository in the image. + type: string + image: + description: Image containing the packaged feature repository. + type: string + required: + - featureRepoPath + type: object + x-kubernetes-validations: + - message: FeatureRepoPath must be a canonical absolute, non-root + path without dot segments or repeated separators. + rule: self.featureRepoPath.startsWith('/') && self.featureRepoPath + != '/' && !self.featureRepoPath.contains('//') && !self.featureRepoPath.endsWith('/') + && !self.featureRepoPath.contains('/./') && !self.featureRepoPath.endsWith('/.') + && !self.featureRepoPath.contains('/../') && !self.featureRepoPath.endsWith('/..') type: object x-kubernetes-validations: - - message: One selection required between init or git. - rule: '[has(self.git), has(self.init)].exists_one(c, c)' + - message: One selection required between init, git, or packaged. + rule: '[has(self.git), has(self.init), has(self.packaged)].exists_one(c, + c)' materialization: description: |- Materialization controls feature materialization behavior (batch size, pull strategy). @@ -6799,6 +7194,59 @@ spec: type: string type: object x-kubernetes-map-type: atomic + consumer: + description: |- + Consumer configures the OpenLineage consumer (event receiver) that enables + Feast to receive and display lineage from... + properties: + apiKeySecretRef: + description: |- + Reference to a Secret containing the key "api_key" that producers must + provide in the X-API-Key header when sending... + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. + type: string + type: object + x-kubernetes-map-type: atomic + connectionStringSecretRef: + description: |- + Reference to a Secret containing the key "connection_string" for a separate + lineage database. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. + type: string + type: object + x-kubernetes-map-type: atomic + enabled: + description: Enable the OpenLineage consumer. + type: boolean + namespaceMapping: + additionalProperties: + type: string + description: |- + NamespaceMapping maps OpenLineage namespaces to Feast projects for + RBAC-based filtering of lineage data in the UI. + type: object + storeType: + default: sql + description: StoreType is the storage backend for lineage + events. Currently only "sql" is supported. + enum: + - sql + type: string + required: + - enabled + type: object enabled: description: Enable OpenLineage integration. type: boolean @@ -7625,6 +8073,10 @@ spec: disableInitContainers: description: Disable the 'feast repo initialization' initContainer type: boolean + initImage: + description: InitImage overrides the image for init containers + (feast-init, feast-apply). + type: string offlineStore: description: OfflineStore configures the offline store service properties: @@ -7778,8 +8230,9 @@ spec: present in a Container. properties: name: - description: Name of the environment variable. - Must be a C_IDENTIFIER. + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. type: string value: description: |- @@ -7829,6 +8282,36 @@ spec: - fieldPath type: object x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: The key within the env + file. An invalid key will prevent + the pod from starting. + type: string + optional: + default: false + description: Specify whether the file + or its key must be defined. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '.. + type: string + volumeName: + description: The name of the volume + mount containing the env file. + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic resourceFieldRef: description: |- Selects a resource of the container: only resources limits and requests @@ -7907,8 +8390,7 @@ spec: x-kubernetes-map-type: atomic prefix: description: Optional text to prepend to the - name of each environment variable. Must be - a C_IDENTIFIER. + name of each environment variable. type: string secretRef: description: The Secret to select from @@ -8137,6 +8619,11 @@ spec: onlineStore: description: OnlineStore configures the online store service properties: + disabled: + description: |- + Disabled skips deploying the online store service entirely, including its + serving pod and persistence. + type: boolean persistence: description: OnlineStorePersistence configures the persistence settings for the online store service @@ -8281,6 +8768,8 @@ spec: - milvus - hybrid - mongodb + - aerospike + - scylladb type: string required: - secretRef @@ -8300,8 +8789,9 @@ spec: present in a Container. properties: name: - description: Name of the environment variable. - Must be a C_IDENTIFIER. + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. type: string value: description: |- @@ -8351,6 +8841,36 @@ spec: - fieldPath type: object x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: The key within the env + file. An invalid key will prevent + the pod from starting. + type: string + optional: + default: false + description: Specify whether the file + or its key must be defined. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '.. + type: string + volumeName: + description: The name of the volume + mount containing the env file. + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic resourceFieldRef: description: |- Selects a resource of the container: only resources limits and requests @@ -8429,8 +8949,7 @@ spec: x-kubernetes-map-type: atomic prefix: description: Optional text to prepend to the - name of each environment variable. Must be - a C_IDENTIFIER. + name of each environment variable. type: string secretRef: description: The Secret to select from @@ -8945,8 +9464,9 @@ spec: variable present in a Container. properties: name: - description: Name of the environment variable. - Must be a C_IDENTIFIER. + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. type: string value: description: |- @@ -8997,6 +9517,36 @@ spec: - fieldPath type: object x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: The key within the + env file. An invalid key will + prevent the pod from starting. + type: string + optional: + default: false + description: Specify whether the + file or its key must be defined. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '.. + type: string + volumeName: + description: The name of the volume + mount containing the env file. + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic resourceFieldRef: description: |- Selects a resource of the container: only resources limits and requests @@ -9077,7 +9627,6 @@ spec: prefix: description: Optional text to prepend to the name of each environment variable. - Must be a C_IDENTIFIER. type: string secretRef: description: The Secret to select from @@ -10268,8 +10817,9 @@ spec: present in a Container. properties: name: - description: Name of the environment variable. Must - be a C_IDENTIFIER. + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. type: string value: description: |- @@ -10318,6 +10868,36 @@ spec: - fieldPath type: object x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: The key within the env file. + An invalid key will prevent the pod from + starting. + type: string + optional: + default: false + description: Specify whether the file or + its key must be defined. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '.. + type: string + volumeName: + description: The name of the volume mount + containing the env file. + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic resourceFieldRef: description: |- Selects a resource of the container: only resources limits and requests @@ -10394,7 +10974,7 @@ spec: x-kubernetes-map-type: atomic prefix: description: Optional text to prepend to the name - of each environment variable. Must be a C_IDENTIFIER. + of each environment variable. type: string secretRef: description: The Secret to select from @@ -11296,9 +11876,8 @@ spec: on the host that shares a pod's lifetime. properties: endpoints: - description: |- - endpoints is the endpoint name that details Glusterfs topology. - More info: https://examples.k8s. + description: endpoints is the endpoint name that + details Glusterfs topology. type: string path: description: |- @@ -11715,6 +12294,52 @@ spec: type: array x-kubernetes-list-type: atomic type: object + podCertificate: + description: |- + Projects an auto-rotating credential bundle (private key and certificate + chain) that the pod can use either as a TLS... + properties: + certificateChainPath: + description: |- + Write the certificate chain at this path in the projected volume. + + Most applications should use credentialBundlePath. + type: string + credentialBundlePath: + description: Write the credential bundle + at this path in the projected volume. + type: string + keyPath: + description: |- + Write the key at this path in the projected volume. + + Most applications should use credentialBundlePath. + type: string + keyType: + description: The type of keypair Kubelet + will generate for the pod. + type: string + maxExpirationSeconds: + description: |- + maxExpirationSeconds is the maximum lifetime permitted for the + certificate. + format: int32 + type: integer + signerName: + description: Kubelet's generated CSRs + will be addressed to this signer. + type: string + userAnnotations: + additionalProperties: + type: string + description: |- + userAnnotations allow pod authors to pass additional information to + the signer implementation. + type: object + required: + - keyType + - signerName + type: object secret: description: secret information about the secret data to project @@ -12313,8 +12938,9 @@ spec: in a Container. properties: name: - description: Name of the environment variable. Must - be a C_IDENTIFIER. + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. type: string value: description: |- @@ -12363,6 +12989,35 @@ spec: - fieldPath type: object x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: The key within the env file. An + invalid key will prevent the pod from starting. + type: string + optional: + default: false + description: Specify whether the file or its + key must be defined. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '.. + type: string + volumeName: + description: The name of the volume mount containing + the env file. + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic resourceFieldRef: description: |- Selects a resource of the container: only resources limits and requests @@ -12438,7 +13093,7 @@ spec: x-kubernetes-map-type: atomic prefix: description: Optional text to prepend to the name of - each environment variable. Must be a C_IDENTIFIER. + each environment variable. type: string secretRef: description: The Secret to select from @@ -12623,7 +13278,6 @@ spec: type. type: string required: - - status - type type: object type: array @@ -12706,8 +13360,9 @@ spec: in a Container. properties: name: - description: Name of the environment variable. Must - be a C_IDENTIFIER. + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. type: string value: description: |- @@ -12756,6 +13411,35 @@ spec: - fieldPath type: object x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: The key within the env file. An + invalid key will prevent the pod from starting. + type: string + optional: + default: false + description: Specify whether the file or its + key must be defined. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '.. + type: string + volumeName: + description: The name of the volume mount containing + the env file. + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic resourceFieldRef: description: |- Selects a resource of the container: only resources limits and requests @@ -12831,7 +13515,7 @@ spec: x-kubernetes-map-type: atomic prefix: description: Optional text to prepend to the name of - each environment variable. Must be a C_IDENTIFIER. + each environment variable. type: string secretRef: description: The Secret to select from @@ -12893,10 +13577,32 @@ spec: - pytorch_nlp type: string type: object + packaged: + description: FeastPackagedOptions describes a feature repository + packaged in a feature server image. + properties: + featureRepoPath: + description: FeatureRepoPath is the canonical absolute path + to the feature repository in the image. + type: string + image: + description: Image containing the packaged feature repository. + type: string + required: + - featureRepoPath + type: object + x-kubernetes-validations: + - message: FeatureRepoPath must be a canonical absolute, non-root + path without dot segments or repeated separators. + rule: self.featureRepoPath.startsWith('/') && self.featureRepoPath + != '/' && !self.featureRepoPath.contains('//') && !self.featureRepoPath.endsWith('/') + && !self.featureRepoPath.contains('/./') && !self.featureRepoPath.endsWith('/.') + && !self.featureRepoPath.contains('/../') && !self.featureRepoPath.endsWith('/..') type: object x-kubernetes-validations: - - message: One selection required between init or git. - rule: '[has(self.git), has(self.init)].exists_one(c, c)' + - message: One selection required between init, git, or packaged. + rule: '[has(self.git), has(self.init), has(self.packaged)].exists_one(c, + c)' services: description: FeatureStoreServices defines the desired feast services. An ephemeral onlineStore feature server is deployed by default. @@ -13083,8 +13789,9 @@ spec: present in a Container. properties: name: - description: Name of the environment variable. Must - be a C_IDENTIFIER. + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. type: string value: description: |- @@ -13133,6 +13840,36 @@ spec: - fieldPath type: object x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: The key within the env file. + An invalid key will prevent the pod from + starting. + type: string + optional: + default: false + description: Specify whether the file or + its key must be defined. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '.. + type: string + volumeName: + description: The name of the volume mount + containing the env file. + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic resourceFieldRef: description: |- Selects a resource of the container: only resources limits and requests @@ -13209,7 +13946,7 @@ spec: x-kubernetes-map-type: atomic prefix: description: Optional text to prepend to the name - of each environment variable. Must be a C_IDENTIFIER. + of each environment variable. type: string secretRef: description: The Secret to select from @@ -13578,6 +14315,8 @@ spec: - milvus - hybrid - mongodb + - aerospike + - scylladb type: string required: - secretRef @@ -13596,8 +14335,9 @@ spec: present in a Container. properties: name: - description: Name of the environment variable. Must - be a C_IDENTIFIER. + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. type: string value: description: |- @@ -13646,6 +14386,36 @@ spec: - fieldPath type: object x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: The key within the env file. + An invalid key will prevent the pod from + starting. + type: string + optional: + default: false + description: Specify whether the file or + its key must be defined. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '.. + type: string + volumeName: + description: The name of the volume mount + containing the env file. + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic resourceFieldRef: description: |- Selects a resource of the container: only resources limits and requests @@ -13722,7 +14492,7 @@ spec: x-kubernetes-map-type: atomic prefix: description: Optional text to prepend to the name - of each environment variable. Must be a C_IDENTIFIER. + of each environment variable. type: string secretRef: description: The Secret to select from @@ -14127,8 +14897,9 @@ spec: present in a Container. properties: name: - description: Name of the environment variable. - Must be a C_IDENTIFIER. + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. type: string value: description: |- @@ -14178,6 +14949,36 @@ spec: - fieldPath type: object x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: The key within the env + file. An invalid key will prevent + the pod from starting. + type: string + optional: + default: false + description: Specify whether the file + or its key must be defined. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '.. + type: string + volumeName: + description: The name of the volume + mount containing the env file. + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic resourceFieldRef: description: |- Selects a resource of the container: only resources limits and requests @@ -14256,8 +15057,7 @@ spec: x-kubernetes-map-type: atomic prefix: description: Optional text to prepend to the - name of each environment variable. Must be - a C_IDENTIFIER. + name of each environment variable. type: string secretRef: description: The Secret to select from @@ -14701,8 +15501,9 @@ spec: in a Container. properties: name: - description: Name of the environment variable. Must - be a C_IDENTIFIER. + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. type: string value: description: |- @@ -14751,6 +15552,35 @@ spec: - fieldPath type: object x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: The key within the env file. An + invalid key will prevent the pod from starting. + type: string + optional: + default: false + description: Specify whether the file or its + key must be defined. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '.. + type: string + volumeName: + description: The name of the volume mount containing + the env file. + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic resourceFieldRef: description: |- Selects a resource of the container: only resources limits and requests @@ -14826,7 +15656,7 @@ spec: x-kubernetes-map-type: atomic prefix: description: Optional text to prepend to the name of - each environment variable. Must be a C_IDENTIFIER. + each environment variable. type: string secretRef: description: The Secret to select from @@ -15720,9 +16550,8 @@ spec: host that shares a pod's lifetime. properties: endpoints: - description: |- - endpoints is the endpoint name that details Glusterfs topology. - More info: https://examples.k8s. + description: endpoints is the endpoint name that details + Glusterfs topology. type: string path: description: |- @@ -16135,6 +16964,52 @@ spec: type: array x-kubernetes-list-type: atomic type: object + podCertificate: + description: |- + Projects an auto-rotating credential bundle (private key and certificate + chain) that the pod can use either as a TLS... + properties: + certificateChainPath: + description: |- + Write the certificate chain at this path in the projected volume. + + Most applications should use credentialBundlePath. + type: string + credentialBundlePath: + description: Write the credential bundle at + this path in the projected volume. + type: string + keyPath: + description: |- + Write the key at this path in the projected volume. + + Most applications should use credentialBundlePath. + type: string + keyType: + description: The type of keypair Kubelet will + generate for the pod. + type: string + maxExpirationSeconds: + description: |- + maxExpirationSeconds is the maximum lifetime permitted for the + certificate. + format: int32 + type: integer + signerName: + description: Kubelet's generated CSRs will + be addressed to this signer. + type: string + userAnnotations: + additionalProperties: + type: string + description: |- + userAnnotations allow pod authors to pass additional information to + the signer implementation. + type: object + required: + - keyType + - signerName + type: object secret: description: secret information about the secret data to project @@ -16573,8 +17448,9 @@ spec: present in a Container. properties: name: - description: Name of the environment variable. Must - be a C_IDENTIFIER. + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. type: string value: description: |- @@ -16623,6 +17499,36 @@ spec: - fieldPath type: object x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: The key within the env file. + An invalid key will prevent the pod from + starting. + type: string + optional: + default: false + description: Specify whether the file or + its key must be defined. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '.. + type: string + volumeName: + description: The name of the volume mount + containing the env file. + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic resourceFieldRef: description: |- Selects a resource of the container: only resources limits and requests @@ -16699,7 +17605,7 @@ spec: x-kubernetes-map-type: atomic prefix: description: Optional text to prepend to the name - of each environment variable. Must be a C_IDENTIFIER. + of each environment variable. type: string secretRef: description: The Secret to select from @@ -16886,7 +17792,6 @@ spec: condition type. type: string required: - - status - type type: object type: array @@ -16971,8 +17876,9 @@ spec: present in a Container. properties: name: - description: Name of the environment variable. Must - be a C_IDENTIFIER. + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. type: string value: description: |- @@ -17021,6 +17927,36 @@ spec: - fieldPath type: object x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: The key within the env file. + An invalid key will prevent the pod from + starting. + type: string + optional: + default: false + description: Specify whether the file or + its key must be defined. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '.. + type: string + volumeName: + description: The name of the volume mount + containing the env file. + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic resourceFieldRef: description: |- Selects a resource of the container: only resources limits and requests @@ -17097,7 +18033,7 @@ spec: x-kubernetes-map-type: atomic prefix: description: Optional text to prepend to the name - of each environment variable. Must be a C_IDENTIFIER. + of each environment variable. type: string secretRef: description: The Secret to select from @@ -17160,10 +18096,32 @@ spec: - pytorch_nlp type: string type: object + packaged: + description: FeastPackagedOptions describes a feature repository + packaged in a feature server image. + properties: + featureRepoPath: + description: FeatureRepoPath is the canonical absolute + path to the feature repository in the image. + type: string + image: + description: Image containing the packaged feature repository. + type: string + required: + - featureRepoPath + type: object + x-kubernetes-validations: + - message: FeatureRepoPath must be a canonical absolute, non-root + path without dot segments or repeated separators. + rule: self.featureRepoPath.startsWith('/') && self.featureRepoPath + != '/' && !self.featureRepoPath.contains('//') && !self.featureRepoPath.endsWith('/') + && !self.featureRepoPath.contains('/./') && !self.featureRepoPath.endsWith('/.') + && !self.featureRepoPath.contains('/../') && !self.featureRepoPath.endsWith('/..') type: object x-kubernetes-validations: - - message: One selection required between init or git. - rule: '[has(self.git), has(self.init)].exists_one(c, c)' + - message: One selection required between init, git, or packaged. + rule: '[has(self.git), has(self.init), has(self.packaged)].exists_one(c, + c)' services: description: FeatureStoreServices defines the desired feast services. An ephemeral onlineStore feature server is deployed by default. @@ -17353,8 +18311,9 @@ spec: present in a Container. properties: name: - description: Name of the environment variable. - Must be a C_IDENTIFIER. + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. type: string value: description: |- @@ -17404,6 +18363,36 @@ spec: - fieldPath type: object x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: The key within the env + file. An invalid key will prevent + the pod from starting. + type: string + optional: + default: false + description: Specify whether the file + or its key must be defined. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '.. + type: string + volumeName: + description: The name of the volume + mount containing the env file. + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic resourceFieldRef: description: |- Selects a resource of the container: only resources limits and requests @@ -17482,8 +18471,7 @@ spec: x-kubernetes-map-type: atomic prefix: description: Optional text to prepend to the - name of each environment variable. Must be - a C_IDENTIFIER. + name of each environment variable. type: string secretRef: description: The Secret to select from @@ -17856,6 +18844,8 @@ spec: - milvus - hybrid - mongodb + - aerospike + - scylladb type: string required: - secretRef @@ -17875,8 +18865,9 @@ spec: present in a Container. properties: name: - description: Name of the environment variable. - Must be a C_IDENTIFIER. + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. type: string value: description: |- @@ -17926,6 +18917,36 @@ spec: - fieldPath type: object x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: The key within the env + file. An invalid key will prevent + the pod from starting. + type: string + optional: + default: false + description: Specify whether the file + or its key must be defined. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '.. + type: string + volumeName: + description: The name of the volume + mount containing the env file. + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic resourceFieldRef: description: |- Selects a resource of the container: only resources limits and requests @@ -18004,8 +19025,7 @@ spec: x-kubernetes-map-type: atomic prefix: description: Optional text to prepend to the - name of each environment variable. Must be - a C_IDENTIFIER. + name of each environment variable. type: string secretRef: description: The Secret to select from @@ -18419,8 +19439,9 @@ spec: variable present in a Container. properties: name: - description: Name of the environment variable. - Must be a C_IDENTIFIER. + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. type: string value: description: |- @@ -18471,6 +19492,36 @@ spec: - fieldPath type: object x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: The key within the + env file. An invalid key will + prevent the pod from starting. + type: string + optional: + default: false + description: Specify whether the + file or its key must be defined. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '.. + type: string + volumeName: + description: The name of the volume + mount containing the env file. + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic resourceFieldRef: description: |- Selects a resource of the container: only resources limits and requests @@ -18551,7 +19602,6 @@ spec: prefix: description: Optional text to prepend to the name of each environment variable. - Must be a C_IDENTIFIER. type: string secretRef: description: The Secret to select from @@ -19006,8 +20056,9 @@ spec: present in a Container. properties: name: - description: Name of the environment variable. Must - be a C_IDENTIFIER. + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. type: string value: description: |- @@ -19056,6 +20107,36 @@ spec: - fieldPath type: object x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: The key within the env file. + An invalid key will prevent the pod from + starting. + type: string + optional: + default: false + description: Specify whether the file or + its key must be defined. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '.. + type: string + volumeName: + description: The name of the volume mount + containing the env file. + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic resourceFieldRef: description: |- Selects a resource of the container: only resources limits and requests @@ -19132,7 +20213,7 @@ spec: x-kubernetes-map-type: atomic prefix: description: Optional text to prepend to the name - of each environment variable. Must be a C_IDENTIFIER. + of each environment variable. type: string secretRef: description: The Secret to select from @@ -20034,9 +21115,8 @@ spec: on the host that shares a pod's lifetime. properties: endpoints: - description: |- - endpoints is the endpoint name that details Glusterfs topology. - More info: https://examples.k8s. + description: endpoints is the endpoint name that + details Glusterfs topology. type: string path: description: |- @@ -20453,6 +21533,52 @@ spec: type: array x-kubernetes-list-type: atomic type: object + podCertificate: + description: |- + Projects an auto-rotating credential bundle (private key and certificate + chain) that the pod can use either as a TLS... + properties: + certificateChainPath: + description: |- + Write the certificate chain at this path in the projected volume. + + Most applications should use credentialBundlePath. + type: string + credentialBundlePath: + description: Write the credential bundle + at this path in the projected volume. + type: string + keyPath: + description: |- + Write the key at this path in the projected volume. + + Most applications should use credentialBundlePath. + type: string + keyType: + description: The type of keypair Kubelet + will generate for the pod. + type: string + maxExpirationSeconds: + description: |- + maxExpirationSeconds is the maximum lifetime permitted for the + certificate. + format: int32 + type: integer + signerName: + description: Kubelet's generated CSRs + will be addressed to this signer. + type: string + userAnnotations: + additionalProperties: + type: string + description: |- + userAnnotations allow pod authors to pass additional information to + the signer implementation. + type: object + required: + - keyType + - signerName + type: object secret: description: secret information about the secret data to project diff --git a/infra/feast-operator/config/default/related_image_fs_patch.yaml b/infra/feast-operator/config/default/related_image_fs_patch.yaml index d7f0617fb5b..5ab07fd91e1 100644 --- a/infra/feast-operator/config/default/related_image_fs_patch.yaml +++ b/infra/feast-operator/config/default/related_image_fs_patch.yaml @@ -9,6 +9,6 @@ spec: - name: manager env: - name: RELATED_IMAGE_FEATURE_SERVER - value: quay.io/feastdev/feature-server:0.63.0 + value: quay.io/feastdev/feature-server:0.65.0 - name: RELATED_IMAGE_CRON_JOB value: quay.io/openshift/origin-cli:4.17 diff --git a/infra/feast-operator/config/manager/kustomization.yaml b/infra/feast-operator/config/manager/kustomization.yaml index 8a25a8041b3..5f3ce6cadda 100644 --- a/infra/feast-operator/config/manager/kustomization.yaml +++ b/infra/feast-operator/config/manager/kustomization.yaml @@ -5,4 +5,4 @@ kind: Kustomization images: - name: controller newName: quay.io/feastdev/feast-operator - newTag: 0.63.0 + newTag: 0.65.0 diff --git a/infra/feast-operator/config/overlays/odh/params.env b/infra/feast-operator/config/overlays/odh/params.env index 3a37a22adef..b0d55d6bd70 100644 --- a/infra/feast-operator/config/overlays/odh/params.env +++ b/infra/feast-operator/config/overlays/odh/params.env @@ -1,5 +1,5 @@ -RELATED_IMAGE_FEAST_OPERATOR=quay.io/feastdev/feast-operator:0.63.0 -RELATED_IMAGE_FEATURE_SERVER=quay.io/feastdev/feature-server:0.63.0 +RELATED_IMAGE_FEAST_OPERATOR=quay.io/feastdev/feast-operator:0.65.0 +RELATED_IMAGE_FEATURE_SERVER=quay.io/feastdev/feature-server:0.65.0 RELATED_IMAGE_CRON_JOB=quay.io/openshift/origin-cli:4.17 # Set at deploy time by the Open Data Hub operator from GatewayConfig (external OIDC). OIDC_ISSUER_URL= diff --git a/infra/feast-operator/config/overlays/rhoai/params.env b/infra/feast-operator/config/overlays/rhoai/params.env index c19204cedd2..dabacfd458c 100644 --- a/infra/feast-operator/config/overlays/rhoai/params.env +++ b/infra/feast-operator/config/overlays/rhoai/params.env @@ -1,5 +1,5 @@ -RELATED_IMAGE_FEAST_OPERATOR=quay.io/feastdev/feast-operator:0.63.0 -RELATED_IMAGE_FEATURE_SERVER=quay.io/feastdev/feature-server:0.63.0 +RELATED_IMAGE_FEAST_OPERATOR=quay.io/feastdev/feast-operator:0.65.0 +RELATED_IMAGE_FEATURE_SERVER=quay.io/feastdev/feature-server:0.65.0 RELATED_IMAGE_CRON_JOB=registry.redhat.io/openshift4/ose-cli@sha256:bc35a9fc663baf0d6493cc57e89e77a240a36c43cf38fb78d8e61d3b87cf5cc5 # Set at deploy time by the Open Data Hub operator from GatewayConfig (external OIDC). OIDC_ISSUER_URL= \ No newline at end of file diff --git a/infra/feast-operator/config/rbac/role.yaml b/infra/feast-operator/config/rbac/role.yaml index 0c1bd7be84b..a79dca283ed 100644 --- a/infra/feast-operator/config/rbac/role.yaml +++ b/infra/feast-operator/config/rbac/role.yaml @@ -9,11 +9,11 @@ rules: resources: - configmaps - persistentvolumeclaims - - serviceaccounts - services verbs: - create - delete + - deletecollection - get - list - update @@ -22,18 +22,45 @@ rules: - "" resources: - namespaces - - pods - secrets verbs: - get - list - watch +- apiGroups: + - "" + resources: + - pods + verbs: + - create + - delete + - deletecollection + - get + - list + - watch - apiGroups: - "" resources: - pods/exec verbs: - create +- apiGroups: + - "" + resources: + - pods/log + verbs: + - get +- apiGroups: + - "" + resources: + - serviceaccounts + verbs: + - create + - delete + - get + - list + - update + - watch - apiGroups: - apps resources: @@ -75,6 +102,14 @@ rules: - patch - update - watch +- apiGroups: + - config.openshift.io + resources: + - apiservers + verbs: + - get + - list + - watch - apiGroups: - feast.dev resources: @@ -150,3 +185,11 @@ rules: - list - update - watch +- apiGroups: + - sparkoperator.k8s.io + resources: + - sparkapplications + verbs: + - create + - delete + - get diff --git a/infra/feast-operator/config/samples/v1_featurestore_oidc_auth.yaml b/infra/feast-operator/config/samples/v1_featurestore_oidc_auth.yaml index 7ef676d0297..97f325bb0bd 100644 --- a/infra/feast-operator/config/samples/v1_featurestore_oidc_auth.yaml +++ b/infra/feast-operator/config/samples/v1_featurestore_oidc_auth.yaml @@ -19,3 +19,7 @@ stringData: client_secret: client_secret username: username password: password + # Optional: enable audience/issuer claim verification on the servers. + # Values must match the claims in the tokens your IdP issues. + # audience: api://feast-feature-server + # issuer: https://idp.example.com/realms/feast diff --git a/infra/feast-operator/config/samples/v1_featurestore_openlineage_consumer.yaml b/infra/feast-operator/config/samples/v1_featurestore_openlineage_consumer.yaml new file mode 100644 index 00000000000..9e242896135 --- /dev/null +++ b/infra/feast-operator/config/samples/v1_featurestore_openlineage_consumer.yaml @@ -0,0 +1,63 @@ +apiVersion: v1 +kind: Secret +metadata: + name: openlineage-producer-secret + namespace: feast +stringData: + api_key: "your-marquez-api-key" #pragma: allowlist secret +--- +apiVersion: v1 +kind: Secret +metadata: + name: openlineage-consumer-secret + namespace: feast +stringData: + api_key: "consumer-api-key-for-producers" #pragma: allowlist secret +--- +apiVersion: feast.dev/v1 +kind: FeatureStore +metadata: + name: sample-openlineage-consumer + namespace: feast +spec: + feastProject: my_project + services: + registry: + local: + persistence: + store: + type: sql + secretRef: + name: registry-db-secret + openlineage: + enabled: true + transportType: http + transportUrl: "http://localhost:8080/api" + transportEndpoint: "v1/lineage" + apiKeySecretRef: + name: openlineage-producer-secret + extraConfig: + namespace: "my_project" + producer: "feast-operator" + emit_on_apply: "true" + emit_on_materialize: "true" + # consumer enables Feast as an OpenLineage event receiver. + # External producers (Airflow, Spark, dbt) can POST events to + # the Feast REST server at POST /api/v1/lineage. + # The Feast UI then displays lineage from all producers in + # Registry, OpenLineage, and Merged views. + consumer: + enabled: true + storeType: sql + # Optional: use a separate database for lineage storage. + # If omitted, the SQL registry database is reused. + # connectionStringSecretRef: + # name: lineage-db-secret + apiKeySecretRef: + name: openlineage-consumer-secret + # namespaceMapping maps OL namespaces to Feast projects + # for RBAC-scoped visibility in the UI. + namespaceMapping: + airflow_production: my_project + spark_etl: my_project + dbt_analytics: my_project diff --git a/infra/feast-operator/config/samples/v1_featurestore_packaged.yaml b/infra/feast-operator/config/samples/v1_featurestore_packaged.yaml new file mode 100644 index 00000000000..4b334b9d3d8 --- /dev/null +++ b/infra/feast-operator/config/samples/v1_featurestore_packaged.yaml @@ -0,0 +1,10 @@ +apiVersion: feast.dev/v1 +kind: FeatureStore +metadata: + name: sample-packaged +spec: + feastProject: sample_packaged + feastProjectDir: + packaged: + image: registry.example.com/feature-server@sha256:0123456789abcdef + featureRepoPath: /opt/feast/feature_repo diff --git a/infra/feast-operator/dist/install.yaml b/infra/feast-operator/dist/install.yaml index 85c46859d0a..be85a29a7b2 100644 --- a/infra/feast-operator/dist/install.yaml +++ b/infra/feast-operator/dist/install.yaml @@ -169,8 +169,9 @@ spec: in a Container. properties: name: - description: Name of the environment variable. Must - be a C_IDENTIFIER. + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. type: string value: description: |- @@ -219,6 +220,35 @@ spec: - fieldPath type: object x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: The key within the env file. An + invalid key will prevent the pod from starting. + type: string + optional: + default: false + description: Specify whether the file or its + key must be defined. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '.. + type: string + volumeName: + description: The name of the volume mount containing + the env file. + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic resourceFieldRef: description: |- Selects a resource of the container: only resources limits and requests @@ -294,7 +324,7 @@ spec: x-kubernetes-map-type: atomic prefix: description: Optional text to prepend to the name of - each environment variable. Must be a C_IDENTIFIER. + each environment variable. type: string secretRef: description: The Secret to select from @@ -479,7 +509,6 @@ spec: type. type: string required: - - status - type type: object type: array @@ -572,8 +601,9 @@ spec: in a Container. properties: name: - description: Name of the environment variable. Must - be a C_IDENTIFIER. + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. type: string value: description: |- @@ -622,6 +652,35 @@ spec: - fieldPath type: object x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: The key within the env file. An + invalid key will prevent the pod from starting. + type: string + optional: + default: false + description: Specify whether the file or its + key must be defined. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '.. + type: string + volumeName: + description: The name of the volume mount containing + the env file. + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic resourceFieldRef: description: |- Selects a resource of the container: only resources limits and requests @@ -697,7 +756,7 @@ spec: x-kubernetes-map-type: atomic prefix: description: Optional text to prepend to the name of - each environment variable. Must be a C_IDENTIFIER. + each environment variable. type: string secretRef: description: The Secret to select from @@ -759,10 +818,32 @@ spec: - pytorch_nlp type: string type: object + packaged: + description: FeastPackagedOptions describes a feature repository + packaged in a feature server image. + properties: + featureRepoPath: + description: FeatureRepoPath is the canonical absolute path + to the feature repository in the image. + type: string + image: + description: Image containing the packaged feature repository. + type: string + required: + - featureRepoPath + type: object + x-kubernetes-validations: + - message: FeatureRepoPath must be a canonical absolute, non-root + path without dot segments or repeated separators. + rule: self.featureRepoPath.startsWith('/') && self.featureRepoPath + != '/' && !self.featureRepoPath.contains('//') && !self.featureRepoPath.endsWith('/') + && !self.featureRepoPath.contains('/./') && !self.featureRepoPath.endsWith('/.') + && !self.featureRepoPath.contains('/../') && !self.featureRepoPath.endsWith('/..') type: object x-kubernetes-validations: - - message: One selection required between init or git. - rule: '[has(self.git), has(self.init)].exists_one(c, c)' + - message: One selection required between init, git, or packaged. + rule: '[has(self.git), has(self.init), has(self.packaged)].exists_one(c, + c)' materialization: description: |- Materialization controls feature materialization behavior (batch size, pull strategy). @@ -801,6 +882,59 @@ spec: type: string type: object x-kubernetes-map-type: atomic + consumer: + description: |- + Consumer configures the OpenLineage consumer (event receiver) that enables + Feast to receive and display lineage from... + properties: + apiKeySecretRef: + description: |- + Reference to a Secret containing the key "api_key" that producers must + provide in the X-API-Key header when sending... + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. + type: string + type: object + x-kubernetes-map-type: atomic + connectionStringSecretRef: + description: |- + Reference to a Secret containing the key "connection_string" for a separate + lineage database. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. + type: string + type: object + x-kubernetes-map-type: atomic + enabled: + description: Enable the OpenLineage consumer. + type: boolean + namespaceMapping: + additionalProperties: + type: string + description: |- + NamespaceMapping maps OpenLineage namespaces to Feast projects for + RBAC-based filtering of lineage data in the UI. + type: object + storeType: + default: sql + description: StoreType is the storage backend for lineage + events. Currently only "sql" is supported. + enum: + - sql + type: string + required: + - enabled + type: object enabled: description: Enable OpenLineage integration. type: boolean @@ -1619,6 +1753,10 @@ spec: disableInitContainers: description: Disable the 'feast repo initialization' initContainer type: boolean + initImage: + description: InitImage overrides the image for init containers + (feast-init, feast-apply). + type: string offlineStore: description: OfflineStore configures the offline store service properties: @@ -1769,8 +1907,9 @@ spec: present in a Container. properties: name: - description: Name of the environment variable. Must - be a C_IDENTIFIER. + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. type: string value: description: |- @@ -1819,6 +1958,36 @@ spec: - fieldPath type: object x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: The key within the env file. + An invalid key will prevent the pod from + starting. + type: string + optional: + default: false + description: Specify whether the file or + its key must be defined. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '.. + type: string + volumeName: + description: The name of the volume mount + containing the env file. + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic resourceFieldRef: description: |- Selects a resource of the container: only resources limits and requests @@ -1895,7 +2064,7 @@ spec: x-kubernetes-map-type: atomic prefix: description: Optional text to prepend to the name - of each environment variable. Must be a C_IDENTIFIER. + of each environment variable. type: string secretRef: description: The Secret to select from @@ -2123,6 +2292,11 @@ spec: onlineStore: description: OnlineStore configures the online store service properties: + disabled: + description: |- + Disabled skips deploying the online store service entirely, including its + serving pod and persistence. + type: boolean persistence: description: OnlineStorePersistence configures the persistence settings for the online store service @@ -2264,6 +2438,8 @@ spec: - milvus - hybrid - mongodb + - aerospike + - scylladb type: string required: - secretRef @@ -2282,8 +2458,9 @@ spec: present in a Container. properties: name: - description: Name of the environment variable. Must - be a C_IDENTIFIER. + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. type: string value: description: |- @@ -2332,6 +2509,36 @@ spec: - fieldPath type: object x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: The key within the env file. + An invalid key will prevent the pod from + starting. + type: string + optional: + default: false + description: Specify whether the file or + its key must be defined. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '.. + type: string + volumeName: + description: The name of the volume mount + containing the env file. + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic resourceFieldRef: description: |- Selects a resource of the container: only resources limits and requests @@ -2408,7 +2615,7 @@ spec: x-kubernetes-map-type: atomic prefix: description: Optional text to prepend to the name - of each environment variable. Must be a C_IDENTIFIER. + of each environment variable. type: string secretRef: description: The Secret to select from @@ -2912,8 +3119,9 @@ spec: present in a Container. properties: name: - description: Name of the environment variable. - Must be a C_IDENTIFIER. + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. type: string value: description: |- @@ -2963,6 +3171,36 @@ spec: - fieldPath type: object x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: The key within the env + file. An invalid key will prevent + the pod from starting. + type: string + optional: + default: false + description: Specify whether the file + or its key must be defined. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '.. + type: string + volumeName: + description: The name of the volume + mount containing the env file. + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic resourceFieldRef: description: |- Selects a resource of the container: only resources limits and requests @@ -3041,8 +3279,7 @@ spec: x-kubernetes-map-type: atomic prefix: description: Optional text to prepend to the - name of each environment variable. Must be - a C_IDENTIFIER. + name of each environment variable. type: string secretRef: description: The Secret to select from @@ -4217,8 +4454,9 @@ spec: in a Container. properties: name: - description: Name of the environment variable. Must - be a C_IDENTIFIER. + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. type: string value: description: |- @@ -4267,6 +4505,35 @@ spec: - fieldPath type: object x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: The key within the env file. An + invalid key will prevent the pod from starting. + type: string + optional: + default: false + description: Specify whether the file or its + key must be defined. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '.. + type: string + volumeName: + description: The name of the volume mount containing + the env file. + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic resourceFieldRef: description: |- Selects a resource of the container: only resources limits and requests @@ -4342,7 +4609,7 @@ spec: x-kubernetes-map-type: atomic prefix: description: Optional text to prepend to the name of - each environment variable. Must be a C_IDENTIFIER. + each environment variable. type: string secretRef: description: The Secret to select from @@ -5236,9 +5503,8 @@ spec: host that shares a pod's lifetime. properties: endpoints: - description: |- - endpoints is the endpoint name that details Glusterfs topology. - More info: https://examples.k8s. + description: endpoints is the endpoint name that details + Glusterfs topology. type: string path: description: |- @@ -5651,6 +5917,52 @@ spec: type: array x-kubernetes-list-type: atomic type: object + podCertificate: + description: |- + Projects an auto-rotating credential bundle (private key and certificate + chain) that the pod can use either as a TLS... + properties: + certificateChainPath: + description: |- + Write the certificate chain at this path in the projected volume. + + Most applications should use credentialBundlePath. + type: string + credentialBundlePath: + description: Write the credential bundle at + this path in the projected volume. + type: string + keyPath: + description: |- + Write the key at this path in the projected volume. + + Most applications should use credentialBundlePath. + type: string + keyType: + description: The type of keypair Kubelet will + generate for the pod. + type: string + maxExpirationSeconds: + description: |- + maxExpirationSeconds is the maximum lifetime permitted for the + certificate. + format: int32 + type: integer + signerName: + description: Kubelet's generated CSRs will + be addressed to this signer. + type: string + userAnnotations: + additionalProperties: + type: string + description: |- + userAnnotations allow pod authors to pass additional information to + the signer implementation. + type: object + required: + - keyType + - signerName + type: object secret: description: secret information about the secret data to project @@ -6168,8 +6480,9 @@ spec: present in a Container. properties: name: - description: Name of the environment variable. Must - be a C_IDENTIFIER. + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. type: string value: description: |- @@ -6218,6 +6531,36 @@ spec: - fieldPath type: object x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: The key within the env file. + An invalid key will prevent the pod from + starting. + type: string + optional: + default: false + description: Specify whether the file or + its key must be defined. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '.. + type: string + volumeName: + description: The name of the volume mount + containing the env file. + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic resourceFieldRef: description: |- Selects a resource of the container: only resources limits and requests @@ -6294,7 +6637,7 @@ spec: x-kubernetes-map-type: atomic prefix: description: Optional text to prepend to the name - of each environment variable. Must be a C_IDENTIFIER. + of each environment variable. type: string secretRef: description: The Secret to select from @@ -6481,7 +6824,6 @@ spec: condition type. type: string required: - - status - type type: object type: array @@ -6576,8 +6918,9 @@ spec: present in a Container. properties: name: - description: Name of the environment variable. Must - be a C_IDENTIFIER. + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. type: string value: description: |- @@ -6626,6 +6969,36 @@ spec: - fieldPath type: object x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: The key within the env file. + An invalid key will prevent the pod from + starting. + type: string + optional: + default: false + description: Specify whether the file or + its key must be defined. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '.. + type: string + volumeName: + description: The name of the volume mount + containing the env file. + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic resourceFieldRef: description: |- Selects a resource of the container: only resources limits and requests @@ -6702,7 +7075,7 @@ spec: x-kubernetes-map-type: atomic prefix: description: Optional text to prepend to the name - of each environment variable. Must be a C_IDENTIFIER. + of each environment variable. type: string secretRef: description: The Secret to select from @@ -6765,10 +7138,32 @@ spec: - pytorch_nlp type: string type: object + packaged: + description: FeastPackagedOptions describes a feature repository + packaged in a feature server image. + properties: + featureRepoPath: + description: FeatureRepoPath is the canonical absolute + path to the feature repository in the image. + type: string + image: + description: Image containing the packaged feature repository. + type: string + required: + - featureRepoPath + type: object + x-kubernetes-validations: + - message: FeatureRepoPath must be a canonical absolute, non-root + path without dot segments or repeated separators. + rule: self.featureRepoPath.startsWith('/') && self.featureRepoPath + != '/' && !self.featureRepoPath.contains('//') && !self.featureRepoPath.endsWith('/') + && !self.featureRepoPath.contains('/./') && !self.featureRepoPath.endsWith('/.') + && !self.featureRepoPath.contains('/../') && !self.featureRepoPath.endsWith('/..') type: object x-kubernetes-validations: - - message: One selection required between init or git. - rule: '[has(self.git), has(self.init)].exists_one(c, c)' + - message: One selection required between init, git, or packaged. + rule: '[has(self.git), has(self.init), has(self.packaged)].exists_one(c, + c)' materialization: description: |- Materialization controls feature materialization behavior (batch size, pull strategy). @@ -6807,6 +7202,59 @@ spec: type: string type: object x-kubernetes-map-type: atomic + consumer: + description: |- + Consumer configures the OpenLineage consumer (event receiver) that enables + Feast to receive and display lineage from... + properties: + apiKeySecretRef: + description: |- + Reference to a Secret containing the key "api_key" that producers must + provide in the X-API-Key header when sending... + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. + type: string + type: object + x-kubernetes-map-type: atomic + connectionStringSecretRef: + description: |- + Reference to a Secret containing the key "connection_string" for a separate + lineage database. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. + type: string + type: object + x-kubernetes-map-type: atomic + enabled: + description: Enable the OpenLineage consumer. + type: boolean + namespaceMapping: + additionalProperties: + type: string + description: |- + NamespaceMapping maps OpenLineage namespaces to Feast projects for + RBAC-based filtering of lineage data in the UI. + type: object + storeType: + default: sql + description: StoreType is the storage backend for lineage + events. Currently only "sql" is supported. + enum: + - sql + type: string + required: + - enabled + type: object enabled: description: Enable OpenLineage integration. type: boolean @@ -7633,6 +8081,10 @@ spec: disableInitContainers: description: Disable the 'feast repo initialization' initContainer type: boolean + initImage: + description: InitImage overrides the image for init containers + (feast-init, feast-apply). + type: string offlineStore: description: OfflineStore configures the offline store service properties: @@ -7786,8 +8238,9 @@ spec: present in a Container. properties: name: - description: Name of the environment variable. - Must be a C_IDENTIFIER. + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. type: string value: description: |- @@ -7837,6 +8290,36 @@ spec: - fieldPath type: object x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: The key within the env + file. An invalid key will prevent + the pod from starting. + type: string + optional: + default: false + description: Specify whether the file + or its key must be defined. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '.. + type: string + volumeName: + description: The name of the volume + mount containing the env file. + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic resourceFieldRef: description: |- Selects a resource of the container: only resources limits and requests @@ -7915,8 +8398,7 @@ spec: x-kubernetes-map-type: atomic prefix: description: Optional text to prepend to the - name of each environment variable. Must be - a C_IDENTIFIER. + name of each environment variable. type: string secretRef: description: The Secret to select from @@ -8145,6 +8627,11 @@ spec: onlineStore: description: OnlineStore configures the online store service properties: + disabled: + description: |- + Disabled skips deploying the online store service entirely, including its + serving pod and persistence. + type: boolean persistence: description: OnlineStorePersistence configures the persistence settings for the online store service @@ -8289,6 +8776,8 @@ spec: - milvus - hybrid - mongodb + - aerospike + - scylladb type: string required: - secretRef @@ -8308,8 +8797,9 @@ spec: present in a Container. properties: name: - description: Name of the environment variable. - Must be a C_IDENTIFIER. + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. type: string value: description: |- @@ -8359,6 +8849,36 @@ spec: - fieldPath type: object x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: The key within the env + file. An invalid key will prevent + the pod from starting. + type: string + optional: + default: false + description: Specify whether the file + or its key must be defined. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '.. + type: string + volumeName: + description: The name of the volume + mount containing the env file. + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic resourceFieldRef: description: |- Selects a resource of the container: only resources limits and requests @@ -8437,8 +8957,7 @@ spec: x-kubernetes-map-type: atomic prefix: description: Optional text to prepend to the - name of each environment variable. Must be - a C_IDENTIFIER. + name of each environment variable. type: string secretRef: description: The Secret to select from @@ -8953,8 +9472,9 @@ spec: variable present in a Container. properties: name: - description: Name of the environment variable. - Must be a C_IDENTIFIER. + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. type: string value: description: |- @@ -9005,6 +9525,36 @@ spec: - fieldPath type: object x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: The key within the + env file. An invalid key will + prevent the pod from starting. + type: string + optional: + default: false + description: Specify whether the + file or its key must be defined. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '.. + type: string + volumeName: + description: The name of the volume + mount containing the env file. + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic resourceFieldRef: description: |- Selects a resource of the container: only resources limits and requests @@ -9085,7 +9635,6 @@ spec: prefix: description: Optional text to prepend to the name of each environment variable. - Must be a C_IDENTIFIER. type: string secretRef: description: The Secret to select from @@ -10276,8 +10825,9 @@ spec: present in a Container. properties: name: - description: Name of the environment variable. Must - be a C_IDENTIFIER. + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. type: string value: description: |- @@ -10326,6 +10876,36 @@ spec: - fieldPath type: object x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: The key within the env file. + An invalid key will prevent the pod from + starting. + type: string + optional: + default: false + description: Specify whether the file or + its key must be defined. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '.. + type: string + volumeName: + description: The name of the volume mount + containing the env file. + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic resourceFieldRef: description: |- Selects a resource of the container: only resources limits and requests @@ -10402,7 +10982,7 @@ spec: x-kubernetes-map-type: atomic prefix: description: Optional text to prepend to the name - of each environment variable. Must be a C_IDENTIFIER. + of each environment variable. type: string secretRef: description: The Secret to select from @@ -11304,9 +11884,8 @@ spec: on the host that shares a pod's lifetime. properties: endpoints: - description: |- - endpoints is the endpoint name that details Glusterfs topology. - More info: https://examples.k8s. + description: endpoints is the endpoint name that + details Glusterfs topology. type: string path: description: |- @@ -11723,6 +12302,52 @@ spec: type: array x-kubernetes-list-type: atomic type: object + podCertificate: + description: |- + Projects an auto-rotating credential bundle (private key and certificate + chain) that the pod can use either as a TLS... + properties: + certificateChainPath: + description: |- + Write the certificate chain at this path in the projected volume. + + Most applications should use credentialBundlePath. + type: string + credentialBundlePath: + description: Write the credential bundle + at this path in the projected volume. + type: string + keyPath: + description: |- + Write the key at this path in the projected volume. + + Most applications should use credentialBundlePath. + type: string + keyType: + description: The type of keypair Kubelet + will generate for the pod. + type: string + maxExpirationSeconds: + description: |- + maxExpirationSeconds is the maximum lifetime permitted for the + certificate. + format: int32 + type: integer + signerName: + description: Kubelet's generated CSRs + will be addressed to this signer. + type: string + userAnnotations: + additionalProperties: + type: string + description: |- + userAnnotations allow pod authors to pass additional information to + the signer implementation. + type: object + required: + - keyType + - signerName + type: object secret: description: secret information about the secret data to project @@ -12321,8 +12946,9 @@ spec: in a Container. properties: name: - description: Name of the environment variable. Must - be a C_IDENTIFIER. + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. type: string value: description: |- @@ -12371,6 +12997,35 @@ spec: - fieldPath type: object x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: The key within the env file. An + invalid key will prevent the pod from starting. + type: string + optional: + default: false + description: Specify whether the file or its + key must be defined. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '.. + type: string + volumeName: + description: The name of the volume mount containing + the env file. + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic resourceFieldRef: description: |- Selects a resource of the container: only resources limits and requests @@ -12446,7 +13101,7 @@ spec: x-kubernetes-map-type: atomic prefix: description: Optional text to prepend to the name of - each environment variable. Must be a C_IDENTIFIER. + each environment variable. type: string secretRef: description: The Secret to select from @@ -12631,7 +13286,6 @@ spec: type. type: string required: - - status - type type: object type: array @@ -12714,8 +13368,9 @@ spec: in a Container. properties: name: - description: Name of the environment variable. Must - be a C_IDENTIFIER. + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. type: string value: description: |- @@ -12761,7 +13416,36 @@ spec: the specified API version. type: string required: - - fieldPath + - fieldPath + type: object + x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: The key within the env file. An + invalid key will prevent the pod from starting. + type: string + optional: + default: false + description: Specify whether the file or its + key must be defined. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '.. + type: string + volumeName: + description: The name of the volume mount containing + the env file. + type: string + required: + - key + - path + - volumeName type: object x-kubernetes-map-type: atomic resourceFieldRef: @@ -12839,7 +13523,7 @@ spec: x-kubernetes-map-type: atomic prefix: description: Optional text to prepend to the name of - each environment variable. Must be a C_IDENTIFIER. + each environment variable. type: string secretRef: description: The Secret to select from @@ -12901,10 +13585,32 @@ spec: - pytorch_nlp type: string type: object + packaged: + description: FeastPackagedOptions describes a feature repository + packaged in a feature server image. + properties: + featureRepoPath: + description: FeatureRepoPath is the canonical absolute path + to the feature repository in the image. + type: string + image: + description: Image containing the packaged feature repository. + type: string + required: + - featureRepoPath + type: object + x-kubernetes-validations: + - message: FeatureRepoPath must be a canonical absolute, non-root + path without dot segments or repeated separators. + rule: self.featureRepoPath.startsWith('/') && self.featureRepoPath + != '/' && !self.featureRepoPath.contains('//') && !self.featureRepoPath.endsWith('/') + && !self.featureRepoPath.contains('/./') && !self.featureRepoPath.endsWith('/.') + && !self.featureRepoPath.contains('/../') && !self.featureRepoPath.endsWith('/..') type: object x-kubernetes-validations: - - message: One selection required between init or git. - rule: '[has(self.git), has(self.init)].exists_one(c, c)' + - message: One selection required between init, git, or packaged. + rule: '[has(self.git), has(self.init), has(self.packaged)].exists_one(c, + c)' services: description: FeatureStoreServices defines the desired feast services. An ephemeral onlineStore feature server is deployed by default. @@ -13091,8 +13797,9 @@ spec: present in a Container. properties: name: - description: Name of the environment variable. Must - be a C_IDENTIFIER. + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. type: string value: description: |- @@ -13141,6 +13848,36 @@ spec: - fieldPath type: object x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: The key within the env file. + An invalid key will prevent the pod from + starting. + type: string + optional: + default: false + description: Specify whether the file or + its key must be defined. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '.. + type: string + volumeName: + description: The name of the volume mount + containing the env file. + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic resourceFieldRef: description: |- Selects a resource of the container: only resources limits and requests @@ -13217,7 +13954,7 @@ spec: x-kubernetes-map-type: atomic prefix: description: Optional text to prepend to the name - of each environment variable. Must be a C_IDENTIFIER. + of each environment variable. type: string secretRef: description: The Secret to select from @@ -13586,6 +14323,8 @@ spec: - milvus - hybrid - mongodb + - aerospike + - scylladb type: string required: - secretRef @@ -13604,8 +14343,9 @@ spec: present in a Container. properties: name: - description: Name of the environment variable. Must - be a C_IDENTIFIER. + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. type: string value: description: |- @@ -13654,6 +14394,36 @@ spec: - fieldPath type: object x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: The key within the env file. + An invalid key will prevent the pod from + starting. + type: string + optional: + default: false + description: Specify whether the file or + its key must be defined. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '.. + type: string + volumeName: + description: The name of the volume mount + containing the env file. + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic resourceFieldRef: description: |- Selects a resource of the container: only resources limits and requests @@ -13730,7 +14500,7 @@ spec: x-kubernetes-map-type: atomic prefix: description: Optional text to prepend to the name - of each environment variable. Must be a C_IDENTIFIER. + of each environment variable. type: string secretRef: description: The Secret to select from @@ -14135,8 +14905,9 @@ spec: present in a Container. properties: name: - description: Name of the environment variable. - Must be a C_IDENTIFIER. + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. type: string value: description: |- @@ -14186,6 +14957,36 @@ spec: - fieldPath type: object x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: The key within the env + file. An invalid key will prevent + the pod from starting. + type: string + optional: + default: false + description: Specify whether the file + or its key must be defined. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '.. + type: string + volumeName: + description: The name of the volume + mount containing the env file. + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic resourceFieldRef: description: |- Selects a resource of the container: only resources limits and requests @@ -14264,8 +15065,7 @@ spec: x-kubernetes-map-type: atomic prefix: description: Optional text to prepend to the - name of each environment variable. Must be - a C_IDENTIFIER. + name of each environment variable. type: string secretRef: description: The Secret to select from @@ -14709,8 +15509,9 @@ spec: in a Container. properties: name: - description: Name of the environment variable. Must - be a C_IDENTIFIER. + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. type: string value: description: |- @@ -14759,6 +15560,35 @@ spec: - fieldPath type: object x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: The key within the env file. An + invalid key will prevent the pod from starting. + type: string + optional: + default: false + description: Specify whether the file or its + key must be defined. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '.. + type: string + volumeName: + description: The name of the volume mount containing + the env file. + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic resourceFieldRef: description: |- Selects a resource of the container: only resources limits and requests @@ -14834,7 +15664,7 @@ spec: x-kubernetes-map-type: atomic prefix: description: Optional text to prepend to the name of - each environment variable. Must be a C_IDENTIFIER. + each environment variable. type: string secretRef: description: The Secret to select from @@ -15728,9 +16558,8 @@ spec: host that shares a pod's lifetime. properties: endpoints: - description: |- - endpoints is the endpoint name that details Glusterfs topology. - More info: https://examples.k8s. + description: endpoints is the endpoint name that details + Glusterfs topology. type: string path: description: |- @@ -16143,6 +16972,52 @@ spec: type: array x-kubernetes-list-type: atomic type: object + podCertificate: + description: |- + Projects an auto-rotating credential bundle (private key and certificate + chain) that the pod can use either as a TLS... + properties: + certificateChainPath: + description: |- + Write the certificate chain at this path in the projected volume. + + Most applications should use credentialBundlePath. + type: string + credentialBundlePath: + description: Write the credential bundle at + this path in the projected volume. + type: string + keyPath: + description: |- + Write the key at this path in the projected volume. + + Most applications should use credentialBundlePath. + type: string + keyType: + description: The type of keypair Kubelet will + generate for the pod. + type: string + maxExpirationSeconds: + description: |- + maxExpirationSeconds is the maximum lifetime permitted for the + certificate. + format: int32 + type: integer + signerName: + description: Kubelet's generated CSRs will + be addressed to this signer. + type: string + userAnnotations: + additionalProperties: + type: string + description: |- + userAnnotations allow pod authors to pass additional information to + the signer implementation. + type: object + required: + - keyType + - signerName + type: object secret: description: secret information about the secret data to project @@ -16581,8 +17456,9 @@ spec: present in a Container. properties: name: - description: Name of the environment variable. Must - be a C_IDENTIFIER. + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. type: string value: description: |- @@ -16631,6 +17507,36 @@ spec: - fieldPath type: object x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: The key within the env file. + An invalid key will prevent the pod from + starting. + type: string + optional: + default: false + description: Specify whether the file or + its key must be defined. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '.. + type: string + volumeName: + description: The name of the volume mount + containing the env file. + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic resourceFieldRef: description: |- Selects a resource of the container: only resources limits and requests @@ -16707,7 +17613,7 @@ spec: x-kubernetes-map-type: atomic prefix: description: Optional text to prepend to the name - of each environment variable. Must be a C_IDENTIFIER. + of each environment variable. type: string secretRef: description: The Secret to select from @@ -16894,7 +17800,6 @@ spec: condition type. type: string required: - - status - type type: object type: array @@ -16979,8 +17884,9 @@ spec: present in a Container. properties: name: - description: Name of the environment variable. Must - be a C_IDENTIFIER. + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. type: string value: description: |- @@ -17029,6 +17935,36 @@ spec: - fieldPath type: object x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: The key within the env file. + An invalid key will prevent the pod from + starting. + type: string + optional: + default: false + description: Specify whether the file or + its key must be defined. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '.. + type: string + volumeName: + description: The name of the volume mount + containing the env file. + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic resourceFieldRef: description: |- Selects a resource of the container: only resources limits and requests @@ -17105,7 +18041,7 @@ spec: x-kubernetes-map-type: atomic prefix: description: Optional text to prepend to the name - of each environment variable. Must be a C_IDENTIFIER. + of each environment variable. type: string secretRef: description: The Secret to select from @@ -17168,10 +18104,32 @@ spec: - pytorch_nlp type: string type: object + packaged: + description: FeastPackagedOptions describes a feature repository + packaged in a feature server image. + properties: + featureRepoPath: + description: FeatureRepoPath is the canonical absolute + path to the feature repository in the image. + type: string + image: + description: Image containing the packaged feature repository. + type: string + required: + - featureRepoPath + type: object + x-kubernetes-validations: + - message: FeatureRepoPath must be a canonical absolute, non-root + path without dot segments or repeated separators. + rule: self.featureRepoPath.startsWith('/') && self.featureRepoPath + != '/' && !self.featureRepoPath.contains('//') && !self.featureRepoPath.endsWith('/') + && !self.featureRepoPath.contains('/./') && !self.featureRepoPath.endsWith('/.') + && !self.featureRepoPath.contains('/../') && !self.featureRepoPath.endsWith('/..') type: object x-kubernetes-validations: - - message: One selection required between init or git. - rule: '[has(self.git), has(self.init)].exists_one(c, c)' + - message: One selection required between init, git, or packaged. + rule: '[has(self.git), has(self.init), has(self.packaged)].exists_one(c, + c)' services: description: FeatureStoreServices defines the desired feast services. An ephemeral onlineStore feature server is deployed by default. @@ -17361,8 +18319,9 @@ spec: present in a Container. properties: name: - description: Name of the environment variable. - Must be a C_IDENTIFIER. + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. type: string value: description: |- @@ -17412,6 +18371,36 @@ spec: - fieldPath type: object x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: The key within the env + file. An invalid key will prevent + the pod from starting. + type: string + optional: + default: false + description: Specify whether the file + or its key must be defined. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '.. + type: string + volumeName: + description: The name of the volume + mount containing the env file. + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic resourceFieldRef: description: |- Selects a resource of the container: only resources limits and requests @@ -17490,8 +18479,7 @@ spec: x-kubernetes-map-type: atomic prefix: description: Optional text to prepend to the - name of each environment variable. Must be - a C_IDENTIFIER. + name of each environment variable. type: string secretRef: description: The Secret to select from @@ -17864,6 +18852,8 @@ spec: - milvus - hybrid - mongodb + - aerospike + - scylladb type: string required: - secretRef @@ -17883,8 +18873,9 @@ spec: present in a Container. properties: name: - description: Name of the environment variable. - Must be a C_IDENTIFIER. + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. type: string value: description: |- @@ -17934,6 +18925,36 @@ spec: - fieldPath type: object x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: The key within the env + file. An invalid key will prevent + the pod from starting. + type: string + optional: + default: false + description: Specify whether the file + or its key must be defined. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '.. + type: string + volumeName: + description: The name of the volume + mount containing the env file. + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic resourceFieldRef: description: |- Selects a resource of the container: only resources limits and requests @@ -18012,8 +19033,7 @@ spec: x-kubernetes-map-type: atomic prefix: description: Optional text to prepend to the - name of each environment variable. Must be - a C_IDENTIFIER. + name of each environment variable. type: string secretRef: description: The Secret to select from @@ -18427,8 +19447,9 @@ spec: variable present in a Container. properties: name: - description: Name of the environment variable. - Must be a C_IDENTIFIER. + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. type: string value: description: |- @@ -18479,6 +19500,36 @@ spec: - fieldPath type: object x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: The key within the + env file. An invalid key will + prevent the pod from starting. + type: string + optional: + default: false + description: Specify whether the + file or its key must be defined. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '.. + type: string + volumeName: + description: The name of the volume + mount containing the env file. + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic resourceFieldRef: description: |- Selects a resource of the container: only resources limits and requests @@ -18559,7 +19610,6 @@ spec: prefix: description: Optional text to prepend to the name of each environment variable. - Must be a C_IDENTIFIER. type: string secretRef: description: The Secret to select from @@ -19014,8 +20064,9 @@ spec: present in a Container. properties: name: - description: Name of the environment variable. Must - be a C_IDENTIFIER. + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. type: string value: description: |- @@ -19064,6 +20115,36 @@ spec: - fieldPath type: object x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: The key within the env file. + An invalid key will prevent the pod from + starting. + type: string + optional: + default: false + description: Specify whether the file or + its key must be defined. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '.. + type: string + volumeName: + description: The name of the volume mount + containing the env file. + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic resourceFieldRef: description: |- Selects a resource of the container: only resources limits and requests @@ -19140,7 +20221,7 @@ spec: x-kubernetes-map-type: atomic prefix: description: Optional text to prepend to the name - of each environment variable. Must be a C_IDENTIFIER. + of each environment variable. type: string secretRef: description: The Secret to select from @@ -20042,9 +21123,8 @@ spec: on the host that shares a pod's lifetime. properties: endpoints: - description: |- - endpoints is the endpoint name that details Glusterfs topology. - More info: https://examples.k8s. + description: endpoints is the endpoint name that + details Glusterfs topology. type: string path: description: |- @@ -20461,6 +21541,52 @@ spec: type: array x-kubernetes-list-type: atomic type: object + podCertificate: + description: |- + Projects an auto-rotating credential bundle (private key and certificate + chain) that the pod can use either as a TLS... + properties: + certificateChainPath: + description: |- + Write the certificate chain at this path in the projected volume. + + Most applications should use credentialBundlePath. + type: string + credentialBundlePath: + description: Write the credential bundle + at this path in the projected volume. + type: string + keyPath: + description: |- + Write the key at this path in the projected volume. + + Most applications should use credentialBundlePath. + type: string + keyType: + description: The type of keypair Kubelet + will generate for the pod. + type: string + maxExpirationSeconds: + description: |- + maxExpirationSeconds is the maximum lifetime permitted for the + certificate. + format: int32 + type: integer + signerName: + description: Kubelet's generated CSRs + will be addressed to this signer. + type: string + userAnnotations: + additionalProperties: + type: string + description: |- + userAnnotations allow pod authors to pass additional information to + the signer implementation. + type: object + required: + - keyType + - signerName + type: object secret: description: secret information about the secret data to project @@ -21017,11 +22143,11 @@ rules: resources: - configmaps - persistentvolumeclaims - - serviceaccounts - services verbs: - create - delete + - deletecollection - get - list - update @@ -21030,18 +22156,45 @@ rules: - "" resources: - namespaces - - pods - secrets verbs: - get - list - watch +- apiGroups: + - "" + resources: + - pods + verbs: + - create + - delete + - deletecollection + - get + - list + - watch - apiGroups: - "" resources: - pods/exec verbs: - create +- apiGroups: + - "" + resources: + - pods/log + verbs: + - get +- apiGroups: + - "" + resources: + - serviceaccounts + verbs: + - create + - delete + - get + - list + - update + - watch - apiGroups: - apps resources: @@ -21083,6 +22236,14 @@ rules: - patch - update - watch +- apiGroups: + - config.openshift.io + resources: + - apiservers + verbs: + - get + - list + - watch - apiGroups: - feast.dev resources: @@ -21158,6 +22319,14 @@ rules: - list - update - watch +- apiGroups: + - sparkoperator.k8s.io + resources: + - sparkapplications + verbs: + - create + - delete + - get --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole @@ -21293,14 +22462,14 @@ spec: - /manager env: - name: RELATED_IMAGE_FEATURE_SERVER - value: quay.io/feastdev/feature-server:0.63.0 + value: quay.io/feastdev/feature-server:0.65.0 - name: RELATED_IMAGE_CRON_JOB value: quay.io/openshift/origin-cli:4.17 - name: GOMEMLIMIT value: 230MiB - name: OIDC_ISSUER_URL value: "" - image: quay.io/feastdev/feast-operator:0.63.0 + image: quay.io/feastdev/feast-operator:0.65.0 livenessProbe: httpGet: path: /healthz diff --git a/infra/feast-operator/dist/operator-e2e-tests b/infra/feast-operator/dist/operator-e2e-tests index 8a4bbe95ca8..0d5ff42aef8 100755 Binary files a/infra/feast-operator/dist/operator-e2e-tests and b/infra/feast-operator/dist/operator-e2e-tests differ diff --git a/infra/feast-operator/docs/api/markdown/ref.md b/infra/feast-operator/docs/api/markdown/ref.md index dd7acf55fb3..cb911ffae22 100644 --- a/infra/feast-operator/docs/api/markdown/ref.md +++ b/infra/feast-operator/docs/api/markdown/ref.md @@ -188,6 +188,23 @@ _Appears in:_ | `template` _string_ | Template for the created project | +#### FeastPackagedOptions + + + +FeastPackagedOptions describes a feature repository packaged in a feature server image. + +_Appears in:_ +- [FeastProjectDir](#feastprojectdir) + +| Field | Description | +| --- | --- | +| `image` _string_ | Image containing the packaged feature repository. When set, this image is used by the +repository initialization and feast apply containers and as the default service image. +When omitted, the operator's configured feature server image is used. | +| `featureRepoPath` _string_ | FeatureRepoPath is the canonical absolute path to the feature repository in the image. | + + #### FeastProjectDir @@ -201,6 +218,7 @@ _Appears in:_ | --- | --- | | `git` _[GitCloneOptions](#gitcloneoptions)_ | | | `init` _[FeastInitOptions](#feastinitoptions)_ | | +| `packaged` _[FeastPackagedOptions](#feastpackagedoptions)_ | | #### FeatureStore @@ -256,6 +274,8 @@ _Appears in:_ This enables annotation-driven integrations like OpenTelemetry auto-instrumentation, Istio sidecar injection, Vault agent injection, etc. | | `disableInitContainers` _boolean_ | Disable the 'feast repo initialization' initContainer | +| `initImage` _string_ | InitImage overrides the image for init containers (feast-init, feast-apply). +Resolution order: InitImage → FeastProjectDir.Packaged.Image → RELATED_IMAGE_FEATURE_SERVER → DefaultImage. | | `runFeastApplyOnInit` _boolean_ | Runs feast apply on pod start to populate the registry. Defaults to true. Ignored when DisableInitContainers is true. | | `volumes` _[Volume](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.30/#volume-v1-core) array_ | Volumes specifies the volumes to mount in the FeatureStore deployment. A corresponding `VolumeMount` should be added to whichever feast service(s) require access to said volume(s). | | `scaling` _[ScalingConfig](#scalingconfig)_ | Scaling configures horizontal scaling for the FeatureStore deployment (e.g. HPA autoscaling). @@ -654,6 +674,9 @@ _Appears in:_ | `persistence` _[OnlineStorePersistence](#onlinestorepersistence)_ | | | `serving` _[ServingConfig](#servingconfig)_ | Serving configures the Feast feature_server section written into feature_store.yaml for the online serve pod. Controls metrics granularity, offline push batching, and MCP. | +| `disabled` _boolean_ | Disabled skips deploying the online store service entirely, including its +serving pod and persistence. Omitting the online store block, or setting +this to false, deploys the online store with defaults as before. | #### OnlineStoreDBStorePersistence @@ -726,6 +749,31 @@ emit_on_materialize) and transport-specific options (e.g. kafka bootstrap_servers, topic; file path). Boolean values ("true"/"false") and integer values are automatically coerced to their native YAML types. Keys must be valid Feast OpenLineageConfig YAML field names. | +| `consumer` _[OpenLineageConsumerConfig](#openlineageconsumerconfig)_ | Consumer configures the OpenLineage consumer (event receiver) that enables +Feast to receive and display lineage from external producers (Airflow, Spark, dbt, etc.). | + + +#### OpenLineageConsumerConfig + + + +OpenLineageConsumerConfig configures the OpenLineage consumer (event receiver). +When enabled, the Feast REST server exposes POST /api/v1/lineage to receive +OpenLineage events from any producer, storing them for visualization in the Feast UI. + +_Appears in:_ +- [OpenLineageConfig](#openlineageconfig) + +| Field | Description | +| --- | --- | +| `enabled` _boolean_ | Enable the OpenLineage consumer. | +| `storeType` _string_ | StoreType is the storage backend for lineage events. Currently only "sql" is supported. | +| `connectionStringSecretRef` _[LocalObjectReference](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.30/#localobjectreference-v1-core)_ | Reference to a Secret containing the key "connection_string" for a separate +lineage database. If omitted, the SQL registry database is reused. | +| `apiKeySecretRef` _[LocalObjectReference](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.30/#localobjectreference-v1-core)_ | Reference to a Secret containing the key "api_key" that producers must +provide in the X-API-Key header when sending events. | +| `namespaceMapping` _object (keys:string, values:string)_ | NamespaceMapping maps OpenLineage namespaces to Feast projects for +RBAC-based filtering of lineage data in the UI. | #### OptionalCtrConfigs diff --git a/infra/feast-operator/go.mod b/infra/feast-operator/go.mod index 021e1a1b020..ab19a1de20a 100644 --- a/infra/feast-operator/go.mod +++ b/infra/feast-operator/go.mod @@ -3,25 +3,28 @@ module github.com/feast-dev/feast/infra/feast-operator go 1.25.0 require ( - github.com/onsi/ginkgo/v2 v2.22.2 - github.com/onsi/gomega v1.36.2 - github.com/openshift/api v0.0.0-20240912201240-0a8800162826 // release-4.17 + github.com/onsi/ginkgo/v2 v2.28.1 + github.com/onsi/gomega v1.39.1 + github.com/openshift/api v0.0.0-20260317165824-54a3998d81eb // release-4.17 gopkg.in/yaml.v3 v3.0.1 - k8s.io/api v0.33.1 - k8s.io/apimachinery v0.33.1 - k8s.io/client-go v0.33.1 - sigs.k8s.io/controller-runtime v0.21.0 + k8s.io/api v0.35.2 + k8s.io/apimachinery v0.35.2 + k8s.io/client-go v0.35.2 + sigs.k8s.io/controller-runtime v0.23.3 ) require ( + github.com/openshift/controller-runtime-common v0.0.0-20260428152732-64ee174f5e2e github.com/prometheus-operator/prometheus-operator/pkg/client v0.83.0 - github.com/prometheus/client_golang v1.22.0 - github.com/prometheus/client_model v0.6.1 - k8s.io/utils v0.0.0-20250502105355-0f33e8f1c979 + github.com/prometheus/client_golang v1.23.2 + github.com/prometheus/client_model v0.6.2 + k8s.io/apiextensions-apiserver v0.35.1 + k8s.io/utils v0.0.0-20260210185600-b8788abfbbc2 ) require ( cel.dev/expr v0.25.1 // indirect + github.com/Masterminds/semver/v3 v3.4.0 // indirect github.com/antlr4-go/antlr/v4 v4.13.0 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/blang/semver/v4 v4.0.0 // indirect @@ -31,8 +34,8 @@ require ( github.com/emicklei/go-restful/v3 v3.12.2 // indirect github.com/evanphx/json-patch/v5 v5.9.11 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect - github.com/fsnotify/fsnotify v1.7.0 // indirect - github.com/fxamacker/cbor/v2 v2.8.0 // indirect + github.com/fsnotify/fsnotify v1.9.0 // indirect + github.com/fxamacker/cbor/v2 v2.9.0 // indirect github.com/go-logr/logr v1.4.3 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/go-logr/zapr v1.3.0 // indirect @@ -40,41 +43,44 @@ require ( github.com/go-openapi/jsonreference v0.21.0 // indirect github.com/go-openapi/swag v0.23.1 // indirect github.com/go-task/slim-sprig/v3 v3.0.0 // indirect - github.com/gogo/protobuf v1.3.2 // indirect github.com/google/btree v1.1.3 // indirect - github.com/google/cel-go v0.23.2 // indirect - github.com/google/gnostic-models v0.6.9 // indirect + github.com/google/cel-go v0.26.0 // indirect + github.com/google/gnostic-models v0.7.0 // indirect github.com/google/go-cmp v0.7.0 // indirect - github.com/google/pprof v0.0.0-20241210010833-40e02aabc2ad // indirect + github.com/google/pprof v0.0.0-20260115054156-294ebfa9ad83 // indirect github.com/google/uuid v1.6.0 // indirect - github.com/grpc-ecosystem/grpc-gateway/v2 v2.24.0 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.3 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/mailru/easyjson v0.9.0 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect - github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect - github.com/pkg/errors v0.9.1 // indirect + github.com/openshift/library-go v0.0.0-20260213153706-03f1709971c5 // indirect + github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring v0.83.0 // indirect - github.com/prometheus/common v0.62.0 // indirect - github.com/prometheus/procfs v0.15.1 // indirect - github.com/spf13/cobra v1.8.1 // indirect - github.com/spf13/pflag v1.0.5 // indirect + github.com/prometheus/common v0.66.1 // indirect + github.com/prometheus/procfs v0.16.1 // indirect + github.com/spf13/cobra v1.10.0 // indirect + github.com/spf13/pflag v1.0.9 // indirect github.com/stoewer/go-strcase v1.3.0 // indirect github.com/x448/float16 v0.8.4 // indirect go.opentelemetry.io/auto/sdk v1.2.1 // indirect - go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.58.0 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0 // indirect go.opentelemetry.io/otel v1.44.0 // indirect - go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.33.0 // indirect - go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.33.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.34.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.34.0 // indirect go.opentelemetry.io/otel/metric v1.44.0 // indirect go.opentelemetry.io/otel/sdk v1.44.0 // indirect go.opentelemetry.io/otel/trace v1.44.0 // indirect - go.opentelemetry.io/proto/otlp v1.4.0 // indirect + go.opentelemetry.io/proto/otlp v1.5.0 // indirect go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.27.0 // indirect + go.yaml.in/yaml/v2 v2.4.3 // indirect + go.yaml.in/yaml/v3 v3.0.4 // indirect golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 // indirect + golang.org/x/mod v0.35.0 // indirect golang.org/x/net v0.55.0 // indirect golang.org/x/oauth2 v0.34.0 // indirect golang.org/x/sync v0.20.0 // indirect @@ -88,16 +94,15 @@ require ( google.golang.org/genproto/googleapis/rpc v0.0.0-20251202230838-ff82c1b0f217 // indirect google.golang.org/grpc v1.79.3 // indirect google.golang.org/protobuf v1.36.10 // indirect - gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect + gopkg.in/evanphx/json-patch.v4 v4.13.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect - k8s.io/apiextensions-apiserver v0.33.1 // indirect - k8s.io/apiserver v0.33.1 // indirect - k8s.io/component-base v0.33.1 // indirect + k8s.io/apiserver v0.35.1 // indirect + k8s.io/component-base v0.35.1 // indirect k8s.io/klog/v2 v2.130.1 // indirect - k8s.io/kube-openapi v0.0.0-20250318190949-c8a335a9a2ff // indirect + k8s.io/kube-openapi v0.0.0-20250910181357-589584f1c912 // indirect sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.31.2 // indirect - sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8 // indirect + sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 // indirect sigs.k8s.io/randfill v1.0.0 // indirect - sigs.k8s.io/structured-merge-diff/v4 v4.7.0 // indirect - sigs.k8s.io/yaml v1.4.0 // indirect + sigs.k8s.io/structured-merge-diff/v6 v6.3.2-0.20260122202528-d9cc6641c482 // indirect + sigs.k8s.io/yaml v1.6.0 // indirect ) diff --git a/infra/feast-operator/go.sum b/infra/feast-operator/go.sum index 6c80ee96e61..b642252f7d3 100644 --- a/infra/feast-operator/go.sum +++ b/infra/feast-operator/go.sum @@ -1,5 +1,7 @@ cel.dev/expr v0.25.1 h1:1KrZg61W6TWSxuNZ37Xy49ps13NUovb66QLprthtwi4= cel.dev/expr v0.25.1/go.mod h1:hrXvqGP6G6gyx8UAHSHJ5RGk//1Oj5nXQ2NI02Nrsg4= +github.com/Masterminds/semver/v3 v3.4.0 h1:Zog+i5UMtVoCU8oKka5P7i9q9HgrJeGzI9SA1Xbatp0= +github.com/Masterminds/semver/v3 v3.4.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM= github.com/antlr4-go/antlr/v4 v4.13.0 h1:lxCg3LAv+EUK6t1i0y1V6/SLeUi0eKEKdhQAlS8TVTI= github.com/antlr4-go/antlr/v4 v4.13.0/go.mod h1:pfChB/xh/Unjila75QW7+VU4TSnWnnk9UTnmpPaOR2g= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= @@ -10,7 +12,7 @@ github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK3 github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= @@ -23,10 +25,16 @@ github.com/evanphx/json-patch/v5 v5.9.11 h1:/8HVnzMq13/3x9TPvjG08wUGqBTmZBsCWzjT github.com/evanphx/json-patch/v5 v5.9.11/go.mod h1:3j+LviiESTElxA4p3EMKAB9HXj3/XEtnUf6OZxqIQTM= github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= -github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= -github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= -github.com/fxamacker/cbor/v2 v2.8.0 h1:fFtUGXUzXPHTIUdne5+zzMPTfffl3RD5qYnkY40vtxU= -github.com/fxamacker/cbor/v2 v2.8.0/go.mod h1:vM4b+DJCtHn+zz7h3FFp/hDAI9WNWCsZj23V5ytsSxQ= +github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k= +github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= +github.com/fxamacker/cbor/v2 v2.9.0 h1:NpKPmjDBgUfBms6tr6JZkTHtfFGcMKsw3eGcmD/sapM= +github.com/fxamacker/cbor/v2 v2.9.0/go.mod h1:vM4b+DJCtHn+zz7h3FFp/hDAI9WNWCsZj23V5ytsSxQ= +github.com/gkampitakis/ciinfo v0.3.2 h1:JcuOPk8ZU7nZQjdUhctuhQofk7BGHuIy0c9Ez8BNhXs= +github.com/gkampitakis/ciinfo v0.3.2/go.mod h1:1NIwaOcFChN4fa/B0hEBdAb6npDlFL8Bwx4dfRLRqAo= +github.com/gkampitakis/go-diff v1.3.2 h1:Qyn0J9XJSDTgnsgHRdz9Zp24RaJeKMUHg2+PDZZdC4M= +github.com/gkampitakis/go-diff v1.3.2/go.mod h1:LLgOrpqleQe26cte8s36HTWcTmMEur6OPYerdAAS9tk= +github.com/gkampitakis/go-snaps v0.5.15 h1:amyJrvM1D33cPHwVrjo9jQxX8g/7E2wYdZ+01KS3zGE= +github.com/gkampitakis/go-snaps v0.5.15/go.mod h1:HNpx/9GoKisdhw9AFOBT1N7DBs9DiHo/hGheFGBZ+mc= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= @@ -42,36 +50,35 @@ github.com/go-openapi/swag v0.23.1 h1:lpsStH0n2ittzTnbaSloVZLuB5+fvSY/+hnagBjSNZ github.com/go-openapi/swag v0.23.1/go.mod h1:STZs8TbRvEQQKUA+JZNAm3EWlgaOBGpyFDqQnDHMef0= github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= -github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= -github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/goccy/go-yaml v1.18.0 h1:8W7wMFS12Pcas7KU+VVkaiCng+kG8QiFeFwzFb+rwuw= +github.com/goccy/go-yaml v1.18.0/go.mod h1:XBurs7gK8ATbW4ZPGKgcbrY1Br56PdM69F7LkFRi1kA= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/google/btree v1.1.3 h1:CVpQJjYgC4VbzxeGVHfvZrv1ctoYCAI8vbl07Fcxlyg= github.com/google/btree v1.1.3/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4= -github.com/google/cel-go v0.23.2 h1:UdEe3CvQh3Nv+E/j9r1Y//WO0K0cSyD7/y0bzyLIMI4= -github.com/google/cel-go v0.23.2/go.mod h1:52Pb6QsDbC5kvgxvZhiL9QX1oZEkcUF/ZqaPx1J5Wwo= -github.com/google/gnostic-models v0.6.9 h1:MU/8wDLif2qCXZmzncUQ/BOfxWfthHi63KqpoNbWqVw= -github.com/google/gnostic-models v0.6.9/go.mod h1:CiWsm0s6BSQd1hRn8/QmxqB6BesYcbSZxsz9b0KuDBw= -github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/cel-go v0.26.0 h1:DPGjXackMpJWH680oGY4lZhYjIameYmR+/6RBdDGmaI= +github.com/google/cel-go v0.26.0/go.mod h1:A9O8OU9rdvrK5MQyrqfIxo1a0u4g3sF8KB6PUIaryMM= +github.com/google/gnostic-models v0.7.0 h1:qwTtogB15McXDaNqTZdzPJRHvaVJlAl+HVQnLmJEJxo= +github.com/google/gnostic-models v0.7.0/go.mod h1:whL5G0m6dmc5cPxKc5bdKdEN3UjI7OUGxBlw57miDrQ= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/pprof v0.0.0-20241210010833-40e02aabc2ad h1:a6HEuzUHeKH6hwfN/ZoQgRgVIWFJljSWa/zetS2WTvg= -github.com/google/pprof v0.0.0-20241210010833-40e02aabc2ad/go.mod h1:vavhavw2zAxS5dIdcRluK6cSGGPlZynqzFM8NdvU144= +github.com/google/pprof v0.0.0-20260115054156-294ebfa9ad83 h1:z2ogiKUYzX5Is6zr/vP9vJGqPwcdqsWjOt+V8J7+bTc= +github.com/google/pprof v0.0.0-20260115054156-294ebfa9ad83/go.mod h1:MxpfABSjhmINe3F1It9d+8exIHFvUqtLIRCdOGNXqiI= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.24.0 h1:TmHmbvxPmaegwhDubVz0lICL0J5Ka2vwTzhoePEXsGE= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.24.0/go.mod h1:qztMSjm835F2bXf+5HKAPIS5qsmQDqZna/PgVt4rWtI= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.3 h1:5ZPtiqj0JL5oKWmcsq4VMaAW5ukBEgSGXEN89zeH1Jo= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.3/go.mod h1:ndYquD05frm2vACXE1nsccT4oJzjhw2arTS2cpUD1PI= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= +github.com/joshdk/go-junit v1.0.0 h1:S86cUKIdwBHWwA6xCmFlf3RTLfVXYQfvanM5Uh+K6GE= +github.com/joshdk/go-junit v1.0.0/go.mod h1:TiiV0PqkaNfFXjEiyjWM3XXrhVyCa1K4Zfga6W52ung= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= -github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= -github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= @@ -82,42 +89,53 @@ github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0 github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/mailru/easyjson v0.9.0 h1:PrnmzHw7262yW8sTBwxi1PdJA3Iw/EKBa8psRf7d9a4= github.com/mailru/easyjson v0.9.0/go.mod h1:1+xMtQp2MRNVL/V1bOzuP3aP8VNwRW55fQUto+XFtTU= +github.com/maruel/natural v1.1.1 h1:Hja7XhhmvEFhcByqDoHz9QZbkWey+COd9xWfCfn1ioo= +github.com/maruel/natural v1.1.1/go.mod h1:v+Rfd79xlw1AgVBjbO0BEQmptqb5HvL/k9GRHB7ZKEg= +github.com/mfridman/tparse v0.18.0 h1:wh6dzOKaIwkUGyKgOntDW4liXSo37qg5AXbIhkMV3vE= +github.com/mfridman/tparse v0.18.0/go.mod h1:gEvqZTuCgEhPbYk/2lS3Kcxg1GmTxxU7kTC8DvP0i/A= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee h1:W5t00kpgFdJifH4BDsTlE89Zl93FEloxaWZfGcifgq8= +github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= -github.com/onsi/ginkgo/v2 v2.22.2 h1:/3X8Panh8/WwhU/3Ssa6rCKqPLuAkVY2I0RoyDLySlU= -github.com/onsi/ginkgo/v2 v2.22.2/go.mod h1:oeMosUL+8LtarXBHu/c0bx2D/K9zyQ6uX3cTyztHwsk= -github.com/onsi/gomega v1.36.2 h1:koNYke6TVk6ZmnyHrCXba/T/MoLBXFjeC1PtvYgw0A8= -github.com/onsi/gomega v1.36.2/go.mod h1:DdwyADRjrc825LhMEkD76cHR5+pUnjhUN8GlHlRPHzY= -github.com/openshift/api v0.0.0-20240912201240-0a8800162826 h1:A8D9SN/hJUwAbdO0rPCVTqmuBOctdgurr53gK701SYo= -github.com/openshift/api v0.0.0-20240912201240-0a8800162826/go.mod h1:OOh6Qopf21pSzqNVCB5gomomBXb8o5sGKZxG2KNpaXM= +github.com/onsi/ginkgo/v2 v2.28.1 h1:S4hj+HbZp40fNKuLUQOYLDgZLwNUVn19N3Atb98NCyI= +github.com/onsi/ginkgo/v2 v2.28.1/go.mod h1:CLtbVInNckU3/+gC8LzkGUb9oF+e8W8TdUsxPwvdOgE= +github.com/onsi/gomega v1.39.1 h1:1IJLAad4zjPn2PsnhH70V4DKRFlrCzGBNrNaru+Vf28= +github.com/onsi/gomega v1.39.1/go.mod h1:hL6yVALoTOxeWudERyfppUcZXjMwIMLnuSfruD2lcfg= +github.com/openshift/api v0.0.0-20260317165824-54a3998d81eb h1:iwBR3mzmyE3EMFx7R3CQ9lOccTS0dNht8TW82aGITg0= +github.com/openshift/api v0.0.0-20260317165824-54a3998d81eb/go.mod h1:pyVjK0nZ4sRs4fuQVQ4rubsJdahI1PB94LnQ8sGdvxo= +github.com/openshift/controller-runtime-common v0.0.0-20260428152732-64ee174f5e2e h1:k89oIo2EjX0PRSdi1kesktCyWp50SC9WwKurvupvRGs= +github.com/openshift/controller-runtime-common v0.0.0-20260428152732-64ee174f5e2e/go.mod h1:XGabTMnNbz0M5Oa7IbscZp/jmcc7aHobvOCUWwkzKvM= +github.com/openshift/library-go v0.0.0-20260213153706-03f1709971c5 h1:9Pe6iVOMjt9CdA/vaKBNUSoEIjIe1po5Ha3ABRYXLJI= +github.com/openshift/library-go v0.0.0-20260213153706-03f1709971c5/go.mod h1:K3FoNLgNBFYbFuG+Kr8usAnQxj1w84XogyUp2M8rK8k= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring v0.83.0 h1:j9Ce3W6X6Tzi0QnSap+YzGwpqJLJGP/7xV6P9f86jjM= github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring v0.83.0/go.mod h1:sSxwdmprUfmRfTknPc4KIjUd2ZIc/kirw4UdXNhOauM= github.com/prometheus-operator/prometheus-operator/pkg/client v0.83.0 h1:odshP0+Jo6iUNGpK8MOFA6p5Yj0QOV4yLgiqFU5MVuI= github.com/prometheus-operator/prometheus-operator/pkg/client v0.83.0/go.mod h1:6Ndhfow0psSp7dV1qp9zK5h++CDKz4eSFWPbrHd5Iic= -github.com/prometheus/client_golang v1.22.0 h1:rb93p9lokFEsctTys46VnV1kLCDpVZ0a/Y92Vm0Zc6Q= -github.com/prometheus/client_golang v1.22.0/go.mod h1:R7ljNsLXhuQXYZYtw6GAE9AZg8Y7vEW5scdCXrWRXC0= -github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= -github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= -github.com/prometheus/common v0.62.0 h1:xasJaQlnWAeyHdUBeGjXmutelfJHWMRr+Fg4QszZ2Io= -github.com/prometheus/common v0.62.0/go.mod h1:vyBcEuLSvWos9B1+CyL7JZ2up+uFzXhkqml0W5zIY1I= -github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= -github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= +github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o= +github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UHKeFTEQ1YCr+0Gyqmg= +github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk= +github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE= +github.com/prometheus/common v0.66.1 h1:h5E0h5/Y8niHc5DlaLlWLArTQI7tMrsfQjHV+d9ZoGs= +github.com/prometheus/common v0.66.1/go.mod h1:gcaUsgf3KfRSwHY4dIMXLPV0K/Wg1oZ8+SbZk/HH/dA= +github.com/prometheus/procfs v0.16.1 h1:hZ15bTNuirocR6u0JZ6BAHHmwS1p8B4P6MRqxtzMyRg= +github.com/prometheus/procfs v0.16.1/go.mod h1:teAbpZRB1iIAJYREa1LsoWUXykVXA1KlTmWl8x/U+Is= github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM= -github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y= -github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= -github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/cobra v1.10.0 h1:a5/WeUlSDCvV5a45ljW2ZFtV0bTDpkfSAj3uqB6Sc+0= +github.com/spf13/cobra v1.10.0/go.mod h1:9dhySC7dnTtEiqzmqfkLj47BslqLCUPMXjG2lj/NgoE= +github.com/spf13/pflag v1.0.8/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/pflag v1.0.9 h1:9exaQaMOCwffKiiiYk6/BndUBv+iRViNW+4lEMi0PvY= +github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stoewer/go-strcase v1.3.0 h1:g0eASXYtp+yvN9fK8sH94oCIk0fau9uV1/ZdJ0AVEzs= github.com/stoewer/go-strcase v1.3.0/go.mod h1:fAH5hQ5pehh+j3nZfvwdk2RgEgQjAoM8wodgtPmh1xo= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -131,20 +149,26 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/tidwall/gjson v1.18.0 h1:FIDeeyB800efLX89e5a8Y0BNH+LOngJyGrIWxG2FKQY= +github.com/tidwall/gjson v1.18.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= +github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= +github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= +github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4= +github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= +github.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY= +github.com/tidwall/sjson v1.2.5/go.mod h1:Fvgq9kS/6ociJEDnK0Fk1cpYF4FIW6ZF7LAe+6jwd28= github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= -github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.58.0 h1:yd02MEjBdJkG3uabWP9apV+OuWRIXGDuJEUJbOHmCFU= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.58.0/go.mod h1:umTcuxiv1n/s/S6/c2AT/g2CQ7u5C59sHDNmfSwgz7Q= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0 h1:F7Jx+6hwnZ41NSFTO5q4LYDtJRXBf2PD0rNBkeB/lus= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0/go.mod h1:UHB22Z8QsdRDrnAtX4PntOl36ajSxcdUMt1sF7Y6E7Q= go.opentelemetry.io/otel v1.44.0 h1:JjwHmHpA4iZ3wBxluu2fbbE7j4kqlE8jXyAyPXH7HqU= go.opentelemetry.io/otel v1.44.0/go.mod h1:BMgjTHL9WPRlRjL2oZCBTL4whCGtXch2H4BhOPIAyYc= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.33.0 h1:Vh5HayB/0HHfOQA7Ctx69E/Y/DcQSMPpKANYVMQ7fBA= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.33.0/go.mod h1:cpgtDBaqD/6ok/UG0jT15/uKjAY8mRA53diogHBg3UI= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.33.0 h1:5pojmb1U1AogINhN3SurB+zm/nIcusopeBNp42f45QM= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.33.0/go.mod h1:57gTHJSE5S1tqg+EKsLPlTWhpHMsWlVmer+LA926XiA= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.34.0 h1:OeNbIYk/2C15ckl7glBlOBp5+WlYsOElzTNmiPW/x60= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.34.0/go.mod h1:7Bept48yIeqxP2OZ9/AqIpYS94h2or0aB4FypJTc8ZM= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.34.0 h1:tgJ0uaNS4c98WRNUEx5U3aDlrDOI5Rs+1Vifcw4DJ8U= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.34.0/go.mod h1:U7HYyW0zt/a9x5J1Kjs+r1f/d4ZHnYFclhYY2+YbeoE= go.opentelemetry.io/otel/metric v1.44.0 h1:1w0gILTcHdr3YI+ixLyjemwrVnsMURbTZFrSYCdDdmc= go.opentelemetry.io/otel/metric v1.44.0/go.mod h1:8O7hanEPBNgEMmybD3s2VBKcgWOCsA6tzHBPODAiquo= go.opentelemetry.io/otel/sdk v1.44.0 h1:nHYwb9lK+fJPU/dnT6s7W7Z8itMWyqrnVfbheVYrZ58= @@ -153,57 +177,38 @@ go.opentelemetry.io/otel/sdk/metric v1.44.0 h1:3LlKgI+VjbVsjNRFZJZAJ30WjXC5VkNRk go.opentelemetry.io/otel/sdk/metric v1.44.0/go.mod h1:5B5pMARnXxKhltooO4xUuCBorl65a4EpnTalObqOigA= go.opentelemetry.io/otel/trace v1.44.0 h1:jxF5CsGYCe74MCRx2X4g7WsY/VBKRqqpNvXlX/6gtIk= go.opentelemetry.io/otel/trace v1.44.0/go.mod h1:oLl1jrMQAVo6v3GAggN+1VH9VIz9iUSvW53sW1Q8PIE= -go.opentelemetry.io/proto/otlp v1.4.0 h1:TA9WRvW6zMwP+Ssb6fLoUIuirti1gGbP28GcKG1jgeg= -go.opentelemetry.io/proto/otlp v1.4.0/go.mod h1:PPBWZIP98o2ElSqI35IHfu7hIhSwvc5N38Jw8pXuGFY= +go.opentelemetry.io/proto/otlp v1.5.0 h1:xJvq7gMzB31/d406fB8U5CBdyQGw4P399D1aQWU/3i4= +go.opentelemetry.io/proto/otlp v1.5.0/go.mod h1:keN8WnHxOy8PG0rQZjJJ5A2ebUoafqWp0eVQ4yIXvJ4= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= -golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +go.yaml.in/yaml/v2 v2.4.3 h1:6gvOSjQoTB3vt1l+CU+tSyi/HOjfOjRLJ4YwYZGwRO0= +go.yaml.in/yaml/v2 v2.4.3/go.mod h1:zSxWcmIDjOzPXpjlTTbAsKokqkDNAVtZO0WOMiT90s8= +go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= +go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 h1:2dVuKD2vS7b0QIHQbpyTISPd0LeHDbnYEryqj5Q1ug8= golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56/go.mod h1:M4RDyNAINzryxdtnbRXRL/OHtkFuWGRjvuhBJpk2IlY= -golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/mod v0.35.0 h1:Ww1D637e6Pg+Zb2KrWfHQUnH2dQRLBQyAtpr/haaJeM= +golang.org/x/mod v0.35.0/go.mod h1:+GwiRhIInF8wPm+4AoT6L0FA1QWAad3OMdTRx4tFYlU= golang.org/x/net v0.55.0 h1:bcvxaJn3e1U6InsFWt1JUq1aSjnRxLzT2rtD2KfkDF8= golang.org/x/net v0.55.0/go.mod h1:L5U2KuzuOe1lY7Z+aWVIKK6qEeJXnXV9yzGA+WCHJww= golang.org/x/oauth2 v0.34.0 h1:hqK/t4AKgbqWkdkcAeI8XLmbK+4m4G5YeQRrmiotGlw= golang.org/x/oauth2 v0.34.0/go.mod h1:lzm5WQJQwKZ3nwavOZ3IS5Aulzxi68dUSgRHujetwEA= -golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4= golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= -golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.45.0 h1:dO4czNzziLiiXplLQgBCEpCvXQ3dnkn0SdaZSYdQ+FY= golang.org/x/sys v0.45.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/term v0.43.0 h1:S4RLU2sB31O/NCl+zFN9Aru9A/Cq2aqKpTZJ6B+DwT4= golang.org/x/term v0.43.0/go.mod h1:lrhlHNdQJHO+1qVYiHfFKVuVioJIheAc3fBSMFYEIsk= -golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.37.0 h1:Cqjiwd9eSg8e0QAkyCaQTNHFIIzWtidPahFWR83rTrc= golang.org/x/text v0.37.0/go.mod h1:a5sjxXGs9hsn/AJVwuElvCAo9v8QYLzvavO5z2PiM38= golang.org/x/time v0.11.0 h1:/bpjEDfN9tkoN/ryeYHnv5hcMlc8ncjMcM4XBk5NWV0= golang.org/x/time v0.11.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg= -golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.44.0 h1:UP4ajHPIcuMjT1GqzDWRlalUEoY+uzoZKnhOjbIPD2c= golang.org/x/tools v0.44.0/go.mod h1:KA0AfVErSdxRZIsOVipbv3rQhVXTnlU6UhKxHd1seDI= -golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gomodules.xyz/jsonpatch/v2 v2.4.0 h1:Ci3iUJyx9UeRx7CeFN8ARgGbkESwJK+KB9lLcWxY/Zw= gomodules.xyz/jsonpatch/v2 v2.4.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuBJT4moAY= gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk= @@ -219,41 +224,40 @@ google.golang.org/protobuf v1.36.10/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= -gopkg.in/evanphx/json-patch.v4 v4.12.0 h1:n6jtcsulIzXPJaxegRbvFNNrZDjbij7ny3gmSPG+6V4= -gopkg.in/evanphx/json-patch.v4 v4.12.0/go.mod h1:p8EYWUEYMpynmqDbY58zCKCFZw8pRWMG4EsWvDvM72M= +gopkg.in/evanphx/json-patch.v4 v4.13.0 h1:czT3CmqEaQ1aanPc5SdlgQrrEIb8w/wwCvWWnfEbYzo= +gopkg.in/evanphx/json-patch.v4 v4.13.0/go.mod h1:p8EYWUEYMpynmqDbY58zCKCFZw8pRWMG4EsWvDvM72M= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -k8s.io/api v0.33.1 h1:tA6Cf3bHnLIrUK4IqEgb2v++/GYUtqiu9sRVk3iBXyw= -k8s.io/api v0.33.1/go.mod h1:87esjTn9DRSRTD4fWMXamiXxJhpOIREjWOSjsW1kEHw= -k8s.io/apiextensions-apiserver v0.33.1 h1:N7ccbSlRN6I2QBcXevB73PixX2dQNIW0ZRuguEE91zI= -k8s.io/apiextensions-apiserver v0.33.1/go.mod h1:uNQ52z1A1Gu75QSa+pFK5bcXc4hq7lpOXbweZgi4dqA= -k8s.io/apimachinery v0.33.1 h1:mzqXWV8tW9Rw4VeW9rEkqvnxj59k1ezDUl20tFK/oM4= -k8s.io/apimachinery v0.33.1/go.mod h1:BHW0YOu7n22fFv/JkYOEfkUYNRN0fj0BlvMFWA7b+SM= -k8s.io/apiserver v0.33.1 h1:yLgLUPDVC6tHbNcw5uE9mo1T6ELhJj7B0geifra3Qdo= -k8s.io/apiserver v0.33.1/go.mod h1:VMbE4ArWYLO01omz+k8hFjAdYfc3GVAYPrhP2tTKccs= -k8s.io/client-go v0.33.1 h1:ZZV/Ks2g92cyxWkRRnfUDsnhNn28eFpt26aGc8KbXF4= -k8s.io/client-go v0.33.1/go.mod h1:JAsUrl1ArO7uRVFWfcj6kOomSlCv+JpvIsp6usAGefA= -k8s.io/component-base v0.33.1 h1:EoJ0xA+wr77T+G8p6T3l4efT2oNwbqBVKR71E0tBIaI= -k8s.io/component-base v0.33.1/go.mod h1:guT/w/6piyPfTgq7gfvgetyXMIh10zuXA6cRRm3rDuY= +k8s.io/api v0.35.2 h1:tW7mWc2RpxW7HS4CoRXhtYHSzme1PN1UjGHJ1bdrtdw= +k8s.io/api v0.35.2/go.mod h1:7AJfqGoAZcwSFhOjcGM7WV05QxMMgUaChNfLTXDRE60= +k8s.io/apiextensions-apiserver v0.35.1 h1:p5vvALkknlOcAqARwjS20kJffgzHqwyQRM8vHLwgU7w= +k8s.io/apiextensions-apiserver v0.35.1/go.mod h1:2CN4fe1GZ3HMe4wBr25qXyJnJyZaquy4nNlNmb3R7AQ= +k8s.io/apimachinery v0.35.2 h1:NqsM/mmZA7sHW02JZ9RTtk3wInRgbVxL8MPfzSANAK8= +k8s.io/apimachinery v0.35.2/go.mod h1:jQCgFZFR1F4Ik7hvr2g84RTJSZegBc8yHgFWKn//hns= +k8s.io/apiserver v0.35.1 h1:potxdhhTL4i6AYAa2QCwtlhtB1eCdWQFvJV6fXgJzxs= +k8s.io/apiserver v0.35.1/go.mod h1:BiL6Dd3A2I/0lBnteXfWmCFobHM39vt5+hJQd7Lbpi4= +k8s.io/client-go v0.35.2 h1:YUfPefdGJA4aljDdayAXkc98DnPkIetMl4PrKX97W9o= +k8s.io/client-go v0.35.2/go.mod h1:4QqEwh4oQpeK8AaefZ0jwTFJw/9kIjdQi0jpKeYvz7g= +k8s.io/component-base v0.35.1 h1:XgvpRf4srp037QWfGBLFsYMUQJkE5yMa94UsJU7pmcE= +k8s.io/component-base v0.35.1/go.mod h1:HI/6jXlwkiOL5zL9bqA3en1Ygv60F03oEpnuU1G56Bs= k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk= k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= -k8s.io/kube-openapi v0.0.0-20250318190949-c8a335a9a2ff h1:/usPimJzUKKu+m+TE36gUyGcf03XZEP0ZIKgKj35LS4= -k8s.io/kube-openapi v0.0.0-20250318190949-c8a335a9a2ff/go.mod h1:5jIi+8yX4RIb8wk3XwBo5Pq2ccx4FP10ohkbSKCZoK8= -k8s.io/utils v0.0.0-20250502105355-0f33e8f1c979 h1:jgJW5IePPXLGB8e/1wvd0Ich9QE97RvvF3a8J3fP/Lg= -k8s.io/utils v0.0.0-20250502105355-0f33e8f1c979/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +k8s.io/kube-openapi v0.0.0-20250910181357-589584f1c912 h1:Y3gxNAuB0OBLImH611+UDZcmKS3g6CthxToOb37KgwE= +k8s.io/kube-openapi v0.0.0-20250910181357-589584f1c912/go.mod h1:kdmbQkyfwUagLfXIad1y2TdrjPFWp2Q89B3qkRwf/pQ= +k8s.io/utils v0.0.0-20260210185600-b8788abfbbc2 h1:AZYQSJemyQB5eRxqcPky+/7EdBj0xi3g0ZcxxJ7vbWU= +k8s.io/utils v0.0.0-20260210185600-b8788abfbbc2/go.mod h1:xDxuJ0whA3d0I4mf/C4ppKHxXynQ+fxnkmQH0vTHnuk= sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.31.2 h1:jpcvIRr3GLoUoEKRkHKSmGjxb6lWwrBlJsXc+eUYQHM= sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.31.2/go.mod h1:Ve9uj1L+deCXFrPOk1LpFXqTg7LCFzFso6PA48q/XZw= -sigs.k8s.io/controller-runtime v0.21.0 h1:CYfjpEuicjUecRk+KAeyYh+ouUBn4llGyDYytIGcJS8= -sigs.k8s.io/controller-runtime v0.21.0/go.mod h1:OSg14+F65eWqIu4DceX7k/+QRAbTTvxeQSNSOQpukWM= -sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8 h1:gBQPwqORJ8d8/YNZWEjoZs7npUVDpVXUUOFfW6CgAqE= -sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8/go.mod h1:mdzfpAEoE6DHQEN0uh9ZbOCuHbLK5wOm7dK4ctXE9Tg= -sigs.k8s.io/randfill v0.0.0-20250304075658-069ef1bbf016/go.mod h1:XeLlZ/jmk4i1HRopwe7/aU3H5n1zNUcX6TM94b3QxOY= +sigs.k8s.io/controller-runtime v0.23.3 h1:VjB/vhoPoA9l1kEKZHBMnQF33tdCLQKJtydy4iqwZ80= +sigs.k8s.io/controller-runtime v0.23.3/go.mod h1:B6COOxKptp+YaUT5q4l6LqUJTRpizbgf9KSRNdQGns0= +sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 h1:IpInykpT6ceI+QxKBbEflcR5EXP7sU1kvOlxwZh5txg= +sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730/go.mod h1:mdzfpAEoE6DHQEN0uh9ZbOCuHbLK5wOm7dK4ctXE9Tg= sigs.k8s.io/randfill v1.0.0 h1:JfjMILfT8A6RbawdsK2JXGBR5AQVfd+9TbzrlneTyrU= sigs.k8s.io/randfill v1.0.0/go.mod h1:XeLlZ/jmk4i1HRopwe7/aU3H5n1zNUcX6TM94b3QxOY= -sigs.k8s.io/structured-merge-diff/v4 v4.7.0 h1:qPeWmscJcXP0snki5IYF79Z8xrl8ETFxgMd7wez1XkI= -sigs.k8s.io/structured-merge-diff/v4 v4.7.0/go.mod h1:dDy58f92j70zLsuZVuUX5Wp9vtxXpaZnkPGWeqDfCps= -sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E= -sigs.k8s.io/yaml v1.4.0/go.mod h1:Ejl7/uTz7PSA4eKMyQCUTnhZYNmLIl+5c2lQPGR2BPY= +sigs.k8s.io/structured-merge-diff/v6 v6.3.2-0.20260122202528-d9cc6641c482 h1:2WOzJpHUBVrrkDjU4KBT8n5LDcj824eX0I5UKcgeRUs= +sigs.k8s.io/structured-merge-diff/v6 v6.3.2-0.20260122202528-d9cc6641c482/go.mod h1:M3W8sfWvn2HhQDIbGWj3S099YozAsymCo/wrT5ohRUE= +sigs.k8s.io/yaml v1.6.0 h1:G8fkbMSAFqgEFgh4b1wmtzDnioxFCUgTZhlbj5P9QYs= +sigs.k8s.io/yaml v1.6.0/go.mod h1:796bPqUfzR/0jLAl6XjHl3Ck7MiyVv8dbTdyT3/pMf4= diff --git a/infra/feast-operator/internal/controller/featurestore_controller.go b/infra/feast-operator/internal/controller/featurestore_controller.go index ae877447ddb..b94808c0df5 100644 --- a/infra/feast-operator/internal/controller/featurestore_controller.go +++ b/infra/feast-operator/internal/controller/featurestore_controller.go @@ -60,14 +60,19 @@ type FeatureStoreReconciler struct { Metrics *feastmetrics.FeatureStoreMetrics } +// +kubebuilder:rbac:groups=config.openshift.io,resources=apiservers,verbs=get;list;watch // +kubebuilder:rbac:groups=feast.dev,resources=featurestores,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=feast.dev,resources=featurestores/status,verbs=get;update;patch // +kubebuilder:rbac:groups=feast.dev,resources=featurestores/finalizers,verbs=update // +kubebuilder:rbac:groups=apps,resources=deployments,verbs=get;list;create;update;watch;delete -// +kubebuilder:rbac:groups=core,resources=services;configmaps;persistentvolumeclaims;serviceaccounts,verbs=get;list;create;update;watch;delete +// +kubebuilder:rbac:groups=core,resources=services;configmaps;persistentvolumeclaims,verbs=get;list;create;update;watch;delete;deletecollection +// +kubebuilder:rbac:groups=core,resources=serviceaccounts,verbs=get;list;create;update;watch;delete // +kubebuilder:rbac:groups=rbac.authorization.k8s.io,resources=roles;rolebindings;clusterroles;clusterrolebindings;subjectaccessreviews,verbs=get;list;create;update;watch;delete -// +kubebuilder:rbac:groups=core,resources=secrets;pods;namespaces,verbs=get;list;watch +// +kubebuilder:rbac:groups=core,resources=secrets;namespaces,verbs=get;list;watch +// +kubebuilder:rbac:groups=core,resources=pods,verbs=get;list;watch;create;delete;deletecollection // +kubebuilder:rbac:groups=core,resources=pods/exec,verbs=create +// +kubebuilder:rbac:groups=core,resources=pods/log,verbs=get +// +kubebuilder:rbac:groups=sparkoperator.k8s.io,resources=sparkapplications,verbs=create;get;delete // +kubebuilder:rbac:groups=authentication.k8s.io,resources=tokenreviews,verbs=create // +kubebuilder:rbac:groups=route.openshift.io,resources=routes,verbs=get;list;create;update;watch;delete // +kubebuilder:rbac:groups=batch,resources=cronjobs,verbs=get;list;watch;create;update;patch;delete diff --git a/infra/feast-operator/internal/controller/featurestore_controller_oidc_auth_test.go b/infra/feast-operator/internal/controller/featurestore_controller_oidc_auth_test.go index e15f8ecfa8a..0f99f6d0479 100644 --- a/infra/feast-operator/internal/controller/featurestore_controller_oidc_auth_test.go +++ b/infra/feast-operator/internal/controller/featurestore_controller_oidc_auth_test.go @@ -493,6 +493,8 @@ func expectedServerOidcAuthorizConfig() services.AuthzConfig { string(services.OidcClientSecret): "client-secret", string(services.OidcUsername): "username", string(services.OidcPassword): "password", + string(services.OidcAudience): "api://feast-feature-server", + string(services.OidcIssuer): "https://keycloak.example.com/realms/test", }, } } @@ -509,6 +511,8 @@ func validOidcSecretMap() map[string]string { string(services.OidcClientSecret): "client-secret", string(services.OidcUsername): "username", string(services.OidcPassword): "password", + string(services.OidcAudience): "api://feast-feature-server", + string(services.OidcIssuer): "https://keycloak.example.com/realms/test", } } diff --git a/infra/feast-operator/internal/controller/featurestore_controller_packaged_test.go b/infra/feast-operator/internal/controller/featurestore_controller_packaged_test.go new file mode 100644 index 00000000000..9c534a624bd --- /dev/null +++ b/infra/feast-operator/internal/controller/featurestore_controller_packaged_test.go @@ -0,0 +1,237 @@ +/* +Copyright 2026 Feast Community. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller + +import ( + "context" + + feastdevv1 "github.com/feast-dev/feast/infra/feast-operator/api/v1" + "github.com/feast-dev/feast/infra/feast-operator/internal/controller/handler" + "github.com/feast-dev/feast/infra/feast-operator/internal/controller/services" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/reconcile" +) + +var _ = Describe("Packaged feature repositories", func() { + const ( + resourceName = "packaged-feature-repo" + packagedImage = "registry.example.com/feature-server@sha256:0123456789abcdef" + packagedRepoDir = "/opt/feast/feature_repo" + ) + + ctx := context.Background() + key := types.NamespacedName{Name: resourceName, Namespace: "default"} + + newFeatureStore := func() *feastdevv1.FeatureStore { + return &feastdevv1.FeatureStore{ + ObjectMeta: metav1.ObjectMeta{Name: key.Name, Namespace: key.Namespace}, + Spec: feastdevv1.FeatureStoreSpec{ + FeastProject: feastProject, + FeastProjectDir: &feastdevv1.FeastProjectDir{ + Packaged: &feastdevv1.FeastPackagedOptions{ + Image: packagedImage, + FeatureRepoPath: packagedRepoDir, + }, + }, + }, + } + } + + reconcileFeatureStore := func() (*feastdevv1.FeatureStore, *appsv1.Deployment) { + reconciler := &FeatureStoreReconciler{Client: k8sClient, Scheme: k8sClient.Scheme()} + _, err := reconciler.Reconcile(ctx, reconcile.Request{NamespacedName: key}) + Expect(err).NotTo(HaveOccurred()) + + featureStore := &feastdevv1.FeatureStore{} + Expect(k8sClient.Get(ctx, key, featureStore)).To(Succeed()) + feastServices := services.FeastServices{ + Handler: handler.FeastHandler{ + Client: k8sClient, + Context: ctx, + Scheme: k8sClient.Scheme(), + FeatureStore: featureStore, + }, + } + deployment := &appsv1.Deployment{} + meta := feastServices.GetObjectMeta() + Expect(k8sClient.Get(ctx, types.NamespacedName{Name: meta.Name, Namespace: meta.Namespace}, deployment)).To(Succeed()) + return featureStore, deployment + } + + BeforeEach(func() { + Expect(k8sClient.Create(ctx, newFeatureStore())).To(Succeed()) + }) + + AfterEach(func() { + featureStore := &feastdevv1.FeatureStore{} + Expect(k8sClient.Get(ctx, key, featureStore)).To(Succeed()) + Expect(k8sClient.Delete(ctx, featureStore)).To(Succeed()) + }) + + It("stages the packaged repository and applies it from the shared directory", func() { + featureStore, deployment := reconcileFeatureStore() + + canonicalRepoDir := services.EphemeralPath + "/" + feastProject + "/" + services.FeatureRepoDir + Expect(deployment.Spec.Template.Spec.InitContainers).To(HaveLen(2)) + initContainer := deployment.Spec.Template.Spec.InitContainers[0] + Expect(initContainer.Name).To(Equal("feast-init")) + Expect(initContainer.Image).To(Equal(packagedImage)) + Expect(initContainer.WorkingDir).To(Equal(services.EphemeralPath)) + Expect(initContainer.Env).To(ContainElements( + corev1.EnvVar{Name: "FEAST_PACKAGED_FEATURE_REPO_PATH", Value: packagedRepoDir}, + corev1.EnvVar{Name: "FEAST_STAGED_FEATURE_REPO_PATH", Value: canonicalRepoDir}, + )) + Expect(initContainer.Args).To(HaveLen(1)) + Expect(initContainer.Args[0]).To(ContainSubstring(`rm -rf -- "${FEAST_STAGED_FEATURE_REPO_PATH}"`)) + Expect(initContainer.Args[0]).To(ContainSubstring(`cp -a -- "${FEAST_PACKAGED_FEATURE_REPO_PATH}/." "${FEAST_STAGED_FEATURE_REPO_PATH}/"`)) + Expect(initContainer.Args[0]).To(ContainSubstring(`printf '%s' "${TMP_FEATURE_STORE_YAML_BASE64}" | base64 -d`)) + Expect(initContainer.Args[0]).To(ContainSubstring(`"${FEAST_STAGED_FEATURE_REPO_PATH}/feature_store.yaml"`)) + + applyContainer := deployment.Spec.Template.Spec.InitContainers[1] + Expect(applyContainer.Name).To(Equal("feast-apply")) + Expect(applyContainer.Image).To(Equal(packagedImage)) + Expect(applyContainer.Command).To(Equal([]string{"feast", "apply"})) + Expect(applyContainer.WorkingDir).To(Equal(canonicalRepoDir)) + + online := services.GetOnlineContainer(*deployment) + Expect(online.Image).To(Equal(packagedImage)) + Expect(online.WorkingDir).To(Equal(canonicalRepoDir)) + Expect(*featureStore.Status.Applied.Services.OnlineStore.Server.Image).To(Equal(packagedImage)) + }) + + It("supports staging without applying and direct use of the baked repository", func() { + featureStore := &feastdevv1.FeatureStore{} + Expect(k8sClient.Get(ctx, key, featureStore)).To(Succeed()) + featureStore.Spec.Services = &feastdevv1.FeatureStoreServices{RunFeastApplyOnInit: ptr(false)} + Expect(k8sClient.Update(ctx, featureStore)).To(Succeed()) + + featureStore, deployment := reconcileFeatureStore() + Expect(deployment.Spec.Template.Spec.InitContainers).To(HaveLen(1)) + Expect(deployment.Spec.Template.Spec.InitContainers[0].Name).To(Equal("feast-init")) + + featureStore.Spec.Services.DisableInitContainers = true + Expect(k8sClient.Update(ctx, featureStore)).To(Succeed()) + _, deployment = reconcileFeatureStore() + Expect(deployment.Spec.Template.Spec.InitContainers).To(BeEmpty()) + online := services.GetOnlineContainer(*deployment) + Expect(online.Image).To(Equal(packagedImage)) + Expect(online.WorkingDir).To(Equal(packagedRepoDir)) + }) + + It("keeps explicit service images ahead of the packaged image", func() { + const serviceImage = "registry.example.com/online-server:custom" + featureStore := &feastdevv1.FeatureStore{} + Expect(k8sClient.Get(ctx, key, featureStore)).To(Succeed()) + featureStore.Spec.Services = &feastdevv1.FeatureStoreServices{ + OnlineStore: &feastdevv1.OnlineStore{ + Server: &feastdevv1.ServerConfigs{ + ContainerConfigs: feastdevv1.ContainerConfigs{ + DefaultCtrConfigs: feastdevv1.DefaultCtrConfigs{Image: ptr(serviceImage)}, + }, + }, + }, + } + Expect(k8sClient.Update(ctx, featureStore)).To(Succeed()) + + _, deployment := reconcileFeatureStore() + Expect(deployment.Spec.Template.Spec.InitContainers[0].Image).To(Equal(packagedImage)) + Expect(services.GetOnlineContainer(*deployment).Image).To(Equal(serviceImage)) + }) + + It("keeps an explicit init image ahead of the packaged image", func() { + const initImage = "registry.example.com/feast-init:custom" + featureStore := &feastdevv1.FeatureStore{} + Expect(k8sClient.Get(ctx, key, featureStore)).To(Succeed()) + featureStore.Spec.Services = &feastdevv1.FeatureStoreServices{ + InitImage: ptr(initImage), + } + Expect(k8sClient.Update(ctx, featureStore)).To(Succeed()) + + _, deployment := reconcileFeatureStore() + Expect(deployment.Spec.Template.Spec.InitContainers).To(HaveLen(2)) + Expect(deployment.Spec.Template.Spec.InitContainers[0].Image).To(Equal(initImage)) + Expect(deployment.Spec.Template.Spec.InitContainers[1].Image).To(Equal(initImage)) + Expect(services.GetOnlineContainer(*deployment).Image).To(Equal(packagedImage)) + }) + + It("supports path-only direct mode with an explicit service image", func() { + const serviceImage = "registry.example.com/online-server:air-gapped" + featureStore := &feastdevv1.FeatureStore{} + Expect(k8sClient.Get(ctx, key, featureStore)).To(Succeed()) + featureStore.Spec.FeastProjectDir.Packaged.Image = "" + featureStore.Spec.Services = &feastdevv1.FeatureStoreServices{ + DisableInitContainers: true, + OnlineStore: &feastdevv1.OnlineStore{ + Server: &feastdevv1.ServerConfigs{ + ContainerConfigs: feastdevv1.ContainerConfigs{ + DefaultCtrConfigs: feastdevv1.DefaultCtrConfigs{Image: ptr(serviceImage)}, + }, + }, + }, + } + Expect(k8sClient.Update(ctx, featureStore)).To(Succeed()) + + _, deployment := reconcileFeatureStore() + Expect(deployment.Spec.Template.Spec.InitContainers).To(BeEmpty()) + online := services.GetOnlineContainer(*deployment) + Expect(online.Image).To(Equal(serviceImage)) + Expect(online.WorkingDir).To(Equal(packagedRepoDir)) + }) + + It("retains the operator image fallback when the packaged image is omitted", func() { + featureStore := &feastdevv1.FeatureStore{} + Expect(k8sClient.Get(ctx, key, featureStore)).To(Succeed()) + featureStore.Spec.FeastProjectDir.Packaged.Image = "" + Expect(k8sClient.Update(ctx, featureStore)).To(Succeed()) + + _, deployment := reconcileFeatureStore() + initImage := deployment.Spec.Template.Spec.InitContainers[0].Image + Expect(initImage).NotTo(BeEmpty()) + Expect(services.GetOnlineContainer(*deployment).Image).To(Equal(initImage)) + }) + + DescribeTable("rejects packaged and staged repository path overlap", + func(featureRepoPath string) { + featureStore := &feastdevv1.FeatureStore{} + Expect(k8sClient.Get(ctx, key, featureStore)).To(Succeed()) + featureStore.Spec.FeastProjectDir.Packaged.FeatureRepoPath = featureRepoPath + Expect(k8sClient.Update(ctx, featureStore)).To(Succeed()) + + reconciler := &FeatureStoreReconciler{Client: k8sClient, Scheme: k8sClient.Scheme()} + _, err := reconciler.Reconcile(ctx, reconcile.Request{NamespacedName: key}) + Expect(err).To(MatchError(ContainSubstring("overlaps staged repository path"))) + }, + Entry("equal paths", services.EphemeralPath+"/"+feastProject+"/"+services.FeatureRepoDir), + Entry("packaged path is an ancestor", services.EphemeralPath+"/"+feastProject), + Entry("packaged path is a descendant", services.EphemeralPath+"/"+feastProject+"/"+services.FeatureRepoDir+"/baked"), + ) + + It("allows similar path prefixes that do not overlap", func() { + featureStore := &feastdevv1.FeatureStore{} + Expect(k8sClient.Get(ctx, key, featureStore)).To(Succeed()) + featureStore.Spec.FeastProjectDir.Packaged.FeatureRepoPath = + services.EphemeralPath + "/" + feastProject + "/" + services.FeatureRepoDir + "-image" + Expect(k8sClient.Update(ctx, featureStore)).To(Succeed()) + + reconcileFeatureStore() + }) +}) diff --git a/infra/feast-operator/internal/controller/featurestore_controller_test.go b/infra/feast-operator/internal/controller/featurestore_controller_test.go index a9ac5235eda..712644f7a0c 100644 --- a/infra/feast-operator/internal/controller/featurestore_controller_test.go +++ b/infra/feast-operator/internal/controller/featurestore_controller_test.go @@ -299,6 +299,41 @@ var _ = Describe("FeatureStore Controller", func() { Expect(err).NotTo(HaveOccurred()) Expect(deploy.Spec.Template.Spec.InitContainers).To(HaveLen(2)) Expect(deploy.Spec.Template.Spec.InitContainers[0].Args[0]).To(ContainSubstring("feast init -t spark")) + + // initImage is independent of server images: init containers use initImage, + // main containers keep their own server.image. + initImage := "quay.io/org/feast-init:custom" + serverImage := "quay.io/org/feast-online:server" + if resource.Spec.Services == nil { + resource.Spec.Services = &feastdevv1.FeatureStoreServices{} + } + resource.Spec.Services.InitImage = &initImage + if resource.Spec.Services.OnlineStore == nil { + resource.Spec.Services.OnlineStore = &feastdevv1.OnlineStore{} + } + if resource.Spec.Services.OnlineStore.Server == nil { + resource.Spec.Services.OnlineStore.Server = &feastdevv1.ServerConfigs{} + } + resource.Spec.Services.OnlineStore.Server.Image = &serverImage + err = k8sClient.Update(ctx, resource) + Expect(err).NotTo(HaveOccurred()) + _, err = controllerReconciler.Reconcile(ctx, reconcile.Request{ + NamespacedName: typeNamespacedName, + }) + Expect(err).NotTo(HaveOccurred()) + + err = k8sClient.Get(ctx, types.NamespacedName{ + Name: objMeta.Name, + Namespace: objMeta.Namespace, + }, deploy) + Expect(err).NotTo(HaveOccurred()) + Expect(deploy.Spec.Template.Spec.InitContainers).To(HaveLen(2)) + Expect(deploy.Spec.Template.Spec.InitContainers[0].Image).To(Equal(initImage)) + Expect(deploy.Spec.Template.Spec.InitContainers[1].Image).To(Equal(initImage)) + online = services.GetOnlineContainer(*deploy) + Expect(online).NotTo(BeNil()) + Expect(online.Image).To(Equal(serverImage)) + Expect(online.Image).NotTo(Equal(initImage)) }) It("should properly encode a feature_store.yaml config", func() { diff --git a/infra/feast-operator/internal/controller/services/batch_engine_rbac.go b/infra/feast-operator/internal/controller/services/batch_engine_rbac.go new file mode 100644 index 00000000000..97afcd3b48d --- /dev/null +++ b/infra/feast-operator/internal/controller/services/batch_engine_rbac.go @@ -0,0 +1,285 @@ +/* +Copyright 2024 Feast Community. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package services + +import ( + "embed" + "fmt" + + feastdevv1 "github.com/feast-dev/feast/infra/feast-operator/api/v1" + corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/yaml" +) + +const ( + BatchEngineFeastType FeastServiceType = "batch-engine" + BatchDriverFeastType FeastServiceType = "batch-driver" +) + +//go:embed rbac_templates/*.yaml +var batchEngineRBACTemplates embed.FS + +// BatchEngineRBACTemplate declares RBAC requirements for a batch compute engine. +type BatchEngineRBACTemplate struct { + EngineType string `json:"engine_type" yaml:"engine_type"` + Server *RBACRoleSpec `json:"server,omitempty" yaml:"server,omitempty"` + Driver *DriverRBACSpec `json:"driver,omitempty" yaml:"driver,omitempty"` +} + +// RBACRoleSpec defines policy rules for a Role. +type RBACRoleSpec struct { + Rules []rbacv1.PolicyRule `json:"rules" yaml:"rules"` +} + +// DriverRBACSpec defines policy rules and optional SA creation for a driver Role. +type DriverRBACSpec struct { + CreateServiceAccount bool `json:"create_service_account" yaml:"create_service_account"` + Rules []rbacv1.PolicyRule `json:"rules" yaml:"rules"` +} + +func loadBatchEngineTemplate(engineType string) (*BatchEngineRBACTemplate, error) { + data, err := batchEngineRBACTemplates.ReadFile( + "rbac_templates/" + engineType + ".yaml", + ) + if err != nil { + return nil, nil + } + var tmpl BatchEngineRBACTemplate + if err := yaml.Unmarshal(data, &tmpl); err != nil { + return nil, fmt.Errorf("failed to parse RBAC template for engine %q: %w", engineType, err) + } + return &tmpl, nil +} + +func (feast *FeastServices) reconcileBatchEngineRBAC() error { + config, ok := feast.getBatchEngineConfig() + if !ok { + return feast.deleteBatchEngineRBAC() + } + + engineType, _ := config["type"].(string) + if engineType == "" { + return feast.deleteBatchEngineRBAC() + } + + tmpl, err := loadBatchEngineTemplate(engineType) + if err != nil { + return err + } + if tmpl == nil { + return feast.deleteBatchEngineRBAC() + } + + if tmpl.Server != nil { + if err := feast.ensureBatchEngineRole(BatchEngineFeastType, tmpl.Server.Rules); err != nil { + return err + } + if err := feast.ensureBatchEngineRoleBinding(BatchEngineFeastType, feast.initFeastSA().Name); err != nil { + return err + } + } + + if tmpl.Driver != nil { + driverSAName := resolveBatchDriverSAName(feast.Handler.FeatureStore, config) + if tmpl.Driver.CreateServiceAccount { + if err := feast.ensureBatchDriverServiceAccount(driverSAName); err != nil { + return err + } + } + if err := feast.ensureBatchEngineRole(BatchDriverFeastType, tmpl.Driver.Rules); err != nil { + return err + } + if err := feast.ensureBatchEngineRoleBinding(BatchDriverFeastType, driverSAName); err != nil { + return err + } + } + + return nil +} + +// getBatchEngineConfig returns the parsed batch-engine ConfigMap data. +// ok=false means no batch engine is configured or the ConfigMap is unreadable. +func (feast *FeastServices) getBatchEngineConfig() (map[string]interface{}, bool) { + appliedSpec := feast.Handler.FeatureStore.Status.Applied + if appliedSpec.BatchEngine == nil || appliedSpec.BatchEngine.ConfigMapRef == nil { + return nil, false + } + + configMapKey := appliedSpec.BatchEngine.ConfigMapKey + if configMapKey == "" { + configMapKey = "config" + } + + cm, err := feast.getConfigMap(appliedSpec.BatchEngine.ConfigMapRef.Name) + if err != nil { + return nil, false + } + + data, found := cm.Data[configMapKey] + if !found { + return nil, false + } + + var config map[string]interface{} + if err := yaml.Unmarshal([]byte(data), &config); err != nil { + return nil, false + } + return config, true +} + +// resolveBatchDriverSAName returns the ServiceAccount name for the Spark driver. +// If batch engine config sets a non-empty service_account, that value wins. +// Otherwise defaults to feast--batch-driver (same name used for RBAC). +func resolveBatchDriverSAName(featureStore *feastdevv1.FeatureStore, config map[string]interface{}) string { + if sa, ok := config["service_account"].(string); ok && sa != "" { + return sa + } + return GetFeastServiceName(featureStore, BatchDriverFeastType) +} + +func (feast *FeastServices) ensureBatchEngineRole(feastType FeastServiceType, rules []rbacv1.PolicyRule) error { + logger := log.FromContext(feast.Handler.Context) + role := &rbacv1.Role{ + ObjectMeta: metav1.ObjectMeta{ + Name: feast.GetFeastServiceName(feastType), + Namespace: feast.Handler.FeatureStore.Namespace, + }, + } + role.SetGroupVersionKind(rbacv1.SchemeGroupVersion.WithKind("Role")) + + op, err := controllerutil.CreateOrUpdate(feast.Handler.Context, feast.Handler.Client, role, func() error { + role.Labels = feast.getFeastTypeLabels(feastType) + role.Rules = rules + return controllerutil.SetControllerReference(feast.Handler.FeatureStore, role, feast.Handler.Scheme) + }) + if err != nil { + return err + } + if op == controllerutil.OperationResultCreated || op == controllerutil.OperationResultUpdated { + logger.Info("Successfully reconciled", "Role", role.Name, "operation", op) + } + return nil +} + +func (feast *FeastServices) ensureBatchEngineRoleBinding(feastType FeastServiceType, saName string) error { + logger := log.FromContext(feast.Handler.Context) + roleBinding := &rbacv1.RoleBinding{ + ObjectMeta: metav1.ObjectMeta{ + Name: feast.GetFeastServiceName(feastType), + Namespace: feast.Handler.FeatureStore.Namespace, + }, + } + roleBinding.SetGroupVersionKind(rbacv1.SchemeGroupVersion.WithKind("RoleBinding")) + + op, err := controllerutil.CreateOrUpdate(feast.Handler.Context, feast.Handler.Client, roleBinding, func() error { + roleBinding.Labels = feast.getFeastTypeLabels(feastType) + roleBinding.Subjects = []rbacv1.Subject{{ + Kind: rbacv1.ServiceAccountKind, + Name: saName, + Namespace: feast.Handler.FeatureStore.Namespace, + }} + roleBinding.RoleRef = rbacv1.RoleRef{ + APIGroup: rbacv1.GroupName, + Kind: "Role", + Name: feast.GetFeastServiceName(feastType), + } + return controllerutil.SetControllerReference(feast.Handler.FeatureStore, roleBinding, feast.Handler.Scheme) + }) + if err != nil { + return err + } + if op == controllerutil.OperationResultCreated || op == controllerutil.OperationResultUpdated { + logger.Info("Successfully reconciled", "RoleBinding", roleBinding.Name, "operation", op) + } + return nil +} + +func (feast *FeastServices) ensureBatchDriverServiceAccount(saName string) error { + logger := log.FromContext(feast.Handler.Context) + sa := &corev1.ServiceAccount{ + ObjectMeta: metav1.ObjectMeta{ + Name: saName, + Namespace: feast.Handler.FeatureStore.Namespace, + }, + } + sa.SetGroupVersionKind(corev1.SchemeGroupVersion.WithKind("ServiceAccount")) + + op, err := controllerutil.CreateOrUpdate(feast.Handler.Context, feast.Handler.Client, sa, func() error { + if sa.Labels == nil { + sa.Labels = map[string]string{} + } + for k, v := range feast.getFeastTypeLabels(BatchDriverFeastType) { + sa.Labels[k] = v + } + return controllerutil.SetControllerReference(feast.Handler.FeatureStore, sa, feast.Handler.Scheme) + }) + if err != nil { + return err + } + if op == controllerutil.OperationResultCreated || op == controllerutil.OperationResultUpdated { + logger.Info("Successfully reconciled", "ServiceAccount", sa.Name, "operation", op) + } + return nil +} + +func (feast *FeastServices) deleteBatchEngineRBAC() error { + serverRoleName := feast.GetFeastServiceName(BatchEngineFeastType) + driverRoleName := feast.GetFeastServiceName(BatchDriverFeastType) + ns := feast.Handler.FeatureStore.Namespace + + serverRoleBinding := &rbacv1.RoleBinding{ + ObjectMeta: metav1.ObjectMeta{Name: serverRoleName, Namespace: ns}, + } + serverRoleBinding.SetGroupVersionKind(rbacv1.SchemeGroupVersion.WithKind("RoleBinding")) + if err := feast.Handler.DeleteOwnedFeastObj(serverRoleBinding); err != nil { + return err + } + + serverRole := &rbacv1.Role{ + ObjectMeta: metav1.ObjectMeta{Name: serverRoleName, Namespace: ns}, + } + serverRole.SetGroupVersionKind(rbacv1.SchemeGroupVersion.WithKind("Role")) + if err := feast.Handler.DeleteOwnedFeastObj(serverRole); err != nil { + return err + } + + driverRoleBinding := &rbacv1.RoleBinding{ + ObjectMeta: metav1.ObjectMeta{Name: driverRoleName, Namespace: ns}, + } + driverRoleBinding.SetGroupVersionKind(rbacv1.SchemeGroupVersion.WithKind("RoleBinding")) + if err := feast.Handler.DeleteOwnedFeastObj(driverRoleBinding); err != nil { + return err + } + + driverRole := &rbacv1.Role{ + ObjectMeta: metav1.ObjectMeta{Name: driverRoleName, Namespace: ns}, + } + driverRole.SetGroupVersionKind(rbacv1.SchemeGroupVersion.WithKind("Role")) + if err := feast.Handler.DeleteOwnedFeastObj(driverRole); err != nil { + return err + } + + driverSA := &corev1.ServiceAccount{ + ObjectMeta: metav1.ObjectMeta{Name: driverRoleName, Namespace: ns}, + } + driverSA.SetGroupVersionKind(corev1.SchemeGroupVersion.WithKind("ServiceAccount")) + return feast.Handler.DeleteOwnedFeastObj(driverSA) +} diff --git a/infra/feast-operator/internal/controller/services/batch_engine_rbac_test.go b/infra/feast-operator/internal/controller/services/batch_engine_rbac_test.go new file mode 100644 index 00000000000..9d792c6f923 --- /dev/null +++ b/infra/feast-operator/internal/controller/services/batch_engine_rbac_test.go @@ -0,0 +1,195 @@ +/* +Copyright 2024 Feast Community. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package services + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + rbacv1 "k8s.io/api/rbac/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/yaml" + + feastdevv1 "github.com/feast-dev/feast/infra/feast-operator/api/v1" +) + +var _ = Describe("Batch Engine RBAC", func() { + + Describe("loadBatchEngineTemplate", func() { + It("should load spark_application template", func() { + tmpl, err := loadBatchEngineTemplate("spark_application") + Expect(err).NotTo(HaveOccurred()) + Expect(tmpl).NotTo(BeNil()) + Expect(tmpl.EngineType).To(Equal("spark_application")) + Expect(tmpl.Server).NotTo(BeNil()) + Expect(tmpl.Server.Rules).NotTo(BeEmpty()) + Expect(tmpl.Driver).NotTo(BeNil()) + Expect(tmpl.Driver.CreateServiceAccount).To(BeTrue()) + Expect(tmpl.Driver.Rules).NotTo(BeEmpty()) + }) + + It("should return nil for unknown engine type", func() { + tmpl, err := loadBatchEngineTemplate("nonexistent_engine") + Expect(err).NotTo(HaveOccurred()) + Expect(tmpl).To(BeNil()) + }) + + It("should contain correct server rules for spark_application", func() { + tmpl, err := loadBatchEngineTemplate("spark_application") + Expect(err).NotTo(HaveOccurred()) + + serverRules := tmpl.Server.Rules + Expect(serverRules).To(HaveLen(4)) + + hasConfigMapRule := false + hasSparkAppRule := false + hasPodListRule := false + hasPodLogRule := false + + for _, rule := range serverRules { + if containsResource(rule, "configmaps") && containsVerb(rule, "create") && containsVerb(rule, "delete") { + hasConfigMapRule = true + } + if containsResource(rule, "sparkapplications") && containsVerb(rule, "create") && containsVerb(rule, "get") && containsVerb(rule, "delete") { + hasSparkAppRule = true + } + if containsResource(rule, "pods") && containsVerb(rule, "list") { + hasPodListRule = true + } + if containsResource(rule, "pods/log") && containsVerb(rule, "get") { + hasPodLogRule = true + } + } + + Expect(hasConfigMapRule).To(BeTrue(), "should have configmaps create/delete rule") + Expect(hasSparkAppRule).To(BeTrue(), "should have sparkapplications create/get/delete rule") + Expect(hasPodListRule).To(BeTrue(), "should have pods list rule") + Expect(hasPodLogRule).To(BeTrue(), "should have pods/log get rule") + }) + + It("should contain correct driver rules for spark_application", func() { + tmpl, err := loadBatchEngineTemplate("spark_application") + Expect(err).NotTo(HaveOccurred()) + + driverRules := tmpl.Driver.Rules + Expect(driverRules).To(HaveLen(2)) + + hasPodRule := false + hasResourceRule := false + for _, rule := range driverRules { + if containsResource(rule, "pods") && + containsVerb(rule, "create") && + containsVerb(rule, "deletecollection") { + hasPodRule = true + } + if containsResource(rule, "services") && + containsResource(rule, "configmaps") && + containsResource(rule, "persistentvolumeclaims") && + containsVerb(rule, "deletecollection") { + hasResourceRule = true + } + } + Expect(hasPodRule).To(BeTrue(), "should have pods CRUD + deletecollection rule") + Expect(hasResourceRule).To(BeTrue(), "should have services/configmaps/PVCs CRUD + deletecollection rule") + }) + }) + + Describe("BatchEngineRBACTemplate YAML parsing", func() { + It("should correctly unmarshal a template", func() { + yamlData := ` +engine_type: test_engine +server: + rules: + - apiGroups: [""] + resources: ["secrets"] + verbs: ["get", "list"] +driver: + create_service_account: true + rules: + - apiGroups: [""] + resources: ["pods"] + verbs: ["get"] +` + var tmpl BatchEngineRBACTemplate + err := yaml.Unmarshal([]byte(yamlData), &tmpl) + Expect(err).NotTo(HaveOccurred()) + Expect(tmpl.EngineType).To(Equal("test_engine")) + Expect(tmpl.Server).NotTo(BeNil()) + Expect(tmpl.Server.Rules).To(HaveLen(1)) + Expect(tmpl.Driver).NotTo(BeNil()) + Expect(tmpl.Driver.CreateServiceAccount).To(BeTrue()) + Expect(tmpl.Driver.Rules).To(HaveLen(1)) + }) + + It("should handle server-only template (no driver)", func() { + yamlData := ` +engine_type: server_only +server: + rules: + - apiGroups: ["batch"] + resources: ["jobs"] + verbs: ["create", "delete"] +` + var tmpl BatchEngineRBACTemplate + err := yaml.Unmarshal([]byte(yamlData), &tmpl) + Expect(err).NotTo(HaveOccurred()) + Expect(tmpl.Server).NotTo(BeNil()) + Expect(tmpl.Driver).To(BeNil()) + }) + }) +}) + +var _ = Describe("resolveBatchDriverSAName", func() { + It("defaults to feast--batch-driver when service_account is omitted", func() { + fs := &feastdevv1.FeatureStore{ObjectMeta: metav1.ObjectMeta{Name: "spark-pg-e2e", Namespace: "feast-spark"}} + Expect(resolveBatchDriverSAName(fs, map[string]interface{}{ + "type": "spark_application", + "image": "quay.io/example/driver:v1", + })).To(Equal("feast-spark-pg-e2e-batch-driver")) + }) + + It("defaults when service_account is empty string", func() { + fs := &feastdevv1.FeatureStore{ObjectMeta: metav1.ObjectMeta{Name: "spark-pg-e2e"}} + Expect(resolveBatchDriverSAName(fs, map[string]interface{}{ + "service_account": "", + })).To(Equal("feast-spark-pg-e2e-batch-driver")) + }) + + It("keeps an explicit service_account override", func() { + fs := &feastdevv1.FeatureStore{ObjectMeta: metav1.ObjectMeta{Name: "spark-pg-e2e"}} + Expect(resolveBatchDriverSAName(fs, map[string]interface{}{ + "service_account": "my-custom-driver", + })).To(Equal("my-custom-driver")) + }) +}) + +func containsResource(rule rbacv1.PolicyRule, resource string) bool { + for _, r := range rule.Resources { + if r == resource { + return true + } + } + return false +} + +func containsVerb(rule rbacv1.PolicyRule, verb string) bool { + for _, v := range rule.Verbs { + if v == verb { + return true + } + } + return false +} diff --git a/infra/feast-operator/internal/controller/services/namespace_registry.go b/infra/feast-operator/internal/controller/services/namespace_registry.go index dcea98a5764..122e7ba9e98 100644 --- a/infra/feast-operator/internal/controller/services/namespace_registry.go +++ b/infra/feast-operator/internal/controller/services/namespace_registry.go @@ -36,8 +36,22 @@ type NamespaceRegistryData struct { Namespaces map[string][]string `json:"namespaces"` } +// isProtectedProject checks if this CR is annotated as a protected project +func (feast *FeastServices) isProtectedProject() bool { + annotations := feast.Handler.FeatureStore.GetAnnotations() + return annotations[ProtectedProjectAnnotation] == "true" +} + // deployNamespaceRegistry creates and manages the namespace registry ConfigMap func (feast *FeastServices) deployNamespaceRegistry() error { + // Skip namespace registry for protected projects. + // Protected projects are managed externally and should not be visible to other instances. + if feast.isProtectedProject() { + logger := log.FromContext(feast.Handler.Context) + logger.V(1).Info("Skipping namespace registry for protected project", "project", feast.Handler.FeatureStore.Spec.FeastProject) + return nil + } + // Check if we can determine the target namespace before creating any resources targetNamespace, err := feast.getNamespaceRegistryNamespace() if err != nil { @@ -230,6 +244,11 @@ func (feast *FeastServices) getNamespaceRegistryNamespace() (string, error) { // AddToNamespaceRegistry adds a feature store instance to the namespace registry func (feast *FeastServices) AddToNamespaceRegistry() error { + // Skip for protected projects — they should not appear in the namespace registry. + if feast.isProtectedProject() { + return nil + } + logger := log.FromContext(feast.Handler.Context) targetNamespace, err := feast.getNamespaceRegistryNamespace() if err != nil { diff --git a/infra/feast-operator/internal/controller/services/rbac_templates/spark_application.yaml b/infra/feast-operator/internal/controller/services/rbac_templates/spark_application.yaml new file mode 100644 index 00000000000..c03e0f3db57 --- /dev/null +++ b/infra/feast-operator/internal/controller/services/rbac_templates/spark_application.yaml @@ -0,0 +1,26 @@ +engine_type: spark_application + +server: + rules: + - apiGroups: [""] + resources: ["configmaps"] + verbs: ["create", "delete"] + - apiGroups: ["sparkoperator.k8s.io"] + resources: ["sparkapplications"] + verbs: ["create", "get", "delete"] + - apiGroups: [""] + resources: ["pods"] + verbs: ["list"] + - apiGroups: [""] + resources: ["pods/log"] + verbs: ["get"] + +driver: + create_service_account: true + rules: + - apiGroups: [""] + resources: ["pods"] + verbs: ["create", "get", "list", "watch", "delete", "deletecollection"] + - apiGroups: [""] + resources: ["services", "configmaps", "persistentvolumeclaims"] + verbs: ["create", "get", "list", "watch", "delete", "deletecollection"] diff --git a/infra/feast-operator/internal/controller/services/repo_config.go b/infra/feast-operator/internal/controller/services/repo_config.go index 454dd5b234a..b4a01b312b1 100644 --- a/infra/feast-operator/internal/controller/services/repo_config.go +++ b/infra/feast-operator/internal/controller/services/repo_config.go @@ -85,7 +85,7 @@ func getServiceRepoConfig( } if appliedSpec.BatchEngine != nil { - err := setRepoConfigBatchEngine(appliedSpec.BatchEngine, configMapExtractionFunc, &repoConfig) + err := setRepoConfigBatchEngine(featureStore, appliedSpec.BatchEngine, configMapExtractionFunc, &repoConfig) if err != nil { return repoConfig, err } @@ -138,6 +138,17 @@ func getBaseServiceRepoConfig( } for _, prop := range OidcOptionalSecretProperties { if val, exists := secretProperties[string(prop)]; exists { + // Secret values are YAML-parsed on extraction, so an + // all-digits audience or issuer arrives as an int and + // would render unquoted, which the SDK's OidcAuthConfig + // rejects (Optional[str]). Coerce the claim keys back to + // strings; the five original keys keep their historical + // typing. + if prop == OidcAudience || prop == OidcIssuer { + if _, isString := val.(string); !isString { + val = fmt.Sprintf("%v", val) + } + } oidcParameters[string(prop)] = val } } @@ -342,6 +353,7 @@ func setRepoConfigOffline(services *feastdevv1.FeatureStoreServices, secretExtra } func setRepoConfigBatchEngine( + featureStore *feastdevv1.FeatureStore, batchEngineConfig *feastdevv1.BatchEngineConfig, configMapExtractionFunc func(configMapRef string, configMapKey string) (map[string]interface{}, error), repoConfig *RepoConfig) error { @@ -362,6 +374,12 @@ func setRepoConfigBatchEngine( return fmt.Errorf("batch engine config must contain 'type' field") } delete(config, "type") + // Inject service_account only for spark_application so baked feature_store.yaml + // matches the SA/RoleBinding created by reconcileBatchEngineRBAC. + // Other batch engines are left unchanged. + if engineType == "spark_application" { + config["service_account"] = resolveBatchDriverSAName(featureStore, config) + } repoConfig.BatchEngine = &ComputeEngineConfig{ Type: engineType, Parameters: config, @@ -466,6 +484,54 @@ func setRepoConfigOpenLineage( yamlCfg.ApiKey = &apiKeyStr } + if ol.Consumer != nil { + consumerCfg := &OpenLineageConsumerYamlConfig{ + Enabled: ol.Consumer.Enabled, + StoreType: ol.Consumer.StoreType, + NamespaceMapping: ol.Consumer.NamespaceMapping, + } + + if ol.Consumer.ConnectionStringSecretRef != nil { + params, err := secretExtractionFunc("", ol.Consumer.ConnectionStringSecretRef.Name, "") + if err != nil { + return fmt.Errorf("failed to read consumer connection string from secret %s: %w", + ol.Consumer.ConnectionStringSecretRef.Name, err) + } + connStr, exists := params["connection_string"] + if !exists { + return fmt.Errorf("secret %q does not contain the required key \"connection_string\"", + ol.Consumer.ConnectionStringSecretRef.Name) + } + connStrStr, ok := connStr.(string) + if !ok { + return fmt.Errorf("key \"connection_string\" in secret %q must be a string, got %T", + ol.Consumer.ConnectionStringSecretRef.Name, connStr) + } + consumerCfg.ConnectionString = &connStrStr + } + + if ol.Consumer.ApiKeySecretRef != nil { + params, err := secretExtractionFunc("", ol.Consumer.ApiKeySecretRef.Name, "") + if err != nil { + return fmt.Errorf("failed to read consumer API key from secret %s: %w", + ol.Consumer.ApiKeySecretRef.Name, err) + } + apiKey, exists := params["api_key"] + if !exists { + return fmt.Errorf("secret %q does not contain the required key \"api_key\"", + ol.Consumer.ApiKeySecretRef.Name) + } + apiKeyStr, ok := apiKey.(string) + if !ok { + return fmt.Errorf("key \"api_key\" in secret %q must be a string, got %T", + ol.Consumer.ApiKeySecretRef.Name, apiKey) + } + consumerCfg.ApiKey = &apiKeyStr + } + + yamlCfg.Consumer = consumerCfg + } + repoConfig.OpenLineage = yamlCfg return nil } diff --git a/infra/feast-operator/internal/controller/services/repo_config_test.go b/infra/feast-operator/internal/controller/services/repo_config_test.go index 5ae22a795f6..e87efdf7dec 100644 --- a/infra/feast-operator/internal/controller/services/repo_config_test.go +++ b/infra/feast-operator/internal/controller/services/repo_config_test.go @@ -212,16 +212,20 @@ var _ = Describe("Repo Config", func() { string(OidcClientId): clientIDValue, string(OidcClientSecret): "client-secret", string(OidcUsername): "username", - string(OidcPassword): "password"}) + string(OidcPassword): "password", + string(OidcAudience): "api://feast-feature-server", + string(OidcIssuer): "https://login.example.com/realms/master"}) repoConfig, err = getServiceRepoConfig(featureStore, secretExtractionFunc, emptyMockExtractConfigFromConfigMap, false) Expect(err).NotTo(HaveOccurred()) Expect(repoConfig.AuthzConfig.Type).To(Equal(OidcAuthType)) - Expect(repoConfig.AuthzConfig.OidcParameters).To(HaveLen(5)) + Expect(repoConfig.AuthzConfig.OidcParameters).To(HaveLen(7)) Expect(repoConfig.AuthzConfig.OidcParameters).To(HaveKey(string(OidcClientId))) Expect(repoConfig.AuthzConfig.OidcParameters).To(HaveKey(string(OidcAuthDiscoveryUrl))) Expect(repoConfig.AuthzConfig.OidcParameters).To(HaveKey(string(OidcClientSecret))) Expect(repoConfig.AuthzConfig.OidcParameters).To(HaveKey(string(OidcUsername))) Expect(repoConfig.AuthzConfig.OidcParameters).To(HaveKey(string(OidcPassword))) + Expect(repoConfig.AuthzConfig.OidcParameters).To(HaveKeyWithValue(string(OidcAudience), "api://feast-feature-server")) + Expect(repoConfig.AuthzConfig.OidcParameters).To(HaveKeyWithValue(string(OidcIssuer), "https://login.example.com/realms/master")) Expect(repoConfig.OfflineStore).To(Equal(expectedOfflineConfig)) Expect(repoConfig.OnlineStore).To(Equal(defaultOnlineStoreConfig(featureStore))) Expect(repoConfig.Registry).To(Equal(defaultRegistryConfig(featureStore))) @@ -229,6 +233,19 @@ var _ = Describe("Repo Config", func() { repoConfig = getClientRepoConfig(featureStore, nil) Expect(repoConfig.AuthzConfig.Type).To(Equal(OidcAuthType)) + By("Coercing numeric audience and issuer Secret values to strings") + secretExtractionFunc = mockOidcConfigFromSecret(map[string]interface{}{ + string(OidcAuthDiscoveryUrl): "discovery-url", + string(OidcClientId): clientIDValue, + // Secret extraction YAML-parses values, so an all-digits + // audience/issuer reaches this code as an int. + string(OidcAudience): 1234567890, + string(OidcIssuer): 9876543210}) + repoConfig, err = getServiceRepoConfig(featureStore, secretExtractionFunc, emptyMockExtractConfigFromConfigMap, false) + Expect(err).NotTo(HaveOccurred()) + Expect(repoConfig.AuthzConfig.OidcParameters).To(HaveKeyWithValue(string(OidcAudience), "1234567890")) + Expect(repoConfig.AuthzConfig.OidcParameters).To(HaveKeyWithValue(string(OidcIssuer), "9876543210")) + By("Having oidc authorization with issuerUrl only (no Secret)") featureStore.Spec.AuthzConfig = &feastdevv1.AuthzConfig{ OidcAuthz: &feastdevv1.OidcAuthz{ @@ -673,6 +690,72 @@ var _ = Describe("Repo Config", func() { Expect(repoConfig.Materialization).To(BeNil()) Expect(repoConfig.OpenLineage).To(BeNil()) }) + + It("should inject default batch_engine.service_account when ConfigMap omits it", func() { + featureStore := minimalFeatureStore() + featureStore.Name = "spark-pg-e2e" + featureStore.Spec.BatchEngine = &feastdevv1.BatchEngineConfig{ + ConfigMapRef: &corev1.LocalObjectReference{Name: "spark-pg-batch-engine"}, + } + ApplyDefaultsToStatus(featureStore) + + extractCM := func(configMapRef string, configMapKey string) (map[string]interface{}, error) { + return map[string]interface{}{ + "type": "spark_application", + "image": "quay.io/example/feast-spark-driver:v6", + // service_account intentionally omitted + }, nil + } + + repoConfig, err := getServiceRepoConfig(featureStore, emptyMockExtractConfigFromSecret, extractCM, false) + Expect(err).NotTo(HaveOccurred()) + Expect(repoConfig.BatchEngine).NotTo(BeNil()) + Expect(repoConfig.BatchEngine.Type).To(Equal("spark_application")) + Expect(repoConfig.BatchEngine.Parameters["service_account"]).To(Equal("feast-spark-pg-e2e-batch-driver")) + }) + + It("should preserve explicit batch_engine.service_account from ConfigMap", func() { + featureStore := minimalFeatureStore() + featureStore.Name = "spark-pg-e2e" + featureStore.Spec.BatchEngine = &feastdevv1.BatchEngineConfig{ + ConfigMapRef: &corev1.LocalObjectReference{Name: "spark-pg-batch-engine"}, + } + ApplyDefaultsToStatus(featureStore) + + extractCM := func(configMapRef string, configMapKey string) (map[string]interface{}, error) { + return map[string]interface{}{ + "type": "spark_application", + "service_account": "my-custom-driver", + }, nil + } + + repoConfig, err := getServiceRepoConfig(featureStore, emptyMockExtractConfigFromSecret, extractCM, false) + Expect(err).NotTo(HaveOccurred()) + Expect(repoConfig.BatchEngine.Parameters["service_account"]).To(Equal("my-custom-driver")) + }) + + It("should not inject service_account for non-spark_application batch engines", func() { + featureStore := minimalFeatureStore() + featureStore.Name = "spark-pg-e2e" + featureStore.Spec.BatchEngine = &feastdevv1.BatchEngineConfig{ + ConfigMapRef: &corev1.LocalObjectReference{Name: "other-batch-engine"}, + } + ApplyDefaultsToStatus(featureStore) + + extractCM := func(configMapRef string, configMapKey string) (map[string]interface{}, error) { + return map[string]interface{}{ + "type": "spark", + // no service_account — must stay omitted for non-spark_application + }, nil + } + + repoConfig, err := getServiceRepoConfig(featureStore, emptyMockExtractConfigFromSecret, extractCM, false) + Expect(err).NotTo(HaveOccurred()) + Expect(repoConfig.BatchEngine).NotTo(BeNil()) + Expect(repoConfig.BatchEngine.Type).To(Equal("spark")) + _, hasSA := repoConfig.BatchEngine.Parameters["service_account"] + Expect(hasSA).To(BeFalse()) + }) }) It("should fail to create the repo configs", func() { featureStore := minimalFeatureStore() diff --git a/infra/feast-operator/internal/controller/services/services.go b/infra/feast-operator/internal/controller/services/services.go index c6271c28d3a..6964938c093 100644 --- a/infra/feast-operator/internal/controller/services/services.go +++ b/infra/feast-operator/internal/controller/services/services.go @@ -18,6 +18,7 @@ package services import ( "errors" + "path" "strconv" "strings" @@ -79,6 +80,9 @@ func (feast *FeastServices) Deploy() error { if err := feast.createServiceAccount(); err != nil { return err } + if err := feast.reconcileBatchEngineRBAC(); err != nil { + return err + } if err := feast.createDeployment(); err != nil { return err } @@ -451,6 +455,10 @@ func (feast *FeastServices) setPod(podSpec *corev1.PodSpec) error { } func (feast *FeastServices) setContainers(podSpec *corev1.PodSpec) error { + if err := feast.validatePackagedFeatureRepoPath(); err != nil { + return err + } + fsYamlB64, err := feast.GetServiceFeatureStoreYamlBase64() if err != nil { return err @@ -469,6 +477,20 @@ func (feast *FeastServices) setContainers(podSpec *corev1.PodSpec) error { if feast.isUiServer() { feast.setContainer(&podSpec.Containers, UIFeastType, fsYamlB64) } + + // When the CR is annotated as a protected project, set FEAST_PROTECTED_PROJECT=true + // so the registry server tags its own project in the shared registry. + // Other FeatureStore instances then exclude this project automatically. + if feast.isProtectedProject() { + protectedEnv := corev1.EnvVar{ + Name: "FEAST_PROTECTED_PROJECT", + Value: "true", + } + for i := range podSpec.Containers { + podSpec.Containers[i].Env = append(podSpec.Containers[i].Env, protectedEnv) + } + } + return nil } @@ -687,9 +709,10 @@ func (feast *FeastServices) setInitContainer(podSpec *corev1.PodSpec, fsYamlB64 feastProjectDir := applied.FeastProjectDir workingDir := getOfflineMountPath(feast.Handler.FeatureStore) projectPath := workingDir + "/" + applied.FeastProject + initImage := getInitContainerImage(&applied) container := corev1.Container{ Name: feastInitContainerName, - Image: getFeatureServerImage(), + Image: initImage, Env: []corev1.EnvVar{ { Name: TmpFeatureStoreYamlEnvVar, @@ -700,6 +723,7 @@ func (feast *FeastServices) setInitContainer(podSpec *corev1.PodSpec, fsYamlB64 WorkingDir: workingDir, } + featureRepoDir := feast.getFeatureRepoDir() var createCommand string if feastProjectDir.Init != nil { initSlice := []string{"feast", "init"} @@ -729,20 +753,43 @@ func (feast *FeastServices) setInitContainer(podSpec *corev1.PodSpec, fsYamlB64 if feastProjectDir.Git.EnvFrom != nil { container.EnvFrom = *feastProjectDir.Git.EnvFrom } + } else if feastProjectDir.Packaged != nil { + container.Env = append(container.Env, + corev1.EnvVar{ + Name: packagedFeatureRepoEnvVar, + Value: path.Clean(feastProjectDir.Packaged.FeatureRepoPath), + }, + corev1.EnvVar{ + Name: stagedFeatureRepoEnvVar, + Value: featureRepoDir, + }, + ) + container.Args = []string{ + "set -euo pipefail\n" + + "echo \"Staging packaged feast repository...\"\n" + + "if [[ ! -d \"${" + packagedFeatureRepoEnvVar + "}\" ]]; then " + + "echo \"Packaged feature repository not found: ${" + packagedFeatureRepoEnvVar + "}\" >&2; exit 1; fi\n" + + "rm -rf -- \"${" + stagedFeatureRepoEnvVar + "}\"\n" + + "mkdir -p -- \"${" + stagedFeatureRepoEnvVar + "}\"\n" + + "cp -a -- \"${" + packagedFeatureRepoEnvVar + "}/.\" \"${" + stagedFeatureRepoEnvVar + "}/\"\n" + + "printf '%s' \"${" + TmpFeatureStoreYamlEnvVar + "}\" | base64 -d > \"${" + stagedFeatureRepoEnvVar + "}/feature_store.yaml\"\n" + + "echo \"Packaged feast repository staging complete\"\n", + } } - featureRepoDir := feast.getFeatureRepoDir() - container.Args = []string{ - "echo \"Creating feast repository...\"\necho '" + createCommand + "'\n" + - "if [[ ! -d " + featureRepoDir + " ]]; then " + createCommand + "; fi;\n" + - "echo $" + TmpFeatureStoreYamlEnvVar + " | base64 -d \u003e " + featureRepoDir + "/feature_store.yaml;\necho \"Feast repo creation complete\";\n", + if feastProjectDir.Packaged == nil { + container.Args = []string{ + "echo \"Creating feast repository...\"\necho '" + createCommand + "'\n" + + "if [[ ! -d " + featureRepoDir + " ]]; then " + createCommand + "; fi;\n" + + "echo $" + TmpFeatureStoreYamlEnvVar + " | base64 -d \u003e " + featureRepoDir + "/feature_store.yaml;\necho \"Feast repo creation complete\";\n", + } } podSpec.InitContainers = append(podSpec.InitContainers, container) if applied.Services.RunFeastApplyOnInit != nil && *applied.Services.RunFeastApplyOnInit { applyContainer := corev1.Container{ Name: feastApplyContainerName, - Image: getFeatureServerImage(), + Image: initImage, Command: []string{feastCommand, "apply"}, WorkingDir: featureRepoDir, } @@ -1274,7 +1321,7 @@ func (feast *FeastServices) isOnlineServer() bool { func (feast *FeastServices) isOnlineStore() bool { appliedServices := feast.Handler.FeatureStore.Status.Applied.Services - return appliedServices != nil && appliedServices.OnlineStore != nil + return appliedServices != nil && appliedServices.OnlineStore != nil && !appliedServices.OnlineStore.Disabled } func (feast *FeastServices) noLocalCoreServerConfigured() bool { @@ -1403,6 +1450,9 @@ func (feast *FeastServices) mountEmptyDirVolumes(podSpec *corev1.PodSpec) { func (feast *FeastServices) getFeatureRepoDir() string { applied := feast.Handler.FeatureStore.Status.Applied + if applied.FeastProjectDir != nil && applied.FeastProjectDir.Packaged != nil && applied.Services.DisableInitContainers { + return path.Clean(applied.FeastProjectDir.Packaged.FeatureRepoPath) + } feastProjectDir := getOfflineMountPath(feast.Handler.FeatureStore) + "/" + applied.FeastProject if applied.FeastProjectDir != nil && applied.FeastProjectDir.Git != nil && len(applied.FeastProjectDir.Git.FeatureRepoPath) > 0 { return feastProjectDir + "/" + applied.FeastProjectDir.Git.FeatureRepoPath @@ -1410,6 +1460,39 @@ func (feast *FeastServices) getFeatureRepoDir() string { return feastProjectDir + "/" + FeatureRepoDir } +func (feast *FeastServices) validatePackagedFeatureRepoPath() error { + applied := feast.Handler.FeatureStore.Status.Applied + if applied.FeastProjectDir == nil || applied.FeastProjectDir.Packaged == nil { + return nil + } + + featureRepoPath := applied.FeastProjectDir.Packaged.FeatureRepoPath + cleanFeatureRepoPath := path.Clean(featureRepoPath) + if !path.IsAbs(featureRepoPath) || cleanFeatureRepoPath == "/" || cleanFeatureRepoPath != featureRepoPath { + return errors.New("packaged feature repository path " + strconv.Quote(featureRepoPath) + " must be a canonical absolute, non-root path") + } + + if !applied.Services.DisableInitContainers { + stagedFeatureRepoPath := path.Clean(feast.getFeatureRepoDir()) + if pathsOverlap(cleanFeatureRepoPath, stagedFeatureRepoPath) { + return errors.New( + "packaged feature repository path " + strconv.Quote(cleanFeatureRepoPath) + + " overlaps staged repository path " + strconv.Quote(stagedFeatureRepoPath), + ) + } + } + + return nil +} + +func pathsOverlap(firstPath, secondPath string) bool { + firstPath = path.Clean(firstPath) + secondPath = path.Clean(secondPath) + return firstPath == secondPath || + strings.HasPrefix(firstPath, secondPath+"/") || + strings.HasPrefix(secondPath, firstPath+"/") +} + func mountEmptyDirVolume(podSpec *corev1.PodSpec) { if podSpec != nil { volName := strings.TrimPrefix(EphemeralPath, "/") diff --git a/infra/feast-operator/internal/controller/services/services_types.go b/infra/feast-operator/internal/controller/services/services_types.go index 366f0c8d765..090696eccaf 100644 --- a/infra/feast-operator/internal/controller/services/services_types.go +++ b/infra/feast-operator/internal/controller/services/services_types.go @@ -26,6 +26,8 @@ import ( const ( TmpFeatureStoreYamlEnvVar = "TMP_FEATURE_STORE_YAML_BASE64" + packagedFeatureRepoEnvVar = "FEAST_PACKAGED_FEATURE_REPO_PATH" + stagedFeatureRepoEnvVar = "FEAST_STAGED_FEATURE_REPO_PATH" feastServerImageVar = "RELATED_IMAGE_FEATURE_SERVER" cronJobImageVar = "RELATED_IMAGE_CRON_JOB" FeatureStoreYamlCmKey = "feature_store.yaml" @@ -40,6 +42,14 @@ const ( NamespaceRegistryDataKey = "namespaces" DefaultKubernetesNamespace = "feast-operator-system" + // ProtectedProjectAnnotation is the annotation key on a FeatureStore CR + // that marks its project as protected. Protected projects are excluded + // from project listings and shielded from teardown by other instances. + // When this annotation is "true", the operator sets FEAST_PROTECTED_PROJECT=true + // on the server pods, which causes the server to tag the project in the + // shared registry on startup. + ProtectedProjectAnnotation = "feast.dev/protected-project" + HttpPort = 80 HttpsPort = 443 HttpScheme = "http" @@ -103,6 +113,8 @@ const ( OidcTokenEnvVar OidcPropertyType = "token_env_var" OidcVerifySsl OidcPropertyType = "verify_ssl" OidcCaCertPath OidcPropertyType = "ca_cert_path" + OidcAudience OidcPropertyType = "audience" + OidcIssuer OidcPropertyType = "issuer" OidcMissingSecretError string = "missing OIDC secret: %s" @@ -266,7 +278,7 @@ var ( }, } - OidcOptionalSecretProperties = []OidcPropertyType{OidcAuthDiscoveryUrl, OidcClientId, OidcClientSecret, OidcUsername, OidcPassword} + OidcOptionalSecretProperties = []OidcPropertyType{OidcAuthDiscoveryUrl, OidcClientId, OidcClientSecret, OidcUsername, OidcPassword, OidcAudience, OidcIssuer} ) // Feast server types: Reserved only for server types like Online, Offline, and Registry servers. Should not be used for client types like the UI, etc. @@ -362,12 +374,22 @@ type MaterializationYamlConfig struct { // emit_on_apply, emit_on_materialize, transport-specific options, etc.) appear at // the same YAML level as the typed connection fields. type OpenLineageYamlConfig struct { - Enabled bool `yaml:"enabled"` - TransportType *string `yaml:"transport_type,omitempty"` - TransportUrl *string `yaml:"transport_url,omitempty"` - TransportEndpoint *string `yaml:"transport_endpoint,omitempty"` - ApiKey *string `yaml:"api_key,omitempty"` - ExtraConfig map[string]interface{} `yaml:",inline,omitempty"` + Enabled bool `yaml:"enabled"` + TransportType *string `yaml:"transport_type,omitempty"` + TransportUrl *string `yaml:"transport_url,omitempty"` + TransportEndpoint *string `yaml:"transport_endpoint,omitempty"` + ApiKey *string `yaml:"api_key,omitempty"` + ExtraConfig map[string]interface{} `yaml:",inline,omitempty"` + Consumer *OpenLineageConsumerYamlConfig `yaml:"consumer,omitempty"` +} + +// OpenLineageConsumerYamlConfig maps to the openlineage.consumer section of feature_store.yaml. +type OpenLineageConsumerYamlConfig struct { + Enabled bool `yaml:"enabled"` + StoreType *string `yaml:"store_type,omitempty"` + ConnectionString *string `yaml:"connection_string,omitempty"` + ApiKey *string `yaml:"api_key,omitempty"` + NamespaceMapping map[string]string `yaml:"namespace_mapping,omitempty"` } // OfflineStoreConfig is the configuration that relates to reading from and writing to the Feast offline store. diff --git a/infra/feast-operator/internal/controller/services/util.go b/infra/feast-operator/internal/controller/services/util.go index 387ccdb631b..84951f2077b 100644 --- a/infra/feast-operator/internal/controller/services/util.go +++ b/infra/feast-operator/internal/controller/services/util.go @@ -99,6 +99,7 @@ func ApplyDefaultsToStatus(cr *feastdevv1.FeatureStore) { if applied.Services == nil { applied.Services = &feastdevv1.FeatureStoreServices{} } + defaultFeatureServerImage := getFeatureServerImageForSpec(applied) services := applied.Services if services.RunFeastApplyOnInit == nil { services.RunFeastApplyOnInit = boolPtr(true) @@ -128,7 +129,7 @@ func ApplyDefaultsToStatus(cr *feastdevv1.FeatureStore) { } if services.Registry.Local.Server != nil { - setDefaultCtrConfigs(&services.Registry.Local.Server.ContainerConfigs.DefaultCtrConfigs) + setDefaultCtrConfigs(&services.Registry.Local.Server.ContainerConfigs.DefaultCtrConfigs, defaultFeatureServerImage) // Set default for GRPC: true if nil if services.Registry.Local.Server.GRPC == nil { defaultGRPC := true @@ -159,37 +160,39 @@ func ApplyDefaultsToStatus(cr *feastdevv1.FeatureStore) { } if services.OfflineStore.Server != nil { - setDefaultCtrConfigs(&services.OfflineStore.Server.ContainerConfigs.DefaultCtrConfigs) + setDefaultCtrConfigs(&services.OfflineStore.Server.ContainerConfigs.DefaultCtrConfigs, defaultFeatureServerImage) } } - // default to onlineStore service deployment + // default to onlineStore service deployment unless it is explicitly disabled if services.OnlineStore == nil { services.OnlineStore = &feastdevv1.OnlineStore{} } - if services.OnlineStore.Persistence == nil { - services.OnlineStore.Persistence = &feastdevv1.OnlineStorePersistence{} - } - - if services.OnlineStore.Persistence.DBPersistence == nil { - if services.OnlineStore.Persistence.FilePersistence == nil { - services.OnlineStore.Persistence.FilePersistence = &feastdevv1.OnlineStoreFilePersistence{} + if !services.OnlineStore.Disabled { + if services.OnlineStore.Persistence == nil { + services.OnlineStore.Persistence = &feastdevv1.OnlineStorePersistence{} } - if len(services.OnlineStore.Persistence.FilePersistence.Path) == 0 { - services.OnlineStore.Persistence.FilePersistence.Path = defaultOnlineStorePath(cr) - } + if services.OnlineStore.Persistence.DBPersistence == nil { + if services.OnlineStore.Persistence.FilePersistence == nil { + services.OnlineStore.Persistence.FilePersistence = &feastdevv1.OnlineStoreFilePersistence{} + } - ensurePVCDefaults(services.OnlineStore.Persistence.FilePersistence.PvcConfig, OnlineFeastType) - } + if len(services.OnlineStore.Persistence.FilePersistence.Path) == 0 { + services.OnlineStore.Persistence.FilePersistence.Path = defaultOnlineStorePath(cr) + } + + ensurePVCDefaults(services.OnlineStore.Persistence.FilePersistence.PvcConfig, OnlineFeastType) + } - if services.OnlineStore.Server == nil { - services.OnlineStore.Server = &feastdevv1.ServerConfigs{} + if services.OnlineStore.Server == nil { + services.OnlineStore.Server = &feastdevv1.ServerConfigs{} + } + setDefaultCtrConfigs(&services.OnlineStore.Server.ContainerConfigs.DefaultCtrConfigs, defaultFeatureServerImage) } - setDefaultCtrConfigs(&services.OnlineStore.Server.ContainerConfigs.DefaultCtrConfigs) if services.UI != nil { - setDefaultCtrConfigs(&services.UI.ContainerConfigs.DefaultCtrConfigs) + setDefaultCtrConfigs(&services.UI.ContainerConfigs.DefaultCtrConfigs, defaultFeatureServerImage) } if applied.CronJob == nil { @@ -198,13 +201,20 @@ func ApplyDefaultsToStatus(cr *feastdevv1.FeatureStore) { setDefaultCronJobConfigs(applied.CronJob) } -func setDefaultCtrConfigs(defaultConfigs *feastdevv1.DefaultCtrConfigs) { +func setDefaultCtrConfigs(defaultConfigs *feastdevv1.DefaultCtrConfigs, defaultImage string) { if defaultConfigs.Image == nil { - img := getFeatureServerImage() + img := defaultImage defaultConfigs.Image = &img } } +func getFeatureServerImageForSpec(spec *feastdevv1.FeatureStoreSpec) string { + if spec != nil && spec.FeastProjectDir != nil && spec.FeastProjectDir.Packaged != nil && spec.FeastProjectDir.Packaged.Image != "" { + return spec.FeastProjectDir.Packaged.Image + } + return getFeatureServerImage() +} + func getFeatureServerImage() string { if img, exists := os.LookupEnv(feastServerImageVar); exists { return img @@ -212,6 +222,16 @@ func getFeatureServerImage() string { return DefaultImage } +// getInitContainerImage resolves the image for feast-init / feast-apply. +// Order: spec.services.initImage → spec.feastProjectDir.packaged.image → +// RELATED_IMAGE_FEATURE_SERVER → DefaultImage. +func getInitContainerImage(spec *feastdevv1.FeatureStoreSpec) string { + if spec != nil && spec.Services != nil && spec.Services.InitImage != nil && len(*spec.Services.InitImage) > 0 { + return *spec.Services.InitImage + } + return getFeatureServerImageForSpec(spec) +} + func checkOfflineStoreFilePersistenceType(value string) error { if slices.Contains(feastdevv1.ValidOfflineStoreFilePersistenceTypes, value) { return nil diff --git a/infra/feast-operator/internal/controller/services/util_test.go b/infra/feast-operator/internal/controller/services/util_test.go new file mode 100644 index 00000000000..5a868d2d101 --- /dev/null +++ b/infra/feast-operator/internal/controller/services/util_test.go @@ -0,0 +1,171 @@ +/* +Copyright 2024 Feast Community. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package services + +import ( + "os" + "testing" + + feastdevv1 "github.com/feast-dev/feast/infra/feast-operator/api/v1" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "k8s.io/utils/ptr" +) + +var _ = Describe("ApplyDefaultsToStatus", func() { + It("deploys the online store with defaults when it is not declared", func() { + cr := &feastdevv1.FeatureStore{ + Spec: feastdevv1.FeatureStoreSpec{ + FeastProject: "test_project", + Services: &feastdevv1.FeatureStoreServices{}, + }, + } + + ApplyDefaultsToStatus(cr) + + online := cr.Status.Applied.Services.OnlineStore + Expect(online).ToNot(BeNil()) + Expect(online.Disabled).To(BeFalse()) + Expect(online.Persistence).ToNot(BeNil()) + Expect(online.Server).ToNot(BeNil()) + }) + + It("applies online store defaults when it is declared", func() { + cr := &feastdevv1.FeatureStore{ + Spec: feastdevv1.FeatureStoreSpec{ + FeastProject: "test_project", + Services: &feastdevv1.FeatureStoreServices{ + OnlineStore: &feastdevv1.OnlineStore{}, + }, + }, + } + + ApplyDefaultsToStatus(cr) + + online := cr.Status.Applied.Services.OnlineStore + Expect(online).ToNot(BeNil()) + Expect(online.Persistence).ToNot(BeNil()) + Expect(online.Server).ToNot(BeNil()) + }) + + // #6586: disabling the online store opts out of its persistence and serving + // pod, letting a registry-only or offline-only ViewerStore skip it while + // leaving the default-on behavior unchanged for everyone else. + It("does not apply persistence or server defaults when the online store is disabled", func() { + cr := &feastdevv1.FeatureStore{ + Spec: feastdevv1.FeatureStoreSpec{ + FeastProject: "test_project", + Services: &feastdevv1.FeatureStoreServices{ + OnlineStore: &feastdevv1.OnlineStore{Disabled: true}, + }, + }, + } + + ApplyDefaultsToStatus(cr) + + online := cr.Status.Applied.Services.OnlineStore + Expect(online).ToNot(BeNil()) + Expect(online.Disabled).To(BeTrue()) + Expect(online.Persistence).To(BeNil()) + Expect(online.Server).To(BeNil()) + }) +}) + +func TestGetInitContainerImage(t *testing.T) { + customInit := "quay.io/org/feast-init:custom" + packagedImage := "quay.io/org/feast-packaged:test" + envImage := "quay.io/org/feast-env:test" + + t.Run("uses initImage ahead of packaged and server images", func(t *testing.T) { + t.Setenv(feastServerImageVar, envImage) + got := getInitContainerImage(&feastdevv1.FeatureStoreSpec{ + FeastProjectDir: &feastdevv1.FeastProjectDir{ + Packaged: &feastdevv1.FeastPackagedOptions{Image: packagedImage}, + }, + Services: &feastdevv1.FeatureStoreServices{ + InitImage: ptr.To(customInit), + OfflineStore: &feastdevv1.OfflineStore{ + Server: &feastdevv1.ServerConfigs{ + ContainerConfigs: feastdevv1.ContainerConfigs{ + DefaultCtrConfigs: feastdevv1.DefaultCtrConfigs{ + Image: ptr.To("quay.io/org/offline:v1"), + }, + }, + }, + }, + OnlineStore: &feastdevv1.OnlineStore{ + Server: &feastdevv1.ServerConfigs{ + ContainerConfigs: feastdevv1.ContainerConfigs{ + DefaultCtrConfigs: feastdevv1.DefaultCtrConfigs{ + Image: ptr.To("quay.io/org/online:v1"), + }, + }, + }, + }, + }, + }) + if got != customInit { + t.Fatalf("got %q, want %q (must not inherit server images)", got, customInit) + } + }) + + t.Run("uses packaged image ahead of RELATED_IMAGE_FEATURE_SERVER", func(t *testing.T) { + t.Setenv(feastServerImageVar, envImage) + got := getInitContainerImage(&feastdevv1.FeatureStoreSpec{ + FeastProjectDir: &feastdevv1.FeastProjectDir{ + Packaged: &feastdevv1.FeastPackagedOptions{Image: packagedImage}, + }, + Services: &feastdevv1.FeatureStoreServices{}, + }) + if got != packagedImage { + t.Fatalf("got %q, want %q", got, packagedImage) + } + }) + + t.Run("falls back to RELATED_IMAGE_FEATURE_SERVER", func(t *testing.T) { + t.Setenv(feastServerImageVar, envImage) + got := getInitContainerImage(&feastdevv1.FeatureStoreSpec{ + Services: &feastdevv1.FeatureStoreServices{}, + }) + if got != envImage { + t.Fatalf("got %q, want %q", got, envImage) + } + }) + + t.Run("falls back to DefaultImage", func(t *testing.T) { + _ = os.Unsetenv(feastServerImageVar) + got := getInitContainerImage(nil) + if got != DefaultImage { + t.Fatalf("got %q, want %q", got, DefaultImage) + } + }) + + t.Run("ignores empty initImage", func(t *testing.T) { + t.Setenv(feastServerImageVar, envImage) + got := getInitContainerImage(&feastdevv1.FeatureStoreSpec{ + FeastProjectDir: &feastdevv1.FeastProjectDir{ + Packaged: &feastdevv1.FeastPackagedOptions{Image: packagedImage}, + }, + Services: &feastdevv1.FeatureStoreServices{ + InitImage: ptr.To(""), + }, + }) + if got != packagedImage { + t.Fatalf("got %q, want %q", got, packagedImage) + } + }) +} diff --git a/infra/feast-operator/test/api/featurestore_packaged_types_test.go b/infra/feast-operator/test/api/featurestore_packaged_types_test.go new file mode 100644 index 00000000000..97525ec1abe --- /dev/null +++ b/infra/feast-operator/test/api/featurestore_packaged_types_test.go @@ -0,0 +1,162 @@ +/* +Copyright 2026 Feast Community. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package api + +import ( + "context" + "strings" + + feastdevv1 "github.com/feast-dev/feast/infra/feast-operator/api/v1" + feastdevv1alpha1 "github.com/feast-dev/feast/infra/feast-operator/api/v1alpha1" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +type packagedFeatureStoreFactory func(name, featureRepoPath string) client.Object +type conflictingPackagedFeatureStoreFactory func(name, conflictingMode string) client.Object + +func newV1PackagedFeatureStore(name, featureRepoPath string) client.Object { + return &feastdevv1.FeatureStore{ + ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: namespaceName}, + Spec: feastdevv1.FeatureStoreSpec{ + FeastProject: "test_project", + FeastProjectDir: &feastdevv1.FeastProjectDir{ + Packaged: &feastdevv1.FeastPackagedOptions{FeatureRepoPath: featureRepoPath}, + }, + }, + } +} + +func newV1Alpha1PackagedFeatureStore(name, featureRepoPath string) client.Object { + return &feastdevv1alpha1.FeatureStore{ + ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: namespaceName}, + Spec: feastdevv1alpha1.FeatureStoreSpec{ + FeastProject: "test_project", + FeastProjectDir: &feastdevv1alpha1.FeastProjectDir{ + Packaged: &feastdevv1alpha1.FeastPackagedOptions{FeatureRepoPath: featureRepoPath}, + }, + }, + } +} + +func newV1ConflictingPackagedFeatureStore(name, conflictingMode string) client.Object { + featureStore := newV1PackagedFeatureStore(name, "/opt/feast/feature_repo").(*feastdevv1.FeatureStore) + switch conflictingMode { + case "init": + featureStore.Spec.FeastProjectDir.Init = &feastdevv1.FeastInitOptions{} + case "git": + featureStore.Spec.FeastProjectDir.Git = &feastdevv1.GitCloneOptions{ + URL: "https://example.com/feature-repo.git", + } + } + return featureStore +} + +func newV1Alpha1ConflictingPackagedFeatureStore(name, conflictingMode string) client.Object { + featureStore := newV1Alpha1PackagedFeatureStore(name, "/opt/feast/feature_repo").(*feastdevv1alpha1.FeatureStore) + switch conflictingMode { + case "init": + featureStore.Spec.FeastProjectDir.Init = &feastdevv1alpha1.FeastInitOptions{} + case "git": + featureStore.Spec.FeastProjectDir.Git = &feastdevv1alpha1.GitCloneOptions{ + URL: "https://example.com/feature-repo.git", + } + } + return featureStore +} + +var _ = Describe("Packaged feature repository path validation", func() { + ctx := context.Background() + apiVersions := []struct { + name string + id string + factory packagedFeatureStoreFactory + conflictingFactory conflictingPackagedFeatureStoreFactory + }{ + { + name: "feast.dev/v1", + id: "v1", + factory: newV1PackagedFeatureStore, + conflictingFactory: newV1ConflictingPackagedFeatureStore, + }, + { + name: "feast.dev/v1alpha1", + id: "v1alpha1", + factory: newV1Alpha1PackagedFeatureStore, + conflictingFactory: newV1Alpha1ConflictingPackagedFeatureStore, + }, + } + + for _, apiVersion := range apiVersions { + apiVersion := apiVersion + Context(apiVersion.name, func() { + DescribeTable("accepts canonical absolute non-root paths", + func(nameSuffix, featureRepoPath string) { + featureStore := apiVersion.factory( + "packaged-"+apiVersion.id+"-"+nameSuffix, + featureRepoPath, + ) + Expect(k8sClient.Create(ctx, featureStore)).To(Succeed()) + Expect(k8sClient.Delete(ctx, featureStore)).To(Succeed()) + }, + Entry("standard", "standard", "/opt/feast/feature_repo"), + Entry("hidden component", "hidden", "/opt/.feast/feature_repo"), + Entry("dot in component", "dot-name", "/opt/feature_repo.v2"), + ) + + DescribeTable("rejects non-canonical, relative, or root paths", + func(nameSuffix, featureRepoPath string) { + featureStore := apiVersion.factory( + "packaged-"+apiVersion.id+"-"+nameSuffix, + featureRepoPath, + ) + err := k8sClient.Create(ctx, featureStore) + Expect(err).To(HaveOccurred()) + Expect(apierrors.IsInvalid(err)).To(BeTrue(), "expected invalid error, got %v", err) + Expect(strings.ToLower(err.Error())).To(ContainSubstring("canonical absolute, non-root path")) + }, + Entry("relative", "relative", "opt/feast/feature_repo"), + Entry("root", "root", "/"), + Entry("parent collapses to root", "parent-root", "/opt/.."), + Entry("leading parent traversal", "leading-parent", "/../x"), + Entry("repeated separator", "repeated-separator", "/opt//feature_repo"), + Entry("current-directory component", "current-dir", "/opt/./feature_repo"), + Entry("trailing separator", "trailing-separator", "/opt/feature_repo/"), + Entry("nested traversal", "nested-traversal", "/a/../../etc"), + Entry("repeated root separator", "repeated-root", "//"), + ) + + DescribeTable("rejects packaged together with another project directory mode", + func(nameSuffix, conflictingMode string) { + featureStore := apiVersion.conflictingFactory( + "packaged-"+apiVersion.id+"-"+nameSuffix, + conflictingMode, + ) + err := k8sClient.Create(ctx, featureStore) + Expect(err).To(HaveOccurred()) + Expect(apierrors.IsInvalid(err)).To(BeTrue(), "expected invalid error, got %v", err) + Expect(err.Error()).To(ContainSubstring("One selection required between init, git, or packaged")) + }, + Entry("init", "with-init", "init"), + Entry("git", "with-git", "git"), + ) + }) + } +}) diff --git a/infra/feast-operator/test/api/suite_test.go b/infra/feast-operator/test/api/suite_test.go index 558068a7957..eef4718cf58 100644 --- a/infra/feast-operator/test/api/suite_test.go +++ b/infra/feast-operator/test/api/suite_test.go @@ -26,6 +26,7 @@ import ( . "github.com/onsi/gomega" feastdevv1 "github.com/feast-dev/feast/infra/feast-operator/api/v1" + feastdevv1alpha1 "github.com/feast-dev/feast/infra/feast-operator/api/v1alpha1" "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/rest" "sigs.k8s.io/controller-runtime/pkg/client" @@ -73,6 +74,8 @@ var _ = BeforeSuite(func() { err = feastdevv1.AddToScheme(scheme.Scheme) Expect(err).NotTo(HaveOccurred()) + err = feastdevv1alpha1.AddToScheme(scheme.Scheme) + Expect(err).NotTo(HaveOccurred()) // +kubebuilder:scaffold:scheme diff --git a/infra/scripts/feature_server_docker_smoke.py b/infra/scripts/feature_server_docker_smoke.py index c8b3b440b7f..801decac90c 100644 --- a/infra/scripts/feature_server_docker_smoke.py +++ b/infra/scripts/feature_server_docker_smoke.py @@ -16,6 +16,7 @@ def list_projects(self, allow_cache=True, tags=None): class _FakeStore: def __init__(self): self.config = SimpleNamespace() + self.project = "smoke_test" self.registry = _FakeRegistry() self._provider = SimpleNamespace( async_supported=SimpleNamespace( @@ -32,6 +33,9 @@ async def initialize(self): def refresh_registry(self): return None + def list_feature_views(self): + return [] + async def close(self): return None diff --git a/infra/templates/README.md.jinja2 b/infra/templates/README.md.jinja2 index ccaadc29ff0..2c92401f83d 100644 --- a/infra/templates/README.md.jinja2 +++ b/infra/templates/README.md.jinja2 @@ -17,7 +17,7 @@ ## Join us on Slack! -👋👋👋 [Come say hi on Slack!](https://communityinviter.com/apps/feastopensource/feast-the-open-source-feature-store) +👋👋👋 [Come say hi on Slack!](https://slack.feast.dev/) [Check out our DeepWiki!](https://deepwiki.com/feast-dev/feast) diff --git a/infra/website/docs/blog/feast-agents-mcp.md b/infra/website/docs/blog/feast-agents-mcp.md index 5678cd5a578..bfa46ee8b02 100644 --- a/infra/website/docs/blog/feast-agents-mcp.md +++ b/infra/website/docs/blog/feast-agents-mcp.md @@ -51,7 +51,7 @@ feature_server: mcp_server_version: "1.0.0" ``` -Once enabled, any MCP-compatible agent -- whether built with LangChain, LlamaIndex, CrewAI, AutoGen, or a custom framework -- can connect to `http://your-feast-server/mcp` and discover available tools like `get-online-features` for entity-based retrieval, `retrieve-online-documents` for vector similarity search, and `write-to-online-store` for persisting agent state. +Once enabled, any MCP-compatible agent -- whether built with LangChain, LlamaIndex, CrewAI, AutoGen, or a custom framework -- can connect to `http://your-feast-server/mcp` and discover available tools like `get-online-features` for entity-based retrieval, `search` for vector similarity search, `vector_store_search` for OpenAI-compatible text search, and `write-to-online-store` for persisting agent state. ## A Concrete Example: Customer-Support Agent with Memory @@ -338,7 +338,7 @@ export OPENAI_BASE_URL="http://localhost:11434/v1" export LLM_MODEL="llama3.1:8b" ./run_demo.sh -# Any OpenAI-compatible provider (Azure, vLLM, LiteLLM, etc.) +# Any OpenAI-compatible provider (Azure, vLLM, etc.) export OPENAI_API_KEY="your-key" # pragma: allowlist secret export OPENAI_BASE_URL="https://your-endpoint/v1" export LLM_MODEL="your-model" diff --git a/infra/website/docs/blog/feast-data-quality-monitoring.md b/infra/website/docs/blog/feast-data-quality-monitoring.md new file mode 100644 index 00000000000..2c918c83a63 --- /dev/null +++ b/infra/website/docs/blog/feast-data-quality-monitoring.md @@ -0,0 +1,224 @@ +--- +title: Data Quality Monitoring in Feast 0.64 +description: Feast 0.64 adds native data quality monitoring with baseline metrics, batch and serving-log analysis, REST APIs, CLI workflows, and a built-in monitoring UI. +date: 2026-06-26 +authors: ["Jitendra Yejare", "Nikhil Kathole", "Francisco Javier Arceo"] +--- + +
+ Feast Data Quality Monitoring +
+ +# Data Quality Monitoring in Feast 0.64 + +Serving ML models in production is extremely hard. + +The reason is simple: production models depend on data from many different places. Every source system has some probability of operational error: a delayed pipeline, a schema change, a column that starts producing nulls, a categorical value that changes meaning, a late partition, a silent backfill, or a service that behaves differently under production traffic. + +The more data sources a model depends on, the more chances there are for one of those systems to drift, fail, or change underneath you. That creates a basic tension in ML systems. Models are data hungry and often benefit from orthogonal features from many upstream systems, but every additional upstream dependency increases operational risk. What ML wants for predictive power can conflict with what engineering wants for reliability. + +The only way to manage that tension is to monitor what is actually happening in production. Feature quality problems rarely arrive as neat exceptions. A model may keep serving predictions while one upstream table starts producing nulls, a batch pipeline shifts a numeric distribution, or production requests drift away from the training baseline. By the time these issues show up in model metrics, the debugging path usually crosses feature definitions, data sources, materialization jobs, and serving logs. + +Feast 0.64 adds a native data quality monitoring system that brings those signals directly into the feature store. Instead of relying on a separate validation framework, Feast can now compute, store, serve, and visualize feature-level statistics across batch data and logged serving data. + +The biggest change is that monitoring is now a first-class Feast workflow: + +- `feast apply` can compute baseline metrics for registered feature views +- `feast monitor run` can compute scheduled daily, weekly, biweekly, monthly, and quarterly metrics +- REST endpoints expose monitoring jobs, per-feature metrics, aggregate feature-view and feature-service metrics, baselines, and time series +- the Feast UI includes a Monitoring page with filters, summary tabs, feature drilldowns, histograms, and time-series charts +- compute is pushed into supported offline stores where possible, with a Python fallback for other backends + +## From validation to monitoring + +Feast previously supported an external-library-based validation path for historical retrievals. That integration was useful, but it lived outside the normal feature store workflow: users had to install extra dependencies, write profiler code, and run validation against saved datasets. + +That original integration proved the need for data quality inside Feast. It helped answer an important question: after generating a training dataset, does this dataset satisfy the expectations we care about? + +But production feature quality problems usually happen after the training dataset is generated. A pipeline may keep running while an upstream producer changes a column, shifts a distribution, starts sending nulls, or changes the meaning of a categorical value. In those cases, the feature code may be perfectly correct while the data feeding it has changed. + +Feast needed monitoring that was closer to the system that actually computes and serves features. By coupling DQM to Feast's compute engines and offline stores, Feast can compute quality metrics where the data already lives, reuse feature metadata, compare batch and serving-log distributions, and expose the results through the same CLI, REST API, and UI used to operate the feature store. + +This also helps when teams maintain multiple feature execution paths. For example, a feature may be generated one way for training and another way for low-latency serving or streaming. DQM is not a formal proof that two implementations are equivalent, but distribution metrics, baselines, and serving-log comparisons provide an early warning when those paths start producing meaningfully different values. + +The new system is broader and more operational. It automatically computes statistical profiles for registered features, stores them in monitoring tables, and makes them available to the CLI, REST API, and UI. This gives teams the kind of feature health view they need after features are already in production, not only during one historical retrieval. + +For each feature, Feast can track: + +| Metric family | Examples | +|---|---| +| completeness | row count, null count, null rate | +| numeric profile | mean, standard deviation, min, max | +| percentiles | p50, p75, p90, p95, p99 | +| distributions | numeric histograms or categorical top values | +| aggregate health | feature-view and feature-service summaries | + +## Baselines start at registration + +The simplest way to turn on monitoring is to enable DQM in `feature_store.yaml`: + +```yaml +data_quality_monitoring: + auto_baseline: true +``` + +When `auto_baseline` is enabled, `feast apply` computes baseline metrics for feature views that do not already have one. The baseline is marked as the reference distribution and can be compared with later scheduled metrics. + +That matters because the baseline lives next to the feature definitions. When a feature view is registered, Feast can also capture what "normal" looked like at registration time. Later monitoring runs can answer whether the current data still resembles that baseline. + +For Feast Operator deployments, the same setting is available on the `FeatureStore` custom resource: + +```yaml +apiVersion: feast.dev/v1 +kind: FeatureStore +spec: + feastProject: my_project + dataQualityMonitoring: + autoBaseline: true +``` + +## Scheduled monitoring with the CLI + +For ongoing monitoring, schedule: + +```bash +feast monitor run +``` + +In auto mode, Feast detects the latest event timestamp in the source data and computes metrics across the supported granularities: daily, weekly, biweekly, monthly, and quarterly. + +You can also scope monitoring to a specific feature view: + +```bash +feast monitor run --feature-view driver_stats +``` + +Or compute a specific window and mark it as a baseline: + +```bash +feast monitor run \ + --feature-view driver_stats \ + --start-date 2025-01-01 \ + --end-date 2025-03-31 \ + --granularity daily \ + --set-baseline +``` + +This makes the CLI easy to wire into Airflow, Kubeflow Pipelines, cron, or any scheduler that already runs Feast materialization jobs. + +## Monitoring serving logs + +Batch data tells you whether source features look healthy. Serving logs tell you what your models actually received. + +If a `FeatureService` has logging configured, Feast can compute monitoring metrics from the logged online features: + +```bash +feast monitor run --source-type log +``` + +You can also run batch and log monitoring together: + +```bash +feast monitor run --source-type all +``` + +Log metrics are stored with `data_source_type="log"` alongside batch metrics. Feast normalizes logged feature names back to the feature view and feature name, which lets the UI and API compare batch and serving distributions without forcing users to maintain a separate mapping. + +## The new Monitoring UI + +The most visible 0.64 improvement is the Monitoring page in the Feast UI. It turns DQM from a background job into something feature owners can inspect without leaving Feast. + +The page includes three main tabs: + +| Tab | What it shows | +|---|---| +| Features | per-feature metrics such as null rate, row count, freshness, and health | +| Feature Views | aggregate quality summaries per feature view | +| Feature Services | aggregate quality summaries for model-facing feature services | + +At the top of the page, users can filter by feature view, granularity, source type, and date range. Baseline is treated as its own view because it represents all baseline data rather than a normal date window. The page also includes a Compute Metrics action that triggers DQM computation from the UI, plus Refresh for reloading already computed results. + +
+ Feast DQM Monitoring dashboard showing feature metrics, filters, histograms, and health status +
+ +Clicking a feature opens a detail page with: + +- a distribution chart for numeric histograms or categorical values +- a statistics panel with null rate, mean, standard deviation, min, max, and percentiles +- a granularity selector that can switch between computed windows and baseline +- time-series charts for metric drift, including aggregate statistics and null-rate trends + +
+ Feast DQM numeric feature detail page with distribution chart, statistics, and time-series analysis +
+ +
+ Feast DQM categorical feature detail page with category distribution and statistics +
+ +This is the workflow we wanted: feature owners can start from a table of health signals, filter down to the part of the feature store they care about, and then drill into the exact feature whose distribution changed. + +## How compute engines fit in + +DQM is intentionally tied to Feast's compute and offline-store architecture. The goal is to compute metrics where the data already lives whenever possible, then store the results in backend-specific monitoring tables. + +Supported backends push computation into the underlying system: + +| Backend | Compute path | Storage path | +|---|---|---| +| PostgreSQL | SQL push-down | `INSERT ON CONFLICT` | +| Snowflake | SQL push-down | `MERGE` with JSON metrics | +| BigQuery | SQL push-down | BigQuery `MERGE` | +| Redshift | SQL push-down | Data API-backed writes | +| Spark | SparkSQL push-down | Parquet-backed tables | +| Oracle | SQL through Ibis | `MERGE` | +| DuckDB | in-memory SQL | Parquet files | +| Dask | PyArrow compute | Parquet files | + +For backends without native monitoring support, Feast falls back to pulling data through the offline store and computing metrics with PyArrow and NumPy. That fallback keeps the API consistent while still allowing mature warehouse and distributed engines to do the heavy lifting. + +This design is especially important for larger feature stores. A null-rate or histogram job should not require exporting a warehouse table into a separate monitoring system. If the feature data already lives in Snowflake, BigQuery, Spark, Redshift, or another supported backend, Feast can push the computation closer to that data. + +Feast 0.64 also adds the Apache Flink compute engine, continuing the broader move toward a unified compute-engine model. DQM follows the same direction: feature quality checks should be part of the feature platform's execution model, not a sidecar that every team wires up differently. + +## REST APIs for automation + +The UI and CLI are built on top of monitoring APIs that can also be used by external systems: + +| Method | Endpoint | Use | +|---|---|---| +| `POST` | `/monitoring/compute` | submit a batch DQM job | +| `POST` | `/monitoring/auto_compute` | auto-detect dates and compute all granularities | +| `POST` | `/monitoring/compute/transient` | compute ad hoc metrics without storing them | +| `POST` | `/monitoring/compute/log` | compute metrics from serving logs | +| `POST` | `/monitoring/auto_compute/log` | auto-compute log metrics | +| `GET` | `/monitoring/jobs/{job_id}` | read DQM job status | +| `GET` | `/monitoring/metrics/features` | read per-feature metrics | +| `GET` | `/monitoring/metrics/feature_views` | read feature-view summaries | +| `GET` | `/monitoring/metrics/feature_services` | read feature-service summaries | +| `GET` | `/monitoring/metrics/baseline` | read baseline metrics | +| `GET` | `/monitoring/metrics/timeseries` | read trend data for charts and alerts | + +The transient compute endpoint is useful for exploration. If someone wants to inspect a very specific date range, Feast can compute fresh metrics and return them directly without storing them as part of the scheduled monitoring history. + +## Production shape + +A typical production setup now looks like this: + +1. Add `data_quality_monitoring.auto_baseline: true` to `feature_store.yaml` +2. Run `feast apply` to register features and compute baseline metrics +3. Schedule `feast monitor run` for batch metrics +4. Enable feature-service logging and schedule `feast monitor run --source-type log` for production serving metrics +5. Use the UI to investigate feature health and distribution changes +6. Use REST APIs to connect monitoring results to alerting, orchestration, or custom dashboards + +Monitoring also respects Feast's existing authorization model. Compute operations require update permissions, while reads and transient exploration require describe permissions. That keeps the new DQM surface aligned with the rest of the registry and feature-store API. + +## What's next + +Feast 0.64 makes DQM part of the feature store instead of an integration around it. The release adds the backend compute path, the CLI, the REST API, and the UI surface in one coherent workflow. + +The next step for users is simple: enable baselines, run monitoring jobs on the same cadence as your data pipelines, and use the UI to make feature quality visible to the teams that own production models. + +For setup details, see the [Feature Quality Monitoring guide](/docs/how-to-guides/feature-monitoring) and the [0.64.0 changelog](https://github.com/feast-dev/feast/blob/master/CHANGELOG.md#0640-2026-06-13). diff --git a/infra/website/docs/blog/feast-mlflow-kubeflow.md b/infra/website/docs/blog/feast-mlflow-kubeflow.md index 3e15cbda26a..d0b89ac7138 100644 --- a/infra/website/docs/blog/feast-mlflow-kubeflow.md +++ b/infra/website/docs/blog/feast-mlflow-kubeflow.md @@ -199,43 +199,25 @@ For cross-system lineage that extends beyond Feast into upstream data pipelines ### Data quality monitoring -Feast integrates with data quality frameworks like [Great Expectations](https://greatexpectations.io/) to detect feature drift, stale data, and schema violations before they silently degrade model performance. The workflow centers on Feast's `SavedDataset` and `ValidationReference` APIs: you save a profiled dataset during training, define a profiler using Great Expectations, and then validate new feature data against that reference in subsequent runs. +Feast's native data quality monitoring system automatically computes statistical metrics — null rates, distributions, percentiles, histograms — for every registered feature across both batch data and serving logs. It detects drift by comparing current metrics against baselines computed during `feast apply`. -```python -from feast import FeatureStore -from feast.dqm.profilers.ge_profiler import ge_profiler -from great_expectations.core import ExpectationSuite -from great_expectations.dataset import PandasDataset - -store = FeatureStore(repo_path=".") - -@ge_profiler -def my_profiler(dataset: PandasDataset) -> ExpectationSuite: - dataset.expect_column_values_to_be_between("conv_rate", min_value=0, max_value=1) - dataset.expect_column_values_to_be_between("acc_rate", min_value=0, max_value=1) - return dataset.get_expectation_suite() - -reference_job = store.get_historical_features( - entity_df=entity_df, - features=["driver_hourly_stats:conv_rate", "driver_hourly_stats:acc_rate"], -) - -dataset = store.create_saved_dataset( - from_=reference_job, - name="driver_stats_validation", - storage=storage, -) +```yaml +# feature_store.yaml +data_quality_monitoring: + auto_baseline: true +``` -reference = dataset.as_reference(name="driver_stats_ref", profiler=my_profiler) +```bash +# Compute metrics across all granularities (daily, weekly, monthly, quarterly) +feast monitor run -new_job = store.get_historical_features( - entity_df=new_entity_df, - features=["driver_hourly_stats:conv_rate", "driver_hourly_stats:acc_rate"], -) -new_job.to_df(validation_reference=reference) +# Monitor serving logs +feast monitor run --source-type log ``` -If validation fails, Feast raises a `ValidationFailed` exception with details on which expectations were violated. Monitoring feature distributions over time — and comparing them to the distributions seen during training — allows you to detect training–serving skew early, before it causes silent model degradation in production. +The monitoring UI dashboard (accessible from the sidebar) provides per-feature health status, distribution histograms, time-series drift charts, and configurable filters. Metrics are also available via REST API endpoints for integration with external alerting systems. + +For details, see the [Feature Quality Monitoring guide](/docs/how-to-guides/feature-monitoring). ### Feast Feature Registry vs. MLflow Model Registry diff --git a/infra/website/docs/blog/feast-openai-compatible-api.md b/infra/website/docs/blog/feast-openai-compatible-api.md new file mode 100644 index 00000000000..f228836afd6 --- /dev/null +++ b/infra/website/docs/blog/feast-openai-compatible-api.md @@ -0,0 +1,364 @@ +--- +title: "Using Feast's OpenAI Compatible Search API" +description: "Feast now exposes an OpenAI-compatible vector store search endpoint. Send a plain text query, get results back in the standard OpenAI format. No client-side embeddings required." +date: 2026-07-07 +authors: ["Chaitanya Patel", "Nikhil Kathole"] +--- + +
+ Sequence diagram showing a client sending a text query to Feast, which embeds and searches server-side +
+ +If you've tried to connect an AI agent to Feast's vector search, you've probably hit this wall: the agent needs to search your feature store, but Feast expects a raw embedding vector. The agent doesn't have one. It has a question in English. + +Until now, the workaround was ugly. You'd call an embedding provider (OpenAI, Ollama, whatever) to turn the text into a float array, then pass that array to Feast's vector search endpoint (`POST /search`, formerly `retrieve-online-documents`). Every client had to know both APIs, carry both sets of credentials, and run glue code whose only job was bridging the gap. + +Feast now has a new endpoint: `POST /v1/vector_stores/{vector_store_id}/search`. It follows the [OpenAI Vector Store Search API](https://platform.openai.com/docs/api-reference/vector-stores-search) format, including proper `vs_{hash}` identifiers for vector stores. You send text, Feast handles the embedding internally, and you get results back in the same JSON shape that OpenAI returns. No float arrays, no extra SDK. + +Each feature view with vector search enabled gets a deterministic `vs_` identifier (e.g. `vs_a1b2c3d4e5f6...`). Discover them via `GET /v1/vector_stores`. + +## The two-API tax + +Here's what searching Feast looked like before: + +```python +import openai +import requests + +# Step 1: Call the embedding provider yourself +embed_response = openai.embeddings.create( + model="text-embedding-3-small", + input="wireless noise-cancelling headphones" +) +query_vector = embed_response.data[0].embedding # 1536 floats + +# Step 2: Call Feast's proprietary API with the raw vector +result = requests.post("http://feast-server:6566/search", json={ + "features": [ + "product_catalog:vector", + "product_catalog:name", + "product_catalog:description", + "product_catalog:price", + ], + "query": query_vector, + "top_k": 5, + "api_version": 2, +}) +``` + +This works fine. But it has costs that add up: + +- Every service calling Feast needs an embedding SDK, an API key, and logic to handle the embedding call. Five microservices means five places managing embedding credentials. +- LLM agents can't use it. They discover tools through MCP or function calling, and they know how to call OpenAI-shaped endpoints. They don't know how to compute embeddings and pass raw float arrays to a custom API. +- The embedding model becomes a client-side decision. Different clients might use different models or versions, which means inconsistent search results against the same vector store. +- Feast's filter syntax is its own format. Not something an agent framework knows out of the box. + +## One endpoint, standard format + +With the new endpoint, that same search looks like this: + +```python +import requests + +# First, discover your vector store IDs +stores = requests.get("http://feast-server:6566/v1/vector_stores").json() +vs_id = stores["data"][0]["id"] # e.g. "vs_a1b2c3d4e5f6..." + +# Then search using the vs_ identifier +result = requests.post( + f"http://feast-server:6566/v1/vector_stores/{vs_id}/search", + json={ + "query": "wireless noise-cancelling headphones", + "max_num_results": 5, + }, +) +``` + +No embedding SDK. No raw vectors. The request and response match OpenAI's format, so anything that already talks to OpenAI can talk to Feast. + +### What happens under the hood + +When Feast receives this request, it: + +1. Embeds the query server-side using the model configured in `feature_store.yaml` (via [Sentence Transformers](https://www.sbert.net/) for local inference — no external API key required). +2. Runs vector similarity search against the feature view's online store (Postgres/pgvector, Milvus, Elasticsearch, SQLite, or whatever backend you've configured). +3. Applies filters if you provided any, using string equality, numeric comparisons, or compound AND/OR conditions in the OpenAI filter format. +4. Returns results in OpenAI's `vector_store.search_results.page` format. + +Because the embedding model is a server-side configuration, every client gets consistent results. No more worrying about whether service A is using `text-embedding-3-small` while service B accidentally stuck with `ada-002`. + +## Setting it up + +### Step 1: Configure the embedding model + +Add an `embedding_model` section to your `feature_store.yaml`: + +```yaml +project: my_project +registry: data/registry.db +provider: local + +online_store: + type: postgres + host: localhost + port: 5432 + database: feast + user: feast + password: ${DB_PASSWORD} + pgvector_enabled: true + vector_len: 384 + enable_openai_compatible_store: true + +embedding_model: + provider: sentence_transformers # default; can be omitted + model: all-MiniLM-L6-v2 +``` + +Feast uses [Sentence Transformers](https://www.sbert.net/) for embedding, so everything runs locally — no external API key required. You can use any HuggingFace model compatible with `SentenceTransformer`: + +```yaml +# Default — lightweight, fast +embedding_model: + model: all-MiniLM-L6-v2 + +# Higher quality, larger model +embedding_model: + model: BAAI/bge-small-en-v1.5 +``` + +### Step 2: Define a feature view with vector search + +```python +from feast import Entity, FeatureView, Field +from feast.types import Array, Float32, String, Float64, Int64 +from datetime import timedelta + +product = Entity(name="product_id", join_keys=["product_id"]) + +product_catalog = FeatureView( + name="product_catalog", + entities=[product], + schema=[ + Field( + name="vector", + dtype=Array(Float32), + vector_index=True, + vector_search_metric="COSINE", + ), + Field(name="name", dtype=String), + Field(name="description", dtype=String), + Field(name="category", dtype=String), + Field(name="price", dtype=Float64), + Field(name="rating", dtype=Float64), + ], + source=product_source, + ttl=timedelta(days=7), +) +``` + +### Step 3: Apply, load data, and serve + +```bash +feast apply +feast serve +``` + +### Step 4: Discover your vector store ID + +```bash +curl http://localhost:6566/v1/vector_stores +``` + +```json +{ + "object": "list", + "data": [ + { + "id": "vs_a1b2c3d4e5f6a1b2c3d4e5f6", + "object": "vector_store", + "name": "product_catalog", + "status": "completed", + "created_at": 1717200000 + } + ] +} +``` + +### Step 5: Search + +```bash +curl -X POST http://localhost:6566/v1/vector_stores/vs_a1b2c3d4e5f6a1b2c3d4e5f6/search \ + -H "Content-Type: application/json" \ + -d '{ + "query": "wireless noise-cancelling headphones", + "max_num_results": 3 + }' +``` + +Response: + +```json +{ + "object": "vector_store.search_results.page", + "search_query": ["wireless noise-cancelling headphones"], + "data": [ + { + "file_id": "vs_a1b2c3d4e5f6a1b2c3d4e5f6_42", + "filename": "vs_a1b2c3d4e5f6a1b2c3d4e5f6", + "score": 0.92, + "attributes": { + "name": "Sony WH-1000XM5", + "description": "Premium wireless noise-cancelling headphones", + "category": "Electronics", + "price": 349.99, + "rating": 4.8 + }, + "content": [ + {"type": "text", "text": "Sony WH-1000XM5"}, + {"type": "text", "text": "Premium wireless noise-cancelling headphones"}, + {"type": "text", "text": "Electronics"} + ] + } + ], + "has_more": false, + "next_page": null +} +``` + +The response follows OpenAI's `vector_store.search_results.page` schema. Any client that already parses OpenAI search results can parse this without changes. + +## Filtering + +The endpoint supports OpenAI-style filters for narrowing results beyond vector similarity. Filters work on the metadata stored alongside your vectors. + +### String filters + +```json +{ + "query": "running shoes", + "max_num_results": 5, + "filters": { + "type": "eq", + "key": "category", + "value": "Footwear" + } +} +``` + +### Numeric filters + +```json +{ + "query": "budget laptop", + "max_num_results": 5, + "filters": { + "type": "lt", + "key": "price", + "value": 500.0 + } +} +``` + +### Compound filters (AND / OR) + +```json +{ + "query": "wireless earbuds", + "max_num_results": 5, + "filters": { + "type": "and", + "filters": [ + {"type": "eq", "key": "category", "value": "Electronics"}, + {"type": "gte", "key": "rating", "value": 4.5}, + {"type": "lt", "key": "price", "value": 200.0} + ] + } +} +``` + +Comparison operators: `eq`, `ne`, `gt`, `gte`, `lt`, `lte`, `in`, `nin`. Compound operators: `and`, `or`. These nest to arbitrary depth. + +Numeric and boolean filters require the `enable_openai_compatible_store` flag in your online store config, plus a `feast apply` to add the `value_num` column to existing tables. String filters work on all existing schemas without migration. + +## What this means for AI agents + +We built this with agents in mind. When Feast added [MCP support](./feast-agents-mcp) earlier this year, agents could discover and call Feast tools dynamically. But vector search still had this gap where the agent needed to produce a float array. LLMs can't do that. + +Now the search tool is just text in, structured results out. An agent calls it the same way it calls any other OpenAI-compatible service. The feature server currently exposes these tools: + +| Capability | Endpoint | What it does | +|---|---|---| +| Structured feature lookup | `get-online-features` | Get customer profiles, account data, etc. | +| Vector search | `search` | Search with a pre-computed embedding vector (or text via `api_version: 2`) | +| List vector stores | `GET /v1/vector_stores` | Discover available vector stores and their `vs_` IDs | +| Get vector store | `GET /v1/vector_stores/{id}` | Get metadata for a specific vector store | +| Vector search (OpenAI format) | `POST /v1/vector_stores/{id}/search` | Search with plain text, embedding handled server-side | +| Write features / memory | `write-to-online-store` | Persist agent state, update features | + +`POST /retrieve-online-documents` remains available as a deprecated alias for `POST /search`. + +That last row is what this post is about. Before it existed, agents could read structured features and write state back, but they couldn't search vectors without help from glue code. + +## What this is, and what it isn't + +This makes Feast's vector search speak OpenAI's protocol. It doesn't turn Feast into a general purpose OpenAI-compatible vector database. + +| Works today | Not yet | +|---|---| +| `GET /v1/vector_stores` (list) | Creating vector stores via the API | +| `GET /v1/vector_stores/{id}` (get) | | +| `POST /v1/vector_stores/{id}/search` | | +| Plain text queries with server-side embedding | Client-provided embedding vectors on this endpoint | +| OpenAI-format filters (string, numeric, compound) | `ranking_options.score_threshold`, `ranking_options.ranker`, `rewrite_query: true` (rejected with 422) | +| All Feast online store backends | Standalone `/v1/embeddings` endpoint | + +Feature views are still defined in Python and managed through `feast apply`. Data is still ingested through Feast's existing write paths. The OpenAI-compatible layer is a read API that gives standard access to what's already in your feature store. + +## Deploying on Kubernetes + +Below is an example Kubernetes setup that deploys the feature server with Sentence Transformers for local embedding: + +```yaml +# configmap.yaml (embedding model section) +embedding_model: + provider: sentence_transformers + model: all-MiniLM-L6-v2 +``` + +```yaml +# deployment.yaml +containers: + - name: feast-server + command: ["feast", "serve", "-h", "0.0.0.0", "-p", "6566"] + ports: + - containerPort: 6566 +``` + +With this setup, embedding happens in-cluster. Nothing leaves your network. + +## Try it yourself + +```bash +# Install Feast with Sentence Transformers support +pip install feast sentence-transformers +``` + +Configure your `feature_store.yaml` with an `embedding_model` section, define a feature view with vector search enabled, run `feast apply`, load your data, start the server with `feast serve`, and search: + +```bash +# Discover your vector store IDs +curl -s http://localhost:6566/v1/vector_stores | python -m json.tool + +# Search using the vs_ identifier from the list response +curl -s http://localhost:6566/v1/vector_stores/YOUR_VS_ID/search \ + -H "Content-Type: application/json" \ + -d '{"query": "your search query", "max_num_results": 5}' | python -m json.tool +``` + +## What's next + +Next on the list: wiring up `ranking_options` and `rewrite_query` so they actually do something (right now they're accepted but ignored). We also want a standalone `/v1/embeddings` endpoint for clients that just need embeddings, and eventually the ability to create feature views through the OpenAI vector store API instead of requiring Python + `feast apply`. + +## Join the conversation + +If you're using this or have thoughts on what the OpenAI-compatible layer should support next, come find us on [Slack](https://slack.feast.dev/) or [GitHub](https://github.com/feast-dev/feast). diff --git a/infra/website/docs/blog/feast-ray-llm-posttrain.md b/infra/website/docs/blog/feast-ray-llm-posttrain.md new file mode 100644 index 00000000000..4b5eac30214 --- /dev/null +++ b/infra/website/docs/blog/feast-ray-llm-posttrain.md @@ -0,0 +1,304 @@ +--- +title: "How to Use Feast for SLM/LLM Post-Training with Ray" +description: "Keep conversation features in Feast, retrieve them for training, then stream into your trainer with Ray." +date: 2026-07-14 +authors: ["Chaitanya Patel"] +--- + +# How to Use Feast for SLM/LLM Post-Training with Ray + +Your support bot answers a lot of tickets. It’s fine—but it sounds generic. The team wants a smaller model that talks more like *your* agents: your refund wording, your product names, your tone. + +So someone says: **fine-tune on our real chats.** + +That part sounds easy. The messy part is the data—exports, notebook cleaning, and prompt formatting scattered across training scripts. + +This post walks through the [ray-llm-posttrain example](https://github.com/feast-dev/feast/tree/master/examples/ray-llm-posttrain): + +1. Put conversation features in Feast +2. Retrieve them with `get_historical_features` (entity-less date range) +3. Get rows into your trainer — stream with Ray **or** materialize with `.to_df()` + +You bring your own trainer. GPT-2 in the script is optional smoke only. + +## What’s in the example + +| Name | Type | What it holds | +|---|---|---| +| `web_documents` | [FeatureView](https://docs.feast.dev/getting-started/concepts/feature-view) | `human`, `bot`, `human_repeat_ratio`, `bot_repeat_ratio` | +| `train_example` | [OnDemandFeatureView](https://docs.feast.dev/reference/beta-on-demand-feature-view) | `cleaned_human`, `cleaned_bot`, `char_count`, `is_trainable`, `sft_text` | +| `llm_posttrain` | FeatureService | Bundles `web_documents` + `train_example` | + +Full definitions live in [feature_definitions.py](https://github.com/feast-dev/feast/blob/master/examples/ray-llm-posttrain/feature_repo/feature_definitions.py). Ray is the [offline store](https://docs.feast.dev/reference/offline-stores/ray) and one way to stream rows out—not a separate feature catalog. + +This example stays on **supported Feast APIs only** (no core patches). Conversation rows already include `document_id` and `event_timestamp` before Feast reads them. + +## Step 1: Point Feast at conversation data + +### Ray offline store (local) + +From the example [feature_store.yaml](https://github.com/feast-dev/feast/blob/master/examples/ray-llm-posttrain/feature_repo/feature_store.yaml). Cap Ray resources on a laptop—see [Ray offline store: resource management](https://docs.feast.dev/reference/offline-stores/ray#important-resource-management): + +```yaml +project: ray_llm_posttrain +registry: data/registry.db +provider: local + +offline_store: + type: ray + storage_path: data/ray_storage + enable_ray_logging: false + ray_conf: + num_cpus: 2 + object_store_memory: 104857600 + _memory: 524288000 + +batch_engine: + type: ray.engine + max_workers: 2 + +online_store: + type: sqlite + path: data/online_store.db + +entity_key_serialization_version: 3 +auth: + type: no_auth +``` + +You can also start from the built-in template: + +```bash +feast init -t ray my_ray_project +``` + +See the [Ray template / offline store docs](https://docs.feast.dev/reference/offline-stores/ray#quick-start-with-ray-template) and the related blog [Scaling ML with Feast and Ray](/blog/feast-ray-distributed-processing). + +### Demo seed: prepare parquet, then `RaySource` + +[RaySource](https://docs.feast.dev/reference/data-sources/ray) tells Feast how to load data through Ray. Hugging Face is only used in a **prepare script**—not as a live Feast source that invents timestamps at retrieval time. + +`nampdn-ai/tiny-webtext` has no `document_id` / `event_timestamp`. Entity-less retrieval needs those columns on the source. We add them **outside Feast**, write parquet, then point Feast at that file (supported path): + +```bash +PYTHONPATH=../../sdk/python python scripts/prepare_data.py +# → feature_repo/data/tiny_webtext.parquet +``` + +```python +from feast.infra.offline_stores.contrib.ray_offline_store.ray_source import RaySource + +tiny_web = RaySource( + name="tiny_webtext", + reader_type="parquet", + path="data/tiny_webtext.parquet", + timestamp_field="event_timestamp", +) +``` + +In production you’d skip the HF prepare step and register your real conversation store (warehouse / lake / parquet) that already has join keys and timestamps. + +More reader types are in the [Ray data source reference](https://docs.feast.dev/reference/data-sources/ray#supported-reader_type-values). + +### Feature view + +```python +web_documents = FeatureView( + name="web_documents", + entities=[document], + ttl=timedelta(days=365), + schema=[ + Field(name="human", dtype=String), + Field(name="bot", dtype=String), + Field(name="human_repeat_ratio", dtype=Float64), + Field(name="bot_repeat_ratio", dtype=Float64), + ], + source=tiny_web, + online=False, +) +``` + +### Optional: OnDemandFeatureView for derived training features + +If you want Feast to own `sft_text` / quality gates (same idea as in the [ODFV docs](https://docs.feast.dev/reference/beta-on-demand-feature-view)): + +```python +@on_demand_feature_view( + sources=[web_documents], + schema=[ + Field(name="cleaned_human", dtype=String), + Field(name="cleaned_bot", dtype=String), + Field(name="char_count", dtype=Int64), + Field(name="is_trainable", dtype=Bool), + Field(name="sft_text", dtype=String), + ], + mode="pandas", +) +def train_example(inputs): + cleaned_human = inputs["human"].fillna("").astype(str).str.strip() + cleaned_bot = inputs["bot"].fillna("").astype(str).str.strip() + # ... length + repeat-ratio gate ... + sft_text = ( + "<|im_start|>user\n" + cleaned_human + "<|im_end|>\n" + "<|im_start|>assistant\n" + cleaned_bot + "<|im_end|>" + ) + return pd.DataFrame({...}) +``` + +```python +llm_posttrain = FeatureService( + name="llm_posttrain", + features=[web_documents, train_example], +) +``` + +Apply: + +```bash +cd examples/ray-llm-posttrain/feature_repo +feast apply +``` + +## Step 2: Retrieve for training (entity-less) + +No `entity_df`—just a date window. That pattern is covered in [Historical Features Without Entity IDs](/blog/entity-less-historical-features-retrieval) and the [FAQ](https://docs.feast.dev/getting-started/faq#how-do-i-run-get_historical_features-without-providing-an-entity-dataframe): + +```python +from datetime import datetime, timezone +from feast import FeatureStore + +store = FeatureStore(repo_path="feature_repo") + +job = store.get_historical_features( + features=[ + "web_documents:human", + "web_documents:bot", + "web_documents:human_repeat_ratio", + "web_documents:bot_repeat_ratio", + ], + start_date=datetime(2024, 6, 1, tzinfo=timezone.utc), + end_date=datetime(2024, 7, 1, tzinfo=timezone.utc), +) +``` + +Then choose how you turn that job into training rows. + +## Step 3: Two ways into the trainer + +| Path | ODFV runs? | When to use | +|---|---|---| +| `job.to_ray_dataset()` then preprocess | **No** | Stream FeatureView columns; shape `sft_text` yourself | +| `job.to_df()` / `to_arrow()` | **Yes** | Want `train_example` outputs from Feast | + +Pick **Option A** when you want full control over text formatting or need custom preprocessing (e.g., multi-turn chat templates, tokenization-aware truncation). Pick **Option B** when you want Feast to enforce quality gates consistently across training and serving. + +### Option A — Stream with Ray, preprocess yourself + +`to_ray_dataset()` returns a Ray Dataset of retrieved FeatureView columns. It does **not** apply OnDemandFeatureViews. Build training text with Ray `map_batches` (as in [train_sft.py](https://github.com/feast-dev/feast/blob/master/examples/ray-llm-posttrain/scripts/train_sft.py)): + +```python +ds = job.to_ray_dataset() + +def preprocess_sft(batch): + import pandas as pd + + if not isinstance(batch, pd.DataFrame): + batch = pd.DataFrame(batch) + human = batch["human"].fillna("").astype(str).str.strip() + bot = batch["bot"].fillna("").astype(str).str.strip() + ok = bot.str.len() >= 64 + sft_text = ( + "<|im_start|>user\n" + human + "<|im_end|>\n" + "<|im_start|>assistant\n" + bot + "<|im_end|>" + ) + return pd.DataFrame({"sft_text": sft_text}).loc[ok].reset_index(drop=True) + +train_ds = ds.map_batches(preprocess_sft, batch_format="pandas") +# → hand train_ds to your SLM/LLM trainer +``` + +Run the example default path: + +```bash +PYTHONPATH=../../sdk/python python scripts/train_sft.py --dry-run +``` + +### Option B — Use the ODFV, then train + +Materialize with `.to_df()` so `train_example` runs (same retrieval/serving idea as in the [ODFV overview](https://docs.feast.dev/reference/beta-on-demand-feature-view#why-use-on-demand-feature-views)): + +```python +df = store.get_historical_features( + features=store.get_feature_service("llm_posttrain"), + start_date=datetime(2024, 6, 1, tzinfo=timezone.utc), + end_date=datetime(2024, 7, 1, tzinfo=timezone.utc), +).to_df() + +trainable = df[df["is_trainable"] & df["sft_text"].astype(str).str.len().gt(0)] +# trainable["sft_text"] → your trainer +# or: import ray; ray.data.from_pandas(trainable[["sft_text"]]) +``` + +```bash +PYTHONPATH=../../sdk/python python scripts/train_sft.py --dry-run --via-df +``` + +### The same ODFV at serving time + +The `train_example` ODFV runs identically during online serving — the quality gate and formatting logic stay in one place: + +```python +# At inference time, the same ODFV runs on the fly +features = store.get_online_features( + features=["train_example:sft_text", "train_example:is_trainable"], + entity_rows=[{"document_id": "doc_42"}], +).to_dict() +# features["sft_text"], features["is_trainable"] — same logic as training +``` + +## Try the full example + +```bash +cd examples/ray-llm-posttrain +uv pip install -e "../../sdk/python[ray]" -r requirements.txt +PYTHONPATH=../../sdk/python python scripts/prepare_data.py +cd feature_repo && feast apply && cd .. + +PYTHONPATH=../../sdk/python python scripts/train_sft.py --dry-run +PYTHONPATH=../../sdk/python python scripts/train_sft.py --dry-run --via-df + +# optional GPT-2 smoke +PYTHONPATH=../../sdk/python python scripts/train_sft.py --max-steps 20 +``` + +Details: [ray-llm-posttrain README](https://github.com/feast-dev/feast/tree/master/examples/ray-llm-posttrain). + +## Takeaways + +1. **Keep conversation features in Feast** — this example’s `web_documents`. +2. **Stream with Ray** — `to_ray_dataset()`, then preprocess training text yourself. +3. **Want ODFVs** — `.to_df()` / `.to_arrow()` to materialize, then train. +4. **Bring your own trainer** — GPT-2 in the example is optional. + +## References + +**This example** + +- [ray-llm-posttrain example](https://github.com/feast-dev/feast/tree/master/examples/ray-llm-posttrain) +- [feature_definitions.py](https://github.com/feast-dev/feast/blob/master/examples/ray-llm-posttrain/feature_repo/feature_definitions.py) +- [train_sft.py](https://github.com/feast-dev/feast/blob/master/examples/ray-llm-posttrain/scripts/train_sft.py) + +**Docs** + +- [Ray offline store](https://docs.feast.dev/reference/offline-stores/ray) +- [Ray data source](https://docs.feast.dev/reference/data-sources/ray) +- [Ray compute engine](https://docs.feast.dev/reference/compute-engine/ray) +- [On demand feature views](https://docs.feast.dev/reference/beta-on-demand-feature-view) +- [Feature retrieval](https://docs.feast.dev/getting-started/concepts/feature-retrieval) +- [FAQ: historical features without entity dataframe](https://docs.feast.dev/getting-started/faq#how-do-i-run-get_historical_features-without-providing-an-entity-dataframe) + +**Related blogs & tutorials** + +- [Historical Features Without Entity IDs](/blog/entity-less-historical-features-retrieval) +- [Scaling ML with Feast and Ray](/blog/feast-ray-distributed-processing) +- [Validating historical features](https://docs.feast.dev/tutorials/validating-historical-features) diff --git a/infra/website/docs/blog/feast-unity-catalog-integration.md b/infra/website/docs/blog/feast-unity-catalog-integration.md new file mode 100644 index 00000000000..1ee977a71eb --- /dev/null +++ b/infra/website/docs/blog/feast-unity-catalog-integration.md @@ -0,0 +1,342 @@ +--- +title: "Feast Gets Native Apache Iceberg Support" +description: "Feast now reads features from any Iceberg catalog — REST, SQL, Hive, Glue, DynamoDB. Connect to Unity Catalog, Apache Polaris, Nessie, or your own PyIceberg catalog. Full support for get_historical_features, materialize, and online serving." +date: 2026-07-18 +authors: ["Nikhil Kathole"] +--- + +# Feast Gets Native Apache Iceberg Support + +Apache Iceberg has become the open table format. Your data lake is probably already on it — whether through Databricks, Snowflake, AWS, or self-managed infrastructure. But until now, connecting Feast to Iceberg tables meant either going through Spark (heavyweight, slow to start) or copying data into Feast-managed Parquet files (data duplication, governance gap). + +Feast now ships a native `IcebergSource` that reads directly from any Iceberg catalog. No data copies. Your feature tables live where they already live — in your Iceberg catalog — and Feast reads from them via PyIceberg. With the DuckDB offline store, you don't even need a Spark cluster — reads happen entirely in-process. + +## Why This Matters + +Before this, the path from "data in Iceberg" to "features in Feast" looked like this: + +1. Data engineers build Iceberg tables in their catalog (UC, Glue, Hive) +2. ML engineers copy data to Feast-managed Parquet files, or configure a SparkSource that couples them to a specific compute engine +3. Two copies of the data. Two metadata systems. No connection between them. + +Now the path is: + +1. Data engineers build Iceberg tables in their catalog +2. ML engineers point `IcebergSource` at the table +3. Done. Feast reads directly from the catalog via PyIceberg. One copy. One source of truth. Choose DuckDB for lightweight local reads or Spark when you need distributed compute — the data source definition stays the same either way. + +## What You Get + +### Any Iceberg Catalog + +`IcebergSource` supports every catalog backend that PyIceberg supports: + +| `catalog_type` | Backend | Example Use Case | +|---|---|---| +| `"rest"` | Iceberg REST Catalog | Databricks Unity Catalog, Apache Polaris, Project Nessie, Snowflake Open Catalog | +| `"sql"` | SQL-backed catalog | Local dev with SQLite, CI/CD, PostgreSQL-backed catalogs | +| `"hive"` | Hive Metastore | On-premise Hadoop, EMR | +| `"glue"` | AWS Glue Data Catalog | AWS-native lakehouse | +| `"dynamodb"` | DynamoDB catalog | Serverless AWS | + +### Both Offline Stores + +| Operation | DuckDB | Spark | +|---|---|---| +| `feast apply` | Yes | Yes | +| `get_historical_features` | Yes | Yes | +| `materialize` / `materialize-incremental` | Yes | Yes | +| `get_online_features` | Yes | Yes | + +Both offline stores use PyIceberg for the actual Iceberg table scan — the difference is what happens after. DuckDB processes the Arrow table in-process (no JVM, no cluster), making it ideal for local development and moderate-scale workloads. Spark is there when you need distributed compute over large datasets. The same `IcebergSource` definition works with either offline store — just change `offline_store.type` in your YAML. + +### Full Iceberg Semantics + +Every read goes through PyIceberg's `table.scan().to_arrow()`. This means you get proper Iceberg semantics: schema evolution, partition pruning, and snapshot isolation — not just raw Parquet file reads. + +## Quick Start + +### Install + +```bash +pip install "feast[iceberg]" +``` + +### Define a Source + +```python +from feast.infra.data_sources.contrib.iceberg_catalog import IcebergSource + +driver_stats = IcebergSource( + warehouse="my_catalog", + namespace="ml_features", + table="driver_hourly_stats", + catalog_type="rest", + endpoint="https://my-iceberg-catalog.example.com", + token_env_var="CATALOG_TOKEN", + timestamp_field="event_timestamp", +) +``` + +### Use It + +```python +from datetime import timedelta +from feast import Entity, FeatureView, Field +from feast.types import Float64, Int64 + +driver = Entity(name="driver", join_keys=["driver_id"]) + +driver_stats_fv = FeatureView( + name="driver_hourly_stats", + entities=[driver], + ttl=timedelta(days=365), + schema=[ + Field(name="driver_id", dtype=Int64), + Field(name="conv_rate", dtype=Float64), + Field(name="acc_rate", dtype=Float64), + Field(name="avg_daily_trips", dtype=Int64), + ], + source=driver_stats, + online=True, +) +``` + +```yaml +# feature_store.yaml +project: my_project +registry: data/registry.db +provider: local +online_store: + type: sqlite + path: data/online_store.db +offline_store: + type: duckdb +``` + +```bash +feast apply +``` + +Then use it like any other Feast source: + +```python +from feast import FeatureStore +import pandas as pd +from datetime import datetime, timezone + +store = FeatureStore(repo_path=".") + +# Training data +training_df = store.get_historical_features( + entity_df=pd.DataFrame({ + "driver_id": [1001, 1002, 1003], + "event_timestamp": [datetime(2026, 7, 1, tzinfo=timezone.utc)] * 3, + }), + features=["driver_hourly_stats:conv_rate", "driver_hourly_stats:acc_rate"], +).to_df() + +# Materialize to online store +store.materialize_incremental(end_date=datetime.now(tz=timezone.utc)) + +# Online serving +online = store.get_online_features( + features=["driver_hourly_stats:conv_rate", "driver_hourly_stats:acc_rate"], + entity_rows=[{"driver_id": 1001}], +).to_dict() +``` + +## Catalog Examples + +### AWS Glue + +```python +glue_source = IcebergSource( + warehouse="my_glue_database", + namespace="ml_features", + table="driver_stats", + catalog_type="glue", + catalog_properties={"region_name": "us-east-1"}, + timestamp_field="event_timestamp", +) +``` + +### Hive Metastore + +```python +hive_source = IcebergSource( + endpoint="thrift://hive-metastore:9083", + warehouse="warehouse", + namespace="features", + table="driver_stats", + catalog_type="hive", + timestamp_field="event_timestamp", +) +``` + +### Apache Polaris / Nessie + +```python +polaris_source = IcebergSource( + endpoint="https://polaris.example.com", + warehouse="my_catalog", + namespace="ml", + table="features", + catalog_type="rest", + token_env_var="POLARIS_TOKEN", + timestamp_field="event_timestamp", +) +``` + +### Local Development (SQLite-backed) + +For development and CI/CD, use a local PyIceberg SQL catalog — no external service required: + +```python +local_source = IcebergSource( + warehouse="dev_warehouse", + namespace="default", + table="driver_stats", + catalog_type="sql", + catalog_name="dev_catalog", + catalog_properties={ + "uri": "sqlite:////tmp/iceberg_catalog.db", + "warehouse": "file:///tmp/iceberg_warehouse", + }, + timestamp_field="event_timestamp", +) +``` + +## Use Case: Unity Catalog Integration + +Unity Catalog users get everything above with simpler configuration. `UnityCatalogSource` extends `IcebergSource` with UC-specific defaults: + +- Default connection via `DATABRICKS_HOST` and `DATABRICKS_TOKEN` environment variables — no manual endpoint or token setup +- Three-level naming (`warehouse.namespace.table`) maps directly to UC's catalog/schema/table hierarchy + +### Databricks Setup + +```bash +export DATABRICKS_HOST="https://your-workspace.cloud.databricks.com" +export DATABRICKS_TOKEN="dapi_your_token_here" +``` + +```python +from feast.infra.data_sources.contrib.iceberg_catalog import UnityCatalogSource + +driver_stats_source = UnityCatalogSource( + warehouse="ml_catalog", + namespace="driver_features", + table="driver_hourly_stats", + timestamp_field="event_timestamp", + created_timestamp_column="created", + description="Hourly aggregated driver statistics", +) +``` + +That's it. No endpoint or token parameters needed — they come from the environment variables. + +### Optional: Governance Metadata Sync + +If you want `feast apply` to annotate your UC tables with `feast.*` properties (project name, feature view, primary keys, owner), you can use the `UnityCatalogProvider`: + +```yaml +# feature_store.yaml +provider: unity_catalog +``` + +This is optional. For most users, `provider: local` is sufficient. Reads, materialization, and online serving work the same regardless of which provider you use. + +### OSS Unity Catalog + +The integration also works with the open-source Unity Catalog, with some differences: + +| Capability | Databricks UC | OSS UC | +|---|---|---| +| Read via Iceberg REST | Built-in | Requires `uniform_iceberg_metadata_location` in H2 DB | +| Read via SQL catalog | Yes | Yes | +| Credential vending | Yes | Not available | + +For OSS UC, set `credential_vending=False` and `token_env_var=None`: + +```python +source = UnityCatalogSource( + warehouse="unity", + namespace="default", + table="driver_hourly_stats", + endpoint="http://localhost:8080/api/2.1/unity-catalog/iceberg", + token_env_var=None, + credential_vending=False, + catalog_type="sql", # recommended for OSS UC + catalog_name="my_catalog", + catalog_properties={ + "uri": "sqlite:////tmp/pyiceberg_catalog.db", + "warehouse": "file:///tmp/warehouse", + }, + timestamp_field="event_timestamp", + register_as_feature_table=False, +) +``` + +## How It Works + +### Read Path + +All data reads go through PyIceberg, regardless of catalog type: + +``` +IcebergSource.get_catalog_client() + → PyIceberg catalog (REST / SQL / Hive / Glue / DynamoDB) + → table.scan().to_arrow() + → Arrow table + → DuckDB ibis memtable or Spark DataFrame +``` + +If the catalog is misconfigured, you get a clear error — consistent with how every other Feast data source works. + +### Catalog Name Isolation + +Each source has a `catalog_name` parameter (default: `"feast_iceberg"`). This is the instance name PyIceberg uses when loading the catalog. If you have multiple sources pointing at different catalogs, use different names to avoid collisions: + +```python +production = IcebergSource(catalog_name="prod_catalog", ...) +staging = IcebergSource(catalog_name="staging_catalog", ...) +``` + +## What's Not Supported + +- **Write-back to Iceberg/UC tables.** Feast reads from existing tables; it doesn't write feature data back. The `write_to_offline_store` API only supports `FileSource` (DuckDB) and `SparkSource` (Spark). Your data engineering pipelines create and populate the tables. +- **Table creation.** Tables must exist before Feast can read from them. `feast apply` registers the feature view in Feast's registry, not the table in the catalog. + +## Configuration Reference + +### IcebergSource + +| Parameter | Default | Description | +|---|---|---| +| `warehouse` | *required* | Catalog or warehouse name | +| `namespace` | *required* | Schema or namespace | +| `table` | *required* | Table name | +| `catalog_type` | `"rest"` | Backend: `"rest"`, `"sql"`, `"hive"`, `"glue"`, `"dynamodb"` | +| `catalog_name` | `"feast_iceberg"` | PyIceberg instance name (unique per catalog to avoid collisions) | +| `endpoint` | `None` | Catalog endpoint URL | +| `catalog_properties` | `{}` | Additional catalog config (e.g., `{"uri": "sqlite:///..."}`) | +| `token_env_var` | `None` | Env var containing auth token | +| `credential_vending` | `True` | Request scoped storage credentials | +| `timestamp_field` | `None` | Event timestamp column | +| `created_timestamp_column` | `None` | Creation timestamp for deduplication | + +### UnityCatalogSource (extends IcebergSource) + +All `IcebergSource` parameters plus: + +| Parameter | Default | Description | +|---|---|---| +| `endpoint` | From `DATABRICKS_HOST` | Defaults to `{DATABRICKS_HOST}/api/2.1/unity-catalog/iceberg` | +| `token_env_var` | `"DATABRICKS_TOKEN"` | Defaults to Databricks token env var | +| `register_as_feature_table` | `True` | Sync `feast.*` properties to UC on `feast apply` | +| `sync_lineage` | `True` | Record lineage in UC (Databricks only) | + +--- + +*Native Iceberg support is available in Feast 0.64+. Install with `pip install "feast[iceberg]"` and check the [Iceberg data source documentation](/reference/data-sources/iceberg) for the full API reference.* diff --git a/infra/website/public/images/blog/feast-dqm-monitoring-hero.png b/infra/website/public/images/blog/feast-dqm-monitoring-hero.png new file mode 100644 index 00000000000..86d7125a3c3 Binary files /dev/null and b/infra/website/public/images/blog/feast-dqm-monitoring-hero.png differ diff --git a/infra/website/public/images/blog/feast-dqm-ui-all-features.png b/infra/website/public/images/blog/feast-dqm-ui-all-features.png new file mode 100644 index 00000000000..4e728a2e86e Binary files /dev/null and b/infra/website/public/images/blog/feast-dqm-ui-all-features.png differ diff --git a/infra/website/public/images/blog/feast-dqm-ui-categorical-feature.png b/infra/website/public/images/blog/feast-dqm-ui-categorical-feature.png new file mode 100644 index 00000000000..a3e6f22b74c Binary files /dev/null and b/infra/website/public/images/blog/feast-dqm-ui-categorical-feature.png differ diff --git a/infra/website/public/images/blog/feast-dqm-ui-numeric-feature.png b/infra/website/public/images/blog/feast-dqm-ui-numeric-feature.png new file mode 100644 index 00000000000..0b5e7f0d23d Binary files /dev/null and b/infra/website/public/images/blog/feast-dqm-ui-numeric-feature.png differ diff --git a/infra/website/public/images/blog/feast-openai-compat-flow.png b/infra/website/public/images/blog/feast-openai-compat-flow.png new file mode 100644 index 00000000000..c7dcac8a5ed Binary files /dev/null and b/infra/website/public/images/blog/feast-openai-compat-flow.png differ diff --git a/infra/website/src/components/Navigation.astro b/infra/website/src/components/Navigation.astro index a5987bf6348..143fcbc047f 100644 --- a/infra/website/src/components/Navigation.astro +++ b/infra/website/src/components/Navigation.astro @@ -14,11 +14,29 @@ COMMUNITY - +
@@ -38,7 +56,7 @@ top: 0; left: 0; right: 0; - background-color: white; + background-color: var(--color-nav-bg); z-index: 1000; } @@ -80,6 +98,35 @@ margin-left: 32px; } + .nav-right { + display: flex; + align-items: center; + gap: 4px; + padding-right: var(--content-padding); + } + + .theme-toggle { + background: none; + border: none; + padding: 8px; + cursor: pointer; + color: var(--color-text); + display: flex; + align-items: center; + justify-content: center; + opacity: 0.7; + transition: opacity 0.2s ease; + } + + .theme-toggle:hover { + opacity: 1; + } + + .icon-sun { display: none; } + .icon-moon { display: block; } + :global([data-theme="dark"]) .icon-sun { display: block; } + :global([data-theme="dark"]) .icon-moon { display: none; } + .mobile-menu-button { display: block; background: none; @@ -87,7 +134,6 @@ padding: 8px; cursor: pointer; color: var(--color-text); - margin-right: var(--content-padding); } @media (min-width: 1024px) { @@ -95,7 +141,7 @@ display: flex; align-items: center; } - + .mobile-menu-button { display: none; } @@ -104,9 +150,9 @@ .mobile-menu { display: none; width: 100%; - background: white; + background: var(--color-nav-bg); padding: 16px 0; - border-top: 1px solid #eee; + border-top: 1px solid var(--color-border); position: absolute; top: 52px; left: 0; @@ -122,17 +168,27 @@ } .mobile-menu a:hover { - background-color: #f5f5f5; + background-color: var(--color-nav-hover); } \ No newline at end of file diff --git a/infra/website/src/layouts/BaseLayout.astro b/infra/website/src/layouts/BaseLayout.astro index 05d940005ba..96e30a6aee2 100644 --- a/infra/website/src/layouts/BaseLayout.astro +++ b/infra/website/src/layouts/BaseLayout.astro @@ -52,7 +52,17 @@ const { - + +
diff --git a/infra/website/src/layouts/BlogLayout.astro b/infra/website/src/layouts/BlogLayout.astro index c5f8379fc80..9fe68a9daa7 100644 --- a/infra/website/src/layouts/BlogLayout.astro +++ b/infra/website/src/layouts/BlogLayout.astro @@ -14,7 +14,7 @@ const { frontmatter } = Astro.props;

{frontmatter.title}

{frontmatter.date && ( -