diff --git a/.claude/rules/feast-components.md b/.claude/rules/feast-components.md index 5c03cb0bd3d..02b1c6f4dd4 100644 --- a/.claude/rules/feast-components.md +++ b/.claude/rules/feast-components.md @@ -24,6 +24,7 @@ For testing patterns and debugging, also read `skills/feast-testing/SKILL.md`. - **Unit tests**: add or update tests in `sdk/python/tests/unit/infra//` - **Integration tests**: run `make test-python-integration-local`; add a universal test case in `sdk/python/tests/integration/` if the change affects retrieval or materialization behavior +- **SQL registry binary columns**: in `infra/registry/sql.py`, a new column that stores a serialized proto or blob metadata must use `ProtoBytes`, not `LargeBinary` directly — `LargeBinary` maps to MySQL `BLOB` (64 KB cap) and silently truncates large protos - **Protos**: if you add a field to a proto message, recompile with `make protos` and update serialization helpers in `proto_registry_utils.py` - **Both SDKs**: if the change affects online serving, check whether the Go server (`go/`) also needs updating - **Skills/Rules**: if the change introduces new patterns, interfaces, or conventions that agents should follow, update the relevant section in `skills/feast-architecture/SKILL.md` (and `skills/feast-testing/SKILL.md` if testing patterns changed) diff --git a/.codecov.yaml b/.codecov.yaml new file mode 100644 index 00000000000..2fa599642ec --- /dev/null +++ b/.codecov.yaml @@ -0,0 +1,48 @@ +codecov: + require_ci_to_pass: true + +coverage: + precision: 2 + round: down + range: "50...70" + + status: + project: + default: + informational: true + target: auto + threshold: 1% + patch: + default: + informational: true + target: 70% + +comment: + layout: "reach,diff,flags,files,footer" + behavior: default + require_changes: false + require_base: false + require_head: true + show_carryforward_flags: true + +flags: + python-unit: + paths: + - sdk/python/feast/ + carryforward: true + go-feature-server: + paths: + - go/ + carryforward: true + +ignore: + - "sdk/python/tests/**" + - "**/*_pb2.py" + - "**/*_pb2_grpc.py" + - "sdk/python/feast/protos/**" + - "sdk/python/feast/embedded_go/**" + - "protos/**" + - "docs/**" + - "ui/**" + - "java/**" + - "infra/feast-operator/test/**" diff --git a/.cursor/rules/feast-components.mdc b/.cursor/rules/feast-components.mdc index a474f00fc47..f015619020d 100644 --- a/.cursor/rules/feast-components.mdc +++ b/.cursor/rules/feast-components.mdc @@ -20,6 +20,7 @@ For testing patterns and debugging, also read `skills/feast-testing/SKILL.md`. - **Unit tests**: add or update tests in `sdk/python/tests/unit/infra//` - **Integration tests**: run `make test-python-integration-local`; add a universal test case in `sdk/python/tests/integration/` if the change affects retrieval or materialization behavior +- **SQL registry binary columns**: in `infra/registry/sql.py`, a new column that stores a serialized proto or blob metadata must use `ProtoBytes`, not `LargeBinary` directly — `LargeBinary` maps to MySQL `BLOB` (64 KB cap) and silently truncates large protos - **Protos**: if you add a field to a proto message, recompile with `make protos` and update serialization helpers in `proto_registry_utils.py` - **Both SDKs**: if the change affects online serving, check whether the Go server (`go/`) also needs updating - **Skills/Rules**: if the change introduces new patterns, interfaces, or conventions that agents should follow, update the relevant section in `skills/feast-architecture/SKILL.md` (and `skills/feast-testing/SKILL.md` if testing patterns changed) diff --git a/.github/workflows/master_only.yml b/.github/workflows/master_only.yml index a5ea51879ed..4900135add1 100644 --- a/.github/workflows/master_only.yml +++ b/.github/workflows/master_only.yml @@ -94,16 +94,19 @@ jobs: include: - component: feature-server-dev target: feature-server-dev + image_name: feature-server build_args: DOCKER_PUSH=true DOCKER_PLATFORMS=linux/amd64,linux/arm64 push_mode: imagetools - component: feature-transformation-server target: feature-transformation-server + image_name: feature-transformation-server build_args: "" push_mode: all_tags - component: feast-operator target: feast-operator - build_args: "" - push_mode: all_tags + image_name: feast-operator + build_args: DOCKER_PUSH=true DOCKER_PLATFORMS=linux/amd64,linux/arm64 + push_mode: imagetools env: REGISTRY: quay.io/feastdev-ci steps: @@ -135,7 +138,7 @@ jobs: - name: Push image run: | if [[ "${{ matrix.push_mode }}" == "imagetools" ]]; then - docker buildx imagetools create -t ${REGISTRY}/feature-server:develop ${REGISTRY}/feature-server:${GITHUB_SHA} + docker buildx imagetools create -t ${REGISTRY}/${{ matrix.image_name }}:develop ${REGISTRY}/${{ matrix.image_name }}:${GITHUB_SHA} else - docker tag ${REGISTRY}/${{ matrix.target }}:${GITHUB_SHA} ${REGISTRY}/${{ matrix.target }}:develop && docker push ${REGISTRY}/${{ matrix.target }} --all-tags + docker tag ${REGISTRY}/${{ matrix.image_name }}:${GITHUB_SHA} ${REGISTRY}/${{ matrix.image_name }}:develop && docker push ${REGISTRY}/${{ matrix.image_name }} --all-tags fi diff --git a/.github/workflows/pr_duckdb_integration_tests.yml b/.github/workflows/pr_duckdb_integration_tests.yml index d099d7fa582..862c5e679ea 100644 --- a/.github/workflows/pr_duckdb_integration_tests.yml +++ b/.github/workflows/pr_duckdb_integration_tests.yml @@ -26,9 +26,9 @@ jobs: token: ${{ secrets.GITHUB_TOKEN }} submodules: recursive - name: Setup pixi - uses: prefix-dev/setup-pixi@v0.8.1 + uses: prefix-dev/setup-pixi@v0.10.0 with: - pixi-version: v0.63.1 + pixi-version: v0.75.0 environments: duckdb-tests cache: true - name: Run DuckDB offline store integration tests diff --git a/.github/workflows/pr_integration_tests.yml b/.github/workflows/pr_integration_tests.yml index 88d5c102250..60d0b13ce72 100644 --- a/.github/workflows/pr_integration_tests.yml +++ b/.github/workflows/pr_integration_tests.yml @@ -49,6 +49,7 @@ jobs: ref: refs/pull/${{ github.event.pull_request.number }}/merge submodules: recursive persist-credentials: false + allow-unsafe-pr-checkout: true # Security: gated by label check above - name: Setup Python uses: actions/setup-python@v5 id: setup-python @@ -108,6 +109,7 @@ jobs: ref: refs/pull/${{ github.event.pull_request.number }}/merge submodules: recursive persist-credentials: false + allow-unsafe-pr-checkout: true # Security: gated by label check above - name: Setup Python uses: actions/setup-python@v5 with: diff --git a/.github/workflows/pr_ray_integration_tests.yml b/.github/workflows/pr_ray_integration_tests.yml index 4d54c8e34ed..81c82bf0c49 100644 --- a/.github/workflows/pr_ray_integration_tests.yml +++ b/.github/workflows/pr_ray_integration_tests.yml @@ -26,9 +26,9 @@ jobs: token: ${{ secrets.GITHUB_TOKEN }} submodules: recursive - name: Setup pixi - uses: prefix-dev/setup-pixi@v0.8.1 + uses: prefix-dev/setup-pixi@v0.10.0 with: - pixi-version: v0.63.1 + pixi-version: v0.75.0 environments: ray-tests cache: true - name: Run Ray integration tests (offline store + compute engine) diff --git a/.github/workflows/pr_registration_integration_tests.yml b/.github/workflows/pr_registration_integration_tests.yml index 81801b643b6..ab60cb22b3a 100644 --- a/.github/workflows/pr_registration_integration_tests.yml +++ b/.github/workflows/pr_registration_integration_tests.yml @@ -27,10 +27,11 @@ jobs: ref: refs/pull/${{ github.event.pull_request.number }}/merge submodules: recursive persist-credentials: false + allow-unsafe-pr-checkout: true # Security: gated by label check above - name: Setup pixi - uses: prefix-dev/setup-pixi@v0.8.1 + uses: prefix-dev/setup-pixi@v0.10.0 with: - pixi-version: v0.63.1 + pixi-version: v0.75.0 environments: registration-tests cache: true - name: Run registration integration tests (local) @@ -58,6 +59,7 @@ jobs: ref: refs/pull/${{ github.event.pull_request.number }}/merge submodules: recursive persist-credentials: false + allow-unsafe-pr-checkout: true # Security: gated by label check above - name: Authenticate to Google Cloud uses: 'google-github-actions/auth@v2' with: @@ -80,9 +82,9 @@ jobs: - name: Install Hadoop dependencies run: make install-hadoop-dependencies-ci - name: Setup pixi - uses: prefix-dev/setup-pixi@v0.8.1 + uses: prefix-dev/setup-pixi@v0.10.0 with: - pixi-version: v0.63.1 + pixi-version: v0.75.0 environments: registration-tests cache: true - name: Run registration integration tests (CI) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index a5377982b29..ffa91034d32 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -29,14 +29,16 @@ on: token: description: 'Personal Access Token' required: true - default: "" type: string publish_ui: description: 'Publish to NPM?' required: true - default: true type: boolean +permissions: + contents: read + id-token: write + jobs: publish-python-sdk: uses: ./.github/workflows/publish_python_sdk.yml diff --git a/.github/workflows/publish_images.yml b/.github/workflows/publish_images.yml index 8b9abddcb0e..5ef0a972ae3 100644 --- a/.github/workflows/publish_images.yml +++ b/.github/workflows/publish_images.yml @@ -74,7 +74,7 @@ jobs: env: VERSION_WITHOUT_PREFIX: ${{ steps.get-version.outputs.version_without_prefix }} run: | - if [ "${{ matrix.component }}" = "feature-server" ]; then + if [[ "${{ matrix.component }}" == "feature-server" || "${{ matrix.component }}" == "feast-operator" ]]; then make build-${{ matrix.component }}-docker REGISTRY=${REGISTRY} VERSION=${VERSION_WITHOUT_PREFIX} DOCKER_PUSH=true DOCKER_PLATFORMS=linux/amd64,linux/arm64 else make build-${{ matrix.component }}-docker REGISTRY=${REGISTRY} VERSION=${VERSION_WITHOUT_PREFIX} @@ -84,8 +84,8 @@ jobs: VERSION_WITHOUT_PREFIX: ${{ steps.get-version.outputs.version_without_prefix }} HIGHEST_SEMVER_TAG: ${{ steps.get-version.outputs.highest_semver_tag }} run: | - if [ "${{ matrix.component }}" = "feature-server" ]; then - echo "feature-server image pushed via buildx during build step" + if [[ "${{ matrix.component }}" == "feature-server" || "${{ matrix.component }}" == "feast-operator" ]]; then + echo "${{ matrix.component }} image pushed via buildx during build step" else make push-${{ matrix.component }}-docker REGISTRY=${REGISTRY} VERSION=${VERSION_WITHOUT_PREFIX} fi @@ -93,8 +93,8 @@ jobs: echo "Only push to latest tag if tag is the highest semver version $HIGHEST_SEMVER_TAG" if [ "${VERSION_WITHOUT_PREFIX}" = "${HIGHEST_SEMVER_TAG:1}" ] then - if [ "${{ matrix.component }}" = "feature-server" ]; then - docker buildx imagetools create -t ${REGISTRY}/feature-server:latest ${REGISTRY}/feature-server:${VERSION_WITHOUT_PREFIX} + if [[ "${{ matrix.component }}" == "feature-server" || "${{ matrix.component }}" == "feast-operator" ]]; then + docker buildx imagetools create -t ${REGISTRY}/${{ matrix.component }}:latest ${REGISTRY}/${{ matrix.component }}:${VERSION_WITHOUT_PREFIX} else docker tag ${REGISTRY}/${{ matrix.component }}:${VERSION_WITHOUT_PREFIX} ${REGISTRY}/${{ matrix.component }}:latest docker push ${REGISTRY}/${{ matrix.component }}:latest diff --git a/.github/workflows/publish_web_ui.yml b/.github/workflows/publish_web_ui.yml index f8f52f6a84c..b36165625fd 100644 --- a/.github/workflows/publish_web_ui.yml +++ b/.github/workflows/publish_web_ui.yml @@ -1,26 +1,6 @@ name: publish web ui on: - workflow_dispatch: # Allows manual trigger of the workflow - inputs: - current_version: - description: 'Current version to bump from (e.g., v1.2.3). If not provided, will auto-detect from git tags' - required: false - type: string - custom_version: # Optional input for a custom version - description: 'Custom version to publish (e.g., v1.2.3) -- only edit if you know what you are doing' - required: false - type: string - token: - description: 'Personal Access Token' - required: false - default: "" - type: string - publish_ui: - description: 'Publish to NPM?' - required: true - default: true - type: boolean workflow_call: # Allows trigger of the workflow from another workflow inputs: current_version: @@ -39,16 +19,17 @@ on: publish_ui: description: 'Publish to NPM?' required: true - default: true type: boolean +permissions: + contents: read + id-token: write + jobs: publish-web-ui-npm: if: github.repository == 'feast-dev/feast' runs-on: ubuntu-latest - env: - # This publish is working using an NPM automation token to bypass 2FA - NPM_TOKEN: ${{ secrets.NPM_TOKEN }} + environment: production steps: - uses: actions/checkout@v4 - name: Determine current version @@ -108,8 +89,10 @@ jobs: - name: Setup Node.js uses: actions/setup-node@v4 with: - node-version-file: './ui/.nvmrc' + node-version: '22.14.0' registry-url: 'https://registry.npmjs.org' + - name: Update npm for trusted publishing + run: npm install --global npm@11.5.1 - name: Bump file versions (temporarily for Web UI publish) if: github.event.inputs.custom_version != '' env: @@ -137,6 +120,3 @@ jobs: working-directory: ./ui if: github.event.inputs.publish_ui != 'false' run: npm publish - env: - # This publish is working using an NPM automation token to bypass 2FA - NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} \ No newline at end of file diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index 935051a044e..1dee7f79963 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -79,9 +79,68 @@ jobs: fi make test-python-unit + - name: Upload Python coverage to Codecov + if: matrix.os == 'ubuntu-latest' && matrix.python-version == '3.12' + uses: codecov/codecov-action@b9fd7d16f6d7d1b5d2bec1a2887e65ceed900238 # v4.6.0 + with: + file: ./coverage.xml + flags: python-unit + fail_ci_if_error: false + env: + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} - name: Minimize uv cache run: uv cache prune --ci + unit-test-go: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Setup Go + uses: actions/setup-go@v5 + with: + go-version-file: go.mod + cache: true + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: "3.11" + architecture: x64 + - name: Install the latest version of uv + uses: astral-sh/setup-uv@v5 + with: + enable-cache: true + - name: Install system dependencies + run: | + sudo apt-get update + sudo apt-get install -y make protobuf-compiler libsqlite3-dev + - name: Install Go proto plugins + run: | + go install google.golang.org/protobuf/cmd/protoc-gen-go@latest + go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@latest + - name: Compile Go protobufs + run: make compile-protos-go + - name: Create virtual environment + run: | + uv venv + echo "${{ github.workspace }}/.venv/bin" >> $GITHUB_PATH + - name: Install feast locally + run: make install-feast-locally + - name: Run Go tests with coverage + run: | + CGO_ENABLED=1 go test \ + -coverprofile=go/coverage.out \ + -covermode=atomic \ + -skip "TestGetOnlineFeatures|TestSqliteOnlineRead" \ + ./go/... + - name: Upload Go coverage to Codecov + uses: codecov/codecov-action@b9fd7d16f6d7d1b5d2bec1a2887e65ceed900238 # v4.6.0 + with: + file: ./go/coverage.out + flags: go-feature-server + fail_ci_if_error: false + env: + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} + unit-test-ui: runs-on: ubuntu-latest env: diff --git a/.secrets.baseline b/.secrets.baseline index 0eefabade70..62d7fd33f37 100644 --- a/.secrets.baseline +++ b/.secrets.baseline @@ -142,7 +142,7 @@ "filename": ".github/workflows/publish.yml", "hashed_secret": "3e26d6750975d678acb8fa35a0f69237881576b0", "is_verified": false, - "line_number": 43 + "line_number": 45 } ], ".github/workflows/publish_python_sdk.yml": [ @@ -957,7 +957,7 @@ "filename": "infra/feast-operator/api/v1/featurestore_types.go", "hashed_secret": "44e17306b837162269a410204daaa5ecee4ec22c", "is_verified": false, - "line_number": 936 + "line_number": 958 } ], "infra/feast-operator/api/v1/zz_generated.deepcopy.go": [ @@ -966,21 +966,21 @@ "filename": "infra/feast-operator/api/v1/zz_generated.deepcopy.go", "hashed_secret": "f914fc9324de1bec1ad13dec94a8ea2ddb41fc87", "is_verified": false, - "line_number": 817 + "line_number": 842 }, { "type": "Secret Keyword", "filename": "infra/feast-operator/api/v1/zz_generated.deepcopy.go", "hashed_secret": "44e17306b837162269a410204daaa5ecee4ec22c", "is_verified": false, - "line_number": 878 + "line_number": 903 }, { "type": "Secret Keyword", "filename": "infra/feast-operator/api/v1/zz_generated.deepcopy.go", "hashed_secret": "c2028031c154bbe86fd69bef740855c74b927dcf", "is_verified": false, - "line_number": 1570 + "line_number": 1595 } ], "infra/feast-operator/api/v1alpha1/featurestore_types.go": [ @@ -989,7 +989,7 @@ "filename": "infra/feast-operator/api/v1alpha1/featurestore_types.go", "hashed_secret": "44e17306b837162269a410204daaa5ecee4ec22c", "is_verified": false, - "line_number": 650 + "line_number": 663 } ], "infra/feast-operator/api/v1alpha1/zz_generated.deepcopy.go": [ @@ -998,21 +998,30 @@ "filename": "infra/feast-operator/api/v1alpha1/zz_generated.deepcopy.go", "hashed_secret": "f914fc9324de1bec1ad13dec94a8ea2ddb41fc87", "is_verified": false, - "line_number": 595 + "line_number": 615 }, { "type": "Secret Keyword", "filename": "infra/feast-operator/api/v1alpha1/zz_generated.deepcopy.go", "hashed_secret": "44e17306b837162269a410204daaa5ecee4ec22c", "is_verified": false, - "line_number": 1103 + "line_number": 1123 }, { "type": "Secret Keyword", "filename": "infra/feast-operator/api/v1alpha1/zz_generated.deepcopy.go", "hashed_secret": "c2028031c154bbe86fd69bef740855c74b927dcf", "is_verified": false, - "line_number": 1108 + "line_number": 1128 + } + ], + "infra/feast-operator/bundle/manifests/openlineage-secret_v1_secret.yaml": [ + { + "type": "Secret Keyword", + "filename": "infra/feast-operator/bundle/manifests/openlineage-secret_v1_secret.yaml", + "hashed_secret": "598319ac6aa4a94e72a9d8a8d405cc7bc9e048ee", + "is_verified": false, + "line_number": 6 } ], "infra/feast-operator/config/samples/v1_featurestore_db_persistence.yaml": [ @@ -1163,7 +1172,7 @@ "filename": "infra/feast-operator/internal/controller/services/repo_config.go", "hashed_secret": "e2fb052132fd6a07a56af2013e0b62a1f510572c", "is_verified": false, - "line_number": 224 + "line_number": 235 } ], "infra/feast-operator/internal/controller/services/services.go": [ @@ -1172,7 +1181,7 @@ "filename": "infra/feast-operator/internal/controller/services/services.go", "hashed_secret": "36dc326eb15c7bdd8d91a6b87905bcea20b637d1", "is_verified": false, - "line_number": 180 + "line_number": 184 } ], "infra/feast-operator/internal/controller/services/tls_test.go": [ @@ -1555,5 +1564,5 @@ } ] }, - "generated_at": "2026-06-26T06:19:05Z" + "generated_at": "2026-07-31T05:29:18Z" } diff --git a/CHANGELOG.md b/CHANGELOG.md index e628e8ea7a3..38b88ae86df 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,147 @@ # Changelog +# [0.65.0](https://github.com/feast-dev/feast/compare/v0.64.0...v0.65.0) (2026-07-20) + + +### Bug Fixes + +* add debug logging for FIPS mode detection fallback ([6c1b24e](https://github.com/feast-dev/feast/commit/6c1b24ee6f27c469107269828b623180882de321)) +* Build embedded UI from local source ([#6525](https://github.com/feast-dev/feast/issues/6525)) ([3500349](https://github.com/feast-dev/feast/commit/35003494862f8b4af7f2d7eea321356743df074f)) +* Bump decommissioned Snowflake Python UDF runtime from 3.9 to 3.10 ([#6606](https://github.com/feast-dev/feast/issues/6606)) ([#6608](https://github.com/feast-dev/feast/issues/6608)) ([10341e4](https://github.com/feast-dev/feast/commit/10341e4d9cc05478ef863b33c3eeee3cc8da0162)) +* configure FIPS-compliant gRPC cipher suites for offline server ([6bc80a2](https://github.com/feast-dev/feast/commit/6bc80a2474e013724b1579d6de824c76e5d77f3d)) +* Correct Flink PyArrow dependency constraints ([#6604](https://github.com/feast-dev/feast/issues/6604)) ([70a9751](https://github.com/feast-dev/feast/commit/70a97515b8dc93e992d214e11c2bf9cd9ec65aa7)) +* Fix ValueError in signal handling for Trino worker threads ([#6428](https://github.com/feast-dev/feast/issues/6428)) ([506d919](https://github.com/feast-dev/feast/commit/506d919f3aaaaccdc4ac14cd23d0870302c6b13c)) +* Fixed monitoring page issues ([7946018](https://github.com/feast-dev/feast/commit/7946018c40f482bd82efb9a1c555d47dba5d4e54)) +* Make pytest config compatible with newer pytest ([#5779](https://github.com/feast-dev/feast/issues/5779)) ([a57ea33](https://github.com/feast-dev/feast/commit/a57ea331c53bf48a08e96764ff88fe2104bdb5bc)) +* Replace comma with space in DynamoDB-incompatible label tag value ([51e3a16](https://github.com/feast-dev/feast/commit/51e3a164fabf6e1cb2a1e41ae55328a4778177c4)) +* Resolve UI build warnings ([#6529](https://github.com/feast-dev/feast/issues/6529)) ([abe92af](https://github.com/feast-dev/feast/commit/abe92af5ef31283472a5d220390ed80b35baf3aa)) +* Unblock nightly UI build ([#6570](https://github.com/feast-dev/feast/issues/6570)) ([f296d4b](https://github.com/feast-dev/feast/commit/f296d4ba14c5d512429219b2b7845673e0fe524d)) +* Use LONGBLOB for SQL registry proto columns on MySQL ([#6566](https://github.com/feast-dev/feast/issues/6566)) ([7e4beb2](https://github.com/feast-dev/feast/commit/7e4beb21fdba8ee00afd7d0176989e42c10e31a1)) + + +### Features + +* Add click-to-zoom lightbox for blog post images ([#6575](https://github.com/feast-dev/feast/issues/6575)) ([1cb23fd](https://github.com/feast-dev/feast/commit/1cb23fde61862c6d53b434cd5b3ccbffea58d2a6)) +* Add dark mode support to website and blog ([#6589](https://github.com/feast-dev/feast/issues/6589)) ([7358fb8](https://github.com/feast-dev/feast/commit/7358fb8c9a9f8543f6add0e13b8b4b06ef11916b)) +* Add OnlineStore for Aerospike ([#6532](https://github.com/feast-dev/feast/issues/6532)) ([9cd35e1](https://github.com/feast-dev/feast/commit/9cd35e140a949dc44a9915300f2724dd2e702f03)) +* Add OpenLineage Consumer to Feast - receive, store, and visualize cross-producer lineage ([#6549](https://github.com/feast-dev/feast/issues/6549)) ([a834126](https://github.com/feast-dev/feast/commit/a834126b674356ea1efeafd7006f579c9148c3a1)) +* Add registry list feature views by updated since ([#6092](https://github.com/feast-dev/feast/issues/6092)) ([#6093](https://github.com/feast-dev/feast/issues/6093)) ([006c606](https://github.com/feast-dev/feast/commit/006c606183457373d8c83b1986a9f35e1d764c9a)) +* Add ScyllaDB online store with vector search ([#6508](https://github.com/feast-dev/feast/issues/6508)) ([1669661](https://github.com/feast-dev/feast/commit/1669661e15d3ba3b5ab9a9fffd19248d9c0da211)) +* Added compute and jobs UI ([ba2c05c](https://github.com/feast-dev/feast/commit/ba2c05c731be64aff8e7af0fdeefcbe8aa397308)) +* Added Iceberg REST Catalog data source support ([e0a8573](https://github.com/feast-dev/feast/commit/e0a8573eb453dd7060343c4e474e7fca7e1378f7)) +* Bring Your Own Spark - SparkApplication ([#6550](https://github.com/feast-dev/feast/issues/6550)) ([dcd496f](https://github.com/feast-dev/feast/commit/dcd496f22e109f0f77338d41e057dd71113b67d0)) +* **cassandra:** Add multi-DC support via per-datacenter execution profiles ([#6434](https://github.com/feast-dev/feast/issues/6434)) ([0de9196](https://github.com/feast-dev/feast/commit/0de9196d75a63e1ba3860de051cab40c6eba8efc)) +* Enhanced data source creation as a visual catalog with type-specific forms ([#6557](https://github.com/feast-dev/feast/issues/6557)) ([d6acbba](https://github.com/feast-dev/feast/commit/d6acbba057cde6e1c088d068e39492448c73fea1)) +* Enhanced datasets UI functionality ([de11152](https://github.com/feast-dev/feast/commit/de111525985b542b8bfa61118e1ee949254d8703)) +* Implement RegistryServer.Proto RPC with RBAC-filtered response ([#6558](https://github.com/feast-dev/feast/issues/6558)) ([#6552](https://github.com/feast-dev/feast/issues/6552)) ([0d02614](https://github.com/feast-dev/feast/commit/0d02614edcc6fb71992cbb0b539c4b0e2a50f810)) +* New zoned timestamp feature type ([#6536](https://github.com/feast-dev/feast/issues/6536)) ([#6537](https://github.com/feast-dev/feast/issues/6537)) ([eb042f0](https://github.com/feast-dev/feast/commit/eb042f04f5d9bdd7dafbaf654d5b5ec2a2572d9f)) +* **operator:** Auto-create RBAC for spark_application batch engine ([#6597](https://github.com/feast-dev/feast/issues/6597)) ([f487b37](https://github.com/feast-dev/feast/commit/f487b37fd317c63d0d0060ccf8be5d8238d484dd)) +* **operator:** integrate cluster TLS profile for OCP 5.0 compliance ([43263a6](https://github.com/feast-dev/feast/commit/43263a658abe5e2080241b5819fdd8affb4e5fef)) +* Permissions CRUD UI and OIDC auth integration in UI ([6511da1](https://github.com/feast-dev/feast/commit/6511da1323f5634595b5b2ae4e8a5055599c7885)) +* Retrieve historical features from BigQuery without entity_df ([#6569](https://github.com/feast-dev/feast/issues/6569)) ([cd5f6bb](https://github.com/feast-dev/feast/commit/cd5f6bbbd36f11f1d2e2faf8e5e773076b7a3026)), closes [#6558](https://github.com/feast-dev/feast/issues/6558) [#6552](https://github.com/feast-dev/feast/issues/6552) +* **spark:** SparkSource query+path and pre-computed offline read for BatchFeatureView ([#6440](https://github.com/feast-dev/feast/issues/6440)) ([4dc8757](https://github.com/feast-dev/feast/commit/4dc8757626c69c833a8d8174a6bd1513b1671ad7)) + + +### BREAKING CHANGES + +* total_timeout_ms is renamed to batch_total_timeout_ms. Config files using the old name must be updated. No default value change. + +Docs updated (reference + perf-tuning guide) with a short explainer on the per-attempt vs total deadline distinction. Two new unit tests pin the policy wiring: socket_timeout_ms propagates to all three scopes, and is omitted (not injected as None) when unset. + +Signed-off-by: Valentyn Kahamlyk + +* refactor(aerospike): use MAP_KEY_ORDERED, KEY_DIGEST, and instance-scoped client + +Cheap-win cleanups flagged in review, all touching the same small patch of write-path and lifecycle code. + +* Map CDTs are now created with MAP_KEY_ORDERED. map_get_by_key / map_remove_by_key on an ordered map are O(log N) in the map size instead of O(N); matters on reads of wide feature views and on the update() background scan (which walks every record in the project's set). + +* Writes drop POLICY_KEY_SEND and rely on the client default (POLICY_KEY_DIGEST). The serialized entity key is no longer stored alongside each record, saving per-record storage the read path never consumes (batch_operate preserves request order; results are paired back by zip in online_read). + +* _client moves from a class attribute to an instance attribute (set in __init__). Previously two AerospikeOnlineStore instances could share the cached client through class state until one wrote self._client. With the instance attribute the state is always per-instance from construction. + +* Drop MongoDB references from class docstrings and comments (they referred to how the storage layout was derived rather than documenting current behavior). Also rewrite the _build_batch_writes docstring to describe the policies applied on the write path. + +Unit test assertions for the write-path record are updated: bw.policy is now None (client default applies) and map ops carry map_policy={'map_order': MAP_KEY_ORDERED}. All three docker-backed integration tests still pass end-to-end (cross-FV upsert, update() background scan, full feature-store round-trip), so the read/write shape survives the ordering and policy changes against a real server. + +Signed-off-by: Valentyn Kahamlyk + +* feat(aerospike): add per-FV namespace/set overrides and prewriting hook + +Adds three configuration knobs to AerospikeOnlineStoreConfig: + +- namespace_overrides: pin individual feature views to a different + Aerospike namespace (e.g. RAM-only vs. SSD-backed) without splitting + the project across stores. +- set_overrides: place a feature view in its own set so admin ops on + it (truncate, scan-based deletes during `feast apply`) do not touch + records of other views. +- prewriting_hook: import-string-resolved callable invoked once per + online_write_batch with the rows about to be written, returning the + rows that actually go on the wire. Resolved and cached on first use; + returning [] short-circuits the wire call. + +Read, write, update and teardown paths all honour the per-FV ns/set +resolution. update() groups dropped feature views by their resolved +(ns, set) pair and issues one background scan per group. teardown() +truncates every unique (ns, set) pair the project may have written to, +including the store-level default. + +Adds 22 unit tests for the new behaviour and updates 3 existing call +sites of _build_batch_writes for the new namespace= parameter. Adds a +sample hook module under examples/online_store/aerospike_overrides_and_hooks/ +and corresponding sections in docs/reference/online-stores/aerospike.md. + +Signed-off-by: Valentyn Kahamlyk + +* test: update aerospike image tag + +Signed-off-by: Valentyn Kahamlyk + +* chore: sync README template and secrets baseline after master merge + +Signed-off-by: Valentyn Kahamlyk + +* chore: fix secrets baseline line number for v1 operator types + +Adding aerospike to the feast-operator enum shifted the allowlisted +SecretRef entry in api/v1/featurestore_types.go by one line. + +Signed-off-by: Valentyn Kahamlyk + +* docs: update aerospike docs + +Signed-off-by: Valentyn Kahamlyk + +* fix(aerospike): wire batch max_retries and fix empty projection handling + +Copilot review feedback on PR #6532: + +- Add max_retries to the batch client policy (batch_operate/batch_write path) +- Treat empty projected feature maps as present FV slots (is not None) +- Return {} from _normalize_projected_features([]) instead of None +- Fix projection unit test mock/assertions +- Correct prewriting_hook config docstring + +Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> +Signed-off-by: Valentyn Kahamlyk + +* style(aerospike): format online_read docs assignment for ruff + +Signed-off-by: Valentyn Kahamlyk + +* chore: update pixi.lock for aerospike optional extra + +Regenerate the v6 lockfile with Pixi v0.63.1 after adding the aerospike extra to pyproject.toml. + +Signed-off-by: Valentyn Kahamlyk + +* fix(aerospike): add client init lock and batch chunking + +Guard lazy client creation with a lock to avoid connection leaks under concurrent first use, and chunk batch reads/writes by batch_max_records so large materializations stay under Aerospike server batch limits. + +Signed-off-by: Valentyn Kahamlyk + # [0.64.0](https://github.com/feast-dev/feast/compare/v0.63.0...v0.64.0) (2026-06-13) diff --git a/Makefile b/Makefile index c7496208f96..d5a6aeeaba9 100644 --- a/Makefile +++ b/Makefile @@ -112,7 +112,7 @@ install-python-dependencies-ci: ## Install Python CI dependencies using uv pip s # Install CPU-only torch first to prevent CUDA dependency issues (Linux only) @if [ "$$(uname -s)" = "Linux" ]; then \ echo "Installing dependencies with torch CPU index for Linux..."; \ - uv pip sync --extra-index-url https://download.pytorch.org/whl/cpu --index-strategy unsafe-best-match sdk/python/requirements/py$(PYTHON_VERSION)-ci-requirements.txt; \ + uv pip sync --torch-backend cpu sdk/python/requirements/py$(PYTHON_VERSION)-ci-requirements.txt; \ else \ echo "Installing dependencies from PyPI for macOS..."; \ uv pip sync sdk/python/requirements/py$(PYTHON_VERSION)-ci-requirements.txt; \ @@ -151,9 +151,10 @@ lock-python-dependencies-all: ## Recompile and lock all Python dependency sets f "uv pip compile -p $(ver) --no-strip-extras pyproject.toml --extra minimal-sdist-build \ --no-emit-package milvus-lite \ --no-emit-package pymilvus \ + --no-emit-package faiss-cpu \ --generate-hashes --output-file sdk/python/requirements/py$(ver)-minimal-sdist-requirements.txt" && \ pixi run --environment $(call get_env_name,$(ver)) --manifest-path infra/scripts/pixi/pixi.toml \ - "uv pip install -p $(ver) pybuild-deps==0.5.0 pip==25.0.1 && \ + "uv pip install -p $(ver) pybuild-deps==0.5.0 pip==25.0.1 typing_extensions && \ pybuild-deps compile --generate-hashes \ -o sdk/python/requirements/py$(ver)-minimal-sdist-requirements-build.txt \ sdk/python/requirements/py$(ver)-minimal-sdist-requirements.txt" && \ @@ -172,6 +173,9 @@ benchmark-python-local: ## Run integration + benchmark tests for Python (local d test-python-unit: ## Run Python unit tests (use pattern= to filter tests, e.g., pattern=milvus, pattern=test_online_retrieval.py, pattern=test_online_retrieval.py::test_get_online_features_milvus) uv run python -m pytest -n 8 --color=yes $(if $(pattern),-k "$(pattern)") \ + --cov=feast \ + --cov-report=xml \ + --cov-report=term-missing \ sdk/python/tests/unit # Fast unit tests only @@ -727,10 +731,16 @@ push-feast-operator-docker: ## Push Feast Operator Docker image $(MAKE) docker-push build-feast-operator-docker: ## Build Feast Operator Docker image - cd infra/feast-operator && \ - IMAGE_TAG_BASE=$(REGISTRY)/feast-operator \ - VERSION=$(VERSION) \ - $(MAKE) docker-build + @if [ -n "$(DOCKER_PLATFORMS)" ]; then \ + cd infra/feast-operator && \ + docker buildx build --push --platform=$(DOCKER_PLATFORMS) \ + --tag $(REGISTRY)/feast-operator:$(VERSION) -f Dockerfile .; \ + else \ + cd infra/feast-operator && \ + IMAGE_TAG_BASE=$(REGISTRY)/feast-operator \ + VERSION=$(VERSION) \ + $(MAKE) docker-build; \ + fi build-feast-operator-docker-on-mac: ## Build Feast Operator Docker image on Mac cd infra/feast-operator && \ diff --git a/README.md b/README.md index 3b35344b021..a91faccae83 100644 --- a/README.md +++ b/README.md @@ -19,7 +19,7 @@ ## Join us on Slack! -👋👋👋 [Come say hi on Slack!](https://communityinviter.com/apps/feastopensource/feast-the-open-source-feature-store) +👋👋👋 [Come say hi on Slack!](https://slack.feast.dev/) [Check out our DeepWiki!](https://deepwiki.com/feast-dev/feast) @@ -255,7 +255,6 @@ The list below contains the functionality that contributors are planning to deve * [x] [Offline Feature Server (alpha)](https://docs.feast.dev/reference/feature-servers/offline-feature-server) * [x] [Registry server (alpha)](https://github.com/feast-dev/feast/blob/master/docs/reference/feature-servers/registry-server.md) * **Data Quality Management (See [RFC](https://docs.google.com/document/d/110F72d4NTv80p35wDSONxhhPBqWRwbZXG4f9mNEMd98/edit))** - * [x] ~~Data profiling and validation (Great Expectations)~~ (deprecated) * [x] [Feature Quality Monitoring](https://docs.feast.dev/how-to-guides/feature-monitoring) — built-in metrics, drift detection, serving log monitoring, and UI dashboard * **Feature Discovery and Governance** * [x] Python SDK for browsing feature registry diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 00000000000..a7a6d645642 --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,30 @@ +# Security Policy + +The Feast community takes security bugs seriously, and we appreciate the effort it takes to find and report them. We follow [GitHub's coordinated disclosure process](https://docs.github.com/en/code-security/security-advisories/guidance-on-reporting-and-writing-information-about-vulnerabilities/about-coordinated-disclosure-of-security-vulnerabilities) so that a fix can be prepared before details become public. + +## Reporting a vulnerability + +Report vulnerabilities privately through GitHub, using **[Report a vulnerability](https://github.com/feast-dev/feast/security/advisories/new)** on this repository's Security tab. Only the maintainers can see the report, and you will be credited on the published advisory if you would like to be. + +Before reporting, please check the [published advisories](https://github.com/feast-dev/feast/security/advisories) to confirm the issue has not already been addressed. + +A report needs to show a clear, reproducible security impact. Please include: + +- the affected version or commit, and the configuration involved +- a proof of concept, or steps that reproduce the issue +- the actual impact, rather than a theoretical concern + +Raw scanner or dependency-audit output does not meet that bar on its own, since it does not establish that the issue is reachable in Feast. Reports that have not been manually verified against Feast, including bulk, automated, or AI-generated submissions, may be closed without further response. + +> [!WARNING] +> Do not open a public GitHub issue, pull request, or Slack message for a security vulnerability. Those are visible to everyone and disclose the problem before a fix exists. + +For anything that is not a vulnerability, including hardening suggestions and questions about how Feast's authentication and authorization work, a normal [GitHub issue](https://github.com/feast-dev/feast/issues) is the right place. + +## Supported versions + +Security fixes are applied to the latest release. Feast releases roughly monthly and offers best-effort community support, as described in the [versioning policy](docs/project/versioning-policy.md); there is no long-term support branch, so upgrading to the current release is the supported way to receive a fix. + +## Published advisories + +Past advisories for this project are listed under [Security advisories](https://github.com/feast-dev/feast/security/advisories). diff --git a/docs/README.md b/docs/README.md index 18f9cf8207d..e8588be340f 100644 --- a/docs/README.md +++ b/docs/README.md @@ -71,7 +71,7 @@ Feast helps ML platform/MLOps teams with DevOps experience productionize real-ti * **batch feature engineering**: Feast supports on-demand and streaming transformations. Feast is also investing in supporting batch transformations. * **native streaming feature integration:** Feast enables users to push streaming features, but does not pull from streaming sources or manage streaming pipelines. * **lineage:** Feast helps tie feature values to model versions, but is not a complete solution for capturing end-to-end lineage from raw data sources to model versions. Feast also has community contributed plugins with [DataHub](https://datahubproject.io/docs/generated/ingestion/sources/feast/) and [Amundsen](https://github.com/amundsen-io/amundsen/blob/4a9d60176767c4d68d1cad5b093320ea22e26a49/databuilder/databuilder/extractor/feast\_extractor.py). -* **data quality / drift detection**: Feast now includes built-in [Feature Quality Monitoring](how-to-guides/feature-monitoring.md) that computes statistical metrics (null rates, distributions, percentiles), detects drift across batch data and serving logs, and provides a monitoring UI dashboard. The older Great Expectations integration is deprecated. +* **data quality / drift detection**: Feast includes built-in [Feature Quality Monitoring](how-to-guides/feature-monitoring.md) that computes statistical metrics (null rates, distributions, percentiles), detects drift across batch data and serving logs, and provides a monitoring UI dashboard. ## Example use cases diff --git a/docs/SUMMARY.md b/docs/SUMMARY.md index 547c88acf68..091014d3ead 100644 --- a/docs/SUMMARY.md +++ b/docs/SUMMARY.md @@ -57,7 +57,6 @@ * [Fraud detection on GCP](tutorials/tutorials-overview/fraud-detection.md) * [Real-time credit scoring on AWS](tutorials/tutorials-overview/real-time-credit-scoring-on-aws.md) * [Driver stats on Snowflake](tutorials/tutorials-overview/driver-stats-on-snowflake.md) -* [\[Deprecated\] Validating historical features with Great Expectations](tutorials/validating-historical-features.md) * [Building streaming features](tutorials/building-streaming-features.md) * [Retrieval Augmented Generation (RAG) with Feast](tutorials/rag-with-docling.md) * [RAG Fine Tuning with Feast and Milvus](../examples/rag-retriever/README.md) @@ -150,6 +149,7 @@ * [Snowflake](reference/online-stores/snowflake.md) * [Redis](reference/online-stores/redis.md) * [Dragonfly](reference/online-stores/dragonfly.md) + * [Valkey](reference/online-stores/valkey.md) * [Datastore](reference/online-stores/datastore.md) * [DynamoDB](reference/online-stores/dynamodb.md) * [Bigtable](reference/online-stores/bigtable.md) @@ -188,6 +188,7 @@ * [Snowflake](reference/compute-engine/snowflake.md) * [AWS Lambda (alpha)](reference/compute-engine/lambda.md) * [Spark (contrib)](reference/compute-engine/spark.md) + * [SparkApplication](reference/compute-engine/spark_application.md) * [Apache Flink](reference/compute-engine/flink.md) * [Ray (contrib)](reference/compute-engine/ray.md) * [Feature repository](reference/feature-repository/README.md) @@ -206,6 +207,9 @@ * [\[Beta\] On demand feature view](reference/beta-on-demand-feature-view.md) * [\[Alpha\] Static Artifacts Loading](reference/alpha-static-artifacts.md) * [\[Alpha\] Vector Database](reference/alpha-vector-database.md) +* [\[Alpha\] OpenAI-Compatible Vector Store API](reference/alpha-vector-database.md#alpha-openai-compatible-vector-store-api) + +* [Data Quality Monitoring](reference/dqm.md) * [\[Deprecated\] Data quality monitoring (Great Expectations)](reference/dqm.md) * [\[Alpha\] Streaming feature computation with Denormalized](reference/denormalized.md) * [\[Alpha\] Feature View Versioning](reference/alpha-feature-view-versioning.md) diff --git a/docs/adr/ADR-0011-data-quality-monitoring.md b/docs/adr/ADR-0011-data-quality-monitoring.md index e2e13745a61..657d219c48c 100644 --- a/docs/adr/ADR-0011-data-quality-monitoring.md +++ b/docs/adr/ADR-0011-data-quality-monitoring.md @@ -2,7 +2,7 @@ ## Status -Accepted +Superseded — The original external-library-based validation has been replaced by Feast's native [Feature Quality Monitoring](../how-to-guides/feature-monitoring.md) system (`feast monitor run`). ## Context @@ -12,92 +12,51 @@ Data quality issues can significantly impact ML model performance. Several compl - **Upstream pipeline bugs**: Bugs in upstream pipelines can cause invalid values to overwrite existing valid values in an online store. - **Training/serving skew**: Distribution shift between training and serving data can decrease model performance. -Feast needed a mechanism to validate data at retrieval time to catch these issues before they affect model training or serving. +Feast needed a mechanism to validate data to catch these issues before they affect model training or serving. ## Decision -Introduce a Data Quality Monitoring (DQM) module that validates datasets against user-curated rules, initially targeting historical retrieval (training dataset generation). +Introduce a Data Quality Monitoring (DQM) module that validates datasets against user-curated rules. -### Design +### Original Design (now replaced) -The validation process uses a **reference dataset** and a **profiler** pattern: +The original validation process used a **reference dataset** and a **profiler** pattern: 1. User prepares a reference dataset (saved from a known-good historical retrieval). 2. User defines a profiler function that produces a profile (set of expectations) from a dataset. 3. Validation is performed by comparing the tested dataset against the reference profile. -### Integration with Great Expectations +This approach was limited to historical retrieval only, required additional dependencies, and offered no built-in UI or automation. -The initial implementation uses [Great Expectations](https://greatexpectations.io/) as the validation engine: +### Current Design -```python -from feast.dqm.profilers.ge_profiler import ge_profiler -from great_expectations.dataset import Dataset -from great_expectations.core.expectation_suite import ExpectationSuite +The current system (`feast monitor run`) provides: -@ge_profiler -def my_profiler(dataset: Dataset) -> ExpectationSuite: - dataset.expect_column_max_to_be_between("column", 1, 2) - dataset.expect_column_values_to_not_be_null("important_feature") - return dataset.get_expectation_suite() -``` - -### Usage - -Validation is triggered during historical feature retrieval via a `validation_reference` parameter: - -```python -from feast import FeatureStore - -store = FeatureStore(".") - -job = store.get_historical_features(...) -df = job.to_df( - validation_reference=store - .get_saved_dataset("my_reference_dataset") - .as_reference(profiler=my_profiler) -) -``` - -If validation fails, a `ValidationFailed` exception is raised with details for all expectations that didn't pass. If validation succeeds, the materialized dataset is returned normally. - -### Key Decisions +- Automatic metric computation (null rates, percentiles, histograms) with no external dependencies +- Monitoring across batch data and serving logs +- CLI and REST API for automation +- Built-in UI monitoring dashboard +- Support for all offline store backends via SQL push-down -- **Profiler-based approach**: Users define their own validation rules via profiler functions rather than Feast prescribing fixed validation rules. -- **Great Expectations integration**: Leverages an established data validation framework rather than building custom validation logic. -- **Validation at retrieval time**: Validation is performed when datasets are materialized (`.to_df()` or `.to_arrow()`), not during ingestion. -- **ValidationReference as a registry object**: Saved datasets and their validation references are stored in the Feast registry for reuse. +See [Feature Quality Monitoring](../how-to-guides/feature-monitoring.md) for full documentation. ## Consequences ### Positive - Users can detect data quality issues before they affect model training. -- Flexible profiler pattern allows custom validation rules per use case. -- Integration with Great Expectations provides a rich set of built-in expectations. -- Reference datasets provide a baseline for detecting data drift. +- Native integration requires no extra dependencies. +- Covers both batch data and serving logs. +- Built-in UI provides immediate visibility into feature health. +- Baselines computed automatically on `feast apply`. ### Negative -- Currently limited to historical retrieval; online store write/read validation is planned but not yet implemented. -- Dependency on Great Expectations adds to the install footprint (optional via `feast[ge]`). -- Automatic profiling capabilities are limited; manual expectation crafting is recommended. - -## Superseded - -This ADR documents the original GE-based approach which is now **deprecated**. It has been superseded by Feast's built-in [Feature Quality Monitoring](../how-to-guides/feature-monitoring.md) system (introduced in 2025), which provides: - -- Automatic metric computation (null rates, percentiles, histograms) with no external dependencies -- Monitoring across batch data and serving logs -- CLI (`feast monitor run`) and REST API for automation -- Built-in UI monitoring dashboard -- Support for all offline store backends via SQL push-down - -The GE-based integration may be removed in a future release. +- Migration required from the original profiler-based approach. ## References -- Original RFC: Feast RFC-027: Data Quality Monitoring -- Implementation: `sdk/python/feast/dqm/`, `sdk/python/feast/saved_dataset.py` -- Documentation: [Data Quality Monitoring (deprecated)](../reference/dqm.md) -- **New system:** [Feature Quality Monitoring](../how-to-guides/feature-monitoring.md) +- Original RFC: Feast RFC-027: Data Quality Monitoring +- Implementation: `sdk/python/feast/monitoring/` +- Documentation: [Data Quality Monitoring](../reference/dqm.md) +- [Feature Quality Monitoring guide](../how-to-guides/feature-monitoring.md) diff --git a/docs/blog/feast-0-18-adds-snowflake-support-and-data-quality-monitoring.md b/docs/blog/feast-0-18-adds-snowflake-support-and-data-quality-monitoring.md index 4b4321e3259..8c587bdde9f 100644 --- a/docs/blog/feast-0-18-adds-snowflake-support-and-data-quality-monitoring.md +++ b/docs/blog/feast-0-18-adds-snowflake-support-and-data-quality-monitoring.md @@ -6,7 +6,7 @@ We are delighted to announce the release of Feast [0.18](https://github.com/feas * Snowflake offline store, which allows you to define and use features stored in Snowflake. * [Experimental] Saved Datasets, which allow training datasets to be persisted in an offline store. -* [Experimental] Data quality monitoring, which allows you to validate your training data with Great Expectations. Future work will allow you to detect issues with upstream data pipelines and check for training-serving skew. +* [Experimental] Data quality monitoring, which allows you to validate your training data. This has since been superseded by Feast's native [Feature Quality Monitoring](../how-to-guides/feature-monitoring.md) system. * Python feature server graduation from alpha status. * Performance improvements to on demand feature views, protobuf serialization and deserialization, and the Python feature server. @@ -22,7 +22,7 @@ Training datasets generated via `get_historical_features` can now be persisted i ### [Experimental] Data quality monitoring -Feast 0.18 includes the first milestone of our data quality monitoring work. Many users have requested ways to validate their training and serving data, as well as monitor for training-serving skew. Feast 0.18 allows users to validate their training data through an integration with [Great Expectations](https://greatexpectations.io/). Users can declare one of the previously generated training datasets as a reference for this validation by persisting it as a "saved dataset" (see previous section). More details about future milestones of data quality monitoring can be found [here](https://docs.feastsite.wpenginepowered.com/v/master/reference/data-quality). There's also a [tutorial on validating historical features](https://docs.feastsite.wpenginepowered.com/v/master/how-to-guides/validation/validating-historical-features) that demonstrates all new concepts in action. +Feast 0.18 includes the first milestone of our data quality monitoring work. Many users have requested ways to validate their training and serving data, as well as monitor for training-serving skew. Feast 0.18 allows users to validate their training data by declaring previously generated training datasets as a reference for validation, persisted as "saved datasets" (see previous section). This initial integration has since been superseded by Feast's native [Feature Quality Monitoring](../how-to-guides/feature-monitoring.md) system, which provides built-in metrics computation, drift detection, serving log monitoring, and a UI dashboard. ### Performance improvements diff --git a/docs/community.md b/docs/community.md index 640b5238b8b..c22c4c41dd4 100644 --- a/docs/community.md +++ b/docs/community.md @@ -2,7 +2,7 @@ ## Links & Resources -* [Come say hi on Slack!](https://communityinviter.com/apps/feastopensource/feast-the-open-source-feature-store) +* [Come say hi on Slack!](https://slack.feast.dev/) * As a part of the Linux Foundation, we ask community members to adhere to the [Linux Foundation Code of Conduct](https://events.linuxfoundation.org/about/code-of-conduct/) * [GitHub Repository](https://github.com/feast-dev/feast/): Find the complete Feast codebase on GitHub. * [Community Governance Doc](https://github.com/feast-dev/feast/blob/master/community): See the governance model of Feast, including who the maintainers are and how decisions are made. diff --git a/docs/getting-started/components/authz_manager.md b/docs/getting-started/components/authz_manager.md index eae3fece50b..12fdf2f39e5 100644 --- a/docs/getting-started/components/authz_manager.md +++ b/docs/getting-started/components/authz_manager.md @@ -44,10 +44,11 @@ The server, in turn, uses the same OIDC server to validate the token and extract Some assumptions are made in the OIDC server configuration: * The OIDC token refers to a client with roles matching the RBAC roles of the configured `Permission`s (*) -* The roles are exposed in the access token under `resource_access..roles` -* The JWT token is expected to have a verified signature and not be expired. The Feast OIDC token parser logic validates for `verify_signature` and `verify_exp` so make sure that the given OIDC provider is configured to meet these requirements. -* The `preferred_username` should be part of the JWT token claim. +* The roles are exposed in the access token under `resource_access..roles` (Keycloak) or in the top-level `roles` claim (Entra ID app roles). Roles found in both are merged. +* The JWT token is expected to have a verified signature and not be expired. The Feast OIDC token parser logic validates for `verify_signature` and `verify_exp` so make sure that the given OIDC provider is configured to meet these requirements. The token's audience and issuer claims are **not** verified by default; both checks can be enabled with the `audience` and `issuer` options (see [Server-Side Configuration](#server-side-configuration)). +* The username is read from the first of `preferred_username`, `upn`, `azp`, `appid`, `sub` present in the token. Entra ID client-credentials (app-only) tokens carry no user claim, so they authenticate as the calling application. * For `GroupBasedPolicy` support, the `groups` claim should be present in the access token (requires a "Group Membership" protocol mapper in Keycloak). +* **Entra ID limitation**: Group claims use object IDs (GUIDs) instead of names, and are omitted entirely when a user exceeds the group overage threshold. GroupBasedPolicy must reference GUIDs and cannot be used for principals with large group memberships. (*) Please note that **the role match is case-sensitive**, e.g. the name of the role in the OIDC server and in the `Permission` configuration must be exactly the same. @@ -69,6 +70,16 @@ For example, the access token for a client `app` of a user with `reader` role an } ``` +A Microsoft Entra ID (Azure AD) client-credentials (app-only) token has no user claim; the application authenticates as itself, and its app roles arrive in the top-level `roles` claim: +```json +{ + "azp": "11111111-2222-3333-4444-555555555555", + "roles": [ + "reader" + ] +} +``` + #### Server-Side Configuration The server requires `auth_discovery_url` and `client_id` to validate incoming JWT tokens via JWKS: @@ -94,6 +105,36 @@ auth: Setting `verify_ssl: false` disables TLS certificate verification for all OIDC provider communication (discovery, JWKS, token endpoint). Only use this in development or internal environments where you accept the security risk. {% endhint %} +By default the server verifies only the token's signature and expiry: any validly-signed, unexpired token from the configured provider is accepted regardless of the audience it was minted for, and authorization (role matching) is the only remaining gate. For defense in depth, set `audience` and/or `issuer` to additionally require a matching `aud` / `iss` claim: + +```yaml +auth: + type: oidc + client_id: _CLIENT_ID_ + auth_discovery_url: https://login.example.com/.well-known/openid-configuration + audience: api://feast-feature-server + issuer: https://login.example.com/realms/master +``` + +A token whose `aud` (or `iss`) claim does not match is rejected at authentication. The two options are independent; leave one unset to skip that check. + +{% hint style="warning" %} +Set these to the values your IdP puts **in the token itself**, which are not always the ones in the discovery document. For example, Microsoft Entra ID commonly issues v1.0 tokens (`iss: https://sts.windows.net//`, `aud: api://`) even when `auth_discovery_url` points at the v2.0 endpoint. That setup keeps working with these options unset, or set to the v1.0 values — but copying the v2.0 issuer from the discovery document would reject every v1.0 token. +{% endhint %} + +To validate token signatures the server fetches the provider's JWKS document and caches it, refetching when the cache expires or when a token presents an unknown key id. Two options tune that behavior: + +```yaml +auth: + type: oidc + client_id: _CLIENT_ID_ + auth_discovery_url: https://login.example.com/.well-known/openid-configuration + jwks_cache_lifespan_seconds: 300 # default; how long the fetched key set is reused + jwks_request_timeout_seconds: 10 # default; network timeout for the JWKS fetch +``` + +`jwks_cache_lifespan_seconds` also bounds how long a key the provider has **revoked** continues to validate tokens, so lower it if your provider rotates or revokes aggressively; each reduction costs proportionally more JWKS fetches. Key rotations that introduce a new key id are picked up immediately regardless of this setting, because an unknown key id triggers a refetch. `jwks_request_timeout_seconds` bounds how long an unresponsive provider can block request serving. Both must be greater than zero. + #### Client-Side Configuration The client supports multiple token source modes. The SDK resolves tokens in the following priority order: diff --git a/docs/getting-started/components/feature-server.md b/docs/getting-started/components/feature-server.md index 4d961054ecb..1b5521a2b86 100644 --- a/docs/getting-started/components/feature-server.md +++ b/docs/getting-started/components/feature-server.md @@ -37,6 +37,8 @@ The Feature Server operates as a stateless service backed by two key components: | `/push` | Pushes feature data to the online and/or offline store. | | `/materialize` | Materializes features within a specific time range to the online store. | | `/materialize-incremental` | Incrementally materializes features up to the current timestamp. | -| `/retrieve-online-documents` | Supports Vector Similarity Search for RAG (Alpha end-ponit) | +| `/search` | Vector similarity search for RAG (Alpha endpoint) | +| `/v1/vector_stores/{id}/search` | OpenAI-compatible vector search with server-side embedding | +| `/retrieve-online-documents` | **Deprecated.** Use `/search` instead. | | `/docs` | API Contract for available endpoints | diff --git a/docs/getting-started/concepts/dataset.md b/docs/getting-started/concepts/dataset.md index 061de54ebca..c86c13503ed 100644 --- a/docs/getting-started/concepts/dataset.md +++ b/docs/getting-started/concepts/dataset.md @@ -1,6 +1,6 @@ # \[Alpha] Saved dataset -Feast datasets allow for conveniently saving dataframes that include both features and entities to be subsequently used for data analysis and model training. Data Quality Monitoring was the original motivation for creating the dataset concept. Note that the Great Expectations-based validation that used saved datasets is now deprecated in favor of Feast's built-in [Feature Quality Monitoring](../../how-to-guides/feature-monitoring.md) system, which does not require saved datasets. +Feast datasets allow for conveniently saving dataframes that include both features and entities to be subsequently used for data analysis and model training. Data Quality Monitoring was the original motivation for creating the dataset concept. Dataset's metadata is stored in the Feast registry and raw data (features, entities, additional input keys and timestamp) is stored in the [offline store](../components/offline-store.md). diff --git a/docs/getting-started/concepts/point-in-time-joins.md b/docs/getting-started/concepts/point-in-time-joins.md index 55672209005..9e385774327 100644 --- a/docs/getting-started/concepts/point-in-time-joins.md +++ b/docs/getting-started/concepts/point-in-time-joins.md @@ -62,3 +62,26 @@ Below is the resulting joined training dataframe. It contains both the original Three feature rows were successfully joined to the entity dataframe rows. The first row in the entity dataframe was older than the earliest feature rows in the feature view and could not be joined. The last row in the entity dataframe was outside of the TTL window \(the event happened 11 hours after the feature row\) and also couldn't be joined. +## Retrieving features as of the event time + +By default, point-in-time joins only constrain the feature's event timestamp. If a data source also has a `created_timestamp_column`, it is used to deduplicate rows that share an event timestamp \(the row with the highest created timestamp wins\), but it is not otherwise filtered. This means a value that was backfilled or corrected *after* an entity dataframe timestamp can still be returned for it. + +To restrict retrieval to feature values that were already available at each entity row's timestamp, pass `filter_by_created_timestamp=True`: + +```python +training_df = store.get_historical_features( + entity_df=entity_df, + features = [ + 'driver_hourly_stats:trips_today', + 'driver_hourly_stats:earnings_today' + ], + filter_by_created_timestamp=True, +) +``` + +This adds a `created_timestamp <= entity_timestamp` condition to the join, so each entity dataframe row only sees feature values whose created timestamp is at or before its own timestamp. This is useful to keep backfilled values from leaking into training data, and to reproduce what the online store would have served at each event time \(assuming the created timestamp reflects when the value became available online\). + +{% hint style="info" %} +Rows with a NULL created timestamp are excluded when the flag is enabled, so the column should be non-null. Not all offline stores support this flag yet; unsupported stores raise an error rather than silently ignoring it. +{% endhint %} + diff --git a/docs/getting-started/genai.md b/docs/getting-started/genai.md index f65aeac85e2..d9f682af1f0 100644 --- a/docs/getting-started/genai.md +++ b/docs/getting-started/genai.md @@ -15,6 +15,7 @@ Feast integrates with popular vector databases to store and retrieve embedding v * **Elasticsearch**: Scalable vector search capabilities * **Postgres with PGVector**: SQL-based vector operations * **Qdrant**: Purpose-built vector database integration +* **ScyllaDB**: Native `vector` type with HNSW ANN index, full `retrieve_online_documents_v2` support These integrations allow you to: - Store embeddings as features @@ -225,7 +226,8 @@ The MCP integration uses the `fastapi_mcp` library to automatically transform yo The fastapi_mcp integration automatically exposes your Feast feature server's FastAPI endpoints as MCP tools. This means AI assistants can: * **Call `/get-online-features`** to retrieve features from the feature store -* **Call `/retrieve-online-documents`** to perform vector similarity search +* **Call `/search`** to perform vector similarity search (`/retrieve-online-documents` is a deprecated alias) +* **Call `/v1/vector_stores/{feature_view}/search`** for OpenAI-compatible text search with server-side embedding * **Call `/write-to-online-store`** to persist agent state (memory, notes, interaction history) * **Use `/health`** to check server status diff --git a/docs/how-to-guides/feast-operator/01-project-provisioning.md b/docs/how-to-guides/feast-operator/01-project-provisioning.md index b54ce57eeeb..787ef173403 100644 --- a/docs/how-to-guides/feast-operator/01-project-provisioning.md +++ b/docs/how-to-guides/feast-operator/01-project-provisioning.md @@ -2,7 +2,8 @@ The operator needs a Feast feature repository (a directory containing `feature_store.yaml` and Python feature-view definitions) to work from. `spec.feastProjectDir` controls how that -directory is created inside the pods. Exactly one of `git` or `init` must be set. +directory is created inside the pods. When `feastProjectDir` is specified, exactly one of +`git`, `init`, or `packaged` must be set. --- @@ -89,7 +90,7 @@ feastProjectDir: ### Full `git` field reference | Field | Type | Description | -|-------|------|-------------| +| ------- | ------ | ------------- | | `url` | string | Repository URL (HTTPS or SSH) | | `ref` | string | Branch, tag, or commit SHA. Defaults to the remote HEAD | | `featureRepoPath` | string | Relative path within the repo to the feature repository directory. Default: `feature_repo` | @@ -151,9 +152,90 @@ feastProjectDir: --- +## Option C — Use a repository packaged in an image (`feastProjectDir.packaged`) + +Use `packaged` when the feature repository is built into a feature-server image. This is +useful in air-gapped environments and in release workflows where feature definitions and +their Python dependencies are promoted together as an immutable image. + +```yaml +apiVersion: feast.dev/v1 +kind: FeatureStore +metadata: + name: packaged-feature-store +spec: + feastProject: credit_scoring + feastProjectDir: + packaged: + image: registry.example.com/feature-server@sha256:0123456789abcdef + featureRepoPath: /opt/feast/feature_repo +``` + +The repository can be added to a Feast feature-server image with a Dockerfile such as: + +```dockerfile +FROM quay.io/feastdev/feature-server:latest +COPY feature_repo/ /opt/feast/feature_repo/ +``` + +`featureRepoPath` must be a canonical absolute, non-root path: do not use `.`, `..`, +repeated separators, or a trailing separator. Put it outside operator-mounted locations +such as `/feast-data`; a volume mounted there would hide files baked into the image. When +init containers are enabled, the packaged path also must not equal, contain, or be +contained by the staged repository path. + +With init containers enabled (the default), each Pod starts in this order: + +1. `feast-init` replaces the operator-managed staged repository with a fresh copy of the + repository from `packaged.featureRepoPath`. With the default storage configuration, + for example, it copies `/opt/feast/feature_repo` from the image to + `/feast-data//feature_repo`. +2. In the staged copy only, `feast-init` replaces `feature_store.yaml` (if exists in the + baked image) with the configuration generated from the FeatureStore resource. The file + baked into the image is not modified. The Python feature definitions come from the + packaged repository, while the FeatureStore resource remains authoritative for runtime + configuration. +3. When `services.runFeastApplyOnInit` is omitted or `true` (the default), `feast-apply` + runs `feast apply` from the staged repository using the packaged image. Setting it to + `false` skips only this step; repository staging still occurs. +4. The Feast service containers start with the staged repository as their working + directory. + +The repository baked into the image is therefore the source artifact, while the staged +repository is the runtime copy used by `feast apply` and the Feast services. + +For a baked repository whose own `feature_store.yaml` must remain authoritative, disable +init containers: + +```yaml +services: + disableInitContainers: true +``` + +In that mode, Feast service containers use `featureRepoPath` directly and neither staging +nor `feast apply` runs during pod initialization. The Operator does not update the registry, +so `feast apply` must be handled separately—for example, by CI/CD or a separately managed +Kubernetes Job or CronJob—whenever the packaged feature definitions change. + +The packaged `image` is optional. When set, it is the default for repository initialization, +`feast apply`, and Feast services. `services.initImage` takes precedence for the +`feast-init` and `feast-apply` init containers, while an explicit image on an individual +service takes precedence for that service. When the packaged image is omitted, the operator +uses `RELATED_IMAGE_FEATURE_SERVER` or its built-in feature-server image fallback. + +### Full `packaged` field reference + +| Field | Type | Required | Description | +|-------|------|----------|-------------| +| `featureRepoPath` | string | yes | Canonical absolute, non-root path to the feature repository in the image; it must not overlap the staged repository path | +| `image` | string | no | Image containing the repository; defaults to the operator feature-server image | + +--- + ## `feast apply` on startup -By default, when the init container completes (git clone or `feast init`), the operator runs +By default, when repository initialization completes (git clone, `feast init`, or packaged +repository staging), the operator runs `feast apply` before starting the servers. This registers all feature definitions with the registry. @@ -175,9 +257,8 @@ services: ## When `feastProjectDir` is omitted -If neither `git` nor `init` is set, the operator mounts an empty directory. In this case -you must supply a `feature_store.yaml` through another mechanism (e.g. a ConfigMap volume -mount via `services.volumes` + `volumeMounts`). +If `feastProjectDir` is not set, the operator defaults to `feastProjectDir.init: {}` and +creates a local template repository. --- @@ -188,3 +269,4 @@ mount via `services.volumes` + `volumeMounts`). - [Sample: private git repo with token](https://github.com/feast-dev/feast/blob/stable/infra/feast-operator/config/samples/v1_featurestore_git_token.yaml) - [Sample: monorepo with featureRepoPath](https://github.com/feast-dev/feast/blob/stable/infra/feast-operator/config/samples/v1_featurestore_git_repopath.yaml) - [Sample: feast init](https://github.com/feast-dev/feast/blob/stable/infra/feast-operator/config/samples/v1_featurestore_init.yaml) +- [Sample: packaged feature repository](https://github.com/feast-dev/feast/blob/stable/infra/feast-operator/config/samples/v1_featurestore_packaged.yaml) diff --git a/docs/how-to-guides/feast-operator/05-security.md b/docs/how-to-guides/feast-operator/05-security.md index 4c28aba9805..002c3a9f846 100644 --- a/docs/how-to-guides/feast-operator/05-security.md +++ b/docs/how-to-guides/feast-operator/05-security.md @@ -39,7 +39,7 @@ to subjects using standard Kubernetes `ClusterRoleBinding` or `RoleBinding` reso > Kubernetes auth requires all services to be exposed as servers (the controller rejects > partial configurations where some services are local while RBAC is enabled). -**SDK docs**: [Feast RBAC](../reference/auth/rbac.md) +**SDK docs**: [Feast RBAC](../../getting-started/architecture/rbac.md) --- @@ -62,8 +62,20 @@ stringData: client_secret: username: # used for client-credentials flow password: + audience: # optional: reject tokens whose aud claim differs + issuer: # optional: reject tokens whose iss claim differs ``` +The optional `audience` and `issuer` keys enable audience and issuer claim verification on the standard OIDC/JWKS validation path; when omitted, the `aud` and `iss` claims are not checked. Set them to the values your IdP puts in the token itself, which are not always the ones in the discovery document (see [OIDC Authorization](../../getting-started/components/authz_manager.md#oidc-authorization)). The Secret key `issuer` is distinct from the CR's `issuerUrl`, which selects the discovery endpoint and plays no part in claim verification. Kubernetes ServiceAccount tokens (validated via TokenReview) and intra-server communication follow separate paths and are not subject to these checks. + +{% hint style="warning" %} +Before enabling these, three operational caveats: + +* **Existing Secret keys take effect on operator upgrade.** Keys named `audience` or `issuer` already present in the referenced Secret were previously ignored; after upgrading they are forwarded to every Feast pod. +* **Your IdP must mint matching tokens for Feast's own clients.** Feast's client-credentials flow requests no audience, so in multi-service topologies (e.g. a remote registry) and for the UI's browser tokens, the IdP must be configured to issue tokens carrying the expected claims (e.g. a Keycloak audience mapper), or inter-service calls will be rejected. +* **Secret edits are not watched.** Changes to these keys apply on the next reconcile or pod restart, not immediately. +{% endhint %} + Reference the Secret from the CR: ```yaml @@ -92,7 +104,7 @@ authz: caCertConfigMap: oidc-ca-cert # ConfigMap with CA cert for SSL verification ``` -**SDK docs**: [Feast OIDC Auth](../reference/auth/oidc.md) +**SDK docs**: [Feast OIDC Auth](../../getting-started/components/authz_manager.md#oidc-authorization) --- diff --git a/docs/how-to-guides/feast-operator/06-batch-and-jobs.md b/docs/how-to-guides/feast-operator/06-batch-and-jobs.md index fd513168c54..71e7b6b95dc 100644 --- a/docs/how-to-guides/feast-operator/06-batch-and-jobs.md +++ b/docs/how-to-guides/feast-operator/06-batch-and-jobs.md @@ -50,12 +50,53 @@ spec: configMapKey: config # key inside the ConfigMap (default: "config") ``` +### SparkApplication batch engine (optional) + +For Bring Your Own Spark on Kubernetes, use `spark_application` instead of in-process Spark. +The Feast Operator auto-creates RBAC for this type. See +[SparkApplication](../reference/compute-engine/spark_application.md) for the full config reference. +Build an image from the reference +[Dockerfile](https://github.com/feast-dev/feast/blob/master/sdk/python/feast/infra/compute_engines/spark_application/Dockerfile) +(or equivalent): + +```yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: feast-spark-application-engine +data: + config: | + type: spark_application + image: my-registry.example.com/feast-spark-driver:latest + namespace: feast + executor_instances: 2 + driver_memory: "2g" + executor_memory: "2g" +``` + +```yaml +apiVersion: feast.dev/v1 +kind: FeatureStore +metadata: + name: sample-spark-application +spec: + feastProject: my_project + batchEngine: + configMapRef: + name: feast-spark-application-engine + configMapKey: config + # Optional: use the Spark driver image for feast-apply / init containers + # services: + # initImage: my-registry.example.com/feast-spark-driver:latest +``` + ### Engine types | `type` | Notes | |--------|-------| | `local` | Default; in-process Python, no extra infra | | `spark` | Apache Spark; requires a Spark operator or standalone cluster | +| `spark_application` | Kubeflow Spark Operator `SparkApplication` CRs; requires Spark Operator + custom image; operator auto-creates RBAC | | `ray` | Ray cluster; requires a Ray operator | | `bytewax` | Bytewax streaming engine | | `snowflake.engine` | Snowflake Snowpark compute | diff --git a/docs/how-to-guides/feast-operator/README.md b/docs/how-to-guides/feast-operator/README.md index 26e515f309f..25d71c130c7 100644 --- a/docs/how-to-guides/feast-operator/README.md +++ b/docs/how-to-guides/feast-operator/README.md @@ -23,7 +23,7 @@ look for store-specific YAML options in the Feast SDK docs. | # | Guide | Topic | |---|-------|-------| -| 1 | [Project Provisioning](01-project-provisioning.md) | `feastProjectDir`: cloning a git repo vs `feast init` templates | +| 1 | [Project Provisioning](01-project-provisioning.md) | `feastProjectDir`: git clone, `feast init`, or a repository packaged in an image | | 2 | [Persistence](02-persistence.md) | File (path + PVC) vs DB store for offline/online/registry; Secret format | | 3 | [Serving & Observability](03-serving-and-observability.md) | Feature server workers, log level, Prometheus metrics, offline push batching, MCP | | 4 | [Registry Topology](04-registry-topology.md) | Local vs remote registry, cross-namespace `feastRef`, remote TLS | diff --git a/docs/how-to-guides/production-deployment-topologies.md b/docs/how-to-guides/production-deployment-topologies.md index ee8bb49be54..52dc2f3873b 100644 --- a/docs/how-to-guides/production-deployment-topologies.md +++ b/docs/how-to-guides/production-deployment-topologies.md @@ -1066,12 +1066,15 @@ Production environments in regulated industries (finance, government, defense) o ### Default init container behavior -When `feastProjectDir` is set on the FeatureStore CR, the operator creates up to two init containers: +When `feastProjectDir` is set on the FeatureStore CR, the operator creates up to two init containers unless `services.disableInitContainers` is `true`: -1. **`feast-init`** — bootstraps the feature repository by running either `git clone` (if `feastProjectDir.git` is set) or `feast init` (if `feastProjectDir.init` is set), then writes the generated `feature_store.yaml` into the repo directory. +1. **`feast-init`** — bootstraps the feature repository by running `git clone`, `feast init`, or copying a repository from `feastProjectDir.packaged.featureRepoPath`. It then writes the operator-generated `feature_store.yaml` into the initialized repository. 2. **`feast-apply`** — runs `feast apply` to register feature definitions in the registry. Controlled by `runFeastApplyOnInit` (defaults to `true`). Skipped when `disableInitContainers` is `true`. -In air-gapped environments, `git clone` will fail because the cluster cannot reach external Git repositories. The solution is to **pre-bake** the feature repository into a custom container image and disable the init containers entirely. +In air-gapped environments, use `feastProjectDir.packaged` to identify a feature repository baked into an image. The operator supports two lifecycle modes: + +* Keep init containers enabled to refresh shared storage from the image, generate configuration from the FeatureStore CR, and optionally run `feast apply`. +* Set `services.disableInitContainers: true` to run directly from the baked path and treat its `feature_store.yaml` as authoritative. ### Air-gapped deployment workflow @@ -1086,12 +1089,12 @@ graph TD end subgraph InternalRegistry["Internal Container Registry"] - Mirror["registry.internal.example.com
/feast/feature-server:v0.61"] + Mirror["registry.internal.example.com
/feast/feature-server:release"] end subgraph AirGappedCluster["Air-Gapped Kubernetes Cluster"] SA["ServiceAccount
(imagePullSecrets)"] - CR["FeatureStore CR
disableInitContainers: true
image: registry.internal..."] + CR["FeatureStore CR
feastProjectDir.packaged
disableInitContainers: true"] Deploy["Feast Deployment
(no init containers)"] SA --> Deploy CR --> Deploy @@ -1105,8 +1108,8 @@ graph TD 1. **Build a custom container image** that bundles the feature repository and all Python dependencies into the Feast base image. 2. **Push** the image to your internal container registry. -3. **Set `services.disableInitContainers: true`** on the FeatureStore CR to skip `git clone` / `feast init` and `feast apply`. -4. **Override the image** on each service using the per-service `image` field. +3. **Configure `feastProjectDir.packaged`** with the image and the canonical absolute path to the bundled repository. Do not use `.`, `..`, repeated separators, or a trailing separator, and keep the path outside operator-mounted locations such as `/feast-data` so it cannot overlap the staged repository. +4. **Choose the lifecycle:** leave init containers enabled for operator-managed configuration and `feast apply`, or set `services.disableInitContainers: true` to use the baked repository and configuration directly. 5. **Set `imagePullPolicy: IfNotPresent`** (or `Never` if images are pre-loaded on nodes). 6. **Configure `imagePullSecrets`** on the namespace's ServiceAccount — the FeatureStore CRD does not expose an `imagePullSecrets` field, so use the standard Kubernetes approach of attaching secrets to the ServiceAccount that the pods run under. @@ -1119,6 +1122,10 @@ metadata: name: airgap-production spec: feastProject: my_project + feastProjectDir: + packaged: + image: registry.internal.example.com/feast/feature-server:release + featureRepoPath: /opt/feast/feature_repo services: disableInitContainers: true onlineStore: @@ -1128,7 +1135,6 @@ spec: secretRef: name: feast-online-store server: - image: registry.internal.example.com/feast/feature-server:v0.61 imagePullPolicy: IfNotPresent resources: requests: @@ -1145,10 +1151,14 @@ spec: secretRef: name: feast-registry-store server: - image: registry.internal.example.com/feast/feature-server:v0.61 imagePullPolicy: IfNotPresent ``` +The packaged image is the default for every Feast service and for the `feast-init` and +`feast-apply` init containers. A per-service `image` still takes precedence for that +service, and `services.initImage` takes precedence for both init containers. Remove +`disableInitContainers: true` to use operator-managed staging and startup apply instead. + {% hint style="info" %} **Pre-populating the registry:** With init containers disabled, `feast apply` does not run on pod startup. You can populate the registry by: diff --git a/docs/project/contributing.md b/docs/project/contributing.md index d79291b9aa8..25ccd80703f 100644 --- a/docs/project/contributing.md +++ b/docs/project/contributing.md @@ -5,6 +5,8 @@ After familiarizing yourself with the documentation, the simplest way to get sta 1. Setup your developer environment by following [development guide](development-guide.md). 2. Either create a [GitHub issue](https://github.com/feast-dev/feast/issues) or make a draft PR (following [development guide](development-guide.md)) to get the ball rolling! +> **Reporting a security vulnerability?** Do not open an issue or PR. Report it privately through [GitHub's advisory form](https://github.com/feast-dev/feast/security/advisories/new); see the [security policy](https://github.com/feast-dev/feast/blob/master/SECURITY.md). + ## Decision making process *See [governance](../../community/governance.md) for more details here* diff --git a/docs/reference/alpha-vector-database.md b/docs/reference/alpha-vector-database.md index 861c3fcb114..61da02ce6f4 100644 --- a/docs/reference/alpha-vector-database.md +++ b/docs/reference/alpha-vector-database.md @@ -15,6 +15,7 @@ Below are supported vector databases and implemented features: | Faiss | [ ] | [ ] | [] | [] | | SQLite | [x] | [ ] | [x] | [x] | | Qdrant | [x] | [x] | [] | [] | +| ScyllaDB | [x] | [x] | [x] | [x] | *Note: V2 Support means the SDK supports retrieval of features along with vector embeddings from vector similarity search. @@ -30,7 +31,241 @@ Beyond that, we will then have `retrieve_online_documents` and `retrieve_online_ backwards compatibility and the adopt industry standard naming conventions. {% endhint %} -**Note**: Milvus and SQLite implement the v2 `retrieve_online_documents_v2` method in the SDK. This will be the longer-term solution so that Data Scientists can easily enable vector similarity search by just flipping a flag. +**Note**: Milvus, SQLite, and ScyllaDB implement the v2 `retrieve_online_documents_v2` method in the SDK. This will be the longer-term solution so that Data Scientists can easily enable vector similarity search by just flipping a flag. + +## Feature server search endpoints + +| Endpoint | Use when | +|----------|----------| +| `POST /search` | You have an embedding vector (or use `api_version: 2` with `query_string`) and want Feast's native online-features response format. | +| `GET /v1/vector_stores` | You want to discover available vector stores and their `vs_{hash}` IDs (OpenAI-compatible). | +| `GET /v1/vector_stores/{id}` | You want metadata for a specific vector store (OpenAI-compatible). | +| `POST /v1/vector_stores/{id}/search` | You want plain-text queries with server-side embedding and an OpenAI-compatible response. | + +`POST /retrieve-online-documents` is deprecated; use `POST /search` instead. + +## [Alpha] OpenAI-Compatible Vector Store API + +{% hint style="warning" %} +**Alpha feature.** This API surface is functional and tested, but may change in future releases. Feedback and contributions are welcome. +{% endhint %} + +Feast exposes a set of [OpenAI-compatible vector store endpoints](https://platform.openai.com/docs/api-reference/vector-stores) that let clients discover, inspect, and search vector stores using plain text queries with server-side embedding. This enables integration with AI agents, LLM tool-calling frameworks, and any OpenAI-compatible client without requiring the caller to produce raw embedding vectors. + +### Vector store IDs + +Each feature view with at least one `vector_index=True` field is automatically assigned a deterministic identifier of the form `vs_{hash}`, where `{hash}` is the first 24 characters of `SHA-256(project + ":" + feature_view_name)`. These IDs are stable across server restarts and registry refreshes. + +For example, a feature view named `product_catalog` in project `my_project` always maps to the same `vs_...` identifier. The listing endpoints return these IDs so clients can discover stores at runtime. + +### Endpoints + +| Method | Path | Permission | Description | +|--------|------|------------|-------------| +| `GET` | `/v1/vector_stores` | `DESCRIBE` | List all vector stores the caller has access to | +| `GET` | `/v1/vector_stores/{vector_store_id}` | `DESCRIBE` | Get metadata for a single vector store | +| `POST` | `/v1/vector_stores/{vector_store_id}/search` | `READ_ONLINE` | Search a vector store with a plain text query | + +All endpoints enforce RBAC when authentication is configured. The listing endpoint filters out stores the caller cannot `DESCRIBE`. + +### Requirements + +1. **Embedding model** — an `embedding_model` section in `feature_store.yaml`. Feast uses [Sentence Transformers](https://www.sbert.net/) by default for local embedding — no external API key required (`pip install sentence-transformers`): + + ```yaml + embedding_model: + provider: sentence_transformers # default; can be omitted + model: all-MiniLM-L6-v2 + ``` + +2. **Vector-indexed feature view** — at least one feature view with `vector_index=True` on a vector field, materialized to an online store that supports vector search. + +3. **Numeric filtering (optional)** — for metadata filters that use numeric or boolean comparisons, set `enable_openai_compatible_store: true` on your online store config and run `feast apply` to add the required `value_num` column. + +### Custom embedding providers + +The built-in Sentence Transformers provider works for most use cases. To use a different embedding backend (OpenAI, Cohere, a custom model, etc.), implement the `EmbeddingProvider` protocol and pass an instance to `FeatureStore`: + +```python +from feast.embedder import EmbeddingProvider + +class MyEmbeddingProvider: + def embed(self, texts: list[str]) -> list[list[float]]: + # Call your embedding API here + return my_model.encode(texts) + + async def aembed(self, texts: list[str]) -> list[list[float]]: + return await my_model.aencode(texts) + +store = FeatureStore( + repo_path=".", + embedding_provider=MyEmbeddingProvider(), +) +``` + +### Numeric storage (`enable_openai_compatible_store`) + +By default, feature values are stored as text in the online store. This means string-ordered comparisons apply (e.g., `'9' > '100'` is `true`). When `enable_openai_compatible_store: true` is set on the online store config, Feast adds a `value_num` column that stores `int`, `float`, `double`, and `bool` values natively so that numeric filters produce correct results. + +```yaml +online_store: + type: postgres # or sqlite + # ... connection settings ... + enable_openai_compatible_store: true +``` + +After changing this setting, run `feast apply` to update the database schema. + +### List vector stores + +```bash +curl http://localhost:6566/v1/vector_stores +``` + +```json +{ + "object": "list", + "data": [ + { + "id": "vs_a1b2c3d4e5f6a1b2c3d4e5f6", + "object": "vector_store", + "name": "product_catalog", + "status": "completed", + "created_at": 1717200000 + } + ] +} +``` + +### Get a single vector store + +```bash +curl http://localhost:6566/v1/vector_stores/vs_a1b2c3d4e5f6a1b2c3d4e5f6 +``` + +Returns the same object shape as a single entry in the list response. Returns `404` if the ID does not match any vector-indexed feature view. + +### Search + +Start the feature server with `feast serve`, then send a search request: + +```bash +curl -X POST http://localhost:6566/v1/vector_stores/vs_a1b2c3d4e5f6a1b2c3d4e5f6/search \ + -H "Content-Type: application/json" \ + -d '{ + "query": "wireless noise-cancelling headphones", + "max_num_results": 5 + }' +``` + +#### Request fields + +| Field | Type | Default | Description | +|-------|------|---------|-------------| +| `query` | `string` or `list[string]` | (required) | Plain text search query. Lists are joined with spaces before embedding. | +| `max_num_results` | `int` | `10` | Maximum number of results to return. | +| `filters` | `object` | `null` | OpenAI-style filters (see below). | +| `ranking_options` | `object` | `null` | Accepted for forward compatibility, but currently ignored. Setting `score_threshold` or `ranker` inside it will return a 422 error. | +| `rewrite_query` | `bool` | `null` | `false` (the default/no-op) is accepted. `true` is not yet supported and will return a 422 error. | +| `metadata` | `object` | `null` | Optional. `metadata.features_to_retrieve` selects specific features. | + +### Filters + +The endpoint supports OpenAI-style filters for narrowing results beyond vector similarity. + +**Comparison operators:** `eq`, `ne`, `gt`, `gte`, `lt`, `lte`, `in`, `nin` + +```json +{"type": "eq", "key": "category", "value": "Electronics"} +``` + +**Compound operators:** `and`, `or` (nest to arbitrary depth) + +```json +{ + "type": "and", + "filters": [ + {"type": "eq", "key": "category", "value": "Electronics"}, + {"type": "gte", "key": "rating", "value": 4.5} + ] +} +``` + +For Postgres and SQLite backends, all filtering (including string equality) requires `enable_openai_compatible_store: true` in the online store config. After enabling, run `feast apply` to update the database schema. + +ScyllaDB supports vector retrieval via `retrieve_online_documents_v2`, but OpenAI-style metadata filtering is not implemented yet. Passing `filters` raises `NotImplementedError`. + +### Response format + +Responses follow the OpenAI `vector_store.search_results.page` schema: + +```json +{ + "object": "vector_store.search_results.page", + "search_query": ["wireless noise-cancelling headphones"], + "data": [ + { + "file_id": "vs_a1b2c3d4e5f6a1b2c3d4e5f6_42", + "filename": "vs_a1b2c3d4e5f6a1b2c3d4e5f6", + "score": 0.92, + "attributes": {"name": "...", "category": "..."}, + "content": [ + {"type": "text", "text": "..."} + ] + } + ], + "has_more": false, + "next_page": null +} +``` + +The `file_id` and `filename` fields use the `vs_{hash}` identifier, not raw feature view names. + +The `score` field is a higher-is-better relevance score derived from the raw vector distance using a metric-dependent conversion: + +| Distance metric | Conversion | Range | +|----------------|------------|-------| +| L2 (default) | `1 / (1 + distance)` | (0, 1] | +| Cosine | `1 - distance` | [0, 1] | +| Inner product / dot | `-distance` | varies | + +The metric is determined by `vector_search_metric` on the feature view's vector field, not by an API parameter. When `features_to_retrieve` is omitted, all non-vector features are returned by default (vector embedding columns are excluded). + +Pagination is not yet implemented; `has_more` is always `false`. + +### SDK usage + +The OpenAI-compatible search is also available directly via the Python SDK: + +```python +import asyncio +from feast import FeatureStore + +store = FeatureStore(repo_path=".") + +result = asyncio.run(store.openai_search( + vector_store_id="product_catalog", + query="wireless noise-cancelling headphones", + max_num_results=5, + filters={"type": "eq", "key": "category", "value": "Electronics"}, +)) + +for item in result["data"]: + print(f"{item['score']:.3f} {item['attributes']}") +``` + +### Supported online stores + +The OpenAI-compatible filtering has been implemented for the following online stores: + +| Online Store | Vector Search | Metadata Filtering | Notes | +|-------------|--------------|-------------------|-------| +| Milvus | Yes | Yes | Boolean expressions | +| Elasticsearch | Yes | Yes | Query DSL clauses | +| Postgres (pgvector) | Yes | Yes | Requires `enable_openai_compatible_store: true` | +| SQLite (sqlite-vec) | Yes | Yes | Requires `enable_openai_compatible_store: true` | +| MongoDB | Yes | Yes | Aggregation pipeline | +| ScyllaDB | Yes | No | Vector search only; metadata filters are not supported yet | ## Examples diff --git a/docs/reference/alpha-web-ui.md b/docs/reference/alpha-web-ui.md index 0556482fcf8..3fe8ce052a8 100644 --- a/docs/reference/alpha-web-ui.md +++ b/docs/reference/alpha-web-ui.md @@ -153,3 +153,12 @@ const tabsRegistry = { ``` Examples of custom tabs can be found in the `ui/custom-tabs` folder. + +## Refreshing the registry + +The Feast UI caches registry data (projects, feature views, entities, etc.) using the registry cache. After running `feast apply` to make changes, it may take up to `cache_ttl_seconds` before the updates appear in the UI. + +To see changes faster: + +- **Lower the TTL**: Set `cache_ttl_seconds: 10` (or similar) in your `feature_store.yaml` registry config. This makes all registry consumers — including the UI — pick up changes within 10 seconds. +- **Refresh on demand**: The UI has a **Refresh** button that explicitly invalidates the server-side registry cache (`POST /api/v1/registry/refresh`) and reloads the UI without a full page refresh. diff --git a/docs/reference/codebase-structure.md b/docs/reference/codebase-structure.md index 6c5ad9a141e..4783773c270 100644 --- a/docs/reference/codebase-structure.md +++ b/docs/reference/codebase-structure.md @@ -28,7 +28,7 @@ The majority of Feast logic lives in these Python files: There are also several important submodules: * `infra/` contains all the infrastructure components, such as the provider, offline store, online store, batch materialization engine, and registry. -* `dqm/` covers data quality monitoring. The legacy Great Expectations profiler (`profilers/ge_profiler`) is deprecated; see [`monitoring/`](../../sdk/python/feast/monitoring/) for the current built-in monitoring system. +* `dqm/` covers data quality monitoring. See [`monitoring/`](../../sdk/python/feast/monitoring/) for the built-in monitoring system. * `diff/` covers the logic for determining how to apply infrastructure changes upon feature repo changes (e.g. the output of `feast plan` and `feast apply`). * `embedded_go/` covers the Go feature server. * `ui/` contains the embedded Web UI, to be launched on the `feast ui` command. diff --git a/docs/reference/compute-engine/README.md b/docs/reference/compute-engine/README.md index 920d5761d28..a570e5688ed 100644 --- a/docs/reference/compute-engine/README.md +++ b/docs/reference/compute-engine/README.md @@ -57,6 +57,14 @@ An example of built output from FeatureBuilder: - Supports point-in-time joins and large-scale materialization - Integrates with `SparkOfflineStore` and `SparkMaterializationJob` +### ☸️ SparkApplicationComputeEngine + +{% page-ref page="spark_application.md" %} + +- Batch materialization via Kubeflow Spark Operator `SparkApplication` CRs +- One SparkApplication per materialize call (multi–feature-view batching) +- Requires network-accessible online/offline/registry stores (no file-based backends) + ### 🌊 FlinkComputeEngine {% page-ref page="flink.md" %} diff --git a/docs/reference/compute-engine/snowflake.md b/docs/reference/compute-engine/snowflake.md index e7b0dc5bd63..f6c633a4e40 100644 --- a/docs/reference/compute-engine/snowflake.md +++ b/docs/reference/compute-engine/snowflake.md @@ -24,5 +24,10 @@ batch_engine: role: sysadmin warehouse: demo_wh database: FEAST + python_udf_runtime_version: "3.10" ``` {% endcode %} + +## Configuration + +* `python_udf_runtime_version` *(optional, default: `"3.10"`)* -- The Snowflake Python UDF `RUNTIME_VERSION` used when Feast deploys its materialization UDFs. Snowflake periodically decommissions old Python UDF runtimes (for example, the 3.9 runtime was decommissioned, requiring Feast to bump its default to 3.10 -- see [#6606](https://github.com/feast-dev/feast/issues/6606)). If Snowflake decommissions the 3.10 runtime in the future, set this field to a still-supported version (e.g. `"3.11"`) instead of waiting for a new Feast release. diff --git a/docs/reference/compute-engine/spark_application.md b/docs/reference/compute-engine/spark_application.md new file mode 100644 index 00000000000..0c071d1ed4f --- /dev/null +++ b/docs/reference/compute-engine/spark_application.md @@ -0,0 +1,182 @@ +# SparkApplication Compute Engine + +## Description + +The **SparkApplication** compute engine runs Feast **batch materialization** on Kubernetes by creating a [Kubeflow Spark Operator](https://github.com/kubeflow/spark-operator) `SparkApplication` custom resource for each materialization job. + +Unlike the in-process [`spark.engine`](spark.md) compute engine (which uses a Spark session inside the Feast process), `spark_application` submits work to the Spark Operator. The operator starts a driver pod and executors from your configured image; Feast polls the SparkApplication until it completes. + +| Capability | Supported | +|------------|-----------| +| `materialize` / `materialize-incremental` | Yes | +| Multiple feature views in one job | Yes — one SparkApplication per materialize call | +| `get_historical_features` | Not yet | +| SparkConnect | Separate approach — not this engine | + +### Design + +1. Feast creates a ConfigMap with job tasks and a driver copy of `feature_store.yaml`. +2. Feast creates a `SparkApplication` CR pointing at the driver entrypoint (`main.py` in the image). +3. Inside the pod, the batch engine type is rewritten to `spark.engine` so materialization uses the Spark session created by `spark-submit` (avoids recursive SparkApplication creation). +4. The driver writes features to your configured **online store** and updates the **registry** (same network backends as the server). + +### Requirements + +- Kubeflow Spark Operator installed and watching the target namespace. +- A container **image** that includes the Feast SDK, PySpark, and clients for your stores. See the reference [Dockerfile](https://github.com/feast-dev/feast/blob/master/sdk/python/feast/infra/compute_engines/spark_application/Dockerfile). +- **Network-accessible** online store, offline store, and registry. File-based backends are rejected because Spark pods have an ephemeral filesystem: + +| Rejected | Examples | Use instead | +|----------|----------|-------------| +| File online | `sqlite`, `faiss` | Redis, remote online, etc. | +| File offline | `dask`, `file`, `duckdb` | `spark`, Postgres, Snowflake, BigQuery, etc. | +| File registry | `file` | SQL registry, Snowflake | + +For distributed reads, configure `offline_store.type: spark` (or another store Spark can read efficiently). + +### Kubernetes / Feast Operator notes + +When using the Feast Operator: + +- Point `spec.batchEngine.configMapRef` at a ConfigMap whose `type` is `spark_application` (see [Guide 6 — Batch Engine & Scheduled Jobs](../../how-to-guides/feast-operator/06-batch-and-jobs.md)). +- The operator auto-creates RBAC for the `spark_application` batch engine (server and driver service accounts). +- Set `spec.services.initImage` if init / `feast-apply` containers need the Spark-capable image. + +--- + +## Example + +{% code title="feature_store.yaml" %} +```yaml +project: my_project +registry: + registry_type: sql + path: postgresql+psycopg://feast:****@postgres:5432/feast +online_store: + type: redis + connection_string: redis:6379 +offline_store: + type: spark + spark_conf: + spark.master: local[*] +batch_engine: + type: spark_application + image: my-registry.example.com/feast-spark-driver:latest + namespace: feast + spark_version: "4.0.1" + driver_cores: 1 + driver_memory: "2g" + executor_instances: 2 + executor_cores: 1 + executor_memory: "2g" + spark_conf: + spark.sql.shuffle.partitions: "100" +``` +{% endcode %} + +### Feast Operator ConfigMap + +```yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: feast-spark-batch-engine + namespace: feast +data: + config: | + type: spark_application + image: my-registry.example.com/feast-spark-driver:latest + namespace: feast + executor_instances: 2 + driver_memory: "2g" + executor_memory: "2g" +--- +apiVersion: feast.dev/v1 +kind: FeatureStore +metadata: + name: feast + namespace: feast +spec: + feastProject: my_project + batchEngine: + configMapRef: + name: feast-spark-batch-engine + configMapKey: config +``` + +--- + +## Remote materialization + +If the client uses a **remote** online store (`online_store.type: remote`), `FeatureStore.materialize()` delegates to the feature server HTTP API. The server runs the SparkApplication engine. + +- Default (`run_async=False`): block until the server finishes sync materialization. +- `run_async=True`: accept asynchronously (`?async=true`); poll feature-view state in the registry for completion. +- `force=True` (with `run_async=True`): override stuck `MATERIALIZING` state on the server. + +```python +from datetime import datetime, timedelta +from feast import FeatureStore + +store = FeatureStore(repo_path=".") # client feature_store.yaml with online_store.type: remote + +store.materialize( + start_date=datetime.utcnow() - timedelta(days=1), + end_date=datetime.utcnow(), +) +``` + +--- + +## Configuration reference + +| Field | Type | Default | Description | +|-------|------|---------|-------------| +| `type` | string | `spark_application` | Engine type key | +| `image` | string | **required** | Container image for the Spark driver/executors | +| `image_pull_secrets` | list[str] | `[]` | Image pull secret names | +| `namespace` | string | `default` | Namespace for SparkApplication and ConfigMap | +| `service_account` | string | `""` | Driver service account; empty uses platform/operator default | +| `spark_version` | string | `4.0.1` | Spark version for the CR | +| `driver_cores` | int | `1` | Driver cores | +| `driver_memory` | string | `1g` | Driver memory | +| `executor_instances` | int | `1` | Number of executors | +| `executor_cores` | int | `1` | Cores per executor | +| `executor_memory` | string | `1g` | Memory per executor | +| `spark_conf` | dict | `null` | Extra Spark configuration | +| `hadoop_conf` | dict | `null` | Extra Hadoop configuration | +| `env` | list[dict] | `[]` | Driver env vars (`name` + `value` or `valueFrom`) | +| `env_from` | list[dict] | `[]` | EnvFrom sources | +| `queue_name` | string | `null` | Optional queue / Kueue label | +| `job_timeout_seconds` | int | `3600` | Max wait for SparkApplication completion | +| `poll_interval_seconds` | int | `10` | Status poll interval | +| `ttl_seconds_after_finished` | int | `3600` | CR TTL after finish | +| `restart_policy` | string | `Never` | SparkApplication restart policy | +| `max_retries` | int | `3` | Retries when restart policy allows | +| `concurrency` | int | `1` | Parallel feature views inside one driver | +| `labels` | dict | `{}` | Extra labels on the CR | +| `volumes` / `volume_mounts` | list | `[]` | Extra volumes for the driver | +| `py_files` | list[str] | `[]` | Additional Python files for Spark | +| `node_selector` | dict | `null` | Pod node selector | +| `tolerations` | list | `[]` | Pod tolerations | +| `staging_location` | string | `null` | Reserved for historical retrieval (ignored for materialize) | + +--- + +## Troubleshooting + +| Symptom | What to check | +|---------|----------------| +| SparkApplication Pending / insufficient CPU | Lower resource requests via `spark_conf` (for example `spark.kubernetes.driver.request.cores`) or free cluster capacity | +| ImagePullBackOff | Image name, tag, and `image_pull_secrets` | +| 403 on ConfigMap or SparkApplication | RBAC for the Feast server and Spark driver service accounts | +| Init `ValueError` about file-based stores | Switch online/offline/registry to network backends | +| Init / feast-apply failures missing Spark deps | Use a Spark-capable image (`initImage` with the Feast Operator) | + +--- + +## Related + +- [Spark compute engine (in-process)](spark.md) +- [Feast Operator — batch engine ConfigMap](../../how-to-guides/feast-operator/06-batch-and-jobs.md) +- [Creating a custom compute engine](../../how-to-guides/customizing-feast/creating-a-custom-compute-engine.md) diff --git a/docs/reference/data-sources/README.md b/docs/reference/data-sources/README.md index 24bf18dbe86..33e47672dcc 100644 --- a/docs/reference/data-sources/README.md +++ b/docs/reference/data-sources/README.md @@ -42,6 +42,10 @@ Please see [Data Source](../../getting-started/concepts/data-ingestion.md) for a [spark.md](spark.md) {% endcontent-ref %} +{% content-ref url="iceberg.md" %} +[iceberg.md](iceberg.md) +{% endcontent-ref %} + {% content-ref url="postgres.md" %} [postgres.md](postgres.md) {% endcontent-ref %} diff --git a/docs/reference/data-sources/iceberg.md b/docs/reference/data-sources/iceberg.md new file mode 100644 index 00000000000..6402a7ec419 --- /dev/null +++ b/docs/reference/data-sources/iceberg.md @@ -0,0 +1,161 @@ +# Iceberg source (contrib) + +## Description + +Iceberg data sources are tables managed by any supported Iceberg catalog. The `IcebergSource` class provides a unified interface with a configurable `catalog_type` parameter: + +- **`"rest"`** (default): [Apache Iceberg REST Catalog specification](https://iceberg.apache.org/concepts/catalog/#decoupling-using-the-rest-catalog) — Unity Catalog, Apache Polaris, Nessie, Snowflake Open Catalog +- **`"hive"`**: Hive Metastore catalog +- **`"glue"`**: AWS Glue catalog +- **`"sql"`**: SQL-based (JDBC) catalog +- **`"dynamodb"`**: DynamoDB-based catalog + +The data source carries catalog connection details (catalog_type, endpoint, warehouse, namespace, table, authentication). When the offline store (DuckDB, Spark) encounters this source, it resolves table metadata and credentials via the configured catalog at query time. + +## Examples + +### IcebergSource (REST catalog) + +Works with any Iceberg REST Catalog: + +```python +from feast.infra.data_sources.contrib.iceberg_catalog import IcebergSource + +my_source = IcebergSource( + catalog_type="rest", # default + endpoint="http://localhost:8081/api/2.1/unity-catalog/iceberg", + warehouse="unity", + namespace="default", + table="driver_features", + timestamp_field="event_timestamp", + token_env_var="UC_TOKEN", +) +``` + +### IcebergSource (Hive Metastore) + +```python +from feast.infra.data_sources.contrib.iceberg_catalog import IcebergSource + +my_source = IcebergSource( + catalog_type="hive", + catalog_properties={"uri": "thrift://metastore:9083"}, + warehouse="my_warehouse", + namespace="default", + table="driver_features", + timestamp_field="event_timestamp", +) +``` + +### IcebergSource (AWS Glue) + +```python +from feast.infra.data_sources.contrib.iceberg_catalog import IcebergSource + +my_source = IcebergSource( + catalog_type="glue", + catalog_properties={"region_name": "us-east-1"}, + warehouse="my_account", + namespace="my_database", + table="driver_features", + timestamp_field="event_timestamp", +) +``` + +### UnityCatalogSource (with governance) {#unity-catalog-source} + +Extends `IcebergSource` with Unity Catalog governance: + +```python +from feast.infra.data_sources.contrib.iceberg_catalog import ( + UnityCatalogSource, +) + +my_uc_source = UnityCatalogSource( + warehouse="production", + namespace="ml_features", + table="driver_stats", + timestamp_field="event_timestamp", + register_as_feature_table=True, # Register in UC on feast apply + sync_lineage=True, # Record lineage in UC +) +``` + +When `endpoint` is omitted, it defaults to `{DATABRICKS_HOST}/api/2.1/unity-catalog/iceberg`. +When `token_env_var` is omitted, it defaults to `DATABRICKS_TOKEN`. + +### Full Feature View Example + +```python +from datetime import timedelta + +from feast import Entity, FeatureView, Field +from feast.types import Float64, Int64 + +from feast.infra.data_sources.contrib.iceberg_catalog import ( + UnityCatalogSource, +) + +driver = Entity(name="driver_id", join_keys=["driver_id"]) + +driver_stats_source = UnityCatalogSource( + warehouse="production", + namespace="ml_features", + table="driver_hourly_stats", + timestamp_field="event_timestamp", + created_timestamp_column="created", +) + +driver_stats_fv = FeatureView( + name="driver_hourly_stats", + entities=[driver], + source=driver_stats_source, + schema=[ + Field(name="conv_rate", dtype=Float64), + Field(name="acc_rate", dtype=Float64), + Field(name="avg_daily_trips", dtype=Int64), + ], + ttl=timedelta(days=1), + online=True, +) +``` + +## Configuration Reference + +### IcebergSource + +| Parameter | Type | Description | +| :--- | :--- | :--- | +| `catalog_type` | `str` | Catalog backend: `"rest"` (default), `"hive"`, `"glue"`, `"sql"`, `"dynamodb"` | +| `endpoint` | `str` | Catalog endpoint URL (required for `"rest"`, optional for others) | +| `warehouse` | `str` | Catalog/warehouse name | +| `namespace` | `str` | Schema/namespace within the catalog | +| `table` | `str` | Table name | +| `catalog_properties` | `dict` | Additional catalog-specific properties passed to PyIceberg | +| `timestamp_field` | `str` | Event timestamp column for point-in-time joins | +| `created_timestamp_column` | `str` | Optional column indicating row creation time | +| `token_env_var` | `str` | Environment variable name holding the auth token | +| `credential_vending` | `bool` | Whether to request scoped credentials (default: `True`) | +| `field_mapping` | `dict` | Column name mapping from source to feature names | + +### UnityCatalogSource (additional parameters) + +| Parameter | Type | Description | +| :--- | :--- | :--- | +| `register_as_feature_table` | `bool` | Register as UC feature table on `feast apply` (default: `True`) | +| `sync_lineage` | `bool` | Sync lineage metadata to Unity Catalog (default: `True`) | + +## Supported Types + +| Iceberg Type | Feast Type | +| :--- | :--- | +| `boolean` | `BOOL` | +| `int` | `INT32` | +| `long` | `INT64` | +| `float` | `FLOAT` | +| `double` | `DOUBLE` | +| `string` | `STRING` | +| `binary` | `BYTES` | +| `timestamp` / `timestamptz` | `INT64` | +| `decimal` | `DOUBLE` | +| `uuid` | `STRING` | diff --git a/docs/reference/dqm.md b/docs/reference/dqm.md index 51f98686dee..47090b5dd1c 100644 --- a/docs/reference/dqm.md +++ b/docs/reference/dqm.md @@ -1,89 +1,81 @@ # Data Quality Monitoring -{% hint style="warning" %} -**Deprecated:** The Great Expectations-based validation described on this page is deprecated and will be removed in a future release. It has been superseded by Feast's built-in [Feature Quality Monitoring](../how-to-guides/feature-monitoring.md) system, which provides richer metrics (histograms, percentiles, drift detection), works across batch data and serving logs, requires no external dependencies, and includes a built-in UI dashboard. +Feast's Data Quality Monitoring (DQM) system computes, stores, and serves statistical metrics for every registered feature. It gives you visibility into feature health — distributions, null rates, percentiles, histograms — across batch data and feature serving logs. -Please migrate to the new monitoring system. See the [Feature Quality Monitoring guide](../how-to-guides/feature-monitoring.md) for setup instructions. -{% endhint %} +Its goal is to address several complex data problems: -## Legacy: Great Expectations Integration - -The following documents the deprecated Great Expectations-based validation that was previously the only DQM option in Feast. This integration relied on `pip install 'feast[ge]'` and only supported validation during historical retrieval. - ---- +* **Data consistency** — new training datasets can differ significantly from previous datasets, potentially requiring changes in model architecture. +* **Upstream pipeline bugs** — bugs in upstream pipelines can cause invalid values to overwrite existing valid values in an online store. +* **Training/serving skew** — distribution shift between training and serving data can decrease model performance. ### Overview -The legacy validation process consists of the following steps: -1. User prepares reference dataset (only [saved datasets](../getting-started/concepts/dataset.md) from historical retrieval are supported). -2. User defines a profiler function that produces a profile using [Great Expectations](https://docs.greatexpectations.io). -3. Validation of the tested dataset is performed with the reference dataset and profiler provided as parameters. +Feast's DQM system works natively with your configured offline store — no additional infrastructure or external dependencies are required. The workflow is: -### Installation -```shell -pip install 'feast[ge]' -``` +1. **Register features** — run `feast apply` to register feature views. If `auto_baseline: true` is configured, baseline metrics are computed automatically. +2. **Schedule monitoring** — run `feast monitor run` on a schedule (daily recommended) to compute metrics across multiple time windows. +3. **Read metrics** — query metrics via the REST API or view them in the Feast UI. -### Dataset profile +### Configuration -This integration uses [Great Expectation's](https://greatexpectations.io/) [ExpectationSuite](https://legacy.docs.greatexpectations.io/en/latest/autoapi/great_expectations/core/expectation_suite/index.html#great_expectations.core.expectation_suite.ExpectationSuite) -as the dataset profile format. The user defines a profiler function that receives a dataset and returns an ExpectationSuite. +Enable DQM in your `feature_store.yaml`: + +```yaml +data_quality_monitoring: + auto_baseline: true +``` -```python -from great_expectations.dataset import Dataset -from great_expectations.core.expectation_suite import ExpectationSuite +### Computing Metrics -from feast.dqm.profilers.ge_profiler import ge_profiler +**Auto mode (recommended for production):** -@ge_profiler -def manual_profiler(dataset: Dataset) -> ExpectationSuite: - dataset.expect_column_max_to_be_between("column", 1, 2) - return dataset.get_expectation_suite() +```bash +feast monitor run ``` -### Validating Training Dataset +This detects the latest event timestamp in the source data and computes metrics for 5 time windows: daily, weekly, biweekly, monthly, and quarterly. -During retrieval of historical features, `validation_reference` can be passed as a parameter to methods `.to_df(validation_reference=...)` or `.to_arrow(validation_reference=...)` of RetrievalJob. -If validation is successful, the materialized dataset is returned. Otherwise, `feast.dqm.errors.ValidationFailed` exception is raised with details for expectations that didn't pass. +**Target a specific feature view:** -```python -from feast import FeatureStore +```bash +feast monitor run --feature-view driver_stats +``` -fs = FeatureStore(".") +**Explicit date range:** -job = fs.get_historical_features(...) -job.to_df( - validation_reference=fs - .get_saved_dataset("my_reference_dataset") - .as_reference(profiler=manual_profiler) -) +```bash +feast monitor run \ + --feature-view driver_stats \ + --start-date 2025-01-01 \ + --end-date 2025-01-07 \ + --granularity weekly ``` ---- +**Set a manual baseline:** -## Migration Guide +```bash +feast monitor run \ + --feature-view driver_stats \ + --start-date 2025-01-01 \ + --end-date 2025-03-31 \ + --granularity daily \ + --set-baseline +``` -The new [Feature Quality Monitoring](../how-to-guides/feature-monitoring.md) system replaces this integration with: +### Monitoring Feature Serving Logs -| Capability | GE-based (deprecated) | New DQM | -|---|---|---| -| Scope | Historical retrieval only | Batch data + serving logs | -| Dependencies | `feast[ge]` extra required | No extra dependencies | -| Metrics | User-defined expectations | Automatic: null rates, percentiles, histograms, drift | -| UI | None | Built-in monitoring dashboard | -| Automation | Manual profiler code | `feast monitor run` CLI + REST API | -| Backends | Limited | All offline store backends | +If your feature services have logging configured, you can compute metrics from the actual features served to models in production: -To migrate: +```bash +feast monitor run --source-type log +``` -1. Enable DQM in `feature_store.yaml`: - ```yaml - data_quality_monitoring: - auto_baseline: true - ``` +### Reading Metrics -2. Run `feast apply` to compute baseline metrics automatically. +Metrics are accessible via the REST API: -3. Schedule `feast monitor run` for ongoing monitoring. +``` +GET /monitoring/metrics/features?project=my_project&feature_view_name=driver_stats&granularity=daily +``` -4. Remove the `feast[ge]` dependency from your requirements. +See the [Feature Quality Monitoring guide](../how-to-guides/feature-monitoring.md) for full API reference, UI integration, and orchestrator examples. diff --git a/docs/reference/feast-cli-commands.md b/docs/reference/feast-cli-commands.md index 99a99ab1707..85781f6abc2 100644 --- a/docs/reference/feast-cli-commands.md +++ b/docs/reference/feast-cli-commands.md @@ -28,6 +28,7 @@ Commands: materialize Run a (non-incremental) materialization job to... materialize-incremental Run an incremental materialization job to ingest... permissions Access permissions + registry Manage the feature registry registry-dump Print contents of the metadata registry teardown Tear down deployed feature store infrastructure version Display Feast SDK version @@ -483,6 +484,18 @@ reader driver_hourly_stats_fresh FeatureView DESCRIBE ``` +## Registry + +### create-schema + +Pre-create the SQL registry schema so the application does not need DDL privileges at runtime. Use this with `schema_mode: verify` or `schema_mode: skip` in your `feature_store.yaml`. + +```text +feast registry create-schema +``` + +This command only applies to SQL-based registries (`registry_type: sql`). It is safe to run multiple times — existing tables are not modified. + ## Teardown Tear down deployed feature store infrastructure diff --git a/docs/reference/feature-servers/python-feature-server.md b/docs/reference/feature-servers/python-feature-server.md index 4802599866d..b1b873cc7d2 100644 --- a/docs/reference/feature-servers/python-feature-server.md +++ b/docs/reference/feature-servers/python-feature-server.md @@ -527,6 +527,42 @@ Prometheus adds an `instance` label per pod, so there is no duplication. Use `sum(rate(...))` or `histogram_quantile(...)` across instances as usual. +## Vector Search (`POST /search`) + +The feature server exposes `POST /search` for vector similarity search against online document embeddings. Pass a pre-computed embedding in `query`, or use `api_version: 2` with `query_string` for text-based search when the online store supports it. + +`POST /retrieve-online-documents` is a deprecated alias with the same request body and response; new integrations should use `/search`. + +## [Alpha] OpenAI-Compatible Vector Store API + +{% hint style="warning" %} +**Alpha feature.** This API surface is functional and tested, but may change in future releases. +{% endhint %} + +The feature server exposes OpenAI-compatible vector store endpoints. This allows clients (including LLM agents and tool-calling frameworks) to discover and search vector data with plain text queries, without computing embeddings client-side. + +Each feature view with vector-indexed fields gets a deterministic `vs_{hash}` identifier derived from `SHA-256(project + ":" + feature_view_name)`. These IDs are stable across server restarts. + +### Endpoints + +| Method | Path | RBAC | Description | +|---|---|---|---| +| `GET` | `/v1/vector_stores` | `DESCRIBE` | List all vector stores (filtered by caller permissions) | +| `GET` | `/v1/vector_stores/{vector_store_id}` | `DESCRIBE` | Get metadata for a single vector store | +| `POST` | `/v1/vector_stores/{vector_store_id}/search` | `READ_ONLINE` | Search a vector store with server-side embedding | + +### Configuration + +Add an `embedding_model` section to your `feature_store.yaml`: + +```yaml +embedding_model: + provider: sentence_transformers # default; can be omitted + model: all-MiniLM-L6-v2 +``` + +Feast uses **Sentence Transformers** (default) for local embedding inference — no external API key required. Custom embedding providers can be plugged in by implementing the `EmbeddingProvider` protocol. See [\[Alpha\] Vector Database](../alpha-vector-database.md#alpha-openai-compatible-vector-store-api) for full configuration, custom providers, filter details, and SDK usage. + ## Starting the feature server in TLS(SSL) mode Enabling TLS mode ensures that data between the Feast client and server is transmitted securely. For an ideal production environment, it is recommended to start the feature server in TLS mode. @@ -598,7 +634,11 @@ The [PyTorch NLP template](https://github.com/feast-dev/feast/tree/main/sdk/pyth | Endpoint | Resource Type | Permission | Description | |----------------------------|---------------------------------|-------------------------------------------------------|----------------------------------------------------------------| | /get-online-features | FeatureView,OnDemandFeatureView | Read Online | Get online features from the feature store | -| /retrieve-online-documents | FeatureView | Read Online | Retrieve online documents from the feature store for RAG | +| /search | FeatureView | Read Online | Vector similarity search for RAG (embedding vector or text query) | +| /retrieve-online-documents | FeatureView | Read Online | **Deprecated.** Use `/search` instead. | +| /v1/vector_stores | FeatureView | Describe | [Alpha] List all vector stores | +| /v1/vector_stores/{id} | FeatureView | Describe | [Alpha] Get a single vector store | +| /v1/vector_stores/{id}/search | FeatureView | Read Online | [Alpha] OpenAI-compatible vector search with server-side embedding | | /push | FeatureView | Write Online, Write Offline, Write Online and Offline | Push features to the feature store (online, offline, or both) | | /write-to-online-store | FeatureView | Write Online | Write features to the online store | | /materialize | FeatureView | Write Online | Materialize features within a specified time range | diff --git a/docs/reference/feature-store-yaml.md b/docs/reference/feature-store-yaml.md index 9c8975eb335..1aac166bd8b 100644 --- a/docs/reference/feature-store-yaml.md +++ b/docs/reference/feature-store-yaml.md @@ -66,6 +66,7 @@ registry: |-------|------|---------|-------------| | `registry_type` | string | `file` | Registry backend (`file`, `sql`, etc.) | | `path` | string | — | Connection string or file path | +| `schema_mode` | string | `auto` | SQL registry only. `auto`: create tables on startup; `verify`: check tables exist, error if missing; `skip`: no DDL or verification. See [SQL Registry docs](registries/sql.md#schema-management-schema_mode). | | `mcp.enabled` | bool | `false` | Enable MCP (Model Context Protocol) on the REST registry server | When `registry.mcp.enabled` is `true`, the REST registry server exposes registry diff --git a/docs/reference/online-stores/README.md b/docs/reference/online-stores/README.md index 39294966170..257864b9b30 100644 --- a/docs/reference/online-stores/README.md +++ b/docs/reference/online-stores/README.md @@ -22,6 +22,10 @@ Please see [Online Store](../../getting-started/components/online-store.md) for [dragonfly.md](dragonfly.md) {% endcontent-ref %} +{% content-ref url="valkey.md" %} +[valkey.md](valkey.md) +{% endcontent-ref %} + {% content-ref url="datastore.md" %} [datastore.md](datastore.md) {% endcontent-ref %} @@ -31,7 +35,7 @@ Please see [Online Store](../../getting-started/components/online-store.md) for {% endcontent-ref %} {% content-ref url="bigtable.md" %} -[bigtable.md](mysql.md) +[bigtable.md](bigtable.md) {% endcontent-ref %} {% content-ref url="postgres.md" %} diff --git a/docs/reference/online-stores/overview.md b/docs/reference/online-stores/overview.md index 6ee076b0669..663a48836dc 100644 --- a/docs/reference/online-stores/overview.md +++ b/docs/reference/online-stores/overview.md @@ -29,26 +29,26 @@ See this [issue](https://github.com/feast-dev/feast/issues/2254) for a discussio ## Functionality Matrix There are currently five core online store implementations: `SqliteOnlineStore`, `RedisOnlineStore`, `DynamoDBOnlineStore`, `SnowflakeOnlineStore`, and `DatastoreOnlineStore`. -There are several additional implementations contributed by the Feast community (`PostgreSQLOnlineStore`, `HbaseOnlineStore` and `CassandraOnlineStore`), which are not guaranteed to be stable or to match the functionality of the core implementations. +There are several additional implementations contributed by the Feast community (`PostgreSQLOnlineStore`, `HbaseOnlineStore`, `CassandraOnlineStore` and `ScyllaDBOnlineStore`), which are not guaranteed to be stable or to match the functionality of the core implementations. Details for each specific online store, such as how to configure it in a `feature_store.yaml`, can be found [here](README.md). Below is a matrix indicating which online stores support what functionality. -| | Sqlite | Redis | DynamoDB | Snowflake | Datastore | Postgres | Hbase | [[Cassandra](https://cassandra.apache.org/_/index.html) / [Astra DB](https://www.datastax.com/products/datastax-astra?utm_source=feast)] | Milvus | -| :-------------------------------------------------------- | :-- | :-- | :-- | :-- | :-- | :-- | :-- | :-- |:----| -| write feature values to the online store | yes | yes | yes | yes | yes | yes | yes | yes | yes | -| read feature values from the online store | yes | yes | yes | yes | yes | yes | yes | yes | yes | -| update infrastructure (e.g. tables) in the online store | yes | yes | yes | yes | yes | yes | yes | yes | yes | -| teardown infrastructure (e.g. tables) in the online store | yes | yes | yes | yes | yes | yes | yes | yes | yes | -| generate a plan of infrastructure changes | yes | no | no | no | no | no | no | yes | no | -| support for on-demand transforms | yes | yes | yes | yes | yes | yes | yes | yes | yes | -| readable by Python SDK | yes | yes | yes | yes | yes | yes | yes | yes | yes | -| readable by Java | no | yes | no | no | no | no | no | no | no | -| readable by Go | yes | yes | no | no | no | no | no | no | no | -| support for entityless feature views | yes | yes | yes | yes | yes | yes | yes | yes | yes | -| support for concurrent writing to the same key | no | yes | no | no | no | no | no | no | yes | -| support for ttl (time to live) at retrieval | no | yes | no | no | no | no | no | no | no | -| support for deleting expired data | no | yes | no | no | no | no | no | no | no | -| collocated by feature view | yes | no | yes | yes | yes | yes | yes | yes | no | -| collocated by feature service | no | no | no | no | no | no | no | no | no | -| collocated by entity key | no | yes | no | no | no | no | no | no | yes | +| | Sqlite | Redis | DynamoDB | Snowflake | Datastore | Postgres | Hbase | [[Cassandra](https://cassandra.apache.org/_/index.html) / [Astra DB](https://www.datastax.com/products/datastax-astra?utm_source=feast)] | Milvus | ScyllaDB | +| :-------------------------------------------------------- | :-- | :-- | :-- | :-- | :-- | :-- | :-- | :-- |:----| :-- | +| write feature values to the online store | yes | yes | yes | yes | yes | yes | yes | yes | yes | yes | +| read feature values from the online store | yes | yes | yes | yes | yes | yes | yes | yes | yes | yes | +| update infrastructure (e.g. tables) in the online store | yes | yes | yes | yes | yes | yes | yes | yes | yes | yes | +| teardown infrastructure (e.g. tables) in the online store | yes | yes | yes | yes | yes | yes | yes | yes | yes | yes | +| generate a plan of infrastructure changes | yes | no | no | no | no | no | no | yes | no | no | +| support for on-demand transforms | yes | yes | yes | yes | yes | yes | yes | yes | yes | yes | +| readable by Python SDK | yes | yes | yes | yes | yes | yes | yes | yes | yes | yes | +| readable by Java | no | yes | no | no | no | no | no | no | no | no | +| readable by Go | yes | yes | no | no | no | no | no | no | no | no | +| support for entityless feature views | yes | yes | yes | yes | yes | yes | yes | yes | yes | yes | +| support for concurrent writing to the same key | no | yes | no | no | no | no | no | no | yes | no | +| support for ttl (time to live) at retrieval | no | yes | no | no | no | no | no | no | no | yes | +| support for deleting expired data | no | yes | no | no | no | no | no | no | no | yes | +| collocated by feature view | yes | no | yes | yes | yes | yes | yes | yes | no | yes | +| collocated by feature service | no | no | no | no | no | no | no | no | no | no | +| collocated by entity key | no | yes | no | no | no | no | no | no | yes | no | diff --git a/docs/reference/online-stores/scylladb.md b/docs/reference/online-stores/scylladb.md index c8583ac101a..98dc03b24cb 100644 --- a/docs/reference/online-stores/scylladb.md +++ b/docs/reference/online-stores/scylladb.md @@ -2,20 +2,15 @@ ## Description -ScyllaDB is a low-latency and high-performance Cassandra-compatible (uses CQL) database. You can use the existing Cassandra connector to use ScyllaDB as an online store in Feast. - -The [ScyllaDB](https://www.scylladb.com/) online store provides support for materializing feature values into a ScyllaDB or [ScyllaDB Cloud](https://www.scylladb.com/product/scylla-cloud/) cluster for serving online features real-time. +[ScyllaDB](https://www.scylladb.com/) is a distributed real-time NoSQL database with vector search support. +This integration uses the native **`scylla-driver`** Python driver for optimised performance and supports materializing feature values into a [ScyllaDB Cloud](https://www.scylladb.com/product/scylla-cloud/) cluster for real-time online feature serving. ## Getting started -Install Feast with Cassandra support: -```bash -pip install "feast[cassandra]" -``` +Install Feast with the `scylladb` extra, which pulls in `scylla-driver` automatically: -Create a new Feast project: ```bash -feast init REPO_NAME -t cassandra +pip install feast[scylladb] ``` ### Example (ScyllaDB) @@ -26,7 +21,7 @@ project: scylla_feature_repo registry: data/registry.db provider: local online_store: - type: cassandra + type: scylladb hosts: - 172.17.0.2 keyspace: feast @@ -43,44 +38,106 @@ project: scylla_feature_repo registry: data/registry.db provider: local online_store: - type: cassandra + type: scylladb hosts: - node-0.aws_us_east_1.xxxxxxxx.clusters.scylla.cloud - node-1.aws_us_east_1.xxxxxxxx.clusters.scylla.cloud - node-2.aws_us_east_1.xxxxxxxx.clusters.scylla.cloud keyspace: feast username: scylla - password: password + password: xxxxxx + local_dc: AWS_US_EAST_1 ``` {% endcode %} - -The full set of configuration options is available in [CassandraOnlineStoreConfig](https://rtd.feast.dev/en/master/#feast.infra.online_stores.cassandra_online_store.cassandra_online_store.CassandraOnlineStoreConfig). -For a full explanation of configuration options please look at file -`sdk/python/feast/infra/online_stores/contrib/cassandra_online_store/README.md`. +## Configuration options + +| Parameter | Type | Default | Description | +| :--- | :--- | :--- | :--- | +| `hosts` | list[str] | *(required)* | Contact-point host addresses. | +| `port` | int | `9042` | CQL port. | +| `keyspace` | str | `feast_keyspace` | Target ScyllaDB keyspace. | +| `username` | str | `None` | Auth username. | +| `password` | str | `None` | Auth password. | +| `local_dc` | str | `None` | Local datacenter name for DC-aware load balancing. | +| `request_timeout` | float | `None` | Driver request timeout in seconds. | +| `read_concurrency` | int | `100` | `concurrency` argument passed to the driver's `execute_concurrent_with_args` for reads. Controls how many CQL statements are in-flight at once. | +| `write_concurrency` | int | `100` | `concurrency` argument passed to the driver's `execute_concurrent_with_args` for writes. Controls how many CQL statements are in-flight at once. | +| `vector_similarity_function` | str | `COSINE` | Default similarity function for vector indexes. Supported: `COSINE`, `DOT_PRODUCT`, `EUCLIDEAN`. Can be overridden per-feature via the `similarity_function` Field tag. | Storage specifications can be found at `docs/specs/online_store_format.md`. +## Vector Search + +ScyllaDB Cloud supports approximate nearest-neighbour (ANN) vector search. +To enable it for a feature view, tag the embedding `Field` with `vector_index=true` and specify the number of dimensions: + +{% code title="feature_definitions.py" %} +```python +from feast import FeatureView, Field +from feast.types import Array, Float32, String + +documents_fv = FeatureView( + name="documents", + entities=[item], + schema=[ + Field(name="text", dtype=String), + Field( + name="embedding", + dtype=Array(Float32), + tags={ + "vector_index": "true", + "dimensions": "768", + "similarity_function": "COSINE", # COSINE | DOT_PRODUCT | EUCLIDEAN + }, + ), + ], + online=True, + source=push_source, +) +``` +{% endcode %} + +When `feast apply` runs, the store automatically creates the necessary tables and HNSW ANN index for any feature view with vector-tagged fields. + +To query the top-k most similar documents: + +```python +result = store.retrieve_online_documents_v2( + features=["documents:text", "documents:embedding"], + query=[0.1, 0.2, ...], # your query embedding + top_k=10, + distance_metric="COSINE", +) +``` + +### Metadata filtering (OpenAI-compatible) + +ScyllaDB supports vector similarity search, but OpenAI-style metadata filtering is **not supported yet**. +Passing `filters` to `retrieve_online_documents_v2` or the OpenAI-compatible search endpoint raises `NotImplementedError`. + +For filtered vector search today, use one of the backends that implement metadata filters (for example Milvus, Elasticsearch, Postgres, SQLite, or MongoDB). See [Alpha Vector Database](../alpha-vector-database.md#supported-online-stores). + ## Functionality Matrix The set of functionality supported by online stores is described in detail [here](overview.md#functionality). -Below is a matrix indicating which functionality is supported by the Cassandra plugin. +Below is a matrix indicating which functionality is supported by the ScyllaDB online store. -| | Cassandra | +| | ScyllaDB | | :-------------------------------------------------------- | :-------- | | write feature values to the online store | yes | | read feature values from the online store | yes | | update infrastructure (e.g. tables) in the online store | yes | | teardown infrastructure (e.g. tables) in the online store | yes | -| generate a plan of infrastructure changes | yes | +| generate a plan of infrastructure changes | no | | support for on-demand transforms | yes | | readable by Python SDK | yes | | readable by Java | no | | readable by Go | no | | support for entityless feature views | yes | | support for concurrent writing to the same key | no | -| support for ttl (time to live) at retrieval | no | -| support for deleting expired data | no | +| support for ttl (time to live) at retrieval | yes | +| support for deleting expired data | yes | | collocated by feature view | yes | | collocated by feature service | no | | collocated by entity key | no | @@ -89,6 +146,6 @@ To compare this set of functionality against other online stores, please see the ## Resources -* [Sample application with ScyllaDB](https://feature-store.scylladb.com/stable/) +* [ScyllaDB Vector Search documentation](https://cloud.docs.scylladb.com/stable/vector-search/) * [ScyllaDB website](https://www.scylladb.com/) * [ScyllaDB Cloud documentation](https://cloud.docs.scylladb.com/stable/) diff --git a/docs/reference/online-stores/valkey.md b/docs/reference/online-stores/valkey.md new file mode 100644 index 00000000000..4ede3f65b5d --- /dev/null +++ b/docs/reference/online-stores/valkey.md @@ -0,0 +1,94 @@ +# Valkey online store + +## Description + +[Valkey](https://valkey.io/) is an open source (BSD-3-Clause), high-performance key/value datastore hosted by the Linux Foundation, created as a community fork of Redis. It maintains compatibility with the Redis wire protocol, so it can act as a drop-in replacement for Redis. Valkey is also offered as a managed engine by major cloud providers (for example, Amazon ElastiCache for Valkey). + +Similar to Redis and [Dragonfly](dragonfly.md), Valkey can be used as an online feature store for Feast: Feast's Redis online store only issues core commands (hash reads/writes, scans, key expiry, pipelines), all of which Valkey implements. + +Feast's standard online store operations have been verified against Valkey 8.1: `feast apply`, `feast materialize`, online retrieval via `get_online_features`, `feast teardown`, and key expiry via the `key_ttl_seconds` option. Features that depend on Redis modules (such as vector search) are outside the scope of this page. + +## Using Valkey as a drop-in Feast online store instead of Redis + +Make sure you have Python and `pip` installed. + +Install the Feast SDK and CLI + +`pip install feast` + +In order to use Valkey as the online store, you'll need to install the redis extra: + +`pip install 'feast[redis]'` + +### 1. Create a feature repository + +Bootstrap a new feature repository: + +``` +feast init feast_valkey +cd feast_valkey/feature_repo +``` + +Update `feature_repo/feature_store.yaml` with the below contents: + +``` +project: feast_valkey +registry: data/registry.db +provider: local +online_store: + type: redis + connection_string: "localhost:6379" +``` + +Note that the online store `type` remains `redis`: Feast talks to Valkey over the Redis protocol, and all options of the [Redis online store](redis.md) (such as `key_ttl_seconds`) apply unchanged. + +### 2. Start Valkey + +There are several options available to get Valkey up and running quickly. We will be using Docker for this tutorial. + +`docker run -d -p 6379:6379 valkey/valkey:8.1` + +### 3. Register feature definitions and deploy your feature store + +`feast apply` + +The `apply` command scans python files in the current directory for feature view/entity definitions, registers the objects, and deploys infrastructure. +You should see the following output: + +``` +.... +Created entity driver +Created feature view driver_hourly_stats_fresh +Created feature view driver_hourly_stats +Created on demand feature view transformed_conv_rate +Created on demand feature view transformed_conv_rate_fresh +Created feature service driver_activity_v1 +Created feature service driver_activity_v3 +Created feature service driver_activity_v2 +``` + +## Functionality Matrix + +The set of functionality supported by online stores is described in detail [here](overview.md#functionality). +Below is a matrix indicating which functionality is supported by the Redis online store, which Feast uses to communicate with Valkey. + +| | Redis | +| :-------------------------------------------------------- | :---- | +| write feature values to the online store | yes | +| read feature values from the online store | yes | +| update infrastructure (e.g. tables) in the online store | yes | +| teardown infrastructure (e.g. tables) in the online store | yes | +| generate a plan of infrastructure changes | no | +| support for on-demand transforms | yes | +| readable by Python SDK | yes | +| readable by Java | yes | +| readable by Go | yes | +| support for entityless feature views | yes | +| support for concurrent writing to the same key | yes | +| support for ttl (time to live) at retrieval | yes | +| support for deleting expired data | yes | +| collocated by feature view | no | +| collocated by feature service | no | +| collocated by entity key | yes | + +To compare this set of functionality against other online stores, please see the full [functionality matrix](overview.md#functionality-matrix). diff --git a/docs/reference/openlineage.md b/docs/reference/openlineage.md index 3acc6e7872b..bf9e18750ed 100644 --- a/docs/reference/openlineage.md +++ b/docs/reference/openlineage.md @@ -404,7 +404,7 @@ The consumer automatically links datasets across different producers when they r 2. **SymlinksDatasetFacet** — Producers can declare aliases. For example, Feast can declare that its internal `driver_hourly_stats` is a symlink to the Spark output at `s3://bucket/features/driver_hourly_stats/`. 3. **dataSource URI matching** — Datasets with matching `dataSource.uri` facets are linked even if their namespace or name differ. -Compatible producers include Airflow, Spark, dbt, Flink, Feast, Dagster, and Great Expectations. +Compatible producers include Airflow, Spark, dbt, Flink, Feast, and Dagster. ### RBAC for Lineage diff --git a/docs/reference/registries/sql.md b/docs/reference/registries/sql.md index ef9993c8753..e8d1bcef17a 100644 --- a/docs/reference/registries/sql.md +++ b/docs/reference/registries/sql.md @@ -80,10 +80,114 @@ docker build \ If you are running Feast in Kubernetes, set the `image.repository` and `imagePullSecrets` Helm values accordingly to utilize your custom image. +## Schema management (`schema_mode`) + +By default, the SQL registry creates its tables on every startup (`schema_mode: auto`). In production environments where the application should not have DDL privileges, you can pre-create the schema and configure the registry to only verify it: + +```yaml +registry: + registry_type: sql + path: postgresql://db:5432/feast + schema_mode: verify # or "skip" +``` + +| Value | Behavior | +|---|---| +| `auto` (default) | Creates tables if they don't exist. Current behavior, no breaking change. | +| `verify` | Skips DDL. Checks that all expected tables exist on startup; raises an error listing missing tables if any are absent. When a separate `read_path` is configured, the read replica is also verified — a lagging replica (e.g. mid-migration) will block startup. Note: this is a table-level check only — it does not verify individual columns. A schema created by an older Feast version (missing newer columns) will pass verification but may fail at query time. | +| `skip` | Skips both creation and verification. Use when schema is managed entirely outside Feast (e.g. by a migration tool). | + +### Pre-creating the schema + +When using `verify` or `skip` mode, run the following CLI command with a user that has DDL privileges to create the schema before starting the application: + +```shell +feast registry create-schema +``` + +This reads `feature_store.yaml`, connects to the configured database, and creates all required tables. It is safe to run multiple times — existing tables are not modified. + There are some things to note about how the SQL registry works: -- Once instantiated, the Registry ensures the tables needed to store data exist, and creates them if they do not. -- Upon tearing down the feast project, the registry ensures that the tables are dropped from the database. -- The schema for how data is laid out in tables can be found . It is intentionally simple, storing the serialized protobuf versions of each Feast object keyed by its name. +- When `schema_mode` is `auto` (the default), the Registry ensures the tables needed to store data exist, and creates them if they do not. +- Upon tearing down the feast project, the registry deletes all rows from the registry tables (it does not drop the tables themselves). This runs regardless of `schema_mode` and requires only DML (`DELETE`) privileges, not DDL. +- The schema for how data is laid out in tables can be found in the table definitions in [`sdk/python/feast/infra/registry/sql.py`](https://github.com/feast-dev/feast/blob/master/sdk/python/feast/infra/registry/sql.py). It is intentionally simple, storing the serialized protobuf versions of each Feast object keyed by its name. + +## MySQL: serialized-proto columns use `LONGBLOB` + +The registry stores each Feast object as a serialized protobuf in a binary +column. On MySQL these columns are created as `LONGBLOB` (up to 4 GB). Earlier +versions created them as `BLOB`, which caps at 64 KB — a single `FeatureView` +proto routinely exceeds that, so MySQL would silently truncate the write and the +registry would later fail to load with a protobuf `DecodeError` (for example, +`feast serve` failing to start). Other dialects (PostgreSQL, SQLite) were never +affected. + +New deployments get the correct schema automatically — the registry creates its +tables as `LONGBLOB` on first use. When an existing MySQL/MariaDB registry still +has `BLOB` columns, the registry logs an error at startup listing the affected +columns (it does not refuse to start — a registry whose protos all fit in 64 KB +is unaffected). **Existing deployments are not migrated automatically**: the +registry only creates tables that do not already exist, and it has no +schema-migration step, so previously created `BLOB` columns remain `BLOB`. To +upgrade an existing MySQL registry, alter each serialized-proto column to +`LONGBLOB`, for example: + +> ⚠️ **Run the migration carefully on a live registry.** A `BLOB`→`LONGBLOB` +> change is a column *data-type* change, which MySQL InnoDB performs with +> `ALGORITHM=COPY` — a full table rebuild under a metadata lock that blocks +> readers and writers for the duration (potentially minutes on a large table +> such as `feature_view_version_history`). `ALGORITHM=INPLACE` is **not** +> generally supported for this change and is rejected with +> `ER_ALTER_OPERATION_NOT_SUPPORTED_REASON` on most builds — do not rely on it. +> +> **Before running any `ALTER TABLE`:** +> +> 1. **Stop all `feast apply` and materialization jobs.** This is required, not +> optional — a write of a `>64 KB` proto to a not-yet-widened `BLOB` column +> truncates silently with no error, and concurrent writes also extend the +> `ALTER`'s lock duration. +> 2. Confirm there are no active writers (e.g. `SHOW PROCESSLIST`). +> 3. Verify you have a backup of the registry database. +> +> Then, to minimize the lock window: +> +> - On large tables, or on managed MySQL (AWS RDS, Aurora) without shell access, +> use an online schema-change tool — +> [`pt-online-schema-change`](https://docs.percona.com/percona-toolkit/pt-online-schema-change.html) +> (Percona Toolkit) or [`gh-ost`](https://github.com/github/gh-ost) — which +> rebuild the table without a long-held lock. For small tables a plain +> `ALTER TABLE` in the maintenance window is fine. +> - Apply one table at a time so a failure is easy to isolate and re-run. +> - Resume jobs only after all `ALTER TABLE` statements complete successfully. +> - Rollback is safe (revert `MODIFY ... BLOB`) **only** while no stored proto +> exceeds 64 KB; otherwise a revert re-introduces truncation. + +```sql +ALTER TABLE projects MODIFY project_proto LONGBLOB NOT NULL; +ALTER TABLE entities MODIFY entity_proto LONGBLOB NOT NULL; +ALTER TABLE data_sources MODIFY data_source_proto LONGBLOB NOT NULL; +ALTER TABLE feature_views MODIFY materialized_intervals LONGBLOB, + MODIFY feature_view_proto LONGBLOB NOT NULL, + MODIFY user_metadata LONGBLOB; +ALTER TABLE stream_feature_views MODIFY feature_view_proto LONGBLOB NOT NULL, + MODIFY user_metadata LONGBLOB; +ALTER TABLE on_demand_feature_views MODIFY feature_view_proto LONGBLOB NOT NULL, + MODIFY user_metadata LONGBLOB; +ALTER TABLE label_views MODIFY feature_view_proto LONGBLOB NOT NULL, + MODIFY user_metadata LONGBLOB; +ALTER TABLE feature_services MODIFY feature_service_proto LONGBLOB NOT NULL; +ALTER TABLE saved_datasets MODIFY saved_dataset_proto LONGBLOB NOT NULL; +ALTER TABLE validation_references MODIFY validation_reference_proto LONGBLOB NOT NULL; +ALTER TABLE managed_infra MODIFY infra_proto LONGBLOB NOT NULL; +ALTER TABLE permissions MODIFY permission_proto LONGBLOB NOT NULL; +-- LARGE TABLE: one row per versioned apply — likely the slowest ALTER. Use +-- pt-online-schema-change or gh-ost if this registry has significant history. +ALTER TABLE feature_view_version_history MODIFY feature_view_proto LONGBLOB NOT NULL; +``` + +Any object whose proto already exceeded 64 KB before the upgrade may have been +stored truncated; re-run `feast apply` for those objects after altering the +columns so the full proto is rewritten. ## Example Usage: Concurrent materialization The SQL Registry should be used when materializing feature views concurrently to ensure correctness of data in the registry. This can be achieved by simply running feast materialize or feature_store.materialize multiple times using a correctly configured feature_store.yaml. This will make each materialization process talk to the registry database concurrently, and ensure the metadata updates are serialized. diff --git a/docs/roadmap.md b/docs/roadmap.md index 4eac7e68b3f..d92ffa38f24 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -90,7 +90,6 @@ The list below contains the functionality that contributors are planning to deve * [x] [Offline Feature Server (alpha)](https://docs.feast.dev/reference/feature-servers/offline-feature-server) * [x] [Registry server (alpha)](https://github.com/feast-dev/feast/blob/master/docs/reference/feature-servers/registry-server.md) * **Data Quality Management (See [RFC](https://docs.google.com/document/d/110F72d4NTv80p35wDSONxhhPBqWRwbZXG4f9mNEMd98/edit))** - * [x] ~~Data profiling and validation (Great Expectations)~~ (deprecated) * [x] [Feature Quality Monitoring](https://docs.feast.dev/how-to-guides/feature-monitoring) — built-in metrics, drift detection, serving log monitoring, and UI dashboard * **Feature Discovery and Governance** * [x] Python SDK for browsing feature registry diff --git a/docs/tutorials/validating-historical-features.md b/docs/tutorials/validating-historical-features.md deleted file mode 100644 index f2037f7c9c9..00000000000 --- a/docs/tutorials/validating-historical-features.md +++ /dev/null @@ -1,920 +0,0 @@ -# Validating historical features with Great Expectations - -{% hint style="warning" %} -**Deprecated:** This tutorial demonstrates the legacy Great Expectations-based validation which is deprecated. For new projects, use Feast's built-in [Feature Quality Monitoring](../how-to-guides/feature-monitoring.md) system which provides automatic metrics computation, drift detection, and a monitoring UI — with no external dependencies required. See also the [Monitoring Quickstart notebook](../../examples/monitoring/monitoring-quickstart.ipynb). -{% endhint %} - -In this tutorial, we will use the public dataset of Chicago taxi trips to present data validation capabilities of Feast. -- The original dataset is stored in BigQuery and consists of raw data for each taxi trip (one row per trip) since 2013. -- We will generate several training datasets (aka historical features in Feast) for different periods and evaluate expectations made on one dataset against another. - -Types of features we're ingesting and generating: -- Features that aggregate raw data with daily intervals (eg, trips per day, average fare or speed for a specific day, etc.). -- Features using SQL while pulling data from BigQuery (like total trips time or total miles travelled). -- Features calculated on the fly when requested using Feast's on-demand transformations - -Our plan: - -0. Prepare environment -1. Pull data from BigQuery (optional) -2. Declare & apply features and feature views in Feast -3. Generate reference dataset -4. Develop & test profiler function -5. Run validation on different dataset using reference dataset & profiler - - -> The original notebook and datasets for this tutorial can be found on [GitHub](https://github.com/feast-dev/dqm-tutorial). - -### 0. Setup - -Install Feast Python SDK and great expectations: - - -```python -!pip install 'feast[ge]' -``` - - -### 1. Dataset preparation (Optional) - -**You can skip this step if you don't have GCP account. Please use parquet files that are coming with this tutorial instead** - - -```python -!pip install google-cloud-bigquery -``` - - -```python -import pyarrow.parquet - -from google.cloud.bigquery import Client -``` - - -```python -bq_client = Client(project='kf-feast') -``` - -Running some basic aggregations while pulling data from BigQuery. Grouping by taxi_id and day: - - -```python -data_query = """SELECT - taxi_id, - TIMESTAMP_TRUNC(trip_start_timestamp, DAY) as day, - SUM(trip_miles) as total_miles_travelled, - SUM(trip_seconds) as total_trip_seconds, - SUM(fare) as total_earned, - COUNT(*) as trip_count -FROM `bigquery-public-data.chicago_taxi_trips.taxi_trips` -WHERE - trip_miles > 0 AND trip_seconds > 60 AND - trip_start_timestamp BETWEEN '2019-01-01' and '2020-12-31' AND - trip_total < 1000 -GROUP BY taxi_id, TIMESTAMP_TRUNC(trip_start_timestamp, DAY)""" -``` - - -```python -driver_stats_table = bq_client.query(data_query).to_arrow() - -# Storing resulting dataset into parquet file -pyarrow.parquet.write_table(driver_stats_table, "trips_stats.parquet") -``` - - -```python -def entities_query(year): - return f"""SELECT - distinct taxi_id -FROM `bigquery-public-data.chicago_taxi_trips.taxi_trips` -WHERE - trip_miles > 0 AND trip_seconds > 0 AND - trip_start_timestamp BETWEEN '{year}-01-01' and '{year}-12-31' -""" -``` - - -```python -entities_2019_table = bq_client.query(entities_query(2019)).to_arrow() - -# Storing entities (taxi ids) into parquet file -pyarrow.parquet.write_table(entities_2019_table, "entities.parquet") -``` - - -## 2. Declaring features - - -```python -import pyarrow.parquet -import pandas as pd - -from feast import FeatureView, Entity, FeatureStore, Field, BatchFeatureView -from feast.types import Float64, Int64 -from feast.value_type import ValueType -from feast.data_format import ParquetFormat -from feast.on_demand_feature_view import on_demand_feature_view -from feast.infra.offline_stores.file_source import FileSource -from feast.infra.offline_stores.file import SavedDatasetFileStorage -from datetime import timedelta - -``` - - -```python -batch_source = FileSource( - timestamp_field="day", - path="trips_stats.parquet", # using parquet file that we created on previous step - file_format=ParquetFormat() -) -``` - - -```python -taxi_entity = Entity(name='taxi', join_keys=['taxi_id']) -``` - - -```python -trips_stats_fv = BatchFeatureView( - name='trip_stats', - entities=[taxi_entity], - schema=[ - Field(name="total_miles_travelled", dtype=Float64), - Field(name="total_trip_seconds", dtype=Float64), - Field(name="total_earned", dtype=Float64), - Field(name="trip_count", dtype=Int64), - - ], - ttl=timedelta(seconds=86400), - source=batch_source, -) -``` - -*Read more about feature views in [Feast docs](https://docs.feast.dev/getting-started/concepts/feature-view)* - - -```python -@on_demand_feature_view( - sources=[ - trips_stats_fv, - ], - schema=[ - Field(name="avg_fare", dtype=Float64), - Field(name="avg_speed", dtype=Float64), - Field(name="avg_trip_seconds", dtype=Float64), - Field(name="earned_per_hour", dtype=Float64), - ] -) -def on_demand_stats(inp: pd.DataFrame) -> pd.DataFrame: - out = pd.DataFrame() - out["avg_fare"] = inp["total_earned"] / inp["trip_count"] - out["avg_speed"] = 3600 * inp["total_miles_travelled"] / inp["total_trip_seconds"] - out["avg_trip_seconds"] = inp["total_trip_seconds"] / inp["trip_count"] - out["earned_per_hour"] = 3600 * inp["total_earned"] / inp["total_trip_seconds"] - return out -``` - -*Read more about on demand feature views [here](../reference/beta-on-demand-feature-view.md)* - - -```python -store = FeatureStore(".") # using feature_store.yaml that stored in the same directory -``` - - -```python -store.apply([taxi_entity, trips_stats_fv, on_demand_stats]) # writing to the registry -``` - - -## 3. Generating training (reference) dataset - - -```python -taxi_ids = pyarrow.parquet.read_table("entities.parquet").to_pandas() -``` - -Generating range of timestamps with daily frequency: - - -```python -timestamps = pd.DataFrame() -timestamps["event_timestamp"] = pd.date_range("2019-06-01", "2019-07-01", freq='D') -``` - -Cross merge (aka relation multiplication) produces entity dataframe with each taxi_id repeated for each timestamp: - - -```python -entity_df = pd.merge(taxi_ids, timestamps, how='cross') -entity_df -``` - - - - -
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
taxi_idevent_timestamp
091d5288487e87c5917b813ba6f75ab1c3a9749af906a2d...2019-06-01
191d5288487e87c5917b813ba6f75ab1c3a9749af906a2d...2019-06-02
291d5288487e87c5917b813ba6f75ab1c3a9749af906a2d...2019-06-03
391d5288487e87c5917b813ba6f75ab1c3a9749af906a2d...2019-06-04
491d5288487e87c5917b813ba6f75ab1c3a9749af906a2d...2019-06-05
.........
1569797ebf27414a0c7b128e7925e1da56d51a8b81484f7630cf...2019-06-27
1569807ebf27414a0c7b128e7925e1da56d51a8b81484f7630cf...2019-06-28
1569817ebf27414a0c7b128e7925e1da56d51a8b81484f7630cf...2019-06-29
1569827ebf27414a0c7b128e7925e1da56d51a8b81484f7630cf...2019-06-30
1569837ebf27414a0c7b128e7925e1da56d51a8b81484f7630cf...2019-07-01
-

156984 rows × 2 columns

-
- - - -Retrieving historical features for resulting entity dataframe and persisting output as a saved dataset: - - -```python -job = store.get_historical_features( - entity_df=entity_df, - features=[ - "trip_stats:total_miles_travelled", - "trip_stats:total_trip_seconds", - "trip_stats:total_earned", - "trip_stats:trip_count", - "on_demand_stats:avg_fare", - "on_demand_stats:avg_trip_seconds", - "on_demand_stats:avg_speed", - "on_demand_stats:earned_per_hour", - ] -) - -store.create_saved_dataset( - from_=job, - name='my_training_ds', - storage=SavedDatasetFileStorage(path='my_training_ds.parquet') -) -``` - -```python -, full_feature_names = False, tags = {}, _retrieval_job = , min_event_timestamp = 2019-06-01 00:00:00, max_event_timestamp = 2019-07-01 00:00:00)> -``` - - -## 4. Developing dataset profiler - -Dataset profiler is a function that accepts dataset and generates set of its characteristics. This charasteristics will be then used to evaluate (validate) next datasets. - -**Important: datasets are not compared to each other! -Feast use a reference dataset and a profiler function to generate a reference profile. -This profile will be then used during validation of the tested dataset.** - - -```python -import numpy as np - -from feast.dqm.profilers.ge_profiler import ge_profiler - -from great_expectations.core.expectation_suite import ExpectationSuite -from great_expectations.dataset import PandasDataset -``` - - -Loading saved dataset first and exploring the data: - - -```python -ds = store.get_saved_dataset('my_training_ds') -ds.to_df() -``` - -
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
total_earnedavg_trip_secondstaxi_idtotal_miles_travelledtrip_countearned_per_hourevent_timestamptotal_trip_secondsavg_fareavg_speed
068.252270.00000091d5288487e87c5917b813ba6f75ab1c3a9749af906a2d...24.702.054.1189432019-06-01 00:00:00+00:004540.034.12500019.585903
1221.00560.5000007a4a6162eaf27805aef407d25d5cb21fe779cd962922cb...54.1824.059.1436222019-06-01 00:00:00+00:0013452.09.20833314.499554
2160.501010.769231f4c9d05b215d7cbd08eca76252dae51cdb7aca9651d4ef...41.3013.043.9726032019-06-01 00:00:00+00:0013140.012.34615411.315068
3183.75697.550000c1f533318f8480a59173a9728ea0248c0d3eb187f4b897...37.3020.047.4159562019-06-01 00:00:00+00:0013951.09.1875009.625116
4217.751054.076923455b6b5cae6ca5a17cddd251485f2266d13d6a2c92f07c...69.6913.057.2064512019-06-01 00:00:00+00:0013703.016.75000018.308692
.................................
15697938.001980.0000000cccf0ec1f46d1e0beefcfdeaf5188d67e170cdff92618...14.901.069.0909092019-07-01 00:00:00+00:001980.038.00000027.090909
156980135.00551.250000beefd3462e3f5a8e854942a2796876f6db73ebbd25b435...28.4016.055.1020412019-07-01 00:00:00+00:008820.08.43750011.591837
156981NaNNaN9a3c52aa112f46cf0d129fafbd42051b0fb9b0ff8dcb0e...NaNNaNNaN2019-07-01 00:00:00+00:00NaNNaNNaN
15698263.00815.00000008308c31cd99f495dea73ca276d19a6258d7b4c9c88e43...19.964.069.5705522019-07-01 00:00:00+00:003260.015.75000022.041718
156983NaNNaN7ebf27414a0c7b128e7925e1da56d51a8b81484f7630cf...NaNNaNNaN2019-07-01 00:00:00+00:00NaNNaNNaN
-

156984 rows × 10 columns

-
- - - -Feast uses [Great Expectations](https://docs.greatexpectations.io/docs/) as a validation engine and [ExpectationSuite](https://legacy.docs.greatexpectations.io/en/latest/autoapi/great_expectations/core/expectation_suite/index.html#great_expectations.core.expectation_suite.ExpectationSuite) as a dataset's profile. Hence, we need to develop a function that will generate ExpectationSuite. This function will receive instance of [PandasDataset](https://legacy.docs.greatexpectations.io/en/latest/autoapi/great_expectations/dataset/index.html?highlight=pandasdataset#great_expectations.dataset.PandasDataset) (wrapper around pandas.DataFrame) so we can utilize both Pandas DataFrame API and some helper functions from PandasDataset during profiling. - - -```python -DELTA = 0.1 # controlling allowed window in fraction of the value on scale [0, 1] - -@ge_profiler -def stats_profiler(ds: PandasDataset) -> ExpectationSuite: - # simple checks on data consistency - ds.expect_column_values_to_be_between( - "avg_speed", - min_value=0, - max_value=60, - mostly=0.99 # allow some outliers - ) - - ds.expect_column_values_to_be_between( - "total_miles_travelled", - min_value=0, - max_value=500, - mostly=0.99 # allow some outliers - ) - - # expectation of means based on observed values - observed_mean = ds.trip_count.mean() - ds.expect_column_mean_to_be_between("trip_count", - min_value=observed_mean * (1 - DELTA), - max_value=observed_mean * (1 + DELTA)) - - observed_mean = ds.earned_per_hour.mean() - ds.expect_column_mean_to_be_between("earned_per_hour", - min_value=observed_mean * (1 - DELTA), - max_value=observed_mean * (1 + DELTA)) - - - # expectation of quantiles - qs = [0.5, 0.75, 0.9, 0.95] - observed_quantiles = ds.avg_fare.quantile(qs) - - ds.expect_column_quantile_values_to_be_between( - "avg_fare", - quantile_ranges={ - "quantiles": qs, - "value_ranges": [[None, max_value] for max_value in observed_quantiles] - }) - - return ds.get_expectation_suite() -``` - -Testing our profiler function: - - -```python -ds.get_profile(profiler=stats_profiler) -``` - 02/02/2022 02:43:47 PM INFO: 5 expectation(s) included in expectation_suite. result_format settings filtered. - - - - -**Verify that all expectations that we coded in our profiler are present here. Otherwise (if you can't find some expectations) it means that it failed to pass on the reference dataset (do it silently is default behavior of Great Expectations).** - -Now we can create validation reference from dataset and profiler function: - - -```python -validation_reference = ds.as_reference(name="validation_reference_dataset", profiler=stats_profiler) -``` - -and test it against our existing retrieval job - - -```python -_ = job.to_df(validation_reference=validation_reference) -``` - - 02/02/2022 02:43:52 PM INFO: 5 expectation(s) included in expectation_suite. result_format settings filtered. - 02/02/2022 02:43:53 PM INFO: Validating data_asset_name None with expectation_suite_name default - - -Validation successfully passed as no exceptions were raised. - - -### 5. Validating new historical retrieval - -Creating new timestamps for Dec 2020: - - -```python -from feast.dqm.errors import ValidationFailed -``` - - -```python -timestamps = pd.DataFrame() -timestamps["event_timestamp"] = pd.date_range("2020-12-01", "2020-12-07", freq='D') -``` - - -```python -entity_df = pd.merge(taxi_ids, timestamps, how='cross') -entity_df -``` - -
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
taxi_idevent_timestamp
091d5288487e87c5917b813ba6f75ab1c3a9749af906a2d...2020-12-01
191d5288487e87c5917b813ba6f75ab1c3a9749af906a2d...2020-12-02
291d5288487e87c5917b813ba6f75ab1c3a9749af906a2d...2020-12-03
391d5288487e87c5917b813ba6f75ab1c3a9749af906a2d...2020-12-04
491d5288487e87c5917b813ba6f75ab1c3a9749af906a2d...2020-12-05
.........
354437ebf27414a0c7b128e7925e1da56d51a8b81484f7630cf...2020-12-03
354447ebf27414a0c7b128e7925e1da56d51a8b81484f7630cf...2020-12-04
354457ebf27414a0c7b128e7925e1da56d51a8b81484f7630cf...2020-12-05
354467ebf27414a0c7b128e7925e1da56d51a8b81484f7630cf...2020-12-06
354477ebf27414a0c7b128e7925e1da56d51a8b81484f7630cf...2020-12-07
-

35448 rows × 2 columns

-
- - -```python -job = store.get_historical_features( - entity_df=entity_df, - features=[ - "trip_stats:total_miles_travelled", - "trip_stats:total_trip_seconds", - "trip_stats:total_earned", - "trip_stats:trip_count", - "on_demand_stats:avg_fare", - "on_demand_stats:avg_trip_seconds", - "on_demand_stats:avg_speed", - "on_demand_stats:earned_per_hour", - ] -) -``` - -Execute retrieval job with validation reference: - - -```python -try: - df = job.to_df(validation_reference=validation_reference) -except ValidationFailed as exc: - print(exc.validation_report) -``` - - 02/02/2022 02:43:58 PM INFO: 5 expectation(s) included in expectation_suite. result_format settings filtered. - 02/02/2022 02:43:59 PM INFO: Validating data_asset_name None with expectation_suite_name default - - [ - { - "expectation_config": { - "expectation_type": "expect_column_mean_to_be_between", - "kwargs": { - "column": "trip_count", - "min_value": 10.387244591346153, - "max_value": 12.695521167200855, - "result_format": "COMPLETE" - }, - "meta": {} - }, - "meta": {}, - "result": { - "observed_value": 6.692920555429092, - "element_count": 35448, - "missing_count": 31055, - "missing_percent": 87.6071992778154 - }, - "exception_info": { - "raised_exception": false, - "exception_message": null, - "exception_traceback": null - }, - "success": false - }, - { - "expectation_config": { - "expectation_type": "expect_column_mean_to_be_between", - "kwargs": { - "column": "earned_per_hour", - "min_value": 52.320624975640214, - "max_value": 63.94743052578249, - "result_format": "COMPLETE" - }, - "meta": {} - }, - "meta": {}, - "result": { - "observed_value": 68.99268345164135, - "element_count": 35448, - "missing_count": 31055, - "missing_percent": 87.6071992778154 - }, - "exception_info": { - "raised_exception": false, - "exception_message": null, - "exception_traceback": null - }, - "success": false - }, - { - "expectation_config": { - "expectation_type": "expect_column_quantile_values_to_be_between", - "kwargs": { - "column": "avg_fare", - "quantile_ranges": { - "quantiles": [ - 0.5, - 0.75, - 0.9, - 0.95 - ], - "value_ranges": [ - [ - null, - 16.4 - ], - [ - null, - 26.229166666666668 - ], - [ - null, - 36.4375 - ], - [ - null, - 42.0 - ] - ] - }, - "result_format": "COMPLETE" - }, - "meta": {} - }, - "meta": {}, - "result": { - "observed_value": { - "quantiles": [ - 0.5, - 0.75, - 0.9, - 0.95 - ], - "values": [ - 19.5, - 28.1, - 38.0, - 44.125 - ] - }, - "element_count": 35448, - "missing_count": 31055, - "missing_percent": 87.6071992778154, - "details": { - "success_details": [ - false, - false, - false, - false - ] - } - }, - "exception_info": { - "raised_exception": false, - "exception_message": null, - "exception_traceback": null - }, - "success": false - } - ] - - -Validation failed since several expectations didn't pass: -* Trip count (mean) decreased more than 10% (which is expected when comparing Dec 2020 vs June 2019) -* Average Fare increased - all quantiles are higher than expected -* Earn per hour (mean) increased more than 10% (most probably due to increased fare) - diff --git a/examples/ray-llm-posttrain/.gitignore b/examples/ray-llm-posttrain/.gitignore new file mode 100644 index 00000000000..4bf1a46bf0e --- /dev/null +++ b/examples/ray-llm-posttrain/.gitignore @@ -0,0 +1,13 @@ +# Feast / Ray local artifacts +data/ +.feast/ +ray_storage/ +/tmp/ray/ +ray_results/ +.ray/ + +__pycache__/ +*.py[cod] +.pytest_cache/ +.venv/ +.env diff --git a/examples/ray-llm-posttrain/README.md b/examples/ray-llm-posttrain/README.md new file mode 100644 index 00000000000..64252a5a603 --- /dev/null +++ b/examples/ray-llm-posttrain/README.md @@ -0,0 +1,35 @@ +# How to Use Feast for SLM/LLM Post-Training (with Ray) + +| Name | Type | Fields | +|---|---|---| +| `web_documents` | FeatureView | `human`, `bot`, `human_repeat_ratio`, `bot_repeat_ratio` | +| `train_example` | OnDemandFeatureView | `cleaned_human`, `cleaned_bot`, `char_count`, `is_trainable`, `sft_text` | +| `llm_posttrain` | FeatureService | `web_documents` + `train_example` | + +Source data is **prepared parquet** (`document_id` + `event_timestamp` already present). No Feast core patches. + +## Paths + +| Flag | What happens | +|---|---| +| (default) | `to_ray_dataset()` + preprocess `sft_text` (ODFV does **not** run) | +| `--via-df` | `to_df()` so ODFV `train_example` runs | + +## Setup + +```bash +uv pip install -e "../../sdk/python[ray]" -r requirements.txt +PYTHONPATH=../../sdk/python python scripts/prepare_data.py +cd feature_repo && feast apply && cd .. +``` + +## Run (data load only) + +```bash +PYTHONPATH=../../sdk/python python scripts/train_sft.py --dry-run +PYTHONPATH=../../sdk/python python scripts/train_sft.py --dry-run --via-df +``` + +## Blog + +[How to Use Feast for SLM/LLM Post-Training with Ray](/blog/feast-ray-llm-posttrain) diff --git a/examples/ray-llm-posttrain/feature_repo/feature_definitions.py b/examples/ray-llm-posttrain/feature_repo/feature_definitions.py new file mode 100644 index 00000000000..a85470c3729 --- /dev/null +++ b/examples/ray-llm-posttrain/feature_repo/feature_definitions.py @@ -0,0 +1,108 @@ +"""Feast feature definitions for Ray + ODFV LLM post-training. + +Pipeline (supported Feast APIs only): + scripts/prepare_data.py → parquet with document_id + event_timestamp + → RaySource (parquet) + → FeatureView web_documents + → OnDemandFeatureView train_example + → FeatureService llm_posttrain +""" + +from __future__ import annotations + +from datetime import timedelta +from pathlib import Path + +from feast import Entity, FeatureService, FeatureView, Field, ValueType +from feast.infra.offline_stores.contrib.ray_offline_store.ray_source import RaySource +from feast.on_demand_feature_view import on_demand_feature_view +from feast.types import Bool, Float64, Int64, String + +_REPO_DIR = Path(__file__).resolve().parent +_PARQUET = str(_REPO_DIR / "data" / "tiny_webtext.parquet") + +document = Entity( + name="document", + join_keys=["document_id"], + value_type=ValueType.STRING, + description="Document id (added by scripts/prepare_data.py)", +) + +# Parquet already has document_id + event_timestamp (see prepare_data.py). +# Do not rely on BatchFeatureView UDFs to invent timestamps during entity-less +# retrieval — that path is not supported without Feast core changes. +tiny_web = RaySource( + name="tiny_webtext", + reader_type="parquet", + path=_PARQUET, + timestamp_field="event_timestamp", +) + +web_documents = FeatureView( + name="web_documents", + entities=[document], + ttl=timedelta(days=365), + schema=[ + Field(name="human", dtype=String), + Field(name="bot", dtype=String), + Field(name="human_repeat_ratio", dtype=Float64), + Field(name="bot_repeat_ratio", dtype=Float64), + ], + source=tiny_web, + online=False, + description="Conversation columns from prepared parquet", + tags={"use_case": "llm_posttrain", "source": "parquet"}, +) + + +@on_demand_feature_view( + sources=[web_documents], + schema=[ + Field(name="cleaned_human", dtype=String), + Field(name="cleaned_bot", dtype=String), + Field(name="char_count", dtype=Int64), + Field(name="is_trainable", dtype=Bool), + Field(name="sft_text", dtype=String), + ], + mode="pandas", +) +def train_example(inputs): + """Quality gate + human→bot SFT formatting.""" + import pandas as pd + + min_chars = 64 + max_repeat_ratio = 0.65 + + cleaned_human = inputs["human"].fillna("").astype(str).str.strip() + cleaned_bot = inputs["bot"].fillna("").astype(str).str.strip() + char_count = cleaned_bot.str.len().astype("int64") + + human_ratio = inputs["human_repeat_ratio"].fillna(1.0).astype(float) + bot_ratio = inputs["bot_repeat_ratio"].fillna(1.0).astype(float) + is_trainable = ( + (char_count >= min_chars) + & (human_ratio <= max_repeat_ratio) + & (bot_ratio <= max_repeat_ratio) + ) + + sft_text = ( + "<|im_start|>user\n" + cleaned_human + "<|im_end|>\n" + "<|im_start|>assistant\n" + cleaned_bot + "<|im_end|>" + ) + + return pd.DataFrame( + { + "cleaned_human": cleaned_human, + "cleaned_bot": cleaned_bot, + "char_count": char_count, + "is_trainable": is_trainable, + "sft_text": sft_text, + } + ) + + +llm_posttrain = FeatureService( + name="llm_posttrain", + features=[web_documents, train_example], + tags={"use_case": "llm_posttrain", "model": "gpt2"}, +) diff --git a/examples/ray-llm-posttrain/feature_repo/feature_store.yaml b/examples/ray-llm-posttrain/feature_repo/feature_store.yaml new file mode 100644 index 00000000000..226dbdfb45c --- /dev/null +++ b/examples/ray-llm-posttrain/feature_repo/feature_store.yaml @@ -0,0 +1,25 @@ +project: ray_llm_posttrain +registry: data/registry.db +provider: local + +# Laptop-friendly Ray offline store (no KubeRay) +offline_store: + type: ray + storage_path: data/ray_storage + enable_ray_logging: false + ray_conf: + num_cpus: 2 + object_store_memory: 104857600 + _memory: 524288000 + +batch_engine: + type: ray.engine + max_workers: 2 + +online_store: + type: sqlite + path: data/online_store.db + +entity_key_serialization_version: 3 +auth: + type: no_auth diff --git a/examples/ray-llm-posttrain/requirements.txt b/examples/ray-llm-posttrain/requirements.txt new file mode 100644 index 00000000000..4a82b391c07 --- /dev/null +++ b/examples/ray-llm-posttrain/requirements.txt @@ -0,0 +1,8 @@ +# Feast + Ray offline store / compute engine +feast[ray]>=0.50.0 +datasets>=2.19.0 + +# Short GPT-2 SFT +transformers>=4.40.0 +torch>=2.1.0 +accelerate>=0.30.0 diff --git a/examples/ray-llm-posttrain/scripts/prepare_data.py b/examples/ray-llm-posttrain/scripts/prepare_data.py new file mode 100644 index 00000000000..efd1a5e032b --- /dev/null +++ b/examples/ray-llm-posttrain/scripts/prepare_data.py @@ -0,0 +1,55 @@ +#!/usr/bin/env python3 +"""Prepare a small local parquet seed for the example. + +Hugging Face tiny-webtext has no document_id / event_timestamp. Feast entity-less +retrieval needs those columns on the *source* data. We synthesize them here +(outside Feast) and write parquet — no Feast core changes required. + + PYTHONPATH=../../sdk/python python scripts/prepare_data.py +""" + +from __future__ import annotations + +import hashlib +from pathlib import Path + +import pandas as pd + +REPO_ROOT = Path(__file__).resolve().parents[1] +OUT_PATH = REPO_ROOT / "feature_repo" / "data" / "tiny_webtext.parquet" +SPLIT = "train[:2000]" +DATASET = "nampdn-ai/tiny-webtext" + + +def main() -> int: + from datasets import load_dataset + + print(f"Loading {DATASET} split={SPLIT!r}...") + ds = load_dataset(DATASET, split=SPLIT) + df = ds.to_pandas() + + demo_base_ts = pd.Timestamp("2024-06-01", tz="UTC") + demo_window_seconds = 30 * 24 * 3600 + + humans = df["human"].fillna("").astype(str) + bots = df["bot"].fillna("").astype(str) + doc_ids: list[str] = [] + timestamps: list[pd.Timestamp] = [] + for human, bot in zip(humans, bots, strict=True): + digest = hashlib.sha256(f"{human}\n{bot}".encode()).hexdigest() + doc_ids.append(digest[:16]) + offset = int(digest[:8], 16) % demo_window_seconds + timestamps.append(demo_base_ts + pd.Timedelta(seconds=offset)) + + df = df.copy() + df["document_id"] = doc_ids + df["event_timestamp"] = timestamps + + OUT_PATH.parent.mkdir(parents=True, exist_ok=True) + df.to_parquet(OUT_PATH, index=False) + print(f"Wrote {len(df)} rows → {OUT_PATH}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/examples/ray-llm-posttrain/scripts/train_sft.py b/examples/ray-llm-posttrain/scripts/train_sft.py new file mode 100644 index 00000000000..a3f4939f855 --- /dev/null +++ b/examples/ray-llm-posttrain/scripts/train_sft.py @@ -0,0 +1,217 @@ +#!/usr/bin/env python3 +"""Feast conversation features → training rows (paths match the blog). + +Paths: + A) Default: get_historical_features → to_ray_dataset() → preprocess sft_text + (ODFVs do NOT run on to_ray_dataset) + B) --via-df: get_historical_features → to_df() (ODFV train_example runs) + + PYTHONPATH=../../sdk/python python scripts/train_sft.py --dry-run + PYTHONPATH=../../sdk/python python scripts/train_sft.py --dry-run --via-df +""" + +from __future__ import annotations + +import argparse +import sys +from datetime import datetime, timezone +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parents[1] +FEATURE_REPO = REPO_ROOT / "feature_repo" +DATA_DIR = REPO_ROOT / "data" + +# Matches the blog Option A snippet (length gate) +_MIN_CHARS = 64 + + +def _parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--max-steps", type=int, default=20) + parser.add_argument("--batch-size", type=int, default=2) + parser.add_argument("--max-length", type=int, default=256) + parser.add_argument( + "--output-dir", + type=Path, + default=DATA_DIR / "gpt2-sft", + ) + parser.add_argument( + "--dry-run", + action="store_true", + help="Print a few training rows; skip the optional GPT-2 smoke", + ) + parser.add_argument( + "--via-df", + action="store_true", + help="Use to_df() so OnDemandFeatureView train_example runs", + ) + return parser.parse_args() + + +def _date_window() -> tuple[datetime, datetime]: + return ( + datetime(2024, 6, 1, tzinfo=timezone.utc), + datetime(2024, 7, 1, tzinfo=timezone.utc), + ) + + +def _preprocess_sft_batch(batch): + """Build sft_text from FeatureView columns (blog Option A).""" + import pandas as pd + + if not isinstance(batch, pd.DataFrame): + batch = pd.DataFrame(batch) + + human = batch["human"].fillna("").astype(str).str.strip() + bot = batch["bot"].fillna("").astype(str).str.strip() + ok = bot.str.len() >= _MIN_CHARS + sft_text = ( + "<|im_start|>user\n" + human + "<|im_end|>\n" + "<|im_start|>assistant\n" + bot + "<|im_end|>" + ) + return pd.DataFrame({"sft_text": sft_text}).loc[ok].reset_index(drop=True) + + +def retrieve_via_ray_stream(): + """Option A: to_ray_dataset() + preprocess (ODFV does not run).""" + from feast import FeatureStore + + store = FeatureStore(repo_path=str(FEATURE_REPO)) + start_date, end_date = _date_window() + + print("get_historical_features → to_ray_dataset() (preprocess sft_text on Ray)") + job = store.get_historical_features( + features=[ + "web_documents:human", + "web_documents:bot", + "web_documents:human_repeat_ratio", + "web_documents:bot_repeat_ratio", + ], + start_date=start_date, + end_date=end_date, + ) + ds = job.to_ray_dataset() + return ds.map_batches(_preprocess_sft_batch, batch_format="pandas") + + +def retrieve_via_df(): + """Option B: to_df() so ODFV train_example runs, then Ray from pandas.""" + import ray + from feast import FeatureStore + + store = FeatureStore(repo_path=str(FEATURE_REPO)) + start_date, end_date = _date_window() + + print("get_historical_features → to_df() (ODFV train_example runs)") + df = store.get_historical_features( + features=store.get_feature_service("llm_posttrain"), + start_date=start_date, + end_date=end_date, + ).to_df() + + if "is_trainable" not in df.columns or "sft_text" not in df.columns: + raise RuntimeError("Expected ODFV columns is_trainable / sft_text from to_df()") + + mask = df["is_trainable"].fillna(False).astype(bool) + mask &= df["sft_text"].fillna("").astype(str).str.len() > 0 + slim = df.loc[mask, ["sft_text"]].reset_index(drop=True) + return ray.data.from_pandas(slim) + + +def train_gpt2_optional( + ray_ds, *, max_steps: int, batch_size: int, max_length: int, output_dir: Path +) -> None: + import torch + from transformers import ( + AutoModelForCausalLM, + AutoTokenizer, + DataCollatorForLanguageModeling, + Trainer, + TrainingArguments, + ) + + rows = ray_ds.take(min(500, max(50, max_steps * batch_size * 4))) + texts = [r["sft_text"] for r in rows if r.get("sft_text")] + if not texts: + raise RuntimeError("No trainable SFT rows") + + print(f"[optional] GPT-2 smoke on {len(texts)} rows, {max_steps} steps...") + tokenizer = AutoTokenizer.from_pretrained("gpt2") + if tokenizer.pad_token is None: + tokenizer.pad_token = tokenizer.eos_token + model = AutoModelForCausalLM.from_pretrained("gpt2") + encodings = tokenizer( + texts, + truncation=True, + max_length=max_length, + padding="max_length", + return_tensors="pt", + ) + + class _TextDataset(torch.utils.data.Dataset): + def __len__(self) -> int: + return encodings["input_ids"].shape[0] + + def __getitem__(self, idx: int) -> dict: + return { + "input_ids": encodings["input_ids"][idx], + "attention_mask": encodings["attention_mask"][idx], + "labels": encodings["input_ids"][idx].clone(), + } + + output_dir.mkdir(parents=True, exist_ok=True) + args = TrainingArguments( + output_dir=str(output_dir), + per_device_train_batch_size=batch_size, + max_steps=max_steps, + logging_steps=max(1, max_steps // 5), + save_steps=max_steps, + learning_rate=5e-5, + report_to=[], + remove_unused_columns=False, + ) + trainer = Trainer( + model=model, + args=args, + train_dataset=_TextDataset(), + data_collator=DataCollatorForLanguageModeling(tokenizer=tokenizer, mlm=False), + ) + trainer.train() + trainer.save_model(str(output_dir)) + tokenizer.save_pretrained(str(output_dir)) + print(f"Saved optional checkpoint to {output_dir}") + + +def main() -> int: + args = _parse_args() + if not (FEATURE_REPO / "feature_store.yaml").exists(): + print(f"Missing feature repo at {FEATURE_REPO}", file=sys.stderr) + return 1 + + if args.via_df: + ds = retrieve_via_df() + else: + ds = retrieve_via_ray_stream() + + sample = ds.take(3) + print(f"Sample training rows: {len(sample)}") + for i, row in enumerate(sample): + preview = str(row.get("sft_text", row))[:160].replace("\n", "\\n") + print(f" [{i}] {preview}...") + + if args.dry_run: + print("Done (trainer skipped).") + return 0 + + train_gpt2_optional( + ds, + max_steps=args.max_steps, + batch_size=args.batch_size, + max_length=args.max_length, + output_dir=args.output_dir, + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/infra/charts/feast-feature-server/Chart.yaml b/infra/charts/feast-feature-server/Chart.yaml index fd31828aa05..f77f882a9d2 100644 --- a/infra/charts/feast-feature-server/Chart.yaml +++ b/infra/charts/feast-feature-server/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: feast-feature-server description: Feast Feature Server in Go or Python type: application -version: 0.64.0 +version: 0.65.0 keywords: - machine learning - big data diff --git a/infra/charts/feast-feature-server/README.md b/infra/charts/feast-feature-server/README.md index 595a9bf9ead..cd8cc475031 100644 --- a/infra/charts/feast-feature-server/README.md +++ b/infra/charts/feast-feature-server/README.md @@ -1,6 +1,6 @@ # Feast Python / Go Feature Server Helm Charts -Current chart version is `0.64.0` +Current chart version is `0.65.0` ## Installation @@ -42,7 +42,7 @@ See [here](https://github.com/feast-dev/feast/tree/master/examples/python-helm-d | fullnameOverride | string | `""` | | | image.pullPolicy | string | `"IfNotPresent"` | | | image.repository | string | `"quay.io/feastdev/feature-server"` | Docker image for Feature Server repository | -| image.tag | string | `"0.64.0"` | The Docker image tag (can be overwritten if custom feature server deps are needed for on demand transforms) | +| image.tag | string | `"0.65.0"` | The Docker image tag (can be overwritten if custom feature server deps are needed for on demand transforms) | | imagePullSecrets | list | `[]` | | | livenessProbe.initialDelaySeconds | int | `30` | | | livenessProbe.periodSeconds | int | `30` | | diff --git a/infra/charts/feast-feature-server/values.yaml b/infra/charts/feast-feature-server/values.yaml index 03a2f1f0b1d..f2bc97d7a2b 100644 --- a/infra/charts/feast-feature-server/values.yaml +++ b/infra/charts/feast-feature-server/values.yaml @@ -9,7 +9,7 @@ image: repository: quay.io/feastdev/feature-server pullPolicy: IfNotPresent # image.tag -- The Docker image tag (can be overwritten if custom feature server deps are needed for on demand transforms) - tag: 0.64.0 + tag: 0.65.0 logLevel: "WARNING" # Set log level DEBUG, INFO, WARNING, ERROR, and CRITICAL (case-insensitive) diff --git a/infra/charts/feast/Chart.yaml b/infra/charts/feast/Chart.yaml index 881c595ba97..dc49ff3fb2f 100644 --- a/infra/charts/feast/Chart.yaml +++ b/infra/charts/feast/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v1 description: Feature store for machine learning name: feast -version: 0.64.0 +version: 0.65.0 keywords: - machine learning - big data diff --git a/infra/charts/feast/README.md b/infra/charts/feast/README.md index afa964c7656..2a288bb48aa 100644 --- a/infra/charts/feast/README.md +++ b/infra/charts/feast/README.md @@ -8,7 +8,7 @@ This repo contains Helm charts for Feast Java components that are being installe ## Chart: Feast -Feature store for machine learning Current chart version is `0.64.0` +Feature store for machine learning Current chart version is `0.65.0` ## Installation @@ -65,8 +65,8 @@ See [here](https://github.com/feast-dev/feast/tree/master/examples/java-demo) fo | Repository | Name | Version | |------------|------|---------| | https://charts.helm.sh/stable | redis | 10.5.6 | -| https://feast-helm-charts.storage.googleapis.com | feature-server(feature-server) | 0.64.0 | -| https://feast-helm-charts.storage.googleapis.com | transformation-service(transformation-service) | 0.64.0 | +| https://feast-helm-charts.storage.googleapis.com | feature-server(feature-server) | 0.65.0 | +| https://feast-helm-charts.storage.googleapis.com | transformation-service(transformation-service) | 0.65.0 | ## Values diff --git a/infra/charts/feast/charts/feature-server/Chart.yaml b/infra/charts/feast/charts/feature-server/Chart.yaml index 7b6e40c3da0..b20c1778a18 100644 --- a/infra/charts/feast/charts/feature-server/Chart.yaml +++ b/infra/charts/feast/charts/feature-server/Chart.yaml @@ -1,8 +1,8 @@ apiVersion: v1 description: "Feast Feature Server: Online feature serving service for Feast" name: feature-server -version: 0.64.0 -appVersion: v0.64.0 +version: 0.65.0 +appVersion: v0.65.0 keywords: - machine learning - big data diff --git a/infra/charts/feast/charts/feature-server/README.md b/infra/charts/feast/charts/feature-server/README.md index 04714b14ed2..571449d9009 100644 --- a/infra/charts/feast/charts/feature-server/README.md +++ b/infra/charts/feast/charts/feature-server/README.md @@ -1,6 +1,6 @@ # feature-server -![Version: 0.64.0](https://img.shields.io/badge/Version-0.64.0-informational?style=flat-square) ![AppVersion: v0.64.0](https://img.shields.io/badge/AppVersion-v0.64.0-informational?style=flat-square) +![Version: 0.65.0](https://img.shields.io/badge/Version-0.65.0-informational?style=flat-square) ![AppVersion: v0.65.0](https://img.shields.io/badge/AppVersion-v0.65.0-informational?style=flat-square) Feast Feature Server: Online feature serving service for Feast @@ -17,7 +17,7 @@ Feast Feature Server: Online feature serving service for Feast | envOverrides | object | `{}` | Extra environment variables to set | | image.pullPolicy | string | `"IfNotPresent"` | Image pull policy | | image.repository | string | `"quay.io/feastdev/feature-server-java"` | Docker image for Feature Server repository | -| image.tag | string | `"0.64.0"` | Image tag | +| image.tag | string | `"0.65.0"` | Image tag | | ingress.grpc.annotations | object | `{}` | Extra annotations for the ingress | | ingress.grpc.auth.enabled | bool | `false` | Flag to enable auth | | ingress.grpc.class | string | `"nginx"` | Which ingress controller to use | diff --git a/infra/charts/feast/charts/feature-server/values.yaml b/infra/charts/feast/charts/feature-server/values.yaml index 0051f028279..3367dd665fa 100644 --- a/infra/charts/feast/charts/feature-server/values.yaml +++ b/infra/charts/feast/charts/feature-server/values.yaml @@ -5,7 +5,7 @@ image: # image.repository -- Docker image for Feature Server repository repository: quay.io/feastdev/feature-server-java # image.tag -- Image tag - tag: 0.64.0 + tag: 0.65.0 # image.pullPolicy -- Image pull policy pullPolicy: IfNotPresent diff --git a/infra/charts/feast/charts/transformation-service/Chart.yaml b/infra/charts/feast/charts/transformation-service/Chart.yaml index ca053c674bc..9dbc3f73cb4 100644 --- a/infra/charts/feast/charts/transformation-service/Chart.yaml +++ b/infra/charts/feast/charts/transformation-service/Chart.yaml @@ -1,8 +1,8 @@ apiVersion: v1 description: "Transformation service: to compute on-demand features" name: transformation-service -version: 0.64.0 -appVersion: v0.64.0 +version: 0.65.0 +appVersion: v0.65.0 keywords: - machine learning - big data diff --git a/infra/charts/feast/charts/transformation-service/README.md b/infra/charts/feast/charts/transformation-service/README.md index f6e39356eee..ad1dd75cd65 100644 --- a/infra/charts/feast/charts/transformation-service/README.md +++ b/infra/charts/feast/charts/transformation-service/README.md @@ -1,6 +1,6 @@ # transformation-service -![Version: 0.64.0](https://img.shields.io/badge/Version-0.64.0-informational?style=flat-square) ![AppVersion: v0.64.0](https://img.shields.io/badge/AppVersion-v0.64.0-informational?style=flat-square) +![Version: 0.65.0](https://img.shields.io/badge/Version-0.65.0-informational?style=flat-square) ![AppVersion: v0.65.0](https://img.shields.io/badge/AppVersion-v0.65.0-informational?style=flat-square) Transformation service: to compute on-demand features @@ -13,7 +13,7 @@ Transformation service: to compute on-demand features | envOverrides | object | `{}` | Extra environment variables to set | | image.pullPolicy | string | `"IfNotPresent"` | Image pull policy | | image.repository | string | `"quay.io/feastdev/feature-transformation-server"` | Docker image for Transformation Server repository | -| image.tag | string | `"0.64.0"` | Image tag | +| image.tag | string | `"0.65.0"` | Image tag | | nodeSelector | object | `{}` | Node labels for pod assignment | | podLabels | object | `{}` | Labels to be added to Feast Serving pods | | replicaCount | int | `1` | Number of pods that will be created | diff --git a/infra/charts/feast/charts/transformation-service/values.yaml b/infra/charts/feast/charts/transformation-service/values.yaml index af34cfa486d..266cd4b48aa 100644 --- a/infra/charts/feast/charts/transformation-service/values.yaml +++ b/infra/charts/feast/charts/transformation-service/values.yaml @@ -5,7 +5,7 @@ image: # image.repository -- Docker image for Transformation Server repository repository: quay.io/feastdev/feature-transformation-server # image.tag -- Image tag - tag: 0.64.0 + tag: 0.65.0 # image.pullPolicy -- Image pull policy pullPolicy: IfNotPresent diff --git a/infra/charts/feast/requirements.yaml b/infra/charts/feast/requirements.yaml index 8f610567ded..3f29ad7dfdc 100644 --- a/infra/charts/feast/requirements.yaml +++ b/infra/charts/feast/requirements.yaml @@ -1,12 +1,12 @@ dependencies: - name: feature-server alias: feature-server - version: 0.64.0 + version: 0.65.0 condition: feature-server.enabled repository: https://feast-helm-charts.storage.googleapis.com - name: transformation-service alias: transformation-service - version: 0.64.0 + version: 0.65.0 condition: transformation-service.enabled repository: https://feast-helm-charts.storage.googleapis.com - name: redis diff --git a/infra/feast-operator/Dockerfile b/infra/feast-operator/Dockerfile index 811c4d31c84..a7ad3fa044d 100644 --- a/infra/feast-operator/Dockerfile +++ b/infra/feast-operator/Dockerfile @@ -12,7 +12,7 @@ COPY --chown=1001:0 go.sum go.sum RUN go mod download # Copy the go source -COPY --chown=1001:0 cmd/main.go cmd/main.go +COPY --chown=1001:0 cmd/ cmd/ COPY --chown=1001:0 api/ api/ COPY --chown=1001:0 internal/controller/ internal/controller/ @@ -21,7 +21,7 @@ COPY --chown=1001:0 internal/controller/ internal/controller/ # was called. For example, if we call make docker-build in a local env which has the Apple Silicon M1 SO # the docker BUILDPLATFORM arg will be linux/arm64 when for Apple x86 it will be linux/amd64. Therefore, # by leaving it empty we can ensure that the container and binary shipped on it will have the same platform. -RUN CGO_ENABLED=0 GOOS=${TARGETOS:-linux} GOARCH=${TARGETARCH} go build -a -o manager cmd/main.go +RUN CGO_ENABLED=0 GOOS=${TARGETOS:-linux} GOARCH=${TARGETARCH} go build -a -o manager ./cmd/ FROM registry.access.redhat.com/ubi9/ubi-minimal:9.8 WORKDIR / diff --git a/infra/feast-operator/Makefile b/infra/feast-operator/Makefile index 5b470437397..b70608a389e 100644 --- a/infra/feast-operator/Makefile +++ b/infra/feast-operator/Makefile @@ -3,7 +3,7 @@ # To re-generate a bundle for another specific version without changing the standard setup, you can: # - use the VERSION as arg of the bundle target (e.g make bundle VERSION=0.0.2) # - use environment variables to overwrite this value (e.g export VERSION=0.0.2) -VERSION ?= 0.64.0 +VERSION ?= 0.65.0 # CHANNELS define the bundle channels used in the bundle. # Add a new line here if you would like to change its default config. (E.g CHANNELS = "candidate,fast,stable") diff --git a/infra/feast-operator/README.md b/infra/feast-operator/README.md index c639be54fde..f879dff1cc1 100644 --- a/infra/feast-operator/README.md +++ b/infra/feast-operator/README.md @@ -7,7 +7,7 @@ This is a K8s Operator that can be used to deploy and manage **Feast**, an open | Guide | Topic | |-------|-------| -| [1 — Project Provisioning](https://docs.feast.dev/how-to-guides/feast-operator/01-project-provisioning) | `feastProjectDir`: git clone vs `feast init` templates | +| [1 — Project Provisioning](https://docs.feast.dev/how-to-guides/feast-operator/01-project-provisioning) | `feastProjectDir`: git clone, `feast init`, or a repository packaged in an image | | [2 — Persistence](https://docs.feast.dev/how-to-guides/feast-operator/02-persistence) | File (path + PVC) vs DB store for offline/online/registry; Secret format | | [3 — Serving & Observability](https://docs.feast.dev/how-to-guides/feast-operator/03-serving-and-observability) | Workers, log level, Prometheus metrics, offline push batching, MCP | | [4 — Registry Topology](https://docs.feast.dev/how-to-guides/feast-operator/04-registry-topology) | Local, remote, cross-namespace `feastRef` | diff --git a/infra/feast-operator/api/feastversion/version.go b/infra/feast-operator/api/feastversion/version.go index 742ede75fd5..deabd34ac38 100644 --- a/infra/feast-operator/api/feastversion/version.go +++ b/infra/feast-operator/api/feastversion/version.go @@ -17,4 +17,4 @@ limitations under the License. package feastversion // Feast release version. Keep on line #20, this is critical to release CI -const FeastVersion = "0.64.0" +const FeastVersion = "0.65.0" diff --git a/infra/feast-operator/api/v1/featurestore_types.go b/infra/feast-operator/api/v1/featurestore_types.go index f0331ecb1b4..3372e74f63a 100644 --- a/infra/feast-operator/api/v1/featurestore_types.go +++ b/infra/feast-operator/api/v1/featurestore_types.go @@ -175,10 +175,22 @@ type FeatureStoreSpec struct { } // FeastProjectDir defines how to create the feast project directory. -// +kubebuilder:validation:XValidation:rule="[has(self.git), has(self.init)].exists_one(c, c)",message="One selection required between init or git." +// +kubebuilder:validation:XValidation:rule="[has(self.git), has(self.init), has(self.packaged)].exists_one(c, c)",message="One selection required between init, git, or packaged." type FeastProjectDir struct { - Git *GitCloneOptions `json:"git,omitempty"` - Init *FeastInitOptions `json:"init,omitempty"` + Git *GitCloneOptions `json:"git,omitempty"` + Init *FeastInitOptions `json:"init,omitempty"` + Packaged *FeastPackagedOptions `json:"packaged,omitempty"` +} + +// FeastPackagedOptions describes a feature repository packaged in a feature server image. +// +kubebuilder:validation:XValidation:rule="self.featureRepoPath.startsWith('/') && self.featureRepoPath != '/' && !self.featureRepoPath.contains('//') && !self.featureRepoPath.endsWith('/') && !self.featureRepoPath.contains('/./') && !self.featureRepoPath.endsWith('/.') && !self.featureRepoPath.contains('/../') && !self.featureRepoPath.endsWith('/..')",message="FeatureRepoPath must be a canonical absolute, non-root path without dot segments or repeated separators." +type FeastPackagedOptions struct { + // Image containing the packaged feature repository. When set, this image is used by the + // repository initialization and feast apply containers and as the default service image. + // When omitted, the operator's configured feature server image is used. + Image string `json:"image,omitempty"` + // FeatureRepoPath is the canonical absolute path to the feature repository in the image. + FeatureRepoPath string `json:"featureRepoPath"` } // GitCloneOptions describes how a clone should be performed. @@ -407,6 +419,10 @@ type FeatureStoreServices struct { PodAnnotations map[string]string `json:"podAnnotations,omitempty"` // Disable the 'feast repo initialization' initContainer DisableInitContainers bool `json:"disableInitContainers,omitempty"` + // InitImage overrides the image for init containers (feast-init, feast-apply). + // Resolution order: InitImage → FeastProjectDir.Packaged.Image → RELATED_IMAGE_FEATURE_SERVER → DefaultImage. + // +optional + InitImage *string `json:"initImage,omitempty"` // Runs feast apply on pod start to populate the registry. Defaults to true. Ignored when DisableInitContainers is true. RunFeastApplyOnInit *bool `json:"runFeastApplyOnInit,omitempty"` // Volumes specifies the volumes to mount in the FeatureStore deployment. A corresponding `VolumeMount` should be added to whichever feast service(s) require access to said volume(s). @@ -544,6 +560,11 @@ type OnlineStore struct { // Controls metrics granularity, offline push batching, and MCP. // +optional Serving *ServingConfig `json:"serving,omitempty"` + // Disabled skips deploying the online store service entirely, including its + // serving pod and persistence. Omitting the online store block, or setting + // this to false, deploys the online store with defaults as before. + // +optional + Disabled bool `json:"disabled,omitempty"` } // ServingConfig configures the feature_server section of the generated feature_store.yaml. @@ -626,7 +647,7 @@ type OnlineStoreFilePersistence struct { // OnlineStoreDBStorePersistence configures the DB store persistence for the online store service type OnlineStoreDBStorePersistence struct { // Type of the persistence type you want to use. - // +kubebuilder:validation:Enum=snowflake.online;redis;datastore;dynamodb;bigtable;postgres;cassandra;mysql;hazelcast;singlestore;hbase;elasticsearch;qdrant;couchbase.online;milvus;hybrid;mongodb;aerospike + // +kubebuilder:validation:Enum=snowflake.online;redis;datastore;dynamodb;bigtable;postgres;cassandra;mysql;hazelcast;singlestore;hbase;elasticsearch;qdrant;couchbase.online;milvus;hybrid;mongodb;aerospike;scylladb Type string `json:"type"` // Data store parameters should be placed as-is from the "feature_store.yaml" under the secret key. "registry_type" & "type" fields should be removed. SecretRef corev1.LocalObjectReference `json:"secretRef"` @@ -653,6 +674,7 @@ var ValidOnlineStoreDBStorePersistenceTypes = []string{ "hybrid", "mongodb", "aerospike", + "scylladb", } // LocalRegistryConfig configures the registry service diff --git a/infra/feast-operator/api/v1/zz_generated.deepcopy.go b/infra/feast-operator/api/v1/zz_generated.deepcopy.go index 2a6b6a69266..3035ed066cf 100644 --- a/infra/feast-operator/api/v1/zz_generated.deepcopy.go +++ b/infra/feast-operator/api/v1/zz_generated.deepcopy.go @@ -257,6 +257,21 @@ func (in *FeastInitOptions) DeepCopy() *FeastInitOptions { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *FeastPackagedOptions) DeepCopyInto(out *FeastPackagedOptions) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new FeastPackagedOptions. +func (in *FeastPackagedOptions) DeepCopy() *FeastPackagedOptions { + if in == nil { + return nil + } + out := new(FeastPackagedOptions) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *FeastProjectDir) DeepCopyInto(out *FeastProjectDir) { *out = *in @@ -270,6 +285,11 @@ func (in *FeastProjectDir) DeepCopyInto(out *FeastProjectDir) { *out = new(FeastInitOptions) **out = **in } + if in.Packaged != nil { + in, out := &in.Packaged, &out.Packaged + *out = new(FeastPackagedOptions) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new FeastProjectDir. @@ -396,6 +416,11 @@ func (in *FeatureStoreServices) DeepCopyInto(out *FeatureStoreServices) { (*out)[key] = val } } + if in.InitImage != nil { + in, out := &in.InitImage, &out.InitImage + *out = new(string) + **out = **in + } if in.RunFeastApplyOnInit != nil { in, out := &in.RunFeastApplyOnInit, &out.RunFeastApplyOnInit *out = new(bool) diff --git a/infra/feast-operator/api/v1alpha1/featurestore_types.go b/infra/feast-operator/api/v1alpha1/featurestore_types.go index c165801eda7..8ccde377e77 100644 --- a/infra/feast-operator/api/v1alpha1/featurestore_types.go +++ b/infra/feast-operator/api/v1alpha1/featurestore_types.go @@ -78,10 +78,22 @@ type FeatureStoreSpec struct { } // FeastProjectDir defines how to create the feast project directory. -// +kubebuilder:validation:XValidation:rule="[has(self.git), has(self.init)].exists_one(c, c)",message="One selection required between init or git." +// +kubebuilder:validation:XValidation:rule="[has(self.git), has(self.init), has(self.packaged)].exists_one(c, c)",message="One selection required between init, git, or packaged." type FeastProjectDir struct { - Git *GitCloneOptions `json:"git,omitempty"` - Init *FeastInitOptions `json:"init,omitempty"` + Git *GitCloneOptions `json:"git,omitempty"` + Init *FeastInitOptions `json:"init,omitempty"` + Packaged *FeastPackagedOptions `json:"packaged,omitempty"` +} + +// FeastPackagedOptions describes a feature repository packaged in a feature server image. +// +kubebuilder:validation:XValidation:rule="self.featureRepoPath.startsWith('/') && self.featureRepoPath != '/' && !self.featureRepoPath.contains('//') && !self.featureRepoPath.endsWith('/') && !self.featureRepoPath.contains('/./') && !self.featureRepoPath.endsWith('/.') && !self.featureRepoPath.contains('/../') && !self.featureRepoPath.endsWith('/..')",message="FeatureRepoPath must be a canonical absolute, non-root path without dot segments or repeated separators." +type FeastPackagedOptions struct { + // Image containing the packaged feature repository. When set, this image is used by the + // repository initialization and feast apply containers and as the default service image. + // When omitted, the operator's configured feature server image is used. + Image string `json:"image,omitempty"` + // FeatureRepoPath is the canonical absolute path to the feature repository in the image. + FeatureRepoPath string `json:"featureRepoPath"` } // GitCloneOptions describes how a clone should be performed. @@ -373,7 +385,7 @@ type OnlineStoreFilePersistence struct { // OnlineStoreDBStorePersistence configures the DB store persistence for the online store service type OnlineStoreDBStorePersistence struct { // Type of the persistence type you want to use. - // +kubebuilder:validation:Enum=snowflake.online;redis;datastore;dynamodb;bigtable;postgres;cassandra;mysql;hazelcast;singlestore;hbase;elasticsearch;qdrant;couchbase.online;milvus;hybrid;mongodb;aerospike + // +kubebuilder:validation:Enum=snowflake.online;redis;datastore;dynamodb;bigtable;postgres;cassandra;mysql;hazelcast;singlestore;hbase;elasticsearch;qdrant;couchbase.online;milvus;hybrid;mongodb;aerospike;scylladb Type string `json:"type"` // Data store parameters should be placed as-is from the "feature_store.yaml" under the secret key. "registry_type" & "type" fields should be removed. SecretRef corev1.LocalObjectReference `json:"secretRef"` @@ -400,6 +412,7 @@ var ValidOnlineStoreDBStorePersistenceTypes = []string{ "hybrid", "mongodb", "aerospike", + "scylladb", } // LocalRegistryConfig configures the registry service diff --git a/infra/feast-operator/api/v1alpha1/zz_generated.deepcopy.go b/infra/feast-operator/api/v1alpha1/zz_generated.deepcopy.go index 4033c368c8b..17ae4841966 100644 --- a/infra/feast-operator/api/v1alpha1/zz_generated.deepcopy.go +++ b/infra/feast-operator/api/v1alpha1/zz_generated.deepcopy.go @@ -183,6 +183,21 @@ func (in *FeastInitOptions) DeepCopy() *FeastInitOptions { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *FeastPackagedOptions) DeepCopyInto(out *FeastPackagedOptions) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new FeastPackagedOptions. +func (in *FeastPackagedOptions) DeepCopy() *FeastPackagedOptions { + if in == nil { + return nil + } + out := new(FeastPackagedOptions) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *FeastProjectDir) DeepCopyInto(out *FeastProjectDir) { *out = *in @@ -196,6 +211,11 @@ func (in *FeastProjectDir) DeepCopyInto(out *FeastProjectDir) { *out = new(FeastInitOptions) **out = **in } + if in.Packaged != nil { + in, out := &in.Packaged, &out.Packaged + *out = new(FeastPackagedOptions) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new FeastProjectDir. diff --git a/infra/feast-operator/bundle/manifests/feast-operator.clusterserviceversion.yaml b/infra/feast-operator/bundle/manifests/feast-operator.clusterserviceversion.yaml index 82e1bc57b36..19af99046e9 100644 --- a/infra/feast-operator/bundle/manifests/feast-operator.clusterserviceversion.yaml +++ b/infra/feast-operator/bundle/manifests/feast-operator.clusterserviceversion.yaml @@ -147,10 +147,10 @@ metadata: } ] capabilities: Basic Install - createdAt: "2026-06-13T11:22:06Z" + createdAt: "2026-07-20T13:27:58Z" operators.operatorframework.io/builder: operator-sdk-v1.41.0 operators.operatorframework.io/project_layout: go.kubebuilder.io/v4 - name: feast-operator.v0.64.0 + name: feast-operator.v0.65.0 namespace: placeholder spec: apiservicedefinitions: {} @@ -180,11 +180,11 @@ spec: resources: - configmaps - persistentvolumeclaims - - serviceaccounts - services verbs: - create - delete + - deletecollection - get - list - update @@ -193,18 +193,45 @@ spec: - "" resources: - namespaces - - pods - secrets verbs: - get - list - watch + - apiGroups: + - "" + resources: + - pods + verbs: + - create + - delete + - deletecollection + - get + - list + - watch - apiGroups: - "" resources: - pods/exec verbs: - create + - apiGroups: + - "" + resources: + - pods/log + verbs: + - get + - apiGroups: + - "" + resources: + - serviceaccounts + verbs: + - create + - delete + - get + - list + - update + - watch - apiGroups: - apps resources: @@ -246,6 +273,14 @@ spec: - patch - update - watch + - apiGroups: + - config.openshift.io + resources: + - apiservers + verbs: + - get + - list + - watch - apiGroups: - feast.dev resources: @@ -321,6 +356,14 @@ spec: - list - update - watch + - apiGroups: + - sparkoperator.k8s.io + resources: + - sparkapplications + verbs: + - create + - delete + - get - apiGroups: - authentication.k8s.io resources: @@ -364,13 +407,13 @@ spec: - /manager env: - name: RELATED_IMAGE_FEATURE_SERVER - value: quay.io/feastdev/feature-server:0.64.0 + value: quay.io/feastdev/feature-server:0.65.0 - name: RELATED_IMAGE_CRON_JOB value: quay.io/openshift/origin-cli:4.17 - name: GOMEMLIMIT value: 230MiB - name: OIDC_ISSUER_URL - image: quay.io/feastdev/feast-operator:0.64.0 + image: quay.io/feastdev/feast-operator:0.65.0 livenessProbe: httpGet: path: /healthz @@ -460,8 +503,8 @@ spec: name: Feast Community url: https://lf-aidata.atlassian.net/wiki/spaces/FEAST/ relatedImages: - - image: quay.io/feastdev/feature-server:0.64.0 + - image: quay.io/feastdev/feature-server:0.65.0 name: feature-server - image: quay.io/openshift/origin-cli:4.17 name: cron-job - version: 0.64.0 + version: 0.65.0 diff --git a/infra/feast-operator/bundle/manifests/feast.dev_featurestores.yaml b/infra/feast-operator/bundle/manifests/feast.dev_featurestores.yaml index 1355d65d993..0ab08afef51 100644 --- a/infra/feast-operator/bundle/manifests/feast.dev_featurestores.yaml +++ b/infra/feast-operator/bundle/manifests/feast.dev_featurestores.yaml @@ -161,8 +161,9 @@ spec: in a Container. properties: name: - description: Name of the environment variable. Must - be a C_IDENTIFIER. + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. type: string value: description: |- @@ -211,6 +212,35 @@ spec: - fieldPath type: object x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: The key within the env file. An + invalid key will prevent the pod from starting. + type: string + optional: + default: false + description: Specify whether the file or its + key must be defined. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '.. + type: string + volumeName: + description: The name of the volume mount containing + the env file. + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic resourceFieldRef: description: |- Selects a resource of the container: only resources limits and requests @@ -286,7 +316,7 @@ spec: x-kubernetes-map-type: atomic prefix: description: Optional text to prepend to the name of - each environment variable. Must be a C_IDENTIFIER. + each environment variable. type: string secretRef: description: The Secret to select from @@ -471,7 +501,6 @@ spec: type. type: string required: - - status - type type: object type: array @@ -564,8 +593,9 @@ spec: in a Container. properties: name: - description: Name of the environment variable. Must - be a C_IDENTIFIER. + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. type: string value: description: |- @@ -614,6 +644,35 @@ spec: - fieldPath type: object x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: The key within the env file. An + invalid key will prevent the pod from starting. + type: string + optional: + default: false + description: Specify whether the file or its + key must be defined. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '.. + type: string + volumeName: + description: The name of the volume mount containing + the env file. + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic resourceFieldRef: description: |- Selects a resource of the container: only resources limits and requests @@ -689,7 +748,7 @@ spec: x-kubernetes-map-type: atomic prefix: description: Optional text to prepend to the name of - each environment variable. Must be a C_IDENTIFIER. + each environment variable. type: string secretRef: description: The Secret to select from @@ -751,10 +810,32 @@ spec: - pytorch_nlp type: string type: object + packaged: + description: FeastPackagedOptions describes a feature repository + packaged in a feature server image. + properties: + featureRepoPath: + description: FeatureRepoPath is the canonical absolute path + to the feature repository in the image. + type: string + image: + description: Image containing the packaged feature repository. + type: string + required: + - featureRepoPath + type: object + x-kubernetes-validations: + - message: FeatureRepoPath must be a canonical absolute, non-root + path without dot segments or repeated separators. + rule: self.featureRepoPath.startsWith('/') && self.featureRepoPath + != '/' && !self.featureRepoPath.contains('//') && !self.featureRepoPath.endsWith('/') + && !self.featureRepoPath.contains('/./') && !self.featureRepoPath.endsWith('/.') + && !self.featureRepoPath.contains('/../') && !self.featureRepoPath.endsWith('/..') type: object x-kubernetes-validations: - - message: One selection required between init or git. - rule: '[has(self.git), has(self.init)].exists_one(c, c)' + - message: One selection required between init, git, or packaged. + rule: '[has(self.git), has(self.init), has(self.packaged)].exists_one(c, + c)' materialization: description: |- Materialization controls feature materialization behavior (batch size, pull strategy). @@ -793,6 +874,59 @@ spec: type: string type: object x-kubernetes-map-type: atomic + consumer: + description: |- + Consumer configures the OpenLineage consumer (event receiver) that enables + Feast to receive and display lineage from... + properties: + apiKeySecretRef: + description: |- + Reference to a Secret containing the key "api_key" that producers must + provide in the X-API-Key header when sending... + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. + type: string + type: object + x-kubernetes-map-type: atomic + connectionStringSecretRef: + description: |- + Reference to a Secret containing the key "connection_string" for a separate + lineage database. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. + type: string + type: object + x-kubernetes-map-type: atomic + enabled: + description: Enable the OpenLineage consumer. + type: boolean + namespaceMapping: + additionalProperties: + type: string + description: |- + NamespaceMapping maps OpenLineage namespaces to Feast projects for + RBAC-based filtering of lineage data in the UI. + type: object + storeType: + default: sql + description: StoreType is the storage backend for lineage + events. Currently only "sql" is supported. + enum: + - sql + type: string + required: + - enabled + type: object enabled: description: Enable OpenLineage integration. type: boolean @@ -1611,6 +1745,10 @@ spec: disableInitContainers: description: Disable the 'feast repo initialization' initContainer type: boolean + initImage: + description: InitImage overrides the image for init containers + (feast-init, feast-apply). + type: string offlineStore: description: OfflineStore configures the offline store service properties: @@ -1761,8 +1899,9 @@ spec: present in a Container. properties: name: - description: Name of the environment variable. Must - be a C_IDENTIFIER. + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. type: string value: description: |- @@ -1811,6 +1950,36 @@ spec: - fieldPath type: object x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: The key within the env file. + An invalid key will prevent the pod from + starting. + type: string + optional: + default: false + description: Specify whether the file or + its key must be defined. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '.. + type: string + volumeName: + description: The name of the volume mount + containing the env file. + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic resourceFieldRef: description: |- Selects a resource of the container: only resources limits and requests @@ -1887,7 +2056,7 @@ spec: x-kubernetes-map-type: atomic prefix: description: Optional text to prepend to the name - of each environment variable. Must be a C_IDENTIFIER. + of each environment variable. type: string secretRef: description: The Secret to select from @@ -2115,6 +2284,11 @@ spec: onlineStore: description: OnlineStore configures the online store service properties: + disabled: + description: |- + Disabled skips deploying the online store service entirely, including its + serving pod and persistence. + type: boolean persistence: description: OnlineStorePersistence configures the persistence settings for the online store service @@ -2257,6 +2431,7 @@ spec: - hybrid - mongodb - aerospike + - scylladb type: string required: - secretRef @@ -2275,8 +2450,9 @@ spec: present in a Container. properties: name: - description: Name of the environment variable. Must - be a C_IDENTIFIER. + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. type: string value: description: |- @@ -2325,6 +2501,36 @@ spec: - fieldPath type: object x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: The key within the env file. + An invalid key will prevent the pod from + starting. + type: string + optional: + default: false + description: Specify whether the file or + its key must be defined. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '.. + type: string + volumeName: + description: The name of the volume mount + containing the env file. + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic resourceFieldRef: description: |- Selects a resource of the container: only resources limits and requests @@ -2401,7 +2607,7 @@ spec: x-kubernetes-map-type: atomic prefix: description: Optional text to prepend to the name - of each environment variable. Must be a C_IDENTIFIER. + of each environment variable. type: string secretRef: description: The Secret to select from @@ -2905,8 +3111,9 @@ spec: present in a Container. properties: name: - description: Name of the environment variable. - Must be a C_IDENTIFIER. + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. type: string value: description: |- @@ -2956,6 +3163,36 @@ spec: - fieldPath type: object x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: The key within the env + file. An invalid key will prevent + the pod from starting. + type: string + optional: + default: false + description: Specify whether the file + or its key must be defined. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '.. + type: string + volumeName: + description: The name of the volume + mount containing the env file. + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic resourceFieldRef: description: |- Selects a resource of the container: only resources limits and requests @@ -3034,8 +3271,7 @@ spec: x-kubernetes-map-type: atomic prefix: description: Optional text to prepend to the - name of each environment variable. Must be - a C_IDENTIFIER. + name of each environment variable. type: string secretRef: description: The Secret to select from @@ -4210,8 +4446,9 @@ spec: in a Container. properties: name: - description: Name of the environment variable. Must - be a C_IDENTIFIER. + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. type: string value: description: |- @@ -4260,6 +4497,35 @@ spec: - fieldPath type: object x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: The key within the env file. An + invalid key will prevent the pod from starting. + type: string + optional: + default: false + description: Specify whether the file or its + key must be defined. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '.. + type: string + volumeName: + description: The name of the volume mount containing + the env file. + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic resourceFieldRef: description: |- Selects a resource of the container: only resources limits and requests @@ -4335,7 +4601,7 @@ spec: x-kubernetes-map-type: atomic prefix: description: Optional text to prepend to the name of - each environment variable. Must be a C_IDENTIFIER. + each environment variable. type: string secretRef: description: The Secret to select from @@ -5229,9 +5495,8 @@ spec: host that shares a pod's lifetime. properties: endpoints: - description: |- - endpoints is the endpoint name that details Glusterfs topology. - More info: https://examples.k8s. + description: endpoints is the endpoint name that details + Glusterfs topology. type: string path: description: |- @@ -5644,6 +5909,52 @@ spec: type: array x-kubernetes-list-type: atomic type: object + podCertificate: + description: |- + Projects an auto-rotating credential bundle (private key and certificate + chain) that the pod can use either as a TLS... + properties: + certificateChainPath: + description: |- + Write the certificate chain at this path in the projected volume. + + Most applications should use credentialBundlePath. + type: string + credentialBundlePath: + description: Write the credential bundle at + this path in the projected volume. + type: string + keyPath: + description: |- + Write the key at this path in the projected volume. + + Most applications should use credentialBundlePath. + type: string + keyType: + description: The type of keypair Kubelet will + generate for the pod. + type: string + maxExpirationSeconds: + description: |- + maxExpirationSeconds is the maximum lifetime permitted for the + certificate. + format: int32 + type: integer + signerName: + description: Kubelet's generated CSRs will + be addressed to this signer. + type: string + userAnnotations: + additionalProperties: + type: string + description: |- + userAnnotations allow pod authors to pass additional information to + the signer implementation. + type: object + required: + - keyType + - signerName + type: object secret: description: secret information about the secret data to project @@ -6161,8 +6472,9 @@ spec: present in a Container. properties: name: - description: Name of the environment variable. Must - be a C_IDENTIFIER. + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. type: string value: description: |- @@ -6211,6 +6523,36 @@ spec: - fieldPath type: object x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: The key within the env file. + An invalid key will prevent the pod from + starting. + type: string + optional: + default: false + description: Specify whether the file or + its key must be defined. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '.. + type: string + volumeName: + description: The name of the volume mount + containing the env file. + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic resourceFieldRef: description: |- Selects a resource of the container: only resources limits and requests @@ -6287,7 +6629,7 @@ spec: x-kubernetes-map-type: atomic prefix: description: Optional text to prepend to the name - of each environment variable. Must be a C_IDENTIFIER. + of each environment variable. type: string secretRef: description: The Secret to select from @@ -6474,7 +6816,6 @@ spec: condition type. type: string required: - - status - type type: object type: array @@ -6569,8 +6910,9 @@ spec: present in a Container. properties: name: - description: Name of the environment variable. Must - be a C_IDENTIFIER. + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. type: string value: description: |- @@ -6619,6 +6961,36 @@ spec: - fieldPath type: object x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: The key within the env file. + An invalid key will prevent the pod from + starting. + type: string + optional: + default: false + description: Specify whether the file or + its key must be defined. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '.. + type: string + volumeName: + description: The name of the volume mount + containing the env file. + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic resourceFieldRef: description: |- Selects a resource of the container: only resources limits and requests @@ -6695,7 +7067,7 @@ spec: x-kubernetes-map-type: atomic prefix: description: Optional text to prepend to the name - of each environment variable. Must be a C_IDENTIFIER. + of each environment variable. type: string secretRef: description: The Secret to select from @@ -6758,10 +7130,32 @@ spec: - pytorch_nlp type: string type: object + packaged: + description: FeastPackagedOptions describes a feature repository + packaged in a feature server image. + properties: + featureRepoPath: + description: FeatureRepoPath is the canonical absolute + path to the feature repository in the image. + type: string + image: + description: Image containing the packaged feature repository. + type: string + required: + - featureRepoPath + type: object + x-kubernetes-validations: + - message: FeatureRepoPath must be a canonical absolute, non-root + path without dot segments or repeated separators. + rule: self.featureRepoPath.startsWith('/') && self.featureRepoPath + != '/' && !self.featureRepoPath.contains('//') && !self.featureRepoPath.endsWith('/') + && !self.featureRepoPath.contains('/./') && !self.featureRepoPath.endsWith('/.') + && !self.featureRepoPath.contains('/../') && !self.featureRepoPath.endsWith('/..') type: object x-kubernetes-validations: - - message: One selection required between init or git. - rule: '[has(self.git), has(self.init)].exists_one(c, c)' + - message: One selection required between init, git, or packaged. + rule: '[has(self.git), has(self.init), has(self.packaged)].exists_one(c, + c)' materialization: description: |- Materialization controls feature materialization behavior (batch size, pull strategy). @@ -6800,6 +7194,59 @@ spec: type: string type: object x-kubernetes-map-type: atomic + consumer: + description: |- + Consumer configures the OpenLineage consumer (event receiver) that enables + Feast to receive and display lineage from... + properties: + apiKeySecretRef: + description: |- + Reference to a Secret containing the key "api_key" that producers must + provide in the X-API-Key header when sending... + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. + type: string + type: object + x-kubernetes-map-type: atomic + connectionStringSecretRef: + description: |- + Reference to a Secret containing the key "connection_string" for a separate + lineage database. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. + type: string + type: object + x-kubernetes-map-type: atomic + enabled: + description: Enable the OpenLineage consumer. + type: boolean + namespaceMapping: + additionalProperties: + type: string + description: |- + NamespaceMapping maps OpenLineage namespaces to Feast projects for + RBAC-based filtering of lineage data in the UI. + type: object + storeType: + default: sql + description: StoreType is the storage backend for lineage + events. Currently only "sql" is supported. + enum: + - sql + type: string + required: + - enabled + type: object enabled: description: Enable OpenLineage integration. type: boolean @@ -7626,6 +8073,10 @@ spec: disableInitContainers: description: Disable the 'feast repo initialization' initContainer type: boolean + initImage: + description: InitImage overrides the image for init containers + (feast-init, feast-apply). + type: string offlineStore: description: OfflineStore configures the offline store service properties: @@ -7779,8 +8230,9 @@ spec: present in a Container. properties: name: - description: Name of the environment variable. - Must be a C_IDENTIFIER. + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. type: string value: description: |- @@ -7830,6 +8282,36 @@ spec: - fieldPath type: object x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: The key within the env + file. An invalid key will prevent + the pod from starting. + type: string + optional: + default: false + description: Specify whether the file + or its key must be defined. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '.. + type: string + volumeName: + description: The name of the volume + mount containing the env file. + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic resourceFieldRef: description: |- Selects a resource of the container: only resources limits and requests @@ -7908,8 +8390,7 @@ spec: x-kubernetes-map-type: atomic prefix: description: Optional text to prepend to the - name of each environment variable. Must be - a C_IDENTIFIER. + name of each environment variable. type: string secretRef: description: The Secret to select from @@ -8138,6 +8619,11 @@ spec: onlineStore: description: OnlineStore configures the online store service properties: + disabled: + description: |- + Disabled skips deploying the online store service entirely, including its + serving pod and persistence. + type: boolean persistence: description: OnlineStorePersistence configures the persistence settings for the online store service @@ -8283,6 +8769,7 @@ spec: - hybrid - mongodb - aerospike + - scylladb type: string required: - secretRef @@ -8302,8 +8789,9 @@ spec: present in a Container. properties: name: - description: Name of the environment variable. - Must be a C_IDENTIFIER. + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. type: string value: description: |- @@ -8353,6 +8841,36 @@ spec: - fieldPath type: object x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: The key within the env + file. An invalid key will prevent + the pod from starting. + type: string + optional: + default: false + description: Specify whether the file + or its key must be defined. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '.. + type: string + volumeName: + description: The name of the volume + mount containing the env file. + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic resourceFieldRef: description: |- Selects a resource of the container: only resources limits and requests @@ -8431,8 +8949,7 @@ spec: x-kubernetes-map-type: atomic prefix: description: Optional text to prepend to the - name of each environment variable. Must be - a C_IDENTIFIER. + name of each environment variable. type: string secretRef: description: The Secret to select from @@ -8947,8 +9464,9 @@ spec: variable present in a Container. properties: name: - description: Name of the environment variable. - Must be a C_IDENTIFIER. + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. type: string value: description: |- @@ -8999,6 +9517,36 @@ spec: - fieldPath type: object x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: The key within the + env file. An invalid key will + prevent the pod from starting. + type: string + optional: + default: false + description: Specify whether the + file or its key must be defined. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '.. + type: string + volumeName: + description: The name of the volume + mount containing the env file. + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic resourceFieldRef: description: |- Selects a resource of the container: only resources limits and requests @@ -9079,7 +9627,6 @@ spec: prefix: description: Optional text to prepend to the name of each environment variable. - Must be a C_IDENTIFIER. type: string secretRef: description: The Secret to select from @@ -10270,8 +10817,9 @@ spec: present in a Container. properties: name: - description: Name of the environment variable. Must - be a C_IDENTIFIER. + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. type: string value: description: |- @@ -10320,6 +10868,36 @@ spec: - fieldPath type: object x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: The key within the env file. + An invalid key will prevent the pod from + starting. + type: string + optional: + default: false + description: Specify whether the file or + its key must be defined. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '.. + type: string + volumeName: + description: The name of the volume mount + containing the env file. + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic resourceFieldRef: description: |- Selects a resource of the container: only resources limits and requests @@ -10396,7 +10974,7 @@ spec: x-kubernetes-map-type: atomic prefix: description: Optional text to prepend to the name - of each environment variable. Must be a C_IDENTIFIER. + of each environment variable. type: string secretRef: description: The Secret to select from @@ -11298,9 +11876,8 @@ spec: on the host that shares a pod's lifetime. properties: endpoints: - description: |- - endpoints is the endpoint name that details Glusterfs topology. - More info: https://examples.k8s. + description: endpoints is the endpoint name that + details Glusterfs topology. type: string path: description: |- @@ -11717,6 +12294,52 @@ spec: type: array x-kubernetes-list-type: atomic type: object + podCertificate: + description: |- + Projects an auto-rotating credential bundle (private key and certificate + chain) that the pod can use either as a TLS... + properties: + certificateChainPath: + description: |- + Write the certificate chain at this path in the projected volume. + + Most applications should use credentialBundlePath. + type: string + credentialBundlePath: + description: Write the credential bundle + at this path in the projected volume. + type: string + keyPath: + description: |- + Write the key at this path in the projected volume. + + Most applications should use credentialBundlePath. + type: string + keyType: + description: The type of keypair Kubelet + will generate for the pod. + type: string + maxExpirationSeconds: + description: |- + maxExpirationSeconds is the maximum lifetime permitted for the + certificate. + format: int32 + type: integer + signerName: + description: Kubelet's generated CSRs + will be addressed to this signer. + type: string + userAnnotations: + additionalProperties: + type: string + description: |- + userAnnotations allow pod authors to pass additional information to + the signer implementation. + type: object + required: + - keyType + - signerName + type: object secret: description: secret information about the secret data to project @@ -12315,8 +12938,9 @@ spec: in a Container. properties: name: - description: Name of the environment variable. Must - be a C_IDENTIFIER. + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. type: string value: description: |- @@ -12365,6 +12989,35 @@ spec: - fieldPath type: object x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: The key within the env file. An + invalid key will prevent the pod from starting. + type: string + optional: + default: false + description: Specify whether the file or its + key must be defined. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '.. + type: string + volumeName: + description: The name of the volume mount containing + the env file. + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic resourceFieldRef: description: |- Selects a resource of the container: only resources limits and requests @@ -12440,7 +13093,7 @@ spec: x-kubernetes-map-type: atomic prefix: description: Optional text to prepend to the name of - each environment variable. Must be a C_IDENTIFIER. + each environment variable. type: string secretRef: description: The Secret to select from @@ -12625,7 +13278,6 @@ spec: type. type: string required: - - status - type type: object type: array @@ -12708,8 +13360,9 @@ spec: in a Container. properties: name: - description: Name of the environment variable. Must - be a C_IDENTIFIER. + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. type: string value: description: |- @@ -12758,6 +13411,35 @@ spec: - fieldPath type: object x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: The key within the env file. An + invalid key will prevent the pod from starting. + type: string + optional: + default: false + description: Specify whether the file or its + key must be defined. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '.. + type: string + volumeName: + description: The name of the volume mount containing + the env file. + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic resourceFieldRef: description: |- Selects a resource of the container: only resources limits and requests @@ -12833,7 +13515,7 @@ spec: x-kubernetes-map-type: atomic prefix: description: Optional text to prepend to the name of - each environment variable. Must be a C_IDENTIFIER. + each environment variable. type: string secretRef: description: The Secret to select from @@ -12895,10 +13577,32 @@ spec: - pytorch_nlp type: string type: object + packaged: + description: FeastPackagedOptions describes a feature repository + packaged in a feature server image. + properties: + featureRepoPath: + description: FeatureRepoPath is the canonical absolute path + to the feature repository in the image. + type: string + image: + description: Image containing the packaged feature repository. + type: string + required: + - featureRepoPath + type: object + x-kubernetes-validations: + - message: FeatureRepoPath must be a canonical absolute, non-root + path without dot segments or repeated separators. + rule: self.featureRepoPath.startsWith('/') && self.featureRepoPath + != '/' && !self.featureRepoPath.contains('//') && !self.featureRepoPath.endsWith('/') + && !self.featureRepoPath.contains('/./') && !self.featureRepoPath.endsWith('/.') + && !self.featureRepoPath.contains('/../') && !self.featureRepoPath.endsWith('/..') type: object x-kubernetes-validations: - - message: One selection required between init or git. - rule: '[has(self.git), has(self.init)].exists_one(c, c)' + - message: One selection required between init, git, or packaged. + rule: '[has(self.git), has(self.init), has(self.packaged)].exists_one(c, + c)' services: description: FeatureStoreServices defines the desired feast services. An ephemeral onlineStore feature server is deployed by default. @@ -13085,8 +13789,9 @@ spec: present in a Container. properties: name: - description: Name of the environment variable. Must - be a C_IDENTIFIER. + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. type: string value: description: |- @@ -13135,6 +13840,36 @@ spec: - fieldPath type: object x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: The key within the env file. + An invalid key will prevent the pod from + starting. + type: string + optional: + default: false + description: Specify whether the file or + its key must be defined. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '.. + type: string + volumeName: + description: The name of the volume mount + containing the env file. + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic resourceFieldRef: description: |- Selects a resource of the container: only resources limits and requests @@ -13211,7 +13946,7 @@ spec: x-kubernetes-map-type: atomic prefix: description: Optional text to prepend to the name - of each environment variable. Must be a C_IDENTIFIER. + of each environment variable. type: string secretRef: description: The Secret to select from @@ -13581,6 +14316,7 @@ spec: - hybrid - mongodb - aerospike + - scylladb type: string required: - secretRef @@ -13599,8 +14335,9 @@ spec: present in a Container. properties: name: - description: Name of the environment variable. Must - be a C_IDENTIFIER. + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. type: string value: description: |- @@ -13649,6 +14386,36 @@ spec: - fieldPath type: object x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: The key within the env file. + An invalid key will prevent the pod from + starting. + type: string + optional: + default: false + description: Specify whether the file or + its key must be defined. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '.. + type: string + volumeName: + description: The name of the volume mount + containing the env file. + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic resourceFieldRef: description: |- Selects a resource of the container: only resources limits and requests @@ -13725,7 +14492,7 @@ spec: x-kubernetes-map-type: atomic prefix: description: Optional text to prepend to the name - of each environment variable. Must be a C_IDENTIFIER. + of each environment variable. type: string secretRef: description: The Secret to select from @@ -14130,8 +14897,9 @@ spec: present in a Container. properties: name: - description: Name of the environment variable. - Must be a C_IDENTIFIER. + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. type: string value: description: |- @@ -14181,6 +14949,36 @@ spec: - fieldPath type: object x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: The key within the env + file. An invalid key will prevent + the pod from starting. + type: string + optional: + default: false + description: Specify whether the file + or its key must be defined. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '.. + type: string + volumeName: + description: The name of the volume + mount containing the env file. + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic resourceFieldRef: description: |- Selects a resource of the container: only resources limits and requests @@ -14259,8 +15057,7 @@ spec: x-kubernetes-map-type: atomic prefix: description: Optional text to prepend to the - name of each environment variable. Must be - a C_IDENTIFIER. + name of each environment variable. type: string secretRef: description: The Secret to select from @@ -14704,8 +15501,9 @@ spec: in a Container. properties: name: - description: Name of the environment variable. Must - be a C_IDENTIFIER. + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. type: string value: description: |- @@ -14754,6 +15552,35 @@ spec: - fieldPath type: object x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: The key within the env file. An + invalid key will prevent the pod from starting. + type: string + optional: + default: false + description: Specify whether the file or its + key must be defined. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '.. + type: string + volumeName: + description: The name of the volume mount containing + the env file. + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic resourceFieldRef: description: |- Selects a resource of the container: only resources limits and requests @@ -14829,7 +15656,7 @@ spec: x-kubernetes-map-type: atomic prefix: description: Optional text to prepend to the name of - each environment variable. Must be a C_IDENTIFIER. + each environment variable. type: string secretRef: description: The Secret to select from @@ -15723,9 +16550,8 @@ spec: host that shares a pod's lifetime. properties: endpoints: - description: |- - endpoints is the endpoint name that details Glusterfs topology. - More info: https://examples.k8s. + description: endpoints is the endpoint name that details + Glusterfs topology. type: string path: description: |- @@ -16138,6 +16964,52 @@ spec: type: array x-kubernetes-list-type: atomic type: object + podCertificate: + description: |- + Projects an auto-rotating credential bundle (private key and certificate + chain) that the pod can use either as a TLS... + properties: + certificateChainPath: + description: |- + Write the certificate chain at this path in the projected volume. + + Most applications should use credentialBundlePath. + type: string + credentialBundlePath: + description: Write the credential bundle at + this path in the projected volume. + type: string + keyPath: + description: |- + Write the key at this path in the projected volume. + + Most applications should use credentialBundlePath. + type: string + keyType: + description: The type of keypair Kubelet will + generate for the pod. + type: string + maxExpirationSeconds: + description: |- + maxExpirationSeconds is the maximum lifetime permitted for the + certificate. + format: int32 + type: integer + signerName: + description: Kubelet's generated CSRs will + be addressed to this signer. + type: string + userAnnotations: + additionalProperties: + type: string + description: |- + userAnnotations allow pod authors to pass additional information to + the signer implementation. + type: object + required: + - keyType + - signerName + type: object secret: description: secret information about the secret data to project @@ -16576,8 +17448,9 @@ spec: present in a Container. properties: name: - description: Name of the environment variable. Must - be a C_IDENTIFIER. + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. type: string value: description: |- @@ -16626,6 +17499,36 @@ spec: - fieldPath type: object x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: The key within the env file. + An invalid key will prevent the pod from + starting. + type: string + optional: + default: false + description: Specify whether the file or + its key must be defined. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '.. + type: string + volumeName: + description: The name of the volume mount + containing the env file. + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic resourceFieldRef: description: |- Selects a resource of the container: only resources limits and requests @@ -16702,7 +17605,7 @@ spec: x-kubernetes-map-type: atomic prefix: description: Optional text to prepend to the name - of each environment variable. Must be a C_IDENTIFIER. + of each environment variable. type: string secretRef: description: The Secret to select from @@ -16889,7 +17792,6 @@ spec: condition type. type: string required: - - status - type type: object type: array @@ -16974,8 +17876,9 @@ spec: present in a Container. properties: name: - description: Name of the environment variable. Must - be a C_IDENTIFIER. + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. type: string value: description: |- @@ -17024,6 +17927,36 @@ spec: - fieldPath type: object x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: The key within the env file. + An invalid key will prevent the pod from + starting. + type: string + optional: + default: false + description: Specify whether the file or + its key must be defined. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '.. + type: string + volumeName: + description: The name of the volume mount + containing the env file. + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic resourceFieldRef: description: |- Selects a resource of the container: only resources limits and requests @@ -17100,7 +18033,7 @@ spec: x-kubernetes-map-type: atomic prefix: description: Optional text to prepend to the name - of each environment variable. Must be a C_IDENTIFIER. + of each environment variable. type: string secretRef: description: The Secret to select from @@ -17163,10 +18096,32 @@ spec: - pytorch_nlp type: string type: object + packaged: + description: FeastPackagedOptions describes a feature repository + packaged in a feature server image. + properties: + featureRepoPath: + description: FeatureRepoPath is the canonical absolute + path to the feature repository in the image. + type: string + image: + description: Image containing the packaged feature repository. + type: string + required: + - featureRepoPath + type: object + x-kubernetes-validations: + - message: FeatureRepoPath must be a canonical absolute, non-root + path without dot segments or repeated separators. + rule: self.featureRepoPath.startsWith('/') && self.featureRepoPath + != '/' && !self.featureRepoPath.contains('//') && !self.featureRepoPath.endsWith('/') + && !self.featureRepoPath.contains('/./') && !self.featureRepoPath.endsWith('/.') + && !self.featureRepoPath.contains('/../') && !self.featureRepoPath.endsWith('/..') type: object x-kubernetes-validations: - - message: One selection required between init or git. - rule: '[has(self.git), has(self.init)].exists_one(c, c)' + - message: One selection required between init, git, or packaged. + rule: '[has(self.git), has(self.init), has(self.packaged)].exists_one(c, + c)' services: description: FeatureStoreServices defines the desired feast services. An ephemeral onlineStore feature server is deployed by default. @@ -17356,8 +18311,9 @@ spec: present in a Container. properties: name: - description: Name of the environment variable. - Must be a C_IDENTIFIER. + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. type: string value: description: |- @@ -17407,6 +18363,36 @@ spec: - fieldPath type: object x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: The key within the env + file. An invalid key will prevent + the pod from starting. + type: string + optional: + default: false + description: Specify whether the file + or its key must be defined. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '.. + type: string + volumeName: + description: The name of the volume + mount containing the env file. + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic resourceFieldRef: description: |- Selects a resource of the container: only resources limits and requests @@ -17485,8 +18471,7 @@ spec: x-kubernetes-map-type: atomic prefix: description: Optional text to prepend to the - name of each environment variable. Must be - a C_IDENTIFIER. + name of each environment variable. type: string secretRef: description: The Secret to select from @@ -17860,6 +18845,7 @@ spec: - hybrid - mongodb - aerospike + - scylladb type: string required: - secretRef @@ -17879,8 +18865,9 @@ spec: present in a Container. properties: name: - description: Name of the environment variable. - Must be a C_IDENTIFIER. + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. type: string value: description: |- @@ -17930,6 +18917,36 @@ spec: - fieldPath type: object x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: The key within the env + file. An invalid key will prevent + the pod from starting. + type: string + optional: + default: false + description: Specify whether the file + or its key must be defined. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '.. + type: string + volumeName: + description: The name of the volume + mount containing the env file. + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic resourceFieldRef: description: |- Selects a resource of the container: only resources limits and requests @@ -18008,8 +19025,7 @@ spec: x-kubernetes-map-type: atomic prefix: description: Optional text to prepend to the - name of each environment variable. Must be - a C_IDENTIFIER. + name of each environment variable. type: string secretRef: description: The Secret to select from @@ -18423,8 +19439,9 @@ spec: variable present in a Container. properties: name: - description: Name of the environment variable. - Must be a C_IDENTIFIER. + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. type: string value: description: |- @@ -18475,6 +19492,36 @@ spec: - fieldPath type: object x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: The key within the + env file. An invalid key will + prevent the pod from starting. + type: string + optional: + default: false + description: Specify whether the + file or its key must be defined. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '.. + type: string + volumeName: + description: The name of the volume + mount containing the env file. + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic resourceFieldRef: description: |- Selects a resource of the container: only resources limits and requests @@ -18555,7 +19602,6 @@ spec: prefix: description: Optional text to prepend to the name of each environment variable. - Must be a C_IDENTIFIER. type: string secretRef: description: The Secret to select from @@ -19010,8 +20056,9 @@ spec: present in a Container. properties: name: - description: Name of the environment variable. Must - be a C_IDENTIFIER. + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. type: string value: description: |- @@ -19060,6 +20107,36 @@ spec: - fieldPath type: object x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: The key within the env file. + An invalid key will prevent the pod from + starting. + type: string + optional: + default: false + description: Specify whether the file or + its key must be defined. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '.. + type: string + volumeName: + description: The name of the volume mount + containing the env file. + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic resourceFieldRef: description: |- Selects a resource of the container: only resources limits and requests @@ -19136,7 +20213,7 @@ spec: x-kubernetes-map-type: atomic prefix: description: Optional text to prepend to the name - of each environment variable. Must be a C_IDENTIFIER. + of each environment variable. type: string secretRef: description: The Secret to select from @@ -20038,9 +21115,8 @@ spec: on the host that shares a pod's lifetime. properties: endpoints: - description: |- - endpoints is the endpoint name that details Glusterfs topology. - More info: https://examples.k8s. + description: endpoints is the endpoint name that + details Glusterfs topology. type: string path: description: |- @@ -20457,6 +21533,52 @@ spec: type: array x-kubernetes-list-type: atomic type: object + podCertificate: + description: |- + Projects an auto-rotating credential bundle (private key and certificate + chain) that the pod can use either as a TLS... + properties: + certificateChainPath: + description: |- + Write the certificate chain at this path in the projected volume. + + Most applications should use credentialBundlePath. + type: string + credentialBundlePath: + description: Write the credential bundle + at this path in the projected volume. + type: string + keyPath: + description: |- + Write the key at this path in the projected volume. + + Most applications should use credentialBundlePath. + type: string + keyType: + description: The type of keypair Kubelet + will generate for the pod. + type: string + maxExpirationSeconds: + description: |- + maxExpirationSeconds is the maximum lifetime permitted for the + certificate. + format: int32 + type: integer + signerName: + description: Kubelet's generated CSRs + will be addressed to this signer. + type: string + userAnnotations: + additionalProperties: + type: string + description: |- + userAnnotations allow pod authors to pass additional information to + the signer implementation. + type: object + required: + - keyType + - signerName + type: object secret: description: secret information about the secret data to project diff --git a/infra/feast-operator/bundle/manifests/openlineage-secret_v1_secret.yaml b/infra/feast-operator/bundle/manifests/openlineage-secret_v1_secret.yaml index 85d09af493c..40483cc0c43 100644 --- a/infra/feast-operator/bundle/manifests/openlineage-secret_v1_secret.yaml +++ b/infra/feast-operator/bundle/manifests/openlineage-secret_v1_secret.yaml @@ -3,4 +3,4 @@ kind: Secret metadata: name: openlineage-secret stringData: - api_key: your-marquez-api-key # pragma: allowlist secret + api_key: your-marquez-api-key diff --git a/infra/feast-operator/cmd/main.go b/infra/feast-operator/cmd/main.go index 0d833f1469b..5d2bbece7dc 100644 --- a/infra/feast-operator/cmd/main.go +++ b/infra/feast-operator/cmd/main.go @@ -34,8 +34,6 @@ import ( corev1 "k8s.io/api/core/v1" policyv1 "k8s.io/api/policy/v1" rbacv1 "k8s.io/api/rbac/v1" - apierrors "k8s.io/apimachinery/pkg/api/errors" - apimeta "k8s.io/apimachinery/pkg/api/meta" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" utilruntime "k8s.io/apimachinery/pkg/util/runtime" @@ -102,7 +100,7 @@ func main() { var probeAddr string var secureMetrics bool var featureStoreMetrics bool - var tlsOpts []func(*tls.Config) + tlsOpts := make([]func(*tls.Config), 0, 2) flag.StringVar(&metricsAddr, "metrics-bind-address", "0", "The address the metrics endpoint binds to. "+ "Use :8443 for HTTPS or :8080 for HTTP, or leave as 0 to disable the metrics service.") flag.StringVar(&probeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.") @@ -130,46 +128,12 @@ func main() { os.Exit(1) } - tlsProfileFetched := false - tlsProfile, err := tlspkg.FetchAPIServerTLSProfile(context.Background(), bootstrapClient) + tlsResult, err := bootstrapTLS(context.Background(), bootstrapClient) if err != nil { - switch { - case apimeta.IsNoMatchError(err): - setupLog.Info("TLS profile not available, using hardened defaults (non-OpenShift cluster)") - case apierrors.IsNotFound(err): - setupLog.Info("APIServer resource not found, using hardened defaults") - default: - setupLog.Error(err, "unable to read APIServer TLS profile, refusing to start with unknown TLS posture") - os.Exit(1) - } - } else { - tlsProfileFetched = true - tlsConfigFn, unsupported := tlspkg.NewTLSConfigFromProfile(tlsProfile) - if len(unsupported) > 0 { - setupLog.Info("TLS profile contains ciphers unsupported by Go", "unsupported", unsupported) - } - tlsOpts = append(tlsOpts, tlsConfigFn) - } - - tlsAdherenceFetched := false - tlsAdherence, err := tlspkg.FetchAPIServerTLSAdherencePolicy(context.Background(), bootstrapClient) - if err != nil { - switch { - case apimeta.IsNoMatchError(err): - setupLog.Info("TLS adherence policy not available (non-OpenShift cluster)") - case apierrors.IsNotFound(err): - setupLog.Info("APIServer resource not found, skipping adherence policy") - default: - setupLog.Error(err, "unable to read APIServer TLS adherence policy, refusing to start") - os.Exit(1) - } - } else { - tlsAdherenceFetched = true + setupLog.Error(err, "TLS bootstrap failed") + os.Exit(1) } - - tlsOpts = append(tlsOpts, func(c *tls.Config) { - c.NextProtos = []string{"h2", "http/1.1"} - }) + tlsOpts = append(tlsOpts, tlsResult.TLSOpts...) webhookServer := webhook.NewServer(webhook.Options{ TLSOpts: tlsOpts, @@ -271,17 +235,17 @@ func main() { ctx, cancel := context.WithCancel(ctrl.SetupSignalHandler()) defer cancel() - if tlsProfileFetched { + if tlsResult.ProfileFetched { watcher := &tlspkg.SecurityProfileWatcher{ Client: mgr.GetClient(), - InitialTLSProfileSpec: tlsProfile, + InitialTLSProfileSpec: tlsResult.ProfileSpec, OnProfileChange: func(_ context.Context, _, _ configv1.TLSProfileSpec) { setupLog.Info("TLS profile changed, initiating shutdown to reload") cancel() }, } - if tlsAdherenceFetched { - watcher.InitialTLSAdherencePolicy = tlsAdherence + if tlsResult.AdherenceFetched { + watcher.InitialTLSAdherencePolicy = tlsResult.AdherencePolicy watcher.OnAdherencePolicyChange = func(_ context.Context, _, _ configv1.TLSAdherencePolicy) { setupLog.Info("TLS adherence policy changed, initiating shutdown to reload") cancel() diff --git a/infra/feast-operator/cmd/tls_bootstrap.go b/infra/feast-operator/cmd/tls_bootstrap.go new file mode 100644 index 00000000000..6fe33631e02 --- /dev/null +++ b/infra/feast-operator/cmd/tls_bootstrap.go @@ -0,0 +1,134 @@ +/* +Copyright 2024 Feast Community. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "context" + "crypto/tls" + "errors" + "fmt" + "time" + + configv1 "github.com/openshift/api/config/v1" + tlspkg "github.com/openshift/controller-runtime-common/pkg/tls" + apierrors "k8s.io/apimachinery/pkg/api/errors" + apimeta "k8s.io/apimachinery/pkg/api/meta" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/log" +) + +const ( + tlsFetchTimeout = 10 * time.Second + alpnH2 = "h2" + alpnHTTP11 = "http/1.1" +) + +type tlsBootstrapResult struct { + TLSOpts []func(*tls.Config) + ProfileFetched bool + ProfileSpec configv1.TLSProfileSpec + AdherenceFetched bool + AdherencePolicy configv1.TLSAdherencePolicy + UnsupportedCiphers []string +} + +func fetchTLSProfile(ctx context.Context, k8sClient client.Client) (configv1.TLSProfileSpec, bool, error) { + fetchCtx, cancel := context.WithTimeout(ctx, tlsFetchTimeout) + defer cancel() + + profile, err := tlspkg.FetchAPIServerTLSProfile(fetchCtx, k8sClient) + if err != nil { + return classifyTLSProfileError(err) + } + return profile, true, nil +} + +func classifyTLSProfileError(err error) (configv1.TLSProfileSpec, bool, error) { + intermediate := *configv1.TLSProfiles[configv1.TLSProfileIntermediateType] + + switch { + case apimeta.IsNoMatchError(err): + return intermediate, false, nil + case apierrors.IsNotFound(err): + return intermediate, false, nil + case isTransientError(err): + return intermediate, true, nil + default: + return configv1.TLSProfileSpec{}, false, fmt.Errorf("unable to read APIServer TLS profile: %w", err) + } +} + +func fetchTLSAdherencePolicy(ctx context.Context, k8sClient client.Client) (configv1.TLSAdherencePolicy, bool, error) { + fetchCtx, cancel := context.WithTimeout(ctx, tlsFetchTimeout) + defer cancel() + + policy, err := tlspkg.FetchAPIServerTLSAdherencePolicy(fetchCtx, k8sClient) + if err == nil { + return policy, true, nil + } + + switch { + case apimeta.IsNoMatchError(err), + apierrors.IsNotFound(err), + isTransientError(err): + return "", false, nil + default: + return "", false, fmt.Errorf("unable to read APIServer TLS adherence policy: %w", err) + } +} + +func bootstrapTLS(ctx context.Context, k8sClient client.Client) (*tlsBootstrapResult, error) { + logger := log.FromContext(ctx) + result := &tlsBootstrapResult{ + TLSOpts: make([]func(*tls.Config), 0, 2), + } + + profile, profileFetched, err := fetchTLSProfile(ctx, k8sClient) + if err != nil { + return nil, err + } + result.ProfileFetched = profileFetched + result.ProfileSpec = profile + + tlsConfigFn, unsupported := tlspkg.NewTLSConfigFromProfile(profile) + result.UnsupportedCiphers = unsupported + if len(unsupported) > 0 { + logger.Info("TLS profile contains ciphers unsupported by Go", "unsupported", unsupported) + } + result.TLSOpts = append(result.TLSOpts, tlsConfigFn) + + adherence, adherenceFetched, err := fetchTLSAdherencePolicy(ctx, k8sClient) + if err != nil { + return nil, err + } + result.AdherenceFetched = adherenceFetched + result.AdherencePolicy = adherence + + result.TLSOpts = append(result.TLSOpts, func(c *tls.Config) { + c.NextProtos = []string{alpnH2, alpnHTTP11} + }) + + return result, nil +} + +func isTransientError(err error) bool { + return apierrors.IsServiceUnavailable(err) || + apierrors.IsTimeout(err) || + apierrors.IsServerTimeout(err) || + apierrors.IsTooManyRequests(err) || + errors.Is(err, context.DeadlineExceeded) +} diff --git a/infra/feast-operator/cmd/tls_bootstrap_test.go b/infra/feast-operator/cmd/tls_bootstrap_test.go new file mode 100644 index 00000000000..0bb31c14bf6 --- /dev/null +++ b/infra/feast-operator/cmd/tls_bootstrap_test.go @@ -0,0 +1,347 @@ +/* +Copyright 2024 Feast Community. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "context" + "crypto/tls" + "errors" + "testing" + + configv1 "github.com/openshift/api/config/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/meta" + "k8s.io/apimachinery/pkg/runtime/schema" +) + +func intermediateProfile() configv1.TLSProfileSpec { + return *configv1.TLSProfiles[configv1.TLSProfileIntermediateType] +} + +func TestClassifyTLSProfileError(t *testing.T) { + tests := []struct { + name string + err error + wantProfileFetched bool + wantError bool + wantIntermediate bool + }{ + { + name: "NoMatchError returns Intermediate defaults, profileFetched=false", + err: &meta.NoResourceMatchError{PartialResource: schema.GroupVersionResource{Group: "config.openshift.io"}}, + wantProfileFetched: false, + wantError: false, + wantIntermediate: true, + }, + { + name: "NotFound returns Intermediate defaults, profileFetched=false", + err: apierrors.NewNotFound(schema.GroupResource{Group: "config.openshift.io", Resource: "apiservers"}, "cluster"), + wantProfileFetched: false, + wantError: false, + wantIntermediate: true, + }, + { + name: "ServiceUnavailable is transient, profileFetched=true", + err: apierrors.NewServiceUnavailable("api server down"), + wantProfileFetched: true, + wantError: false, + wantIntermediate: true, + }, + { + name: "Timeout is transient, profileFetched=true", + err: apierrors.NewTimeoutError("timed out", 5), + wantProfileFetched: true, + wantError: false, + wantIntermediate: true, + }, + { + name: "ServerTimeout is transient, profileFetched=true", + err: apierrors.NewServerTimeout(schema.GroupResource{Group: "config.openshift.io", Resource: "apiservers"}, "GET", 5), + wantProfileFetched: true, + wantError: false, + wantIntermediate: true, + }, + { + name: "TooManyRequests is transient, profileFetched=true", + err: apierrors.NewTooManyRequests("throttled", 5), + wantProfileFetched: true, + wantError: false, + wantIntermediate: true, + }, + { + name: "DeadlineExceeded is transient, profileFetched=true", + err: context.DeadlineExceeded, + wantProfileFetched: true, + wantError: false, + wantIntermediate: true, + }, + { + name: "Forbidden is fatal, returns error", + err: apierrors.NewForbidden(schema.GroupResource{Group: "config.openshift.io", Resource: "apiservers"}, "cluster", errors.New("RBAC")), + wantProfileFetched: false, + wantError: true, + wantIntermediate: false, + }, + { + name: "Unauthorized is fatal, returns error", + err: apierrors.NewUnauthorized("no token"), + wantProfileFetched: false, + wantError: true, + wantIntermediate: false, + }, + { + name: "InternalServerError is fatal, returns error", + err: apierrors.NewInternalError(errors.New("crash")), + wantProfileFetched: false, + wantError: true, + wantIntermediate: false, + }, + { + name: "Generic error is fatal, returns error", + err: errors.New("something unexpected"), + wantProfileFetched: false, + wantError: true, + wantIntermediate: false, + }, + { + name: "Wrapped DeadlineExceeded is transient", + err: errors.Join(errors.New("fetch failed"), context.DeadlineExceeded), + wantProfileFetched: true, + wantError: false, + wantIntermediate: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + profile, fetched, err := classifyTLSProfileError(tt.err) + + if tt.wantError && err == nil { + t.Errorf("expected error, got nil") + } + if !tt.wantError && err != nil { + t.Errorf("unexpected error: %v", err) + } + if fetched != tt.wantProfileFetched { + t.Errorf("profileFetched = %v, want %v", fetched, tt.wantProfileFetched) + } + if tt.wantIntermediate { + intermediate := intermediateProfile() + if profile.MinTLSVersion != intermediate.MinTLSVersion { + t.Errorf("MinTLSVersion = %v, want %v (Intermediate)", profile.MinTLSVersion, intermediate.MinTLSVersion) + } + } + }) + } +} + +func TestIsTransientError(t *testing.T) { + tests := []struct { + name string + err error + want bool + }{ + {"ServiceUnavailable", apierrors.NewServiceUnavailable("down"), true}, + {"Timeout", apierrors.NewTimeoutError("slow", 5), true}, + {"ServerTimeout", apierrors.NewServerTimeout(schema.GroupResource{}, "GET", 5), true}, + {"TooManyRequests", apierrors.NewTooManyRequests("throttled", 5), true}, + {"DeadlineExceeded", context.DeadlineExceeded, true}, + {"Wrapped DeadlineExceeded", errors.Join(errors.New("wrapper"), context.DeadlineExceeded), true}, + {"NotFound", apierrors.NewNotFound(schema.GroupResource{}, "x"), false}, + {"Forbidden", apierrors.NewForbidden(schema.GroupResource{}, "x", errors.New("RBAC")), false}, + {"Unauthorized", apierrors.NewUnauthorized("no token"), false}, + {"InternalError", apierrors.NewInternalError(errors.New("crash")), false}, + {"Generic error", errors.New("oops"), false}, + {"Nil", nil, false}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := isTransientError(tt.err); got != tt.want { + t.Errorf("isTransientError() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestIntermediateProfileHasExpectedDefaults(t *testing.T) { + profile := intermediateProfile() + + if profile.MinTLSVersion != configv1.VersionTLS12 { + t.Errorf("Intermediate MinTLSVersion = %v, want %v", profile.MinTLSVersion, configv1.VersionTLS12) + } + if len(profile.Ciphers) == 0 { + t.Error("Intermediate profile should have non-empty cipher list") + } +} + +func TestTLSConfigFromIntermediateProfile(t *testing.T) { + profile := intermediateProfile() + tlsConfigFn := configv1ToTLSConfig(profile) + + cfg := &tls.Config{} + tlsConfigFn(cfg) + + if cfg.MinVersion != tls.VersionTLS12 { + t.Errorf("MinVersion = %v, want %v (TLS 1.2)", cfg.MinVersion, tls.VersionTLS12) + } + if len(cfg.CipherSuites) == 0 { + t.Error("CipherSuites should not be empty for Intermediate profile") + } +} + +func configv1ToTLSConfig(profile configv1.TLSProfileSpec) func(*tls.Config) { + // Thin wrapper to test the actual conversion without importing tlspkg in tests. + // tlspkg.NewTLSConfigFromProfile is what main.go uses. + var minVersion uint16 + switch profile.MinTLSVersion { + case configv1.VersionTLS10: + minVersion = tls.VersionTLS10 + case configv1.VersionTLS11: + minVersion = tls.VersionTLS11 + case configv1.VersionTLS12: + minVersion = tls.VersionTLS12 + case configv1.VersionTLS13: + minVersion = tls.VersionTLS13 + } + + return func(c *tls.Config) { + c.MinVersion = minVersion + c.CipherSuites = mapCiphers(profile.Ciphers) + } +} + +func mapCiphers(names []string) []uint16 { + cipherMap := map[string]uint16{ + "TLS_AES_128_GCM_SHA256": tls.TLS_AES_128_GCM_SHA256, + "TLS_AES_256_GCM_SHA384": tls.TLS_AES_256_GCM_SHA384, + "TLS_CHACHA20_POLY1305_SHA256": tls.TLS_CHACHA20_POLY1305_SHA256, + "TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256": tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256, + "TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256": tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, + "TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384": tls.TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384, + "TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384": tls.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384, + "TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256": tls.TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256, + "TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256": tls.TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256, + } + var ids []uint16 + for _, name := range names { + if id, ok := cipherMap[name]; ok { + ids = append(ids, id) + } + } + return ids +} + +func TestClassifyTLSProfileError_AllTransientErrorsSetProfileFetched(t *testing.T) { + transientErrors := []error{ + apierrors.NewServiceUnavailable("down"), + apierrors.NewTimeoutError("slow", 5), + apierrors.NewServerTimeout(schema.GroupResource{Group: "config.openshift.io", Resource: "apiservers"}, "GET", 5), + apierrors.NewTooManyRequests("throttled", 5), + context.DeadlineExceeded, + } + + for _, err := range transientErrors { + _, fetched, classifyErr := classifyTLSProfileError(err) + if classifyErr != nil { + t.Errorf("transient error %T should not return error, got: %v", err, classifyErr) + } + if !fetched { + t.Errorf("transient error %T should set profileFetched=true", err) + } + } +} + +func TestClassifyTLSProfileError_NonTransientErrorsDoNotSetProfileFetched(t *testing.T) { + nonTransientErrors := []error{ + &meta.NoResourceMatchError{PartialResource: schema.GroupVersionResource{Group: "config.openshift.io"}}, + apierrors.NewNotFound(schema.GroupResource{}, "cluster"), + } + + for _, err := range nonTransientErrors { + _, fetched, classifyErr := classifyTLSProfileError(err) + if classifyErr != nil { + t.Errorf("graceful error %T should not return error, got: %v", err, classifyErr) + } + if fetched { + t.Errorf("graceful error %T should set profileFetched=false", err) + } + } +} + +func TestClassifyTLSProfileError_FatalErrorsReturnError(t *testing.T) { + fatalErrors := []error{ + apierrors.NewForbidden(schema.GroupResource{}, "cluster", errors.New("RBAC")), + apierrors.NewUnauthorized("no token"), + apierrors.NewInternalError(errors.New("crash")), + errors.New("unexpected"), + } + + for _, err := range fatalErrors { + _, _, classifyErr := classifyTLSProfileError(err) + if classifyErr == nil { + t.Errorf("fatal error %T should return error", err) + } + } +} + +func TestClassifyTLSProfileError_IntermediateProfileAlwaysApplied(t *testing.T) { + allNonFatalErrors := []error{ + &meta.NoResourceMatchError{PartialResource: schema.GroupVersionResource{Group: "config.openshift.io"}}, + apierrors.NewNotFound(schema.GroupResource{}, "cluster"), + apierrors.NewServiceUnavailable("down"), + apierrors.NewTimeoutError("slow", 5), + apierrors.NewServerTimeout(schema.GroupResource{}, "GET", 5), + apierrors.NewTooManyRequests("throttled", 5), + context.DeadlineExceeded, + } + + intermediate := intermediateProfile() + for _, err := range allNonFatalErrors { + profile, _, classifyErr := classifyTLSProfileError(err) + if classifyErr != nil { + t.Fatalf("unexpected error for %T: %v", err, classifyErr) + } + if profile.MinTLSVersion != intermediate.MinTLSVersion { + t.Errorf("for error %T: MinTLSVersion = %v, want Intermediate (%v)", err, profile.MinTLSVersion, intermediate.MinTLSVersion) + } + if len(profile.Ciphers) != len(intermediate.Ciphers) { + t.Errorf("for error %T: got %d ciphers, want %d (Intermediate)", err, len(profile.Ciphers), len(intermediate.Ciphers)) + } + } +} + +func TestTLSBootstrapResult_NextProtosAlwaysSet(t *testing.T) { + // Verify that the TLSOpts from bootstrapTLS always include ALPN with h2 and http/1.1. + // We can't call bootstrapTLS without a real client, but we can verify the function + // in tls_bootstrap.go sets NextProtos. + result := &tlsBootstrapResult{ + TLSOpts: make([]func(*tls.Config), 0, 2), + } + result.TLSOpts = append(result.TLSOpts, func(c *tls.Config) { + c.NextProtos = []string{"h2", alpnHTTP11} + }) + + cfg := &tls.Config{} + for _, opt := range result.TLSOpts { + opt(cfg) + } + + if len(cfg.NextProtos) != 2 || cfg.NextProtos[0] != "h2" || cfg.NextProtos[1] != alpnHTTP11 { + t.Errorf("NextProtos = %v, want [h2, %s]", cfg.NextProtos, alpnHTTP11) + } +} diff --git a/infra/feast-operator/config/component_metadata.yaml b/infra/feast-operator/config/component_metadata.yaml index 129d6029155..7ee38fdb165 100644 --- a/infra/feast-operator/config/component_metadata.yaml +++ b/infra/feast-operator/config/component_metadata.yaml @@ -1,5 +1,5 @@ # This file is required to configure Feast release information for ODH/RHOAI Operator releases: - name: Feast - version: 0.64.0 + version: 0.65.0 repoUrl: https://github.com/feast-dev/feast diff --git a/infra/feast-operator/config/crd/bases/feast.dev_featurestores.yaml b/infra/feast-operator/config/crd/bases/feast.dev_featurestores.yaml index 11650995e04..8184906e14d 100644 --- a/infra/feast-operator/config/crd/bases/feast.dev_featurestores.yaml +++ b/infra/feast-operator/config/crd/bases/feast.dev_featurestores.yaml @@ -810,10 +810,32 @@ spec: - pytorch_nlp type: string type: object + packaged: + description: FeastPackagedOptions describes a feature repository + packaged in a feature server image. + properties: + featureRepoPath: + description: FeatureRepoPath is the canonical absolute path + to the feature repository in the image. + type: string + image: + description: Image containing the packaged feature repository. + type: string + required: + - featureRepoPath + type: object + x-kubernetes-validations: + - message: FeatureRepoPath must be a canonical absolute, non-root + path without dot segments or repeated separators. + rule: self.featureRepoPath.startsWith('/') && self.featureRepoPath + != '/' && !self.featureRepoPath.contains('//') && !self.featureRepoPath.endsWith('/') + && !self.featureRepoPath.contains('/./') && !self.featureRepoPath.endsWith('/.') + && !self.featureRepoPath.contains('/../') && !self.featureRepoPath.endsWith('/..') type: object x-kubernetes-validations: - - message: One selection required between init or git. - rule: '[has(self.git), has(self.init)].exists_one(c, c)' + - message: One selection required between init, git, or packaged. + rule: '[has(self.git), has(self.init), has(self.packaged)].exists_one(c, + c)' materialization: description: |- Materialization controls feature materialization behavior (batch size, pull strategy). @@ -1723,6 +1745,10 @@ spec: disableInitContainers: description: Disable the 'feast repo initialization' initContainer type: boolean + initImage: + description: InitImage overrides the image for init containers + (feast-init, feast-apply). + type: string offlineStore: description: OfflineStore configures the offline store service properties: @@ -2258,6 +2284,11 @@ spec: onlineStore: description: OnlineStore configures the online store service properties: + disabled: + description: |- + Disabled skips deploying the online store service entirely, including its + serving pod and persistence. + type: boolean persistence: description: OnlineStorePersistence configures the persistence settings for the online store service @@ -2400,6 +2431,7 @@ spec: - hybrid - mongodb - aerospike + - scylladb type: string required: - secretRef @@ -7098,10 +7130,32 @@ spec: - pytorch_nlp type: string type: object + packaged: + description: FeastPackagedOptions describes a feature repository + packaged in a feature server image. + properties: + featureRepoPath: + description: FeatureRepoPath is the canonical absolute + path to the feature repository in the image. + type: string + image: + description: Image containing the packaged feature repository. + type: string + required: + - featureRepoPath + type: object + x-kubernetes-validations: + - message: FeatureRepoPath must be a canonical absolute, non-root + path without dot segments or repeated separators. + rule: self.featureRepoPath.startsWith('/') && self.featureRepoPath + != '/' && !self.featureRepoPath.contains('//') && !self.featureRepoPath.endsWith('/') + && !self.featureRepoPath.contains('/./') && !self.featureRepoPath.endsWith('/.') + && !self.featureRepoPath.contains('/../') && !self.featureRepoPath.endsWith('/..') type: object x-kubernetes-validations: - - message: One selection required between init or git. - rule: '[has(self.git), has(self.init)].exists_one(c, c)' + - message: One selection required between init, git, or packaged. + rule: '[has(self.git), has(self.init), has(self.packaged)].exists_one(c, + c)' materialization: description: |- Materialization controls feature materialization behavior (batch size, pull strategy). @@ -8019,6 +8073,10 @@ spec: disableInitContainers: description: Disable the 'feast repo initialization' initContainer type: boolean + initImage: + description: InitImage overrides the image for init containers + (feast-init, feast-apply). + type: string offlineStore: description: OfflineStore configures the offline store service properties: @@ -8561,6 +8619,11 @@ spec: onlineStore: description: OnlineStore configures the online store service properties: + disabled: + description: |- + Disabled skips deploying the online store service entirely, including its + serving pod and persistence. + type: boolean persistence: description: OnlineStorePersistence configures the persistence settings for the online store service @@ -8706,6 +8769,7 @@ spec: - hybrid - mongodb - aerospike + - scylladb type: string required: - secretRef @@ -13513,10 +13577,32 @@ spec: - pytorch_nlp type: string type: object + packaged: + description: FeastPackagedOptions describes a feature repository + packaged in a feature server image. + properties: + featureRepoPath: + description: FeatureRepoPath is the canonical absolute path + to the feature repository in the image. + type: string + image: + description: Image containing the packaged feature repository. + type: string + required: + - featureRepoPath + type: object + x-kubernetes-validations: + - message: FeatureRepoPath must be a canonical absolute, non-root + path without dot segments or repeated separators. + rule: self.featureRepoPath.startsWith('/') && self.featureRepoPath + != '/' && !self.featureRepoPath.contains('//') && !self.featureRepoPath.endsWith('/') + && !self.featureRepoPath.contains('/./') && !self.featureRepoPath.endsWith('/.') + && !self.featureRepoPath.contains('/../') && !self.featureRepoPath.endsWith('/..') type: object x-kubernetes-validations: - - message: One selection required between init or git. - rule: '[has(self.git), has(self.init)].exists_one(c, c)' + - message: One selection required between init, git, or packaged. + rule: '[has(self.git), has(self.init), has(self.packaged)].exists_one(c, + c)' services: description: FeatureStoreServices defines the desired feast services. An ephemeral onlineStore feature server is deployed by default. @@ -14230,6 +14316,7 @@ spec: - hybrid - mongodb - aerospike + - scylladb type: string required: - secretRef @@ -18009,10 +18096,32 @@ spec: - pytorch_nlp type: string type: object + packaged: + description: FeastPackagedOptions describes a feature repository + packaged in a feature server image. + properties: + featureRepoPath: + description: FeatureRepoPath is the canonical absolute + path to the feature repository in the image. + type: string + image: + description: Image containing the packaged feature repository. + type: string + required: + - featureRepoPath + type: object + x-kubernetes-validations: + - message: FeatureRepoPath must be a canonical absolute, non-root + path without dot segments or repeated separators. + rule: self.featureRepoPath.startsWith('/') && self.featureRepoPath + != '/' && !self.featureRepoPath.contains('//') && !self.featureRepoPath.endsWith('/') + && !self.featureRepoPath.contains('/./') && !self.featureRepoPath.endsWith('/.') + && !self.featureRepoPath.contains('/../') && !self.featureRepoPath.endsWith('/..') type: object x-kubernetes-validations: - - message: One selection required between init or git. - rule: '[has(self.git), has(self.init)].exists_one(c, c)' + - message: One selection required between init, git, or packaged. + rule: '[has(self.git), has(self.init), has(self.packaged)].exists_one(c, + c)' services: description: FeatureStoreServices defines the desired feast services. An ephemeral onlineStore feature server is deployed by default. @@ -18736,6 +18845,7 @@ spec: - hybrid - mongodb - aerospike + - scylladb type: string required: - secretRef diff --git a/infra/feast-operator/config/default/related_image_fs_patch.yaml b/infra/feast-operator/config/default/related_image_fs_patch.yaml index 4e314795de2..5ab07fd91e1 100644 --- a/infra/feast-operator/config/default/related_image_fs_patch.yaml +++ b/infra/feast-operator/config/default/related_image_fs_patch.yaml @@ -9,6 +9,6 @@ spec: - name: manager env: - name: RELATED_IMAGE_FEATURE_SERVER - value: quay.io/feastdev/feature-server:0.64.0 + value: quay.io/feastdev/feature-server:0.65.0 - name: RELATED_IMAGE_CRON_JOB value: quay.io/openshift/origin-cli:4.17 diff --git a/infra/feast-operator/config/manager/kustomization.yaml b/infra/feast-operator/config/manager/kustomization.yaml index c713f0fe470..5f3ce6cadda 100644 --- a/infra/feast-operator/config/manager/kustomization.yaml +++ b/infra/feast-operator/config/manager/kustomization.yaml @@ -5,4 +5,4 @@ kind: Kustomization images: - name: controller newName: quay.io/feastdev/feast-operator - newTag: 0.64.0 + newTag: 0.65.0 diff --git a/infra/feast-operator/config/overlays/odh/params.env b/infra/feast-operator/config/overlays/odh/params.env index 49bbac59c71..b0d55d6bd70 100644 --- a/infra/feast-operator/config/overlays/odh/params.env +++ b/infra/feast-operator/config/overlays/odh/params.env @@ -1,5 +1,5 @@ -RELATED_IMAGE_FEAST_OPERATOR=quay.io/feastdev/feast-operator:0.64.0 -RELATED_IMAGE_FEATURE_SERVER=quay.io/feastdev/feature-server:0.64.0 +RELATED_IMAGE_FEAST_OPERATOR=quay.io/feastdev/feast-operator:0.65.0 +RELATED_IMAGE_FEATURE_SERVER=quay.io/feastdev/feature-server:0.65.0 RELATED_IMAGE_CRON_JOB=quay.io/openshift/origin-cli:4.17 # Set at deploy time by the Open Data Hub operator from GatewayConfig (external OIDC). OIDC_ISSUER_URL= diff --git a/infra/feast-operator/config/overlays/rhoai/params.env b/infra/feast-operator/config/overlays/rhoai/params.env index 92e02fb51b3..dabacfd458c 100644 --- a/infra/feast-operator/config/overlays/rhoai/params.env +++ b/infra/feast-operator/config/overlays/rhoai/params.env @@ -1,5 +1,5 @@ -RELATED_IMAGE_FEAST_OPERATOR=quay.io/feastdev/feast-operator:0.64.0 -RELATED_IMAGE_FEATURE_SERVER=quay.io/feastdev/feature-server:0.64.0 +RELATED_IMAGE_FEAST_OPERATOR=quay.io/feastdev/feast-operator:0.65.0 +RELATED_IMAGE_FEATURE_SERVER=quay.io/feastdev/feature-server:0.65.0 RELATED_IMAGE_CRON_JOB=registry.redhat.io/openshift4/ose-cli@sha256:bc35a9fc663baf0d6493cc57e89e77a240a36c43cf38fb78d8e61d3b87cf5cc5 # Set at deploy time by the Open Data Hub operator from GatewayConfig (external OIDC). OIDC_ISSUER_URL= \ No newline at end of file diff --git a/infra/feast-operator/config/rbac/role.yaml b/infra/feast-operator/config/rbac/role.yaml index f6e6801dfa8..a79dca283ed 100644 --- a/infra/feast-operator/config/rbac/role.yaml +++ b/infra/feast-operator/config/rbac/role.yaml @@ -9,11 +9,11 @@ rules: resources: - configmaps - persistentvolumeclaims - - serviceaccounts - services verbs: - create - delete + - deletecollection - get - list - update @@ -22,18 +22,45 @@ rules: - "" resources: - namespaces - - pods - secrets verbs: - get - list - watch +- apiGroups: + - "" + resources: + - pods + verbs: + - create + - delete + - deletecollection + - get + - list + - watch - apiGroups: - "" resources: - pods/exec verbs: - create +- apiGroups: + - "" + resources: + - pods/log + verbs: + - get +- apiGroups: + - "" + resources: + - serviceaccounts + verbs: + - create + - delete + - get + - list + - update + - watch - apiGroups: - apps resources: @@ -158,3 +185,11 @@ rules: - list - update - watch +- apiGroups: + - sparkoperator.k8s.io + resources: + - sparkapplications + verbs: + - create + - delete + - get diff --git a/infra/feast-operator/config/samples/v1_featurestore_oidc_auth.yaml b/infra/feast-operator/config/samples/v1_featurestore_oidc_auth.yaml index 7ef676d0297..97f325bb0bd 100644 --- a/infra/feast-operator/config/samples/v1_featurestore_oidc_auth.yaml +++ b/infra/feast-operator/config/samples/v1_featurestore_oidc_auth.yaml @@ -19,3 +19,7 @@ stringData: client_secret: client_secret username: username password: password + # Optional: enable audience/issuer claim verification on the servers. + # Values must match the claims in the tokens your IdP issues. + # audience: api://feast-feature-server + # issuer: https://idp.example.com/realms/feast diff --git a/infra/feast-operator/config/samples/v1_featurestore_packaged.yaml b/infra/feast-operator/config/samples/v1_featurestore_packaged.yaml new file mode 100644 index 00000000000..4b334b9d3d8 --- /dev/null +++ b/infra/feast-operator/config/samples/v1_featurestore_packaged.yaml @@ -0,0 +1,10 @@ +apiVersion: feast.dev/v1 +kind: FeatureStore +metadata: + name: sample-packaged +spec: + feastProject: sample_packaged + feastProjectDir: + packaged: + image: registry.example.com/feature-server@sha256:0123456789abcdef + featureRepoPath: /opt/feast/feature_repo diff --git a/infra/feast-operator/dist/install.yaml b/infra/feast-operator/dist/install.yaml index 57853e4b83f..be85a29a7b2 100644 --- a/infra/feast-operator/dist/install.yaml +++ b/infra/feast-operator/dist/install.yaml @@ -818,10 +818,32 @@ spec: - pytorch_nlp type: string type: object + packaged: + description: FeastPackagedOptions describes a feature repository + packaged in a feature server image. + properties: + featureRepoPath: + description: FeatureRepoPath is the canonical absolute path + to the feature repository in the image. + type: string + image: + description: Image containing the packaged feature repository. + type: string + required: + - featureRepoPath + type: object + x-kubernetes-validations: + - message: FeatureRepoPath must be a canonical absolute, non-root + path without dot segments or repeated separators. + rule: self.featureRepoPath.startsWith('/') && self.featureRepoPath + != '/' && !self.featureRepoPath.contains('//') && !self.featureRepoPath.endsWith('/') + && !self.featureRepoPath.contains('/./') && !self.featureRepoPath.endsWith('/.') + && !self.featureRepoPath.contains('/../') && !self.featureRepoPath.endsWith('/..') type: object x-kubernetes-validations: - - message: One selection required between init or git. - rule: '[has(self.git), has(self.init)].exists_one(c, c)' + - message: One selection required between init, git, or packaged. + rule: '[has(self.git), has(self.init), has(self.packaged)].exists_one(c, + c)' materialization: description: |- Materialization controls feature materialization behavior (batch size, pull strategy). @@ -1731,6 +1753,10 @@ spec: disableInitContainers: description: Disable the 'feast repo initialization' initContainer type: boolean + initImage: + description: InitImage overrides the image for init containers + (feast-init, feast-apply). + type: string offlineStore: description: OfflineStore configures the offline store service properties: @@ -2266,6 +2292,11 @@ spec: onlineStore: description: OnlineStore configures the online store service properties: + disabled: + description: |- + Disabled skips deploying the online store service entirely, including its + serving pod and persistence. + type: boolean persistence: description: OnlineStorePersistence configures the persistence settings for the online store service @@ -2408,6 +2439,7 @@ spec: - hybrid - mongodb - aerospike + - scylladb type: string required: - secretRef @@ -7106,10 +7138,32 @@ spec: - pytorch_nlp type: string type: object + packaged: + description: FeastPackagedOptions describes a feature repository + packaged in a feature server image. + properties: + featureRepoPath: + description: FeatureRepoPath is the canonical absolute + path to the feature repository in the image. + type: string + image: + description: Image containing the packaged feature repository. + type: string + required: + - featureRepoPath + type: object + x-kubernetes-validations: + - message: FeatureRepoPath must be a canonical absolute, non-root + path without dot segments or repeated separators. + rule: self.featureRepoPath.startsWith('/') && self.featureRepoPath + != '/' && !self.featureRepoPath.contains('//') && !self.featureRepoPath.endsWith('/') + && !self.featureRepoPath.contains('/./') && !self.featureRepoPath.endsWith('/.') + && !self.featureRepoPath.contains('/../') && !self.featureRepoPath.endsWith('/..') type: object x-kubernetes-validations: - - message: One selection required between init or git. - rule: '[has(self.git), has(self.init)].exists_one(c, c)' + - message: One selection required between init, git, or packaged. + rule: '[has(self.git), has(self.init), has(self.packaged)].exists_one(c, + c)' materialization: description: |- Materialization controls feature materialization behavior (batch size, pull strategy). @@ -8027,6 +8081,10 @@ spec: disableInitContainers: description: Disable the 'feast repo initialization' initContainer type: boolean + initImage: + description: InitImage overrides the image for init containers + (feast-init, feast-apply). + type: string offlineStore: description: OfflineStore configures the offline store service properties: @@ -8569,6 +8627,11 @@ spec: onlineStore: description: OnlineStore configures the online store service properties: + disabled: + description: |- + Disabled skips deploying the online store service entirely, including its + serving pod and persistence. + type: boolean persistence: description: OnlineStorePersistence configures the persistence settings for the online store service @@ -8714,6 +8777,7 @@ spec: - hybrid - mongodb - aerospike + - scylladb type: string required: - secretRef @@ -13521,10 +13585,32 @@ spec: - pytorch_nlp type: string type: object + packaged: + description: FeastPackagedOptions describes a feature repository + packaged in a feature server image. + properties: + featureRepoPath: + description: FeatureRepoPath is the canonical absolute path + to the feature repository in the image. + type: string + image: + description: Image containing the packaged feature repository. + type: string + required: + - featureRepoPath + type: object + x-kubernetes-validations: + - message: FeatureRepoPath must be a canonical absolute, non-root + path without dot segments or repeated separators. + rule: self.featureRepoPath.startsWith('/') && self.featureRepoPath + != '/' && !self.featureRepoPath.contains('//') && !self.featureRepoPath.endsWith('/') + && !self.featureRepoPath.contains('/./') && !self.featureRepoPath.endsWith('/.') + && !self.featureRepoPath.contains('/../') && !self.featureRepoPath.endsWith('/..') type: object x-kubernetes-validations: - - message: One selection required between init or git. - rule: '[has(self.git), has(self.init)].exists_one(c, c)' + - message: One selection required between init, git, or packaged. + rule: '[has(self.git), has(self.init), has(self.packaged)].exists_one(c, + c)' services: description: FeatureStoreServices defines the desired feast services. An ephemeral onlineStore feature server is deployed by default. @@ -14238,6 +14324,7 @@ spec: - hybrid - mongodb - aerospike + - scylladb type: string required: - secretRef @@ -18017,10 +18104,32 @@ spec: - pytorch_nlp type: string type: object + packaged: + description: FeastPackagedOptions describes a feature repository + packaged in a feature server image. + properties: + featureRepoPath: + description: FeatureRepoPath is the canonical absolute + path to the feature repository in the image. + type: string + image: + description: Image containing the packaged feature repository. + type: string + required: + - featureRepoPath + type: object + x-kubernetes-validations: + - message: FeatureRepoPath must be a canonical absolute, non-root + path without dot segments or repeated separators. + rule: self.featureRepoPath.startsWith('/') && self.featureRepoPath + != '/' && !self.featureRepoPath.contains('//') && !self.featureRepoPath.endsWith('/') + && !self.featureRepoPath.contains('/./') && !self.featureRepoPath.endsWith('/.') + && !self.featureRepoPath.contains('/../') && !self.featureRepoPath.endsWith('/..') type: object x-kubernetes-validations: - - message: One selection required between init or git. - rule: '[has(self.git), has(self.init)].exists_one(c, c)' + - message: One selection required between init, git, or packaged. + rule: '[has(self.git), has(self.init), has(self.packaged)].exists_one(c, + c)' services: description: FeatureStoreServices defines the desired feast services. An ephemeral onlineStore feature server is deployed by default. @@ -18744,6 +18853,7 @@ spec: - hybrid - mongodb - aerospike + - scylladb type: string required: - secretRef @@ -22033,11 +22143,11 @@ rules: resources: - configmaps - persistentvolumeclaims - - serviceaccounts - services verbs: - create - delete + - deletecollection - get - list - update @@ -22046,18 +22156,45 @@ rules: - "" resources: - namespaces - - pods - secrets verbs: - get - list - watch +- apiGroups: + - "" + resources: + - pods + verbs: + - create + - delete + - deletecollection + - get + - list + - watch - apiGroups: - "" resources: - pods/exec verbs: - create +- apiGroups: + - "" + resources: + - pods/log + verbs: + - get +- apiGroups: + - "" + resources: + - serviceaccounts + verbs: + - create + - delete + - get + - list + - update + - watch - apiGroups: - apps resources: @@ -22182,6 +22319,14 @@ rules: - list - update - watch +- apiGroups: + - sparkoperator.k8s.io + resources: + - sparkapplications + verbs: + - create + - delete + - get --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole @@ -22317,14 +22462,14 @@ spec: - /manager env: - name: RELATED_IMAGE_FEATURE_SERVER - value: quay.io/feastdev/feature-server:0.64.0 + value: quay.io/feastdev/feature-server:0.65.0 - name: RELATED_IMAGE_CRON_JOB value: quay.io/openshift/origin-cli:4.17 - name: GOMEMLIMIT value: 230MiB - name: OIDC_ISSUER_URL value: "" - image: quay.io/feastdev/feast-operator:0.64.0 + image: quay.io/feastdev/feast-operator:0.65.0 livenessProbe: httpGet: path: /healthz diff --git a/infra/feast-operator/dist/operator-e2e-tests b/infra/feast-operator/dist/operator-e2e-tests index cb65e549ff8..0d5ff42aef8 100755 Binary files a/infra/feast-operator/dist/operator-e2e-tests and b/infra/feast-operator/dist/operator-e2e-tests differ diff --git a/infra/feast-operator/docs/api/markdown/ref.md b/infra/feast-operator/docs/api/markdown/ref.md index 0a7782feb2a..cb911ffae22 100644 --- a/infra/feast-operator/docs/api/markdown/ref.md +++ b/infra/feast-operator/docs/api/markdown/ref.md @@ -188,6 +188,23 @@ _Appears in:_ | `template` _string_ | Template for the created project | +#### FeastPackagedOptions + + + +FeastPackagedOptions describes a feature repository packaged in a feature server image. + +_Appears in:_ +- [FeastProjectDir](#feastprojectdir) + +| Field | Description | +| --- | --- | +| `image` _string_ | Image containing the packaged feature repository. When set, this image is used by the +repository initialization and feast apply containers and as the default service image. +When omitted, the operator's configured feature server image is used. | +| `featureRepoPath` _string_ | FeatureRepoPath is the canonical absolute path to the feature repository in the image. | + + #### FeastProjectDir @@ -201,6 +218,7 @@ _Appears in:_ | --- | --- | | `git` _[GitCloneOptions](#gitcloneoptions)_ | | | `init` _[FeastInitOptions](#feastinitoptions)_ | | +| `packaged` _[FeastPackagedOptions](#feastpackagedoptions)_ | | #### FeatureStore @@ -256,6 +274,8 @@ _Appears in:_ This enables annotation-driven integrations like OpenTelemetry auto-instrumentation, Istio sidecar injection, Vault agent injection, etc. | | `disableInitContainers` _boolean_ | Disable the 'feast repo initialization' initContainer | +| `initImage` _string_ | InitImage overrides the image for init containers (feast-init, feast-apply). +Resolution order: InitImage → FeastProjectDir.Packaged.Image → RELATED_IMAGE_FEATURE_SERVER → DefaultImage. | | `runFeastApplyOnInit` _boolean_ | Runs feast apply on pod start to populate the registry. Defaults to true. Ignored when DisableInitContainers is true. | | `volumes` _[Volume](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.30/#volume-v1-core) array_ | Volumes specifies the volumes to mount in the FeatureStore deployment. A corresponding `VolumeMount` should be added to whichever feast service(s) require access to said volume(s). | | `scaling` _[ScalingConfig](#scalingconfig)_ | Scaling configures horizontal scaling for the FeatureStore deployment (e.g. HPA autoscaling). @@ -654,6 +674,9 @@ _Appears in:_ | `persistence` _[OnlineStorePersistence](#onlinestorepersistence)_ | | | `serving` _[ServingConfig](#servingconfig)_ | Serving configures the Feast feature_server section written into feature_store.yaml for the online serve pod. Controls metrics granularity, offline push batching, and MCP. | +| `disabled` _boolean_ | Disabled skips deploying the online store service entirely, including its +serving pod and persistence. Omitting the online store block, or setting +this to false, deploys the online store with defaults as before. | #### OnlineStoreDBStorePersistence diff --git a/infra/feast-operator/internal/controller/featurestore_controller.go b/infra/feast-operator/internal/controller/featurestore_controller.go index 1980fb1f089..b94808c0df5 100644 --- a/infra/feast-operator/internal/controller/featurestore_controller.go +++ b/infra/feast-operator/internal/controller/featurestore_controller.go @@ -65,10 +65,14 @@ type FeatureStoreReconciler struct { // +kubebuilder:rbac:groups=feast.dev,resources=featurestores/status,verbs=get;update;patch // +kubebuilder:rbac:groups=feast.dev,resources=featurestores/finalizers,verbs=update // +kubebuilder:rbac:groups=apps,resources=deployments,verbs=get;list;create;update;watch;delete -// +kubebuilder:rbac:groups=core,resources=services;configmaps;persistentvolumeclaims;serviceaccounts,verbs=get;list;create;update;watch;delete +// +kubebuilder:rbac:groups=core,resources=services;configmaps;persistentvolumeclaims,verbs=get;list;create;update;watch;delete;deletecollection +// +kubebuilder:rbac:groups=core,resources=serviceaccounts,verbs=get;list;create;update;watch;delete // +kubebuilder:rbac:groups=rbac.authorization.k8s.io,resources=roles;rolebindings;clusterroles;clusterrolebindings;subjectaccessreviews,verbs=get;list;create;update;watch;delete -// +kubebuilder:rbac:groups=core,resources=secrets;pods;namespaces,verbs=get;list;watch +// +kubebuilder:rbac:groups=core,resources=secrets;namespaces,verbs=get;list;watch +// +kubebuilder:rbac:groups=core,resources=pods,verbs=get;list;watch;create;delete;deletecollection // +kubebuilder:rbac:groups=core,resources=pods/exec,verbs=create +// +kubebuilder:rbac:groups=core,resources=pods/log,verbs=get +// +kubebuilder:rbac:groups=sparkoperator.k8s.io,resources=sparkapplications,verbs=create;get;delete // +kubebuilder:rbac:groups=authentication.k8s.io,resources=tokenreviews,verbs=create // +kubebuilder:rbac:groups=route.openshift.io,resources=routes,verbs=get;list;create;update;watch;delete // +kubebuilder:rbac:groups=batch,resources=cronjobs,verbs=get;list;watch;create;update;patch;delete diff --git a/infra/feast-operator/internal/controller/featurestore_controller_oidc_auth_test.go b/infra/feast-operator/internal/controller/featurestore_controller_oidc_auth_test.go index e15f8ecfa8a..0f99f6d0479 100644 --- a/infra/feast-operator/internal/controller/featurestore_controller_oidc_auth_test.go +++ b/infra/feast-operator/internal/controller/featurestore_controller_oidc_auth_test.go @@ -493,6 +493,8 @@ func expectedServerOidcAuthorizConfig() services.AuthzConfig { string(services.OidcClientSecret): "client-secret", string(services.OidcUsername): "username", string(services.OidcPassword): "password", + string(services.OidcAudience): "api://feast-feature-server", + string(services.OidcIssuer): "https://keycloak.example.com/realms/test", }, } } @@ -509,6 +511,8 @@ func validOidcSecretMap() map[string]string { string(services.OidcClientSecret): "client-secret", string(services.OidcUsername): "username", string(services.OidcPassword): "password", + string(services.OidcAudience): "api://feast-feature-server", + string(services.OidcIssuer): "https://keycloak.example.com/realms/test", } } diff --git a/infra/feast-operator/internal/controller/featurestore_controller_packaged_test.go b/infra/feast-operator/internal/controller/featurestore_controller_packaged_test.go new file mode 100644 index 00000000000..9c534a624bd --- /dev/null +++ b/infra/feast-operator/internal/controller/featurestore_controller_packaged_test.go @@ -0,0 +1,237 @@ +/* +Copyright 2026 Feast Community. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller + +import ( + "context" + + feastdevv1 "github.com/feast-dev/feast/infra/feast-operator/api/v1" + "github.com/feast-dev/feast/infra/feast-operator/internal/controller/handler" + "github.com/feast-dev/feast/infra/feast-operator/internal/controller/services" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/reconcile" +) + +var _ = Describe("Packaged feature repositories", func() { + const ( + resourceName = "packaged-feature-repo" + packagedImage = "registry.example.com/feature-server@sha256:0123456789abcdef" + packagedRepoDir = "/opt/feast/feature_repo" + ) + + ctx := context.Background() + key := types.NamespacedName{Name: resourceName, Namespace: "default"} + + newFeatureStore := func() *feastdevv1.FeatureStore { + return &feastdevv1.FeatureStore{ + ObjectMeta: metav1.ObjectMeta{Name: key.Name, Namespace: key.Namespace}, + Spec: feastdevv1.FeatureStoreSpec{ + FeastProject: feastProject, + FeastProjectDir: &feastdevv1.FeastProjectDir{ + Packaged: &feastdevv1.FeastPackagedOptions{ + Image: packagedImage, + FeatureRepoPath: packagedRepoDir, + }, + }, + }, + } + } + + reconcileFeatureStore := func() (*feastdevv1.FeatureStore, *appsv1.Deployment) { + reconciler := &FeatureStoreReconciler{Client: k8sClient, Scheme: k8sClient.Scheme()} + _, err := reconciler.Reconcile(ctx, reconcile.Request{NamespacedName: key}) + Expect(err).NotTo(HaveOccurred()) + + featureStore := &feastdevv1.FeatureStore{} + Expect(k8sClient.Get(ctx, key, featureStore)).To(Succeed()) + feastServices := services.FeastServices{ + Handler: handler.FeastHandler{ + Client: k8sClient, + Context: ctx, + Scheme: k8sClient.Scheme(), + FeatureStore: featureStore, + }, + } + deployment := &appsv1.Deployment{} + meta := feastServices.GetObjectMeta() + Expect(k8sClient.Get(ctx, types.NamespacedName{Name: meta.Name, Namespace: meta.Namespace}, deployment)).To(Succeed()) + return featureStore, deployment + } + + BeforeEach(func() { + Expect(k8sClient.Create(ctx, newFeatureStore())).To(Succeed()) + }) + + AfterEach(func() { + featureStore := &feastdevv1.FeatureStore{} + Expect(k8sClient.Get(ctx, key, featureStore)).To(Succeed()) + Expect(k8sClient.Delete(ctx, featureStore)).To(Succeed()) + }) + + It("stages the packaged repository and applies it from the shared directory", func() { + featureStore, deployment := reconcileFeatureStore() + + canonicalRepoDir := services.EphemeralPath + "/" + feastProject + "/" + services.FeatureRepoDir + Expect(deployment.Spec.Template.Spec.InitContainers).To(HaveLen(2)) + initContainer := deployment.Spec.Template.Spec.InitContainers[0] + Expect(initContainer.Name).To(Equal("feast-init")) + Expect(initContainer.Image).To(Equal(packagedImage)) + Expect(initContainer.WorkingDir).To(Equal(services.EphemeralPath)) + Expect(initContainer.Env).To(ContainElements( + corev1.EnvVar{Name: "FEAST_PACKAGED_FEATURE_REPO_PATH", Value: packagedRepoDir}, + corev1.EnvVar{Name: "FEAST_STAGED_FEATURE_REPO_PATH", Value: canonicalRepoDir}, + )) + Expect(initContainer.Args).To(HaveLen(1)) + Expect(initContainer.Args[0]).To(ContainSubstring(`rm -rf -- "${FEAST_STAGED_FEATURE_REPO_PATH}"`)) + Expect(initContainer.Args[0]).To(ContainSubstring(`cp -a -- "${FEAST_PACKAGED_FEATURE_REPO_PATH}/." "${FEAST_STAGED_FEATURE_REPO_PATH}/"`)) + Expect(initContainer.Args[0]).To(ContainSubstring(`printf '%s' "${TMP_FEATURE_STORE_YAML_BASE64}" | base64 -d`)) + Expect(initContainer.Args[0]).To(ContainSubstring(`"${FEAST_STAGED_FEATURE_REPO_PATH}/feature_store.yaml"`)) + + applyContainer := deployment.Spec.Template.Spec.InitContainers[1] + Expect(applyContainer.Name).To(Equal("feast-apply")) + Expect(applyContainer.Image).To(Equal(packagedImage)) + Expect(applyContainer.Command).To(Equal([]string{"feast", "apply"})) + Expect(applyContainer.WorkingDir).To(Equal(canonicalRepoDir)) + + online := services.GetOnlineContainer(*deployment) + Expect(online.Image).To(Equal(packagedImage)) + Expect(online.WorkingDir).To(Equal(canonicalRepoDir)) + Expect(*featureStore.Status.Applied.Services.OnlineStore.Server.Image).To(Equal(packagedImage)) + }) + + It("supports staging without applying and direct use of the baked repository", func() { + featureStore := &feastdevv1.FeatureStore{} + Expect(k8sClient.Get(ctx, key, featureStore)).To(Succeed()) + featureStore.Spec.Services = &feastdevv1.FeatureStoreServices{RunFeastApplyOnInit: ptr(false)} + Expect(k8sClient.Update(ctx, featureStore)).To(Succeed()) + + featureStore, deployment := reconcileFeatureStore() + Expect(deployment.Spec.Template.Spec.InitContainers).To(HaveLen(1)) + Expect(deployment.Spec.Template.Spec.InitContainers[0].Name).To(Equal("feast-init")) + + featureStore.Spec.Services.DisableInitContainers = true + Expect(k8sClient.Update(ctx, featureStore)).To(Succeed()) + _, deployment = reconcileFeatureStore() + Expect(deployment.Spec.Template.Spec.InitContainers).To(BeEmpty()) + online := services.GetOnlineContainer(*deployment) + Expect(online.Image).To(Equal(packagedImage)) + Expect(online.WorkingDir).To(Equal(packagedRepoDir)) + }) + + It("keeps explicit service images ahead of the packaged image", func() { + const serviceImage = "registry.example.com/online-server:custom" + featureStore := &feastdevv1.FeatureStore{} + Expect(k8sClient.Get(ctx, key, featureStore)).To(Succeed()) + featureStore.Spec.Services = &feastdevv1.FeatureStoreServices{ + OnlineStore: &feastdevv1.OnlineStore{ + Server: &feastdevv1.ServerConfigs{ + ContainerConfigs: feastdevv1.ContainerConfigs{ + DefaultCtrConfigs: feastdevv1.DefaultCtrConfigs{Image: ptr(serviceImage)}, + }, + }, + }, + } + Expect(k8sClient.Update(ctx, featureStore)).To(Succeed()) + + _, deployment := reconcileFeatureStore() + Expect(deployment.Spec.Template.Spec.InitContainers[0].Image).To(Equal(packagedImage)) + Expect(services.GetOnlineContainer(*deployment).Image).To(Equal(serviceImage)) + }) + + It("keeps an explicit init image ahead of the packaged image", func() { + const initImage = "registry.example.com/feast-init:custom" + featureStore := &feastdevv1.FeatureStore{} + Expect(k8sClient.Get(ctx, key, featureStore)).To(Succeed()) + featureStore.Spec.Services = &feastdevv1.FeatureStoreServices{ + InitImage: ptr(initImage), + } + Expect(k8sClient.Update(ctx, featureStore)).To(Succeed()) + + _, deployment := reconcileFeatureStore() + Expect(deployment.Spec.Template.Spec.InitContainers).To(HaveLen(2)) + Expect(deployment.Spec.Template.Spec.InitContainers[0].Image).To(Equal(initImage)) + Expect(deployment.Spec.Template.Spec.InitContainers[1].Image).To(Equal(initImage)) + Expect(services.GetOnlineContainer(*deployment).Image).To(Equal(packagedImage)) + }) + + It("supports path-only direct mode with an explicit service image", func() { + const serviceImage = "registry.example.com/online-server:air-gapped" + featureStore := &feastdevv1.FeatureStore{} + Expect(k8sClient.Get(ctx, key, featureStore)).To(Succeed()) + featureStore.Spec.FeastProjectDir.Packaged.Image = "" + featureStore.Spec.Services = &feastdevv1.FeatureStoreServices{ + DisableInitContainers: true, + OnlineStore: &feastdevv1.OnlineStore{ + Server: &feastdevv1.ServerConfigs{ + ContainerConfigs: feastdevv1.ContainerConfigs{ + DefaultCtrConfigs: feastdevv1.DefaultCtrConfigs{Image: ptr(serviceImage)}, + }, + }, + }, + } + Expect(k8sClient.Update(ctx, featureStore)).To(Succeed()) + + _, deployment := reconcileFeatureStore() + Expect(deployment.Spec.Template.Spec.InitContainers).To(BeEmpty()) + online := services.GetOnlineContainer(*deployment) + Expect(online.Image).To(Equal(serviceImage)) + Expect(online.WorkingDir).To(Equal(packagedRepoDir)) + }) + + It("retains the operator image fallback when the packaged image is omitted", func() { + featureStore := &feastdevv1.FeatureStore{} + Expect(k8sClient.Get(ctx, key, featureStore)).To(Succeed()) + featureStore.Spec.FeastProjectDir.Packaged.Image = "" + Expect(k8sClient.Update(ctx, featureStore)).To(Succeed()) + + _, deployment := reconcileFeatureStore() + initImage := deployment.Spec.Template.Spec.InitContainers[0].Image + Expect(initImage).NotTo(BeEmpty()) + Expect(services.GetOnlineContainer(*deployment).Image).To(Equal(initImage)) + }) + + DescribeTable("rejects packaged and staged repository path overlap", + func(featureRepoPath string) { + featureStore := &feastdevv1.FeatureStore{} + Expect(k8sClient.Get(ctx, key, featureStore)).To(Succeed()) + featureStore.Spec.FeastProjectDir.Packaged.FeatureRepoPath = featureRepoPath + Expect(k8sClient.Update(ctx, featureStore)).To(Succeed()) + + reconciler := &FeatureStoreReconciler{Client: k8sClient, Scheme: k8sClient.Scheme()} + _, err := reconciler.Reconcile(ctx, reconcile.Request{NamespacedName: key}) + Expect(err).To(MatchError(ContainSubstring("overlaps staged repository path"))) + }, + Entry("equal paths", services.EphemeralPath+"/"+feastProject+"/"+services.FeatureRepoDir), + Entry("packaged path is an ancestor", services.EphemeralPath+"/"+feastProject), + Entry("packaged path is a descendant", services.EphemeralPath+"/"+feastProject+"/"+services.FeatureRepoDir+"/baked"), + ) + + It("allows similar path prefixes that do not overlap", func() { + featureStore := &feastdevv1.FeatureStore{} + Expect(k8sClient.Get(ctx, key, featureStore)).To(Succeed()) + featureStore.Spec.FeastProjectDir.Packaged.FeatureRepoPath = + services.EphemeralPath + "/" + feastProject + "/" + services.FeatureRepoDir + "-image" + Expect(k8sClient.Update(ctx, featureStore)).To(Succeed()) + + reconcileFeatureStore() + }) +}) diff --git a/infra/feast-operator/internal/controller/featurestore_controller_test.go b/infra/feast-operator/internal/controller/featurestore_controller_test.go index a9ac5235eda..712644f7a0c 100644 --- a/infra/feast-operator/internal/controller/featurestore_controller_test.go +++ b/infra/feast-operator/internal/controller/featurestore_controller_test.go @@ -299,6 +299,41 @@ var _ = Describe("FeatureStore Controller", func() { Expect(err).NotTo(HaveOccurred()) Expect(deploy.Spec.Template.Spec.InitContainers).To(HaveLen(2)) Expect(deploy.Spec.Template.Spec.InitContainers[0].Args[0]).To(ContainSubstring("feast init -t spark")) + + // initImage is independent of server images: init containers use initImage, + // main containers keep their own server.image. + initImage := "quay.io/org/feast-init:custom" + serverImage := "quay.io/org/feast-online:server" + if resource.Spec.Services == nil { + resource.Spec.Services = &feastdevv1.FeatureStoreServices{} + } + resource.Spec.Services.InitImage = &initImage + if resource.Spec.Services.OnlineStore == nil { + resource.Spec.Services.OnlineStore = &feastdevv1.OnlineStore{} + } + if resource.Spec.Services.OnlineStore.Server == nil { + resource.Spec.Services.OnlineStore.Server = &feastdevv1.ServerConfigs{} + } + resource.Spec.Services.OnlineStore.Server.Image = &serverImage + err = k8sClient.Update(ctx, resource) + Expect(err).NotTo(HaveOccurred()) + _, err = controllerReconciler.Reconcile(ctx, reconcile.Request{ + NamespacedName: typeNamespacedName, + }) + Expect(err).NotTo(HaveOccurred()) + + err = k8sClient.Get(ctx, types.NamespacedName{ + Name: objMeta.Name, + Namespace: objMeta.Namespace, + }, deploy) + Expect(err).NotTo(HaveOccurred()) + Expect(deploy.Spec.Template.Spec.InitContainers).To(HaveLen(2)) + Expect(deploy.Spec.Template.Spec.InitContainers[0].Image).To(Equal(initImage)) + Expect(deploy.Spec.Template.Spec.InitContainers[1].Image).To(Equal(initImage)) + online = services.GetOnlineContainer(*deploy) + Expect(online).NotTo(BeNil()) + Expect(online.Image).To(Equal(serverImage)) + Expect(online.Image).NotTo(Equal(initImage)) }) It("should properly encode a feature_store.yaml config", func() { diff --git a/infra/feast-operator/internal/controller/services/batch_engine_rbac.go b/infra/feast-operator/internal/controller/services/batch_engine_rbac.go new file mode 100644 index 00000000000..97afcd3b48d --- /dev/null +++ b/infra/feast-operator/internal/controller/services/batch_engine_rbac.go @@ -0,0 +1,285 @@ +/* +Copyright 2024 Feast Community. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package services + +import ( + "embed" + "fmt" + + feastdevv1 "github.com/feast-dev/feast/infra/feast-operator/api/v1" + corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/yaml" +) + +const ( + BatchEngineFeastType FeastServiceType = "batch-engine" + BatchDriverFeastType FeastServiceType = "batch-driver" +) + +//go:embed rbac_templates/*.yaml +var batchEngineRBACTemplates embed.FS + +// BatchEngineRBACTemplate declares RBAC requirements for a batch compute engine. +type BatchEngineRBACTemplate struct { + EngineType string `json:"engine_type" yaml:"engine_type"` + Server *RBACRoleSpec `json:"server,omitempty" yaml:"server,omitempty"` + Driver *DriverRBACSpec `json:"driver,omitempty" yaml:"driver,omitempty"` +} + +// RBACRoleSpec defines policy rules for a Role. +type RBACRoleSpec struct { + Rules []rbacv1.PolicyRule `json:"rules" yaml:"rules"` +} + +// DriverRBACSpec defines policy rules and optional SA creation for a driver Role. +type DriverRBACSpec struct { + CreateServiceAccount bool `json:"create_service_account" yaml:"create_service_account"` + Rules []rbacv1.PolicyRule `json:"rules" yaml:"rules"` +} + +func loadBatchEngineTemplate(engineType string) (*BatchEngineRBACTemplate, error) { + data, err := batchEngineRBACTemplates.ReadFile( + "rbac_templates/" + engineType + ".yaml", + ) + if err != nil { + return nil, nil + } + var tmpl BatchEngineRBACTemplate + if err := yaml.Unmarshal(data, &tmpl); err != nil { + return nil, fmt.Errorf("failed to parse RBAC template for engine %q: %w", engineType, err) + } + return &tmpl, nil +} + +func (feast *FeastServices) reconcileBatchEngineRBAC() error { + config, ok := feast.getBatchEngineConfig() + if !ok { + return feast.deleteBatchEngineRBAC() + } + + engineType, _ := config["type"].(string) + if engineType == "" { + return feast.deleteBatchEngineRBAC() + } + + tmpl, err := loadBatchEngineTemplate(engineType) + if err != nil { + return err + } + if tmpl == nil { + return feast.deleteBatchEngineRBAC() + } + + if tmpl.Server != nil { + if err := feast.ensureBatchEngineRole(BatchEngineFeastType, tmpl.Server.Rules); err != nil { + return err + } + if err := feast.ensureBatchEngineRoleBinding(BatchEngineFeastType, feast.initFeastSA().Name); err != nil { + return err + } + } + + if tmpl.Driver != nil { + driverSAName := resolveBatchDriverSAName(feast.Handler.FeatureStore, config) + if tmpl.Driver.CreateServiceAccount { + if err := feast.ensureBatchDriverServiceAccount(driverSAName); err != nil { + return err + } + } + if err := feast.ensureBatchEngineRole(BatchDriverFeastType, tmpl.Driver.Rules); err != nil { + return err + } + if err := feast.ensureBatchEngineRoleBinding(BatchDriverFeastType, driverSAName); err != nil { + return err + } + } + + return nil +} + +// getBatchEngineConfig returns the parsed batch-engine ConfigMap data. +// ok=false means no batch engine is configured or the ConfigMap is unreadable. +func (feast *FeastServices) getBatchEngineConfig() (map[string]interface{}, bool) { + appliedSpec := feast.Handler.FeatureStore.Status.Applied + if appliedSpec.BatchEngine == nil || appliedSpec.BatchEngine.ConfigMapRef == nil { + return nil, false + } + + configMapKey := appliedSpec.BatchEngine.ConfigMapKey + if configMapKey == "" { + configMapKey = "config" + } + + cm, err := feast.getConfigMap(appliedSpec.BatchEngine.ConfigMapRef.Name) + if err != nil { + return nil, false + } + + data, found := cm.Data[configMapKey] + if !found { + return nil, false + } + + var config map[string]interface{} + if err := yaml.Unmarshal([]byte(data), &config); err != nil { + return nil, false + } + return config, true +} + +// resolveBatchDriverSAName returns the ServiceAccount name for the Spark driver. +// If batch engine config sets a non-empty service_account, that value wins. +// Otherwise defaults to feast--batch-driver (same name used for RBAC). +func resolveBatchDriverSAName(featureStore *feastdevv1.FeatureStore, config map[string]interface{}) string { + if sa, ok := config["service_account"].(string); ok && sa != "" { + return sa + } + return GetFeastServiceName(featureStore, BatchDriverFeastType) +} + +func (feast *FeastServices) ensureBatchEngineRole(feastType FeastServiceType, rules []rbacv1.PolicyRule) error { + logger := log.FromContext(feast.Handler.Context) + role := &rbacv1.Role{ + ObjectMeta: metav1.ObjectMeta{ + Name: feast.GetFeastServiceName(feastType), + Namespace: feast.Handler.FeatureStore.Namespace, + }, + } + role.SetGroupVersionKind(rbacv1.SchemeGroupVersion.WithKind("Role")) + + op, err := controllerutil.CreateOrUpdate(feast.Handler.Context, feast.Handler.Client, role, func() error { + role.Labels = feast.getFeastTypeLabels(feastType) + role.Rules = rules + return controllerutil.SetControllerReference(feast.Handler.FeatureStore, role, feast.Handler.Scheme) + }) + if err != nil { + return err + } + if op == controllerutil.OperationResultCreated || op == controllerutil.OperationResultUpdated { + logger.Info("Successfully reconciled", "Role", role.Name, "operation", op) + } + return nil +} + +func (feast *FeastServices) ensureBatchEngineRoleBinding(feastType FeastServiceType, saName string) error { + logger := log.FromContext(feast.Handler.Context) + roleBinding := &rbacv1.RoleBinding{ + ObjectMeta: metav1.ObjectMeta{ + Name: feast.GetFeastServiceName(feastType), + Namespace: feast.Handler.FeatureStore.Namespace, + }, + } + roleBinding.SetGroupVersionKind(rbacv1.SchemeGroupVersion.WithKind("RoleBinding")) + + op, err := controllerutil.CreateOrUpdate(feast.Handler.Context, feast.Handler.Client, roleBinding, func() error { + roleBinding.Labels = feast.getFeastTypeLabels(feastType) + roleBinding.Subjects = []rbacv1.Subject{{ + Kind: rbacv1.ServiceAccountKind, + Name: saName, + Namespace: feast.Handler.FeatureStore.Namespace, + }} + roleBinding.RoleRef = rbacv1.RoleRef{ + APIGroup: rbacv1.GroupName, + Kind: "Role", + Name: feast.GetFeastServiceName(feastType), + } + return controllerutil.SetControllerReference(feast.Handler.FeatureStore, roleBinding, feast.Handler.Scheme) + }) + if err != nil { + return err + } + if op == controllerutil.OperationResultCreated || op == controllerutil.OperationResultUpdated { + logger.Info("Successfully reconciled", "RoleBinding", roleBinding.Name, "operation", op) + } + return nil +} + +func (feast *FeastServices) ensureBatchDriverServiceAccount(saName string) error { + logger := log.FromContext(feast.Handler.Context) + sa := &corev1.ServiceAccount{ + ObjectMeta: metav1.ObjectMeta{ + Name: saName, + Namespace: feast.Handler.FeatureStore.Namespace, + }, + } + sa.SetGroupVersionKind(corev1.SchemeGroupVersion.WithKind("ServiceAccount")) + + op, err := controllerutil.CreateOrUpdate(feast.Handler.Context, feast.Handler.Client, sa, func() error { + if sa.Labels == nil { + sa.Labels = map[string]string{} + } + for k, v := range feast.getFeastTypeLabels(BatchDriverFeastType) { + sa.Labels[k] = v + } + return controllerutil.SetControllerReference(feast.Handler.FeatureStore, sa, feast.Handler.Scheme) + }) + if err != nil { + return err + } + if op == controllerutil.OperationResultCreated || op == controllerutil.OperationResultUpdated { + logger.Info("Successfully reconciled", "ServiceAccount", sa.Name, "operation", op) + } + return nil +} + +func (feast *FeastServices) deleteBatchEngineRBAC() error { + serverRoleName := feast.GetFeastServiceName(BatchEngineFeastType) + driverRoleName := feast.GetFeastServiceName(BatchDriverFeastType) + ns := feast.Handler.FeatureStore.Namespace + + serverRoleBinding := &rbacv1.RoleBinding{ + ObjectMeta: metav1.ObjectMeta{Name: serverRoleName, Namespace: ns}, + } + serverRoleBinding.SetGroupVersionKind(rbacv1.SchemeGroupVersion.WithKind("RoleBinding")) + if err := feast.Handler.DeleteOwnedFeastObj(serverRoleBinding); err != nil { + return err + } + + serverRole := &rbacv1.Role{ + ObjectMeta: metav1.ObjectMeta{Name: serverRoleName, Namespace: ns}, + } + serverRole.SetGroupVersionKind(rbacv1.SchemeGroupVersion.WithKind("Role")) + if err := feast.Handler.DeleteOwnedFeastObj(serverRole); err != nil { + return err + } + + driverRoleBinding := &rbacv1.RoleBinding{ + ObjectMeta: metav1.ObjectMeta{Name: driverRoleName, Namespace: ns}, + } + driverRoleBinding.SetGroupVersionKind(rbacv1.SchemeGroupVersion.WithKind("RoleBinding")) + if err := feast.Handler.DeleteOwnedFeastObj(driverRoleBinding); err != nil { + return err + } + + driverRole := &rbacv1.Role{ + ObjectMeta: metav1.ObjectMeta{Name: driverRoleName, Namespace: ns}, + } + driverRole.SetGroupVersionKind(rbacv1.SchemeGroupVersion.WithKind("Role")) + if err := feast.Handler.DeleteOwnedFeastObj(driverRole); err != nil { + return err + } + + driverSA := &corev1.ServiceAccount{ + ObjectMeta: metav1.ObjectMeta{Name: driverRoleName, Namespace: ns}, + } + driverSA.SetGroupVersionKind(corev1.SchemeGroupVersion.WithKind("ServiceAccount")) + return feast.Handler.DeleteOwnedFeastObj(driverSA) +} diff --git a/infra/feast-operator/internal/controller/services/batch_engine_rbac_test.go b/infra/feast-operator/internal/controller/services/batch_engine_rbac_test.go new file mode 100644 index 00000000000..9d792c6f923 --- /dev/null +++ b/infra/feast-operator/internal/controller/services/batch_engine_rbac_test.go @@ -0,0 +1,195 @@ +/* +Copyright 2024 Feast Community. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package services + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + rbacv1 "k8s.io/api/rbac/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/yaml" + + feastdevv1 "github.com/feast-dev/feast/infra/feast-operator/api/v1" +) + +var _ = Describe("Batch Engine RBAC", func() { + + Describe("loadBatchEngineTemplate", func() { + It("should load spark_application template", func() { + tmpl, err := loadBatchEngineTemplate("spark_application") + Expect(err).NotTo(HaveOccurred()) + Expect(tmpl).NotTo(BeNil()) + Expect(tmpl.EngineType).To(Equal("spark_application")) + Expect(tmpl.Server).NotTo(BeNil()) + Expect(tmpl.Server.Rules).NotTo(BeEmpty()) + Expect(tmpl.Driver).NotTo(BeNil()) + Expect(tmpl.Driver.CreateServiceAccount).To(BeTrue()) + Expect(tmpl.Driver.Rules).NotTo(BeEmpty()) + }) + + It("should return nil for unknown engine type", func() { + tmpl, err := loadBatchEngineTemplate("nonexistent_engine") + Expect(err).NotTo(HaveOccurred()) + Expect(tmpl).To(BeNil()) + }) + + It("should contain correct server rules for spark_application", func() { + tmpl, err := loadBatchEngineTemplate("spark_application") + Expect(err).NotTo(HaveOccurred()) + + serverRules := tmpl.Server.Rules + Expect(serverRules).To(HaveLen(4)) + + hasConfigMapRule := false + hasSparkAppRule := false + hasPodListRule := false + hasPodLogRule := false + + for _, rule := range serverRules { + if containsResource(rule, "configmaps") && containsVerb(rule, "create") && containsVerb(rule, "delete") { + hasConfigMapRule = true + } + if containsResource(rule, "sparkapplications") && containsVerb(rule, "create") && containsVerb(rule, "get") && containsVerb(rule, "delete") { + hasSparkAppRule = true + } + if containsResource(rule, "pods") && containsVerb(rule, "list") { + hasPodListRule = true + } + if containsResource(rule, "pods/log") && containsVerb(rule, "get") { + hasPodLogRule = true + } + } + + Expect(hasConfigMapRule).To(BeTrue(), "should have configmaps create/delete rule") + Expect(hasSparkAppRule).To(BeTrue(), "should have sparkapplications create/get/delete rule") + Expect(hasPodListRule).To(BeTrue(), "should have pods list rule") + Expect(hasPodLogRule).To(BeTrue(), "should have pods/log get rule") + }) + + It("should contain correct driver rules for spark_application", func() { + tmpl, err := loadBatchEngineTemplate("spark_application") + Expect(err).NotTo(HaveOccurred()) + + driverRules := tmpl.Driver.Rules + Expect(driverRules).To(HaveLen(2)) + + hasPodRule := false + hasResourceRule := false + for _, rule := range driverRules { + if containsResource(rule, "pods") && + containsVerb(rule, "create") && + containsVerb(rule, "deletecollection") { + hasPodRule = true + } + if containsResource(rule, "services") && + containsResource(rule, "configmaps") && + containsResource(rule, "persistentvolumeclaims") && + containsVerb(rule, "deletecollection") { + hasResourceRule = true + } + } + Expect(hasPodRule).To(BeTrue(), "should have pods CRUD + deletecollection rule") + Expect(hasResourceRule).To(BeTrue(), "should have services/configmaps/PVCs CRUD + deletecollection rule") + }) + }) + + Describe("BatchEngineRBACTemplate YAML parsing", func() { + It("should correctly unmarshal a template", func() { + yamlData := ` +engine_type: test_engine +server: + rules: + - apiGroups: [""] + resources: ["secrets"] + verbs: ["get", "list"] +driver: + create_service_account: true + rules: + - apiGroups: [""] + resources: ["pods"] + verbs: ["get"] +` + var tmpl BatchEngineRBACTemplate + err := yaml.Unmarshal([]byte(yamlData), &tmpl) + Expect(err).NotTo(HaveOccurred()) + Expect(tmpl.EngineType).To(Equal("test_engine")) + Expect(tmpl.Server).NotTo(BeNil()) + Expect(tmpl.Server.Rules).To(HaveLen(1)) + Expect(tmpl.Driver).NotTo(BeNil()) + Expect(tmpl.Driver.CreateServiceAccount).To(BeTrue()) + Expect(tmpl.Driver.Rules).To(HaveLen(1)) + }) + + It("should handle server-only template (no driver)", func() { + yamlData := ` +engine_type: server_only +server: + rules: + - apiGroups: ["batch"] + resources: ["jobs"] + verbs: ["create", "delete"] +` + var tmpl BatchEngineRBACTemplate + err := yaml.Unmarshal([]byte(yamlData), &tmpl) + Expect(err).NotTo(HaveOccurred()) + Expect(tmpl.Server).NotTo(BeNil()) + Expect(tmpl.Driver).To(BeNil()) + }) + }) +}) + +var _ = Describe("resolveBatchDriverSAName", func() { + It("defaults to feast--batch-driver when service_account is omitted", func() { + fs := &feastdevv1.FeatureStore{ObjectMeta: metav1.ObjectMeta{Name: "spark-pg-e2e", Namespace: "feast-spark"}} + Expect(resolveBatchDriverSAName(fs, map[string]interface{}{ + "type": "spark_application", + "image": "quay.io/example/driver:v1", + })).To(Equal("feast-spark-pg-e2e-batch-driver")) + }) + + It("defaults when service_account is empty string", func() { + fs := &feastdevv1.FeatureStore{ObjectMeta: metav1.ObjectMeta{Name: "spark-pg-e2e"}} + Expect(resolveBatchDriverSAName(fs, map[string]interface{}{ + "service_account": "", + })).To(Equal("feast-spark-pg-e2e-batch-driver")) + }) + + It("keeps an explicit service_account override", func() { + fs := &feastdevv1.FeatureStore{ObjectMeta: metav1.ObjectMeta{Name: "spark-pg-e2e"}} + Expect(resolveBatchDriverSAName(fs, map[string]interface{}{ + "service_account": "my-custom-driver", + })).To(Equal("my-custom-driver")) + }) +}) + +func containsResource(rule rbacv1.PolicyRule, resource string) bool { + for _, r := range rule.Resources { + if r == resource { + return true + } + } + return false +} + +func containsVerb(rule rbacv1.PolicyRule, verb string) bool { + for _, v := range rule.Verbs { + if v == verb { + return true + } + } + return false +} diff --git a/infra/feast-operator/internal/controller/services/namespace_registry.go b/infra/feast-operator/internal/controller/services/namespace_registry.go index dcea98a5764..122e7ba9e98 100644 --- a/infra/feast-operator/internal/controller/services/namespace_registry.go +++ b/infra/feast-operator/internal/controller/services/namespace_registry.go @@ -36,8 +36,22 @@ type NamespaceRegistryData struct { Namespaces map[string][]string `json:"namespaces"` } +// isProtectedProject checks if this CR is annotated as a protected project +func (feast *FeastServices) isProtectedProject() bool { + annotations := feast.Handler.FeatureStore.GetAnnotations() + return annotations[ProtectedProjectAnnotation] == "true" +} + // deployNamespaceRegistry creates and manages the namespace registry ConfigMap func (feast *FeastServices) deployNamespaceRegistry() error { + // Skip namespace registry for protected projects. + // Protected projects are managed externally and should not be visible to other instances. + if feast.isProtectedProject() { + logger := log.FromContext(feast.Handler.Context) + logger.V(1).Info("Skipping namespace registry for protected project", "project", feast.Handler.FeatureStore.Spec.FeastProject) + return nil + } + // Check if we can determine the target namespace before creating any resources targetNamespace, err := feast.getNamespaceRegistryNamespace() if err != nil { @@ -230,6 +244,11 @@ func (feast *FeastServices) getNamespaceRegistryNamespace() (string, error) { // AddToNamespaceRegistry adds a feature store instance to the namespace registry func (feast *FeastServices) AddToNamespaceRegistry() error { + // Skip for protected projects — they should not appear in the namespace registry. + if feast.isProtectedProject() { + return nil + } + logger := log.FromContext(feast.Handler.Context) targetNamespace, err := feast.getNamespaceRegistryNamespace() if err != nil { diff --git a/infra/feast-operator/internal/controller/services/rbac_templates/spark_application.yaml b/infra/feast-operator/internal/controller/services/rbac_templates/spark_application.yaml new file mode 100644 index 00000000000..c03e0f3db57 --- /dev/null +++ b/infra/feast-operator/internal/controller/services/rbac_templates/spark_application.yaml @@ -0,0 +1,26 @@ +engine_type: spark_application + +server: + rules: + - apiGroups: [""] + resources: ["configmaps"] + verbs: ["create", "delete"] + - apiGroups: ["sparkoperator.k8s.io"] + resources: ["sparkapplications"] + verbs: ["create", "get", "delete"] + - apiGroups: [""] + resources: ["pods"] + verbs: ["list"] + - apiGroups: [""] + resources: ["pods/log"] + verbs: ["get"] + +driver: + create_service_account: true + rules: + - apiGroups: [""] + resources: ["pods"] + verbs: ["create", "get", "list", "watch", "delete", "deletecollection"] + - apiGroups: [""] + resources: ["services", "configmaps", "persistentvolumeclaims"] + verbs: ["create", "get", "list", "watch", "delete", "deletecollection"] diff --git a/infra/feast-operator/internal/controller/services/repo_config.go b/infra/feast-operator/internal/controller/services/repo_config.go index 21d04db7c0e..b4a01b312b1 100644 --- a/infra/feast-operator/internal/controller/services/repo_config.go +++ b/infra/feast-operator/internal/controller/services/repo_config.go @@ -85,7 +85,7 @@ func getServiceRepoConfig( } if appliedSpec.BatchEngine != nil { - err := setRepoConfigBatchEngine(appliedSpec.BatchEngine, configMapExtractionFunc, &repoConfig) + err := setRepoConfigBatchEngine(featureStore, appliedSpec.BatchEngine, configMapExtractionFunc, &repoConfig) if err != nil { return repoConfig, err } @@ -138,6 +138,17 @@ func getBaseServiceRepoConfig( } for _, prop := range OidcOptionalSecretProperties { if val, exists := secretProperties[string(prop)]; exists { + // Secret values are YAML-parsed on extraction, so an + // all-digits audience or issuer arrives as an int and + // would render unquoted, which the SDK's OidcAuthConfig + // rejects (Optional[str]). Coerce the claim keys back to + // strings; the five original keys keep their historical + // typing. + if prop == OidcAudience || prop == OidcIssuer { + if _, isString := val.(string); !isString { + val = fmt.Sprintf("%v", val) + } + } oidcParameters[string(prop)] = val } } @@ -342,6 +353,7 @@ func setRepoConfigOffline(services *feastdevv1.FeatureStoreServices, secretExtra } func setRepoConfigBatchEngine( + featureStore *feastdevv1.FeatureStore, batchEngineConfig *feastdevv1.BatchEngineConfig, configMapExtractionFunc func(configMapRef string, configMapKey string) (map[string]interface{}, error), repoConfig *RepoConfig) error { @@ -362,6 +374,12 @@ func setRepoConfigBatchEngine( return fmt.Errorf("batch engine config must contain 'type' field") } delete(config, "type") + // Inject service_account only for spark_application so baked feature_store.yaml + // matches the SA/RoleBinding created by reconcileBatchEngineRBAC. + // Other batch engines are left unchanged. + if engineType == "spark_application" { + config["service_account"] = resolveBatchDriverSAName(featureStore, config) + } repoConfig.BatchEngine = &ComputeEngineConfig{ Type: engineType, Parameters: config, diff --git a/infra/feast-operator/internal/controller/services/repo_config_test.go b/infra/feast-operator/internal/controller/services/repo_config_test.go index 5ae22a795f6..e87efdf7dec 100644 --- a/infra/feast-operator/internal/controller/services/repo_config_test.go +++ b/infra/feast-operator/internal/controller/services/repo_config_test.go @@ -212,16 +212,20 @@ var _ = Describe("Repo Config", func() { string(OidcClientId): clientIDValue, string(OidcClientSecret): "client-secret", string(OidcUsername): "username", - string(OidcPassword): "password"}) + string(OidcPassword): "password", + string(OidcAudience): "api://feast-feature-server", + string(OidcIssuer): "https://login.example.com/realms/master"}) repoConfig, err = getServiceRepoConfig(featureStore, secretExtractionFunc, emptyMockExtractConfigFromConfigMap, false) Expect(err).NotTo(HaveOccurred()) Expect(repoConfig.AuthzConfig.Type).To(Equal(OidcAuthType)) - Expect(repoConfig.AuthzConfig.OidcParameters).To(HaveLen(5)) + Expect(repoConfig.AuthzConfig.OidcParameters).To(HaveLen(7)) Expect(repoConfig.AuthzConfig.OidcParameters).To(HaveKey(string(OidcClientId))) Expect(repoConfig.AuthzConfig.OidcParameters).To(HaveKey(string(OidcAuthDiscoveryUrl))) Expect(repoConfig.AuthzConfig.OidcParameters).To(HaveKey(string(OidcClientSecret))) Expect(repoConfig.AuthzConfig.OidcParameters).To(HaveKey(string(OidcUsername))) Expect(repoConfig.AuthzConfig.OidcParameters).To(HaveKey(string(OidcPassword))) + Expect(repoConfig.AuthzConfig.OidcParameters).To(HaveKeyWithValue(string(OidcAudience), "api://feast-feature-server")) + Expect(repoConfig.AuthzConfig.OidcParameters).To(HaveKeyWithValue(string(OidcIssuer), "https://login.example.com/realms/master")) Expect(repoConfig.OfflineStore).To(Equal(expectedOfflineConfig)) Expect(repoConfig.OnlineStore).To(Equal(defaultOnlineStoreConfig(featureStore))) Expect(repoConfig.Registry).To(Equal(defaultRegistryConfig(featureStore))) @@ -229,6 +233,19 @@ var _ = Describe("Repo Config", func() { repoConfig = getClientRepoConfig(featureStore, nil) Expect(repoConfig.AuthzConfig.Type).To(Equal(OidcAuthType)) + By("Coercing numeric audience and issuer Secret values to strings") + secretExtractionFunc = mockOidcConfigFromSecret(map[string]interface{}{ + string(OidcAuthDiscoveryUrl): "discovery-url", + string(OidcClientId): clientIDValue, + // Secret extraction YAML-parses values, so an all-digits + // audience/issuer reaches this code as an int. + string(OidcAudience): 1234567890, + string(OidcIssuer): 9876543210}) + repoConfig, err = getServiceRepoConfig(featureStore, secretExtractionFunc, emptyMockExtractConfigFromConfigMap, false) + Expect(err).NotTo(HaveOccurred()) + Expect(repoConfig.AuthzConfig.OidcParameters).To(HaveKeyWithValue(string(OidcAudience), "1234567890")) + Expect(repoConfig.AuthzConfig.OidcParameters).To(HaveKeyWithValue(string(OidcIssuer), "9876543210")) + By("Having oidc authorization with issuerUrl only (no Secret)") featureStore.Spec.AuthzConfig = &feastdevv1.AuthzConfig{ OidcAuthz: &feastdevv1.OidcAuthz{ @@ -673,6 +690,72 @@ var _ = Describe("Repo Config", func() { Expect(repoConfig.Materialization).To(BeNil()) Expect(repoConfig.OpenLineage).To(BeNil()) }) + + It("should inject default batch_engine.service_account when ConfigMap omits it", func() { + featureStore := minimalFeatureStore() + featureStore.Name = "spark-pg-e2e" + featureStore.Spec.BatchEngine = &feastdevv1.BatchEngineConfig{ + ConfigMapRef: &corev1.LocalObjectReference{Name: "spark-pg-batch-engine"}, + } + ApplyDefaultsToStatus(featureStore) + + extractCM := func(configMapRef string, configMapKey string) (map[string]interface{}, error) { + return map[string]interface{}{ + "type": "spark_application", + "image": "quay.io/example/feast-spark-driver:v6", + // service_account intentionally omitted + }, nil + } + + repoConfig, err := getServiceRepoConfig(featureStore, emptyMockExtractConfigFromSecret, extractCM, false) + Expect(err).NotTo(HaveOccurred()) + Expect(repoConfig.BatchEngine).NotTo(BeNil()) + Expect(repoConfig.BatchEngine.Type).To(Equal("spark_application")) + Expect(repoConfig.BatchEngine.Parameters["service_account"]).To(Equal("feast-spark-pg-e2e-batch-driver")) + }) + + It("should preserve explicit batch_engine.service_account from ConfigMap", func() { + featureStore := minimalFeatureStore() + featureStore.Name = "spark-pg-e2e" + featureStore.Spec.BatchEngine = &feastdevv1.BatchEngineConfig{ + ConfigMapRef: &corev1.LocalObjectReference{Name: "spark-pg-batch-engine"}, + } + ApplyDefaultsToStatus(featureStore) + + extractCM := func(configMapRef string, configMapKey string) (map[string]interface{}, error) { + return map[string]interface{}{ + "type": "spark_application", + "service_account": "my-custom-driver", + }, nil + } + + repoConfig, err := getServiceRepoConfig(featureStore, emptyMockExtractConfigFromSecret, extractCM, false) + Expect(err).NotTo(HaveOccurred()) + Expect(repoConfig.BatchEngine.Parameters["service_account"]).To(Equal("my-custom-driver")) + }) + + It("should not inject service_account for non-spark_application batch engines", func() { + featureStore := minimalFeatureStore() + featureStore.Name = "spark-pg-e2e" + featureStore.Spec.BatchEngine = &feastdevv1.BatchEngineConfig{ + ConfigMapRef: &corev1.LocalObjectReference{Name: "other-batch-engine"}, + } + ApplyDefaultsToStatus(featureStore) + + extractCM := func(configMapRef string, configMapKey string) (map[string]interface{}, error) { + return map[string]interface{}{ + "type": "spark", + // no service_account — must stay omitted for non-spark_application + }, nil + } + + repoConfig, err := getServiceRepoConfig(featureStore, emptyMockExtractConfigFromSecret, extractCM, false) + Expect(err).NotTo(HaveOccurred()) + Expect(repoConfig.BatchEngine).NotTo(BeNil()) + Expect(repoConfig.BatchEngine.Type).To(Equal("spark")) + _, hasSA := repoConfig.BatchEngine.Parameters["service_account"] + Expect(hasSA).To(BeFalse()) + }) }) It("should fail to create the repo configs", func() { featureStore := minimalFeatureStore() diff --git a/infra/feast-operator/internal/controller/services/services.go b/infra/feast-operator/internal/controller/services/services.go index c6271c28d3a..6964938c093 100644 --- a/infra/feast-operator/internal/controller/services/services.go +++ b/infra/feast-operator/internal/controller/services/services.go @@ -18,6 +18,7 @@ package services import ( "errors" + "path" "strconv" "strings" @@ -79,6 +80,9 @@ func (feast *FeastServices) Deploy() error { if err := feast.createServiceAccount(); err != nil { return err } + if err := feast.reconcileBatchEngineRBAC(); err != nil { + return err + } if err := feast.createDeployment(); err != nil { return err } @@ -451,6 +455,10 @@ func (feast *FeastServices) setPod(podSpec *corev1.PodSpec) error { } func (feast *FeastServices) setContainers(podSpec *corev1.PodSpec) error { + if err := feast.validatePackagedFeatureRepoPath(); err != nil { + return err + } + fsYamlB64, err := feast.GetServiceFeatureStoreYamlBase64() if err != nil { return err @@ -469,6 +477,20 @@ func (feast *FeastServices) setContainers(podSpec *corev1.PodSpec) error { if feast.isUiServer() { feast.setContainer(&podSpec.Containers, UIFeastType, fsYamlB64) } + + // When the CR is annotated as a protected project, set FEAST_PROTECTED_PROJECT=true + // so the registry server tags its own project in the shared registry. + // Other FeatureStore instances then exclude this project automatically. + if feast.isProtectedProject() { + protectedEnv := corev1.EnvVar{ + Name: "FEAST_PROTECTED_PROJECT", + Value: "true", + } + for i := range podSpec.Containers { + podSpec.Containers[i].Env = append(podSpec.Containers[i].Env, protectedEnv) + } + } + return nil } @@ -687,9 +709,10 @@ func (feast *FeastServices) setInitContainer(podSpec *corev1.PodSpec, fsYamlB64 feastProjectDir := applied.FeastProjectDir workingDir := getOfflineMountPath(feast.Handler.FeatureStore) projectPath := workingDir + "/" + applied.FeastProject + initImage := getInitContainerImage(&applied) container := corev1.Container{ Name: feastInitContainerName, - Image: getFeatureServerImage(), + Image: initImage, Env: []corev1.EnvVar{ { Name: TmpFeatureStoreYamlEnvVar, @@ -700,6 +723,7 @@ func (feast *FeastServices) setInitContainer(podSpec *corev1.PodSpec, fsYamlB64 WorkingDir: workingDir, } + featureRepoDir := feast.getFeatureRepoDir() var createCommand string if feastProjectDir.Init != nil { initSlice := []string{"feast", "init"} @@ -729,20 +753,43 @@ func (feast *FeastServices) setInitContainer(podSpec *corev1.PodSpec, fsYamlB64 if feastProjectDir.Git.EnvFrom != nil { container.EnvFrom = *feastProjectDir.Git.EnvFrom } + } else if feastProjectDir.Packaged != nil { + container.Env = append(container.Env, + corev1.EnvVar{ + Name: packagedFeatureRepoEnvVar, + Value: path.Clean(feastProjectDir.Packaged.FeatureRepoPath), + }, + corev1.EnvVar{ + Name: stagedFeatureRepoEnvVar, + Value: featureRepoDir, + }, + ) + container.Args = []string{ + "set -euo pipefail\n" + + "echo \"Staging packaged feast repository...\"\n" + + "if [[ ! -d \"${" + packagedFeatureRepoEnvVar + "}\" ]]; then " + + "echo \"Packaged feature repository not found: ${" + packagedFeatureRepoEnvVar + "}\" >&2; exit 1; fi\n" + + "rm -rf -- \"${" + stagedFeatureRepoEnvVar + "}\"\n" + + "mkdir -p -- \"${" + stagedFeatureRepoEnvVar + "}\"\n" + + "cp -a -- \"${" + packagedFeatureRepoEnvVar + "}/.\" \"${" + stagedFeatureRepoEnvVar + "}/\"\n" + + "printf '%s' \"${" + TmpFeatureStoreYamlEnvVar + "}\" | base64 -d > \"${" + stagedFeatureRepoEnvVar + "}/feature_store.yaml\"\n" + + "echo \"Packaged feast repository staging complete\"\n", + } } - featureRepoDir := feast.getFeatureRepoDir() - container.Args = []string{ - "echo \"Creating feast repository...\"\necho '" + createCommand + "'\n" + - "if [[ ! -d " + featureRepoDir + " ]]; then " + createCommand + "; fi;\n" + - "echo $" + TmpFeatureStoreYamlEnvVar + " | base64 -d \u003e " + featureRepoDir + "/feature_store.yaml;\necho \"Feast repo creation complete\";\n", + if feastProjectDir.Packaged == nil { + container.Args = []string{ + "echo \"Creating feast repository...\"\necho '" + createCommand + "'\n" + + "if [[ ! -d " + featureRepoDir + " ]]; then " + createCommand + "; fi;\n" + + "echo $" + TmpFeatureStoreYamlEnvVar + " | base64 -d \u003e " + featureRepoDir + "/feature_store.yaml;\necho \"Feast repo creation complete\";\n", + } } podSpec.InitContainers = append(podSpec.InitContainers, container) if applied.Services.RunFeastApplyOnInit != nil && *applied.Services.RunFeastApplyOnInit { applyContainer := corev1.Container{ Name: feastApplyContainerName, - Image: getFeatureServerImage(), + Image: initImage, Command: []string{feastCommand, "apply"}, WorkingDir: featureRepoDir, } @@ -1274,7 +1321,7 @@ func (feast *FeastServices) isOnlineServer() bool { func (feast *FeastServices) isOnlineStore() bool { appliedServices := feast.Handler.FeatureStore.Status.Applied.Services - return appliedServices != nil && appliedServices.OnlineStore != nil + return appliedServices != nil && appliedServices.OnlineStore != nil && !appliedServices.OnlineStore.Disabled } func (feast *FeastServices) noLocalCoreServerConfigured() bool { @@ -1403,6 +1450,9 @@ func (feast *FeastServices) mountEmptyDirVolumes(podSpec *corev1.PodSpec) { func (feast *FeastServices) getFeatureRepoDir() string { applied := feast.Handler.FeatureStore.Status.Applied + if applied.FeastProjectDir != nil && applied.FeastProjectDir.Packaged != nil && applied.Services.DisableInitContainers { + return path.Clean(applied.FeastProjectDir.Packaged.FeatureRepoPath) + } feastProjectDir := getOfflineMountPath(feast.Handler.FeatureStore) + "/" + applied.FeastProject if applied.FeastProjectDir != nil && applied.FeastProjectDir.Git != nil && len(applied.FeastProjectDir.Git.FeatureRepoPath) > 0 { return feastProjectDir + "/" + applied.FeastProjectDir.Git.FeatureRepoPath @@ -1410,6 +1460,39 @@ func (feast *FeastServices) getFeatureRepoDir() string { return feastProjectDir + "/" + FeatureRepoDir } +func (feast *FeastServices) validatePackagedFeatureRepoPath() error { + applied := feast.Handler.FeatureStore.Status.Applied + if applied.FeastProjectDir == nil || applied.FeastProjectDir.Packaged == nil { + return nil + } + + featureRepoPath := applied.FeastProjectDir.Packaged.FeatureRepoPath + cleanFeatureRepoPath := path.Clean(featureRepoPath) + if !path.IsAbs(featureRepoPath) || cleanFeatureRepoPath == "/" || cleanFeatureRepoPath != featureRepoPath { + return errors.New("packaged feature repository path " + strconv.Quote(featureRepoPath) + " must be a canonical absolute, non-root path") + } + + if !applied.Services.DisableInitContainers { + stagedFeatureRepoPath := path.Clean(feast.getFeatureRepoDir()) + if pathsOverlap(cleanFeatureRepoPath, stagedFeatureRepoPath) { + return errors.New( + "packaged feature repository path " + strconv.Quote(cleanFeatureRepoPath) + + " overlaps staged repository path " + strconv.Quote(stagedFeatureRepoPath), + ) + } + } + + return nil +} + +func pathsOverlap(firstPath, secondPath string) bool { + firstPath = path.Clean(firstPath) + secondPath = path.Clean(secondPath) + return firstPath == secondPath || + strings.HasPrefix(firstPath, secondPath+"/") || + strings.HasPrefix(secondPath, firstPath+"/") +} + func mountEmptyDirVolume(podSpec *corev1.PodSpec) { if podSpec != nil { volName := strings.TrimPrefix(EphemeralPath, "/") diff --git a/infra/feast-operator/internal/controller/services/services_types.go b/infra/feast-operator/internal/controller/services/services_types.go index 098362af96b..090696eccaf 100644 --- a/infra/feast-operator/internal/controller/services/services_types.go +++ b/infra/feast-operator/internal/controller/services/services_types.go @@ -26,6 +26,8 @@ import ( const ( TmpFeatureStoreYamlEnvVar = "TMP_FEATURE_STORE_YAML_BASE64" + packagedFeatureRepoEnvVar = "FEAST_PACKAGED_FEATURE_REPO_PATH" + stagedFeatureRepoEnvVar = "FEAST_STAGED_FEATURE_REPO_PATH" feastServerImageVar = "RELATED_IMAGE_FEATURE_SERVER" cronJobImageVar = "RELATED_IMAGE_CRON_JOB" FeatureStoreYamlCmKey = "feature_store.yaml" @@ -40,6 +42,14 @@ const ( NamespaceRegistryDataKey = "namespaces" DefaultKubernetesNamespace = "feast-operator-system" + // ProtectedProjectAnnotation is the annotation key on a FeatureStore CR + // that marks its project as protected. Protected projects are excluded + // from project listings and shielded from teardown by other instances. + // When this annotation is "true", the operator sets FEAST_PROTECTED_PROJECT=true + // on the server pods, which causes the server to tag the project in the + // shared registry on startup. + ProtectedProjectAnnotation = "feast.dev/protected-project" + HttpPort = 80 HttpsPort = 443 HttpScheme = "http" @@ -103,6 +113,8 @@ const ( OidcTokenEnvVar OidcPropertyType = "token_env_var" OidcVerifySsl OidcPropertyType = "verify_ssl" OidcCaCertPath OidcPropertyType = "ca_cert_path" + OidcAudience OidcPropertyType = "audience" + OidcIssuer OidcPropertyType = "issuer" OidcMissingSecretError string = "missing OIDC secret: %s" @@ -266,7 +278,7 @@ var ( }, } - OidcOptionalSecretProperties = []OidcPropertyType{OidcAuthDiscoveryUrl, OidcClientId, OidcClientSecret, OidcUsername, OidcPassword} + OidcOptionalSecretProperties = []OidcPropertyType{OidcAuthDiscoveryUrl, OidcClientId, OidcClientSecret, OidcUsername, OidcPassword, OidcAudience, OidcIssuer} ) // Feast server types: Reserved only for server types like Online, Offline, and Registry servers. Should not be used for client types like the UI, etc. diff --git a/infra/feast-operator/internal/controller/services/util.go b/infra/feast-operator/internal/controller/services/util.go index 387ccdb631b..84951f2077b 100644 --- a/infra/feast-operator/internal/controller/services/util.go +++ b/infra/feast-operator/internal/controller/services/util.go @@ -99,6 +99,7 @@ func ApplyDefaultsToStatus(cr *feastdevv1.FeatureStore) { if applied.Services == nil { applied.Services = &feastdevv1.FeatureStoreServices{} } + defaultFeatureServerImage := getFeatureServerImageForSpec(applied) services := applied.Services if services.RunFeastApplyOnInit == nil { services.RunFeastApplyOnInit = boolPtr(true) @@ -128,7 +129,7 @@ func ApplyDefaultsToStatus(cr *feastdevv1.FeatureStore) { } if services.Registry.Local.Server != nil { - setDefaultCtrConfigs(&services.Registry.Local.Server.ContainerConfigs.DefaultCtrConfigs) + setDefaultCtrConfigs(&services.Registry.Local.Server.ContainerConfigs.DefaultCtrConfigs, defaultFeatureServerImage) // Set default for GRPC: true if nil if services.Registry.Local.Server.GRPC == nil { defaultGRPC := true @@ -159,37 +160,39 @@ func ApplyDefaultsToStatus(cr *feastdevv1.FeatureStore) { } if services.OfflineStore.Server != nil { - setDefaultCtrConfigs(&services.OfflineStore.Server.ContainerConfigs.DefaultCtrConfigs) + setDefaultCtrConfigs(&services.OfflineStore.Server.ContainerConfigs.DefaultCtrConfigs, defaultFeatureServerImage) } } - // default to onlineStore service deployment + // default to onlineStore service deployment unless it is explicitly disabled if services.OnlineStore == nil { services.OnlineStore = &feastdevv1.OnlineStore{} } - if services.OnlineStore.Persistence == nil { - services.OnlineStore.Persistence = &feastdevv1.OnlineStorePersistence{} - } - - if services.OnlineStore.Persistence.DBPersistence == nil { - if services.OnlineStore.Persistence.FilePersistence == nil { - services.OnlineStore.Persistence.FilePersistence = &feastdevv1.OnlineStoreFilePersistence{} + if !services.OnlineStore.Disabled { + if services.OnlineStore.Persistence == nil { + services.OnlineStore.Persistence = &feastdevv1.OnlineStorePersistence{} } - if len(services.OnlineStore.Persistence.FilePersistence.Path) == 0 { - services.OnlineStore.Persistence.FilePersistence.Path = defaultOnlineStorePath(cr) - } + if services.OnlineStore.Persistence.DBPersistence == nil { + if services.OnlineStore.Persistence.FilePersistence == nil { + services.OnlineStore.Persistence.FilePersistence = &feastdevv1.OnlineStoreFilePersistence{} + } - ensurePVCDefaults(services.OnlineStore.Persistence.FilePersistence.PvcConfig, OnlineFeastType) - } + if len(services.OnlineStore.Persistence.FilePersistence.Path) == 0 { + services.OnlineStore.Persistence.FilePersistence.Path = defaultOnlineStorePath(cr) + } + + ensurePVCDefaults(services.OnlineStore.Persistence.FilePersistence.PvcConfig, OnlineFeastType) + } - if services.OnlineStore.Server == nil { - services.OnlineStore.Server = &feastdevv1.ServerConfigs{} + if services.OnlineStore.Server == nil { + services.OnlineStore.Server = &feastdevv1.ServerConfigs{} + } + setDefaultCtrConfigs(&services.OnlineStore.Server.ContainerConfigs.DefaultCtrConfigs, defaultFeatureServerImage) } - setDefaultCtrConfigs(&services.OnlineStore.Server.ContainerConfigs.DefaultCtrConfigs) if services.UI != nil { - setDefaultCtrConfigs(&services.UI.ContainerConfigs.DefaultCtrConfigs) + setDefaultCtrConfigs(&services.UI.ContainerConfigs.DefaultCtrConfigs, defaultFeatureServerImage) } if applied.CronJob == nil { @@ -198,13 +201,20 @@ func ApplyDefaultsToStatus(cr *feastdevv1.FeatureStore) { setDefaultCronJobConfigs(applied.CronJob) } -func setDefaultCtrConfigs(defaultConfigs *feastdevv1.DefaultCtrConfigs) { +func setDefaultCtrConfigs(defaultConfigs *feastdevv1.DefaultCtrConfigs, defaultImage string) { if defaultConfigs.Image == nil { - img := getFeatureServerImage() + img := defaultImage defaultConfigs.Image = &img } } +func getFeatureServerImageForSpec(spec *feastdevv1.FeatureStoreSpec) string { + if spec != nil && spec.FeastProjectDir != nil && spec.FeastProjectDir.Packaged != nil && spec.FeastProjectDir.Packaged.Image != "" { + return spec.FeastProjectDir.Packaged.Image + } + return getFeatureServerImage() +} + func getFeatureServerImage() string { if img, exists := os.LookupEnv(feastServerImageVar); exists { return img @@ -212,6 +222,16 @@ func getFeatureServerImage() string { return DefaultImage } +// getInitContainerImage resolves the image for feast-init / feast-apply. +// Order: spec.services.initImage → spec.feastProjectDir.packaged.image → +// RELATED_IMAGE_FEATURE_SERVER → DefaultImage. +func getInitContainerImage(spec *feastdevv1.FeatureStoreSpec) string { + if spec != nil && spec.Services != nil && spec.Services.InitImage != nil && len(*spec.Services.InitImage) > 0 { + return *spec.Services.InitImage + } + return getFeatureServerImageForSpec(spec) +} + func checkOfflineStoreFilePersistenceType(value string) error { if slices.Contains(feastdevv1.ValidOfflineStoreFilePersistenceTypes, value) { return nil diff --git a/infra/feast-operator/internal/controller/services/util_test.go b/infra/feast-operator/internal/controller/services/util_test.go new file mode 100644 index 00000000000..5a868d2d101 --- /dev/null +++ b/infra/feast-operator/internal/controller/services/util_test.go @@ -0,0 +1,171 @@ +/* +Copyright 2024 Feast Community. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package services + +import ( + "os" + "testing" + + feastdevv1 "github.com/feast-dev/feast/infra/feast-operator/api/v1" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "k8s.io/utils/ptr" +) + +var _ = Describe("ApplyDefaultsToStatus", func() { + It("deploys the online store with defaults when it is not declared", func() { + cr := &feastdevv1.FeatureStore{ + Spec: feastdevv1.FeatureStoreSpec{ + FeastProject: "test_project", + Services: &feastdevv1.FeatureStoreServices{}, + }, + } + + ApplyDefaultsToStatus(cr) + + online := cr.Status.Applied.Services.OnlineStore + Expect(online).ToNot(BeNil()) + Expect(online.Disabled).To(BeFalse()) + Expect(online.Persistence).ToNot(BeNil()) + Expect(online.Server).ToNot(BeNil()) + }) + + It("applies online store defaults when it is declared", func() { + cr := &feastdevv1.FeatureStore{ + Spec: feastdevv1.FeatureStoreSpec{ + FeastProject: "test_project", + Services: &feastdevv1.FeatureStoreServices{ + OnlineStore: &feastdevv1.OnlineStore{}, + }, + }, + } + + ApplyDefaultsToStatus(cr) + + online := cr.Status.Applied.Services.OnlineStore + Expect(online).ToNot(BeNil()) + Expect(online.Persistence).ToNot(BeNil()) + Expect(online.Server).ToNot(BeNil()) + }) + + // #6586: disabling the online store opts out of its persistence and serving + // pod, letting a registry-only or offline-only ViewerStore skip it while + // leaving the default-on behavior unchanged for everyone else. + It("does not apply persistence or server defaults when the online store is disabled", func() { + cr := &feastdevv1.FeatureStore{ + Spec: feastdevv1.FeatureStoreSpec{ + FeastProject: "test_project", + Services: &feastdevv1.FeatureStoreServices{ + OnlineStore: &feastdevv1.OnlineStore{Disabled: true}, + }, + }, + } + + ApplyDefaultsToStatus(cr) + + online := cr.Status.Applied.Services.OnlineStore + Expect(online).ToNot(BeNil()) + Expect(online.Disabled).To(BeTrue()) + Expect(online.Persistence).To(BeNil()) + Expect(online.Server).To(BeNil()) + }) +}) + +func TestGetInitContainerImage(t *testing.T) { + customInit := "quay.io/org/feast-init:custom" + packagedImage := "quay.io/org/feast-packaged:test" + envImage := "quay.io/org/feast-env:test" + + t.Run("uses initImage ahead of packaged and server images", func(t *testing.T) { + t.Setenv(feastServerImageVar, envImage) + got := getInitContainerImage(&feastdevv1.FeatureStoreSpec{ + FeastProjectDir: &feastdevv1.FeastProjectDir{ + Packaged: &feastdevv1.FeastPackagedOptions{Image: packagedImage}, + }, + Services: &feastdevv1.FeatureStoreServices{ + InitImage: ptr.To(customInit), + OfflineStore: &feastdevv1.OfflineStore{ + Server: &feastdevv1.ServerConfigs{ + ContainerConfigs: feastdevv1.ContainerConfigs{ + DefaultCtrConfigs: feastdevv1.DefaultCtrConfigs{ + Image: ptr.To("quay.io/org/offline:v1"), + }, + }, + }, + }, + OnlineStore: &feastdevv1.OnlineStore{ + Server: &feastdevv1.ServerConfigs{ + ContainerConfigs: feastdevv1.ContainerConfigs{ + DefaultCtrConfigs: feastdevv1.DefaultCtrConfigs{ + Image: ptr.To("quay.io/org/online:v1"), + }, + }, + }, + }, + }, + }) + if got != customInit { + t.Fatalf("got %q, want %q (must not inherit server images)", got, customInit) + } + }) + + t.Run("uses packaged image ahead of RELATED_IMAGE_FEATURE_SERVER", func(t *testing.T) { + t.Setenv(feastServerImageVar, envImage) + got := getInitContainerImage(&feastdevv1.FeatureStoreSpec{ + FeastProjectDir: &feastdevv1.FeastProjectDir{ + Packaged: &feastdevv1.FeastPackagedOptions{Image: packagedImage}, + }, + Services: &feastdevv1.FeatureStoreServices{}, + }) + if got != packagedImage { + t.Fatalf("got %q, want %q", got, packagedImage) + } + }) + + t.Run("falls back to RELATED_IMAGE_FEATURE_SERVER", func(t *testing.T) { + t.Setenv(feastServerImageVar, envImage) + got := getInitContainerImage(&feastdevv1.FeatureStoreSpec{ + Services: &feastdevv1.FeatureStoreServices{}, + }) + if got != envImage { + t.Fatalf("got %q, want %q", got, envImage) + } + }) + + t.Run("falls back to DefaultImage", func(t *testing.T) { + _ = os.Unsetenv(feastServerImageVar) + got := getInitContainerImage(nil) + if got != DefaultImage { + t.Fatalf("got %q, want %q", got, DefaultImage) + } + }) + + t.Run("ignores empty initImage", func(t *testing.T) { + t.Setenv(feastServerImageVar, envImage) + got := getInitContainerImage(&feastdevv1.FeatureStoreSpec{ + FeastProjectDir: &feastdevv1.FeastProjectDir{ + Packaged: &feastdevv1.FeastPackagedOptions{Image: packagedImage}, + }, + Services: &feastdevv1.FeatureStoreServices{ + InitImage: ptr.To(""), + }, + }) + if got != packagedImage { + t.Fatalf("got %q, want %q", got, packagedImage) + } + }) +} diff --git a/infra/feast-operator/test/api/featurestore_packaged_types_test.go b/infra/feast-operator/test/api/featurestore_packaged_types_test.go new file mode 100644 index 00000000000..97525ec1abe --- /dev/null +++ b/infra/feast-operator/test/api/featurestore_packaged_types_test.go @@ -0,0 +1,162 @@ +/* +Copyright 2026 Feast Community. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package api + +import ( + "context" + "strings" + + feastdevv1 "github.com/feast-dev/feast/infra/feast-operator/api/v1" + feastdevv1alpha1 "github.com/feast-dev/feast/infra/feast-operator/api/v1alpha1" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +type packagedFeatureStoreFactory func(name, featureRepoPath string) client.Object +type conflictingPackagedFeatureStoreFactory func(name, conflictingMode string) client.Object + +func newV1PackagedFeatureStore(name, featureRepoPath string) client.Object { + return &feastdevv1.FeatureStore{ + ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: namespaceName}, + Spec: feastdevv1.FeatureStoreSpec{ + FeastProject: "test_project", + FeastProjectDir: &feastdevv1.FeastProjectDir{ + Packaged: &feastdevv1.FeastPackagedOptions{FeatureRepoPath: featureRepoPath}, + }, + }, + } +} + +func newV1Alpha1PackagedFeatureStore(name, featureRepoPath string) client.Object { + return &feastdevv1alpha1.FeatureStore{ + ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: namespaceName}, + Spec: feastdevv1alpha1.FeatureStoreSpec{ + FeastProject: "test_project", + FeastProjectDir: &feastdevv1alpha1.FeastProjectDir{ + Packaged: &feastdevv1alpha1.FeastPackagedOptions{FeatureRepoPath: featureRepoPath}, + }, + }, + } +} + +func newV1ConflictingPackagedFeatureStore(name, conflictingMode string) client.Object { + featureStore := newV1PackagedFeatureStore(name, "/opt/feast/feature_repo").(*feastdevv1.FeatureStore) + switch conflictingMode { + case "init": + featureStore.Spec.FeastProjectDir.Init = &feastdevv1.FeastInitOptions{} + case "git": + featureStore.Spec.FeastProjectDir.Git = &feastdevv1.GitCloneOptions{ + URL: "https://example.com/feature-repo.git", + } + } + return featureStore +} + +func newV1Alpha1ConflictingPackagedFeatureStore(name, conflictingMode string) client.Object { + featureStore := newV1Alpha1PackagedFeatureStore(name, "/opt/feast/feature_repo").(*feastdevv1alpha1.FeatureStore) + switch conflictingMode { + case "init": + featureStore.Spec.FeastProjectDir.Init = &feastdevv1alpha1.FeastInitOptions{} + case "git": + featureStore.Spec.FeastProjectDir.Git = &feastdevv1alpha1.GitCloneOptions{ + URL: "https://example.com/feature-repo.git", + } + } + return featureStore +} + +var _ = Describe("Packaged feature repository path validation", func() { + ctx := context.Background() + apiVersions := []struct { + name string + id string + factory packagedFeatureStoreFactory + conflictingFactory conflictingPackagedFeatureStoreFactory + }{ + { + name: "feast.dev/v1", + id: "v1", + factory: newV1PackagedFeatureStore, + conflictingFactory: newV1ConflictingPackagedFeatureStore, + }, + { + name: "feast.dev/v1alpha1", + id: "v1alpha1", + factory: newV1Alpha1PackagedFeatureStore, + conflictingFactory: newV1Alpha1ConflictingPackagedFeatureStore, + }, + } + + for _, apiVersion := range apiVersions { + apiVersion := apiVersion + Context(apiVersion.name, func() { + DescribeTable("accepts canonical absolute non-root paths", + func(nameSuffix, featureRepoPath string) { + featureStore := apiVersion.factory( + "packaged-"+apiVersion.id+"-"+nameSuffix, + featureRepoPath, + ) + Expect(k8sClient.Create(ctx, featureStore)).To(Succeed()) + Expect(k8sClient.Delete(ctx, featureStore)).To(Succeed()) + }, + Entry("standard", "standard", "/opt/feast/feature_repo"), + Entry("hidden component", "hidden", "/opt/.feast/feature_repo"), + Entry("dot in component", "dot-name", "/opt/feature_repo.v2"), + ) + + DescribeTable("rejects non-canonical, relative, or root paths", + func(nameSuffix, featureRepoPath string) { + featureStore := apiVersion.factory( + "packaged-"+apiVersion.id+"-"+nameSuffix, + featureRepoPath, + ) + err := k8sClient.Create(ctx, featureStore) + Expect(err).To(HaveOccurred()) + Expect(apierrors.IsInvalid(err)).To(BeTrue(), "expected invalid error, got %v", err) + Expect(strings.ToLower(err.Error())).To(ContainSubstring("canonical absolute, non-root path")) + }, + Entry("relative", "relative", "opt/feast/feature_repo"), + Entry("root", "root", "/"), + Entry("parent collapses to root", "parent-root", "/opt/.."), + Entry("leading parent traversal", "leading-parent", "/../x"), + Entry("repeated separator", "repeated-separator", "/opt//feature_repo"), + Entry("current-directory component", "current-dir", "/opt/./feature_repo"), + Entry("trailing separator", "trailing-separator", "/opt/feature_repo/"), + Entry("nested traversal", "nested-traversal", "/a/../../etc"), + Entry("repeated root separator", "repeated-root", "//"), + ) + + DescribeTable("rejects packaged together with another project directory mode", + func(nameSuffix, conflictingMode string) { + featureStore := apiVersion.conflictingFactory( + "packaged-"+apiVersion.id+"-"+nameSuffix, + conflictingMode, + ) + err := k8sClient.Create(ctx, featureStore) + Expect(err).To(HaveOccurred()) + Expect(apierrors.IsInvalid(err)).To(BeTrue(), "expected invalid error, got %v", err) + Expect(err.Error()).To(ContainSubstring("One selection required between init, git, or packaged")) + }, + Entry("init", "with-init", "init"), + Entry("git", "with-git", "git"), + ) + }) + } +}) diff --git a/infra/feast-operator/test/api/suite_test.go b/infra/feast-operator/test/api/suite_test.go index 558068a7957..eef4718cf58 100644 --- a/infra/feast-operator/test/api/suite_test.go +++ b/infra/feast-operator/test/api/suite_test.go @@ -26,6 +26,7 @@ import ( . "github.com/onsi/gomega" feastdevv1 "github.com/feast-dev/feast/infra/feast-operator/api/v1" + feastdevv1alpha1 "github.com/feast-dev/feast/infra/feast-operator/api/v1alpha1" "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/rest" "sigs.k8s.io/controller-runtime/pkg/client" @@ -73,6 +74,8 @@ var _ = BeforeSuite(func() { err = feastdevv1.AddToScheme(scheme.Scheme) Expect(err).NotTo(HaveOccurred()) + err = feastdevv1alpha1.AddToScheme(scheme.Scheme) + Expect(err).NotTo(HaveOccurred()) // +kubebuilder:scaffold:scheme diff --git a/infra/scripts/feature_server_docker_smoke.py b/infra/scripts/feature_server_docker_smoke.py index c8b3b440b7f..801decac90c 100644 --- a/infra/scripts/feature_server_docker_smoke.py +++ b/infra/scripts/feature_server_docker_smoke.py @@ -16,6 +16,7 @@ def list_projects(self, allow_cache=True, tags=None): class _FakeStore: def __init__(self): self.config = SimpleNamespace() + self.project = "smoke_test" self.registry = _FakeRegistry() self._provider = SimpleNamespace( async_supported=SimpleNamespace( @@ -32,6 +33,9 @@ async def initialize(self): def refresh_registry(self): return None + def list_feature_views(self): + return [] + async def close(self): return None diff --git a/infra/templates/README.md.jinja2 b/infra/templates/README.md.jinja2 index ccaadc29ff0..2c92401f83d 100644 --- a/infra/templates/README.md.jinja2 +++ b/infra/templates/README.md.jinja2 @@ -17,7 +17,7 @@ ## Join us on Slack! -👋👋👋 [Come say hi on Slack!](https://communityinviter.com/apps/feastopensource/feast-the-open-source-feature-store) +👋👋👋 [Come say hi on Slack!](https://slack.feast.dev/) [Check out our DeepWiki!](https://deepwiki.com/feast-dev/feast) diff --git a/infra/website/docs/blog/feast-agents-mcp.md b/infra/website/docs/blog/feast-agents-mcp.md index 5678cd5a578..bfa46ee8b02 100644 --- a/infra/website/docs/blog/feast-agents-mcp.md +++ b/infra/website/docs/blog/feast-agents-mcp.md @@ -51,7 +51,7 @@ feature_server: mcp_server_version: "1.0.0" ``` -Once enabled, any MCP-compatible agent -- whether built with LangChain, LlamaIndex, CrewAI, AutoGen, or a custom framework -- can connect to `http://your-feast-server/mcp` and discover available tools like `get-online-features` for entity-based retrieval, `retrieve-online-documents` for vector similarity search, and `write-to-online-store` for persisting agent state. +Once enabled, any MCP-compatible agent -- whether built with LangChain, LlamaIndex, CrewAI, AutoGen, or a custom framework -- can connect to `http://your-feast-server/mcp` and discover available tools like `get-online-features` for entity-based retrieval, `search` for vector similarity search, `vector_store_search` for OpenAI-compatible text search, and `write-to-online-store` for persisting agent state. ## A Concrete Example: Customer-Support Agent with Memory @@ -338,7 +338,7 @@ export OPENAI_BASE_URL="http://localhost:11434/v1" export LLM_MODEL="llama3.1:8b" ./run_demo.sh -# Any OpenAI-compatible provider (Azure, vLLM, LiteLLM, etc.) +# Any OpenAI-compatible provider (Azure, vLLM, etc.) export OPENAI_API_KEY="your-key" # pragma: allowlist secret export OPENAI_BASE_URL="https://your-endpoint/v1" export LLM_MODEL="your-model" diff --git a/infra/website/docs/blog/feast-data-quality-monitoring.md b/infra/website/docs/blog/feast-data-quality-monitoring.md index 88e8550f83f..2c918c83a63 100644 --- a/infra/website/docs/blog/feast-data-quality-monitoring.md +++ b/infra/website/docs/blog/feast-data-quality-monitoring.md @@ -31,7 +31,7 @@ The biggest change is that monitoring is now a first-class Feast workflow: ## From validation to monitoring -Feast previously supported a Great Expectations-based DQM path for validating historical retrievals. That integration was useful, but it lived outside the normal feature store workflow: users had to install the `feast[ge]` extra, write profiler code, and run validation against saved datasets. +Feast previously supported an external-library-based validation path for historical retrievals. That integration was useful, but it lived outside the normal feature store workflow: users had to install extra dependencies, write profiler code, and run validation against saved datasets. That original integration proved the need for data quality inside Feast. It helped answer an important question: after generating a training dataset, does this dataset satisfy the expectations we care about? diff --git a/infra/website/docs/blog/feast-mlflow-kubeflow.md b/infra/website/docs/blog/feast-mlflow-kubeflow.md index 5199da61da5..d0b89ac7138 100644 --- a/infra/website/docs/blog/feast-mlflow-kubeflow.md +++ b/infra/website/docs/blog/feast-mlflow-kubeflow.md @@ -199,10 +199,6 @@ For cross-system lineage that extends beyond Feast into upstream data pipelines ### Data quality monitoring -:::note -The Great Expectations integration described in earlier versions of this post is now **deprecated**. Feast includes a built-in [Feature Quality Monitoring](/docs/how-to-guides/feature-monitoring) system that provides richer metrics, requires no external dependencies, and includes a monitoring UI. -::: - Feast's native data quality monitoring system automatically computes statistical metrics — null rates, distributions, percentiles, histograms — for every registered feature across both batch data and serving logs. It detects drift by comparing current metrics against baselines computed during `feast apply`. ```yaml diff --git a/infra/website/docs/blog/feast-openai-compatible-api.md b/infra/website/docs/blog/feast-openai-compatible-api.md new file mode 100644 index 00000000000..f228836afd6 --- /dev/null +++ b/infra/website/docs/blog/feast-openai-compatible-api.md @@ -0,0 +1,364 @@ +--- +title: "Using Feast's OpenAI Compatible Search API" +description: "Feast now exposes an OpenAI-compatible vector store search endpoint. Send a plain text query, get results back in the standard OpenAI format. No client-side embeddings required." +date: 2026-07-07 +authors: ["Chaitanya Patel", "Nikhil Kathole"] +--- + +
+ Sequence diagram showing a client sending a text query to Feast, which embeds and searches server-side +
+ +If you've tried to connect an AI agent to Feast's vector search, you've probably hit this wall: the agent needs to search your feature store, but Feast expects a raw embedding vector. The agent doesn't have one. It has a question in English. + +Until now, the workaround was ugly. You'd call an embedding provider (OpenAI, Ollama, whatever) to turn the text into a float array, then pass that array to Feast's vector search endpoint (`POST /search`, formerly `retrieve-online-documents`). Every client had to know both APIs, carry both sets of credentials, and run glue code whose only job was bridging the gap. + +Feast now has a new endpoint: `POST /v1/vector_stores/{vector_store_id}/search`. It follows the [OpenAI Vector Store Search API](https://platform.openai.com/docs/api-reference/vector-stores-search) format, including proper `vs_{hash}` identifiers for vector stores. You send text, Feast handles the embedding internally, and you get results back in the same JSON shape that OpenAI returns. No float arrays, no extra SDK. + +Each feature view with vector search enabled gets a deterministic `vs_` identifier (e.g. `vs_a1b2c3d4e5f6...`). Discover them via `GET /v1/vector_stores`. + +## The two-API tax + +Here's what searching Feast looked like before: + +```python +import openai +import requests + +# Step 1: Call the embedding provider yourself +embed_response = openai.embeddings.create( + model="text-embedding-3-small", + input="wireless noise-cancelling headphones" +) +query_vector = embed_response.data[0].embedding # 1536 floats + +# Step 2: Call Feast's proprietary API with the raw vector +result = requests.post("http://feast-server:6566/search", json={ + "features": [ + "product_catalog:vector", + "product_catalog:name", + "product_catalog:description", + "product_catalog:price", + ], + "query": query_vector, + "top_k": 5, + "api_version": 2, +}) +``` + +This works fine. But it has costs that add up: + +- Every service calling Feast needs an embedding SDK, an API key, and logic to handle the embedding call. Five microservices means five places managing embedding credentials. +- LLM agents can't use it. They discover tools through MCP or function calling, and they know how to call OpenAI-shaped endpoints. They don't know how to compute embeddings and pass raw float arrays to a custom API. +- The embedding model becomes a client-side decision. Different clients might use different models or versions, which means inconsistent search results against the same vector store. +- Feast's filter syntax is its own format. Not something an agent framework knows out of the box. + +## One endpoint, standard format + +With the new endpoint, that same search looks like this: + +```python +import requests + +# First, discover your vector store IDs +stores = requests.get("http://feast-server:6566/v1/vector_stores").json() +vs_id = stores["data"][0]["id"] # e.g. "vs_a1b2c3d4e5f6..." + +# Then search using the vs_ identifier +result = requests.post( + f"http://feast-server:6566/v1/vector_stores/{vs_id}/search", + json={ + "query": "wireless noise-cancelling headphones", + "max_num_results": 5, + }, +) +``` + +No embedding SDK. No raw vectors. The request and response match OpenAI's format, so anything that already talks to OpenAI can talk to Feast. + +### What happens under the hood + +When Feast receives this request, it: + +1. Embeds the query server-side using the model configured in `feature_store.yaml` (via [Sentence Transformers](https://www.sbert.net/) for local inference — no external API key required). +2. Runs vector similarity search against the feature view's online store (Postgres/pgvector, Milvus, Elasticsearch, SQLite, or whatever backend you've configured). +3. Applies filters if you provided any, using string equality, numeric comparisons, or compound AND/OR conditions in the OpenAI filter format. +4. Returns results in OpenAI's `vector_store.search_results.page` format. + +Because the embedding model is a server-side configuration, every client gets consistent results. No more worrying about whether service A is using `text-embedding-3-small` while service B accidentally stuck with `ada-002`. + +## Setting it up + +### Step 1: Configure the embedding model + +Add an `embedding_model` section to your `feature_store.yaml`: + +```yaml +project: my_project +registry: data/registry.db +provider: local + +online_store: + type: postgres + host: localhost + port: 5432 + database: feast + user: feast + password: ${DB_PASSWORD} + pgvector_enabled: true + vector_len: 384 + enable_openai_compatible_store: true + +embedding_model: + provider: sentence_transformers # default; can be omitted + model: all-MiniLM-L6-v2 +``` + +Feast uses [Sentence Transformers](https://www.sbert.net/) for embedding, so everything runs locally — no external API key required. You can use any HuggingFace model compatible with `SentenceTransformer`: + +```yaml +# Default — lightweight, fast +embedding_model: + model: all-MiniLM-L6-v2 + +# Higher quality, larger model +embedding_model: + model: BAAI/bge-small-en-v1.5 +``` + +### Step 2: Define a feature view with vector search + +```python +from feast import Entity, FeatureView, Field +from feast.types import Array, Float32, String, Float64, Int64 +from datetime import timedelta + +product = Entity(name="product_id", join_keys=["product_id"]) + +product_catalog = FeatureView( + name="product_catalog", + entities=[product], + schema=[ + Field( + name="vector", + dtype=Array(Float32), + vector_index=True, + vector_search_metric="COSINE", + ), + Field(name="name", dtype=String), + Field(name="description", dtype=String), + Field(name="category", dtype=String), + Field(name="price", dtype=Float64), + Field(name="rating", dtype=Float64), + ], + source=product_source, + ttl=timedelta(days=7), +) +``` + +### Step 3: Apply, load data, and serve + +```bash +feast apply +feast serve +``` + +### Step 4: Discover your vector store ID + +```bash +curl http://localhost:6566/v1/vector_stores +``` + +```json +{ + "object": "list", + "data": [ + { + "id": "vs_a1b2c3d4e5f6a1b2c3d4e5f6", + "object": "vector_store", + "name": "product_catalog", + "status": "completed", + "created_at": 1717200000 + } + ] +} +``` + +### Step 5: Search + +```bash +curl -X POST http://localhost:6566/v1/vector_stores/vs_a1b2c3d4e5f6a1b2c3d4e5f6/search \ + -H "Content-Type: application/json" \ + -d '{ + "query": "wireless noise-cancelling headphones", + "max_num_results": 3 + }' +``` + +Response: + +```json +{ + "object": "vector_store.search_results.page", + "search_query": ["wireless noise-cancelling headphones"], + "data": [ + { + "file_id": "vs_a1b2c3d4e5f6a1b2c3d4e5f6_42", + "filename": "vs_a1b2c3d4e5f6a1b2c3d4e5f6", + "score": 0.92, + "attributes": { + "name": "Sony WH-1000XM5", + "description": "Premium wireless noise-cancelling headphones", + "category": "Electronics", + "price": 349.99, + "rating": 4.8 + }, + "content": [ + {"type": "text", "text": "Sony WH-1000XM5"}, + {"type": "text", "text": "Premium wireless noise-cancelling headphones"}, + {"type": "text", "text": "Electronics"} + ] + } + ], + "has_more": false, + "next_page": null +} +``` + +The response follows OpenAI's `vector_store.search_results.page` schema. Any client that already parses OpenAI search results can parse this without changes. + +## Filtering + +The endpoint supports OpenAI-style filters for narrowing results beyond vector similarity. Filters work on the metadata stored alongside your vectors. + +### String filters + +```json +{ + "query": "running shoes", + "max_num_results": 5, + "filters": { + "type": "eq", + "key": "category", + "value": "Footwear" + } +} +``` + +### Numeric filters + +```json +{ + "query": "budget laptop", + "max_num_results": 5, + "filters": { + "type": "lt", + "key": "price", + "value": 500.0 + } +} +``` + +### Compound filters (AND / OR) + +```json +{ + "query": "wireless earbuds", + "max_num_results": 5, + "filters": { + "type": "and", + "filters": [ + {"type": "eq", "key": "category", "value": "Electronics"}, + {"type": "gte", "key": "rating", "value": 4.5}, + {"type": "lt", "key": "price", "value": 200.0} + ] + } +} +``` + +Comparison operators: `eq`, `ne`, `gt`, `gte`, `lt`, `lte`, `in`, `nin`. Compound operators: `and`, `or`. These nest to arbitrary depth. + +Numeric and boolean filters require the `enable_openai_compatible_store` flag in your online store config, plus a `feast apply` to add the `value_num` column to existing tables. String filters work on all existing schemas without migration. + +## What this means for AI agents + +We built this with agents in mind. When Feast added [MCP support](./feast-agents-mcp) earlier this year, agents could discover and call Feast tools dynamically. But vector search still had this gap where the agent needed to produce a float array. LLMs can't do that. + +Now the search tool is just text in, structured results out. An agent calls it the same way it calls any other OpenAI-compatible service. The feature server currently exposes these tools: + +| Capability | Endpoint | What it does | +|---|---|---| +| Structured feature lookup | `get-online-features` | Get customer profiles, account data, etc. | +| Vector search | `search` | Search with a pre-computed embedding vector (or text via `api_version: 2`) | +| List vector stores | `GET /v1/vector_stores` | Discover available vector stores and their `vs_` IDs | +| Get vector store | `GET /v1/vector_stores/{id}` | Get metadata for a specific vector store | +| Vector search (OpenAI format) | `POST /v1/vector_stores/{id}/search` | Search with plain text, embedding handled server-side | +| Write features / memory | `write-to-online-store` | Persist agent state, update features | + +`POST /retrieve-online-documents` remains available as a deprecated alias for `POST /search`. + +That last row is what this post is about. Before it existed, agents could read structured features and write state back, but they couldn't search vectors without help from glue code. + +## What this is, and what it isn't + +This makes Feast's vector search speak OpenAI's protocol. It doesn't turn Feast into a general purpose OpenAI-compatible vector database. + +| Works today | Not yet | +|---|---| +| `GET /v1/vector_stores` (list) | Creating vector stores via the API | +| `GET /v1/vector_stores/{id}` (get) | | +| `POST /v1/vector_stores/{id}/search` | | +| Plain text queries with server-side embedding | Client-provided embedding vectors on this endpoint | +| OpenAI-format filters (string, numeric, compound) | `ranking_options.score_threshold`, `ranking_options.ranker`, `rewrite_query: true` (rejected with 422) | +| All Feast online store backends | Standalone `/v1/embeddings` endpoint | + +Feature views are still defined in Python and managed through `feast apply`. Data is still ingested through Feast's existing write paths. The OpenAI-compatible layer is a read API that gives standard access to what's already in your feature store. + +## Deploying on Kubernetes + +Below is an example Kubernetes setup that deploys the feature server with Sentence Transformers for local embedding: + +```yaml +# configmap.yaml (embedding model section) +embedding_model: + provider: sentence_transformers + model: all-MiniLM-L6-v2 +``` + +```yaml +# deployment.yaml +containers: + - name: feast-server + command: ["feast", "serve", "-h", "0.0.0.0", "-p", "6566"] + ports: + - containerPort: 6566 +``` + +With this setup, embedding happens in-cluster. Nothing leaves your network. + +## Try it yourself + +```bash +# Install Feast with Sentence Transformers support +pip install feast sentence-transformers +``` + +Configure your `feature_store.yaml` with an `embedding_model` section, define a feature view with vector search enabled, run `feast apply`, load your data, start the server with `feast serve`, and search: + +```bash +# Discover your vector store IDs +curl -s http://localhost:6566/v1/vector_stores | python -m json.tool + +# Search using the vs_ identifier from the list response +curl -s http://localhost:6566/v1/vector_stores/YOUR_VS_ID/search \ + -H "Content-Type: application/json" \ + -d '{"query": "your search query", "max_num_results": 5}' | python -m json.tool +``` + +## What's next + +Next on the list: wiring up `ranking_options` and `rewrite_query` so they actually do something (right now they're accepted but ignored). We also want a standalone `/v1/embeddings` endpoint for clients that just need embeddings, and eventually the ability to create feature views through the OpenAI vector store API instead of requiring Python + `feast apply`. + +## Join the conversation + +If you're using this or have thoughts on what the OpenAI-compatible layer should support next, come find us on [Slack](https://slack.feast.dev/) or [GitHub](https://github.com/feast-dev/feast). diff --git a/infra/website/docs/blog/feast-ray-llm-posttrain.md b/infra/website/docs/blog/feast-ray-llm-posttrain.md new file mode 100644 index 00000000000..4b5eac30214 --- /dev/null +++ b/infra/website/docs/blog/feast-ray-llm-posttrain.md @@ -0,0 +1,304 @@ +--- +title: "How to Use Feast for SLM/LLM Post-Training with Ray" +description: "Keep conversation features in Feast, retrieve them for training, then stream into your trainer with Ray." +date: 2026-07-14 +authors: ["Chaitanya Patel"] +--- + +# How to Use Feast for SLM/LLM Post-Training with Ray + +Your support bot answers a lot of tickets. It’s fine—but it sounds generic. The team wants a smaller model that talks more like *your* agents: your refund wording, your product names, your tone. + +So someone says: **fine-tune on our real chats.** + +That part sounds easy. The messy part is the data—exports, notebook cleaning, and prompt formatting scattered across training scripts. + +This post walks through the [ray-llm-posttrain example](https://github.com/feast-dev/feast/tree/master/examples/ray-llm-posttrain): + +1. Put conversation features in Feast +2. Retrieve them with `get_historical_features` (entity-less date range) +3. Get rows into your trainer — stream with Ray **or** materialize with `.to_df()` + +You bring your own trainer. GPT-2 in the script is optional smoke only. + +## What’s in the example + +| Name | Type | What it holds | +|---|---|---| +| `web_documents` | [FeatureView](https://docs.feast.dev/getting-started/concepts/feature-view) | `human`, `bot`, `human_repeat_ratio`, `bot_repeat_ratio` | +| `train_example` | [OnDemandFeatureView](https://docs.feast.dev/reference/beta-on-demand-feature-view) | `cleaned_human`, `cleaned_bot`, `char_count`, `is_trainable`, `sft_text` | +| `llm_posttrain` | FeatureService | Bundles `web_documents` + `train_example` | + +Full definitions live in [feature_definitions.py](https://github.com/feast-dev/feast/blob/master/examples/ray-llm-posttrain/feature_repo/feature_definitions.py). Ray is the [offline store](https://docs.feast.dev/reference/offline-stores/ray) and one way to stream rows out—not a separate feature catalog. + +This example stays on **supported Feast APIs only** (no core patches). Conversation rows already include `document_id` and `event_timestamp` before Feast reads them. + +## Step 1: Point Feast at conversation data + +### Ray offline store (local) + +From the example [feature_store.yaml](https://github.com/feast-dev/feast/blob/master/examples/ray-llm-posttrain/feature_repo/feature_store.yaml). Cap Ray resources on a laptop—see [Ray offline store: resource management](https://docs.feast.dev/reference/offline-stores/ray#important-resource-management): + +```yaml +project: ray_llm_posttrain +registry: data/registry.db +provider: local + +offline_store: + type: ray + storage_path: data/ray_storage + enable_ray_logging: false + ray_conf: + num_cpus: 2 + object_store_memory: 104857600 + _memory: 524288000 + +batch_engine: + type: ray.engine + max_workers: 2 + +online_store: + type: sqlite + path: data/online_store.db + +entity_key_serialization_version: 3 +auth: + type: no_auth +``` + +You can also start from the built-in template: + +```bash +feast init -t ray my_ray_project +``` + +See the [Ray template / offline store docs](https://docs.feast.dev/reference/offline-stores/ray#quick-start-with-ray-template) and the related blog [Scaling ML with Feast and Ray](/blog/feast-ray-distributed-processing). + +### Demo seed: prepare parquet, then `RaySource` + +[RaySource](https://docs.feast.dev/reference/data-sources/ray) tells Feast how to load data through Ray. Hugging Face is only used in a **prepare script**—not as a live Feast source that invents timestamps at retrieval time. + +`nampdn-ai/tiny-webtext` has no `document_id` / `event_timestamp`. Entity-less retrieval needs those columns on the source. We add them **outside Feast**, write parquet, then point Feast at that file (supported path): + +```bash +PYTHONPATH=../../sdk/python python scripts/prepare_data.py +# → feature_repo/data/tiny_webtext.parquet +``` + +```python +from feast.infra.offline_stores.contrib.ray_offline_store.ray_source import RaySource + +tiny_web = RaySource( + name="tiny_webtext", + reader_type="parquet", + path="data/tiny_webtext.parquet", + timestamp_field="event_timestamp", +) +``` + +In production you’d skip the HF prepare step and register your real conversation store (warehouse / lake / parquet) that already has join keys and timestamps. + +More reader types are in the [Ray data source reference](https://docs.feast.dev/reference/data-sources/ray#supported-reader_type-values). + +### Feature view + +```python +web_documents = FeatureView( + name="web_documents", + entities=[document], + ttl=timedelta(days=365), + schema=[ + Field(name="human", dtype=String), + Field(name="bot", dtype=String), + Field(name="human_repeat_ratio", dtype=Float64), + Field(name="bot_repeat_ratio", dtype=Float64), + ], + source=tiny_web, + online=False, +) +``` + +### Optional: OnDemandFeatureView for derived training features + +If you want Feast to own `sft_text` / quality gates (same idea as in the [ODFV docs](https://docs.feast.dev/reference/beta-on-demand-feature-view)): + +```python +@on_demand_feature_view( + sources=[web_documents], + schema=[ + Field(name="cleaned_human", dtype=String), + Field(name="cleaned_bot", dtype=String), + Field(name="char_count", dtype=Int64), + Field(name="is_trainable", dtype=Bool), + Field(name="sft_text", dtype=String), + ], + mode="pandas", +) +def train_example(inputs): + cleaned_human = inputs["human"].fillna("").astype(str).str.strip() + cleaned_bot = inputs["bot"].fillna("").astype(str).str.strip() + # ... length + repeat-ratio gate ... + sft_text = ( + "<|im_start|>user\n" + cleaned_human + "<|im_end|>\n" + "<|im_start|>assistant\n" + cleaned_bot + "<|im_end|>" + ) + return pd.DataFrame({...}) +``` + +```python +llm_posttrain = FeatureService( + name="llm_posttrain", + features=[web_documents, train_example], +) +``` + +Apply: + +```bash +cd examples/ray-llm-posttrain/feature_repo +feast apply +``` + +## Step 2: Retrieve for training (entity-less) + +No `entity_df`—just a date window. That pattern is covered in [Historical Features Without Entity IDs](/blog/entity-less-historical-features-retrieval) and the [FAQ](https://docs.feast.dev/getting-started/faq#how-do-i-run-get_historical_features-without-providing-an-entity-dataframe): + +```python +from datetime import datetime, timezone +from feast import FeatureStore + +store = FeatureStore(repo_path="feature_repo") + +job = store.get_historical_features( + features=[ + "web_documents:human", + "web_documents:bot", + "web_documents:human_repeat_ratio", + "web_documents:bot_repeat_ratio", + ], + start_date=datetime(2024, 6, 1, tzinfo=timezone.utc), + end_date=datetime(2024, 7, 1, tzinfo=timezone.utc), +) +``` + +Then choose how you turn that job into training rows. + +## Step 3: Two ways into the trainer + +| Path | ODFV runs? | When to use | +|---|---|---| +| `job.to_ray_dataset()` then preprocess | **No** | Stream FeatureView columns; shape `sft_text` yourself | +| `job.to_df()` / `to_arrow()` | **Yes** | Want `train_example` outputs from Feast | + +Pick **Option A** when you want full control over text formatting or need custom preprocessing (e.g., multi-turn chat templates, tokenization-aware truncation). Pick **Option B** when you want Feast to enforce quality gates consistently across training and serving. + +### Option A — Stream with Ray, preprocess yourself + +`to_ray_dataset()` returns a Ray Dataset of retrieved FeatureView columns. It does **not** apply OnDemandFeatureViews. Build training text with Ray `map_batches` (as in [train_sft.py](https://github.com/feast-dev/feast/blob/master/examples/ray-llm-posttrain/scripts/train_sft.py)): + +```python +ds = job.to_ray_dataset() + +def preprocess_sft(batch): + import pandas as pd + + if not isinstance(batch, pd.DataFrame): + batch = pd.DataFrame(batch) + human = batch["human"].fillna("").astype(str).str.strip() + bot = batch["bot"].fillna("").astype(str).str.strip() + ok = bot.str.len() >= 64 + sft_text = ( + "<|im_start|>user\n" + human + "<|im_end|>\n" + "<|im_start|>assistant\n" + bot + "<|im_end|>" + ) + return pd.DataFrame({"sft_text": sft_text}).loc[ok].reset_index(drop=True) + +train_ds = ds.map_batches(preprocess_sft, batch_format="pandas") +# → hand train_ds to your SLM/LLM trainer +``` + +Run the example default path: + +```bash +PYTHONPATH=../../sdk/python python scripts/train_sft.py --dry-run +``` + +### Option B — Use the ODFV, then train + +Materialize with `.to_df()` so `train_example` runs (same retrieval/serving idea as in the [ODFV overview](https://docs.feast.dev/reference/beta-on-demand-feature-view#why-use-on-demand-feature-views)): + +```python +df = store.get_historical_features( + features=store.get_feature_service("llm_posttrain"), + start_date=datetime(2024, 6, 1, tzinfo=timezone.utc), + end_date=datetime(2024, 7, 1, tzinfo=timezone.utc), +).to_df() + +trainable = df[df["is_trainable"] & df["sft_text"].astype(str).str.len().gt(0)] +# trainable["sft_text"] → your trainer +# or: import ray; ray.data.from_pandas(trainable[["sft_text"]]) +``` + +```bash +PYTHONPATH=../../sdk/python python scripts/train_sft.py --dry-run --via-df +``` + +### The same ODFV at serving time + +The `train_example` ODFV runs identically during online serving — the quality gate and formatting logic stay in one place: + +```python +# At inference time, the same ODFV runs on the fly +features = store.get_online_features( + features=["train_example:sft_text", "train_example:is_trainable"], + entity_rows=[{"document_id": "doc_42"}], +).to_dict() +# features["sft_text"], features["is_trainable"] — same logic as training +``` + +## Try the full example + +```bash +cd examples/ray-llm-posttrain +uv pip install -e "../../sdk/python[ray]" -r requirements.txt +PYTHONPATH=../../sdk/python python scripts/prepare_data.py +cd feature_repo && feast apply && cd .. + +PYTHONPATH=../../sdk/python python scripts/train_sft.py --dry-run +PYTHONPATH=../../sdk/python python scripts/train_sft.py --dry-run --via-df + +# optional GPT-2 smoke +PYTHONPATH=../../sdk/python python scripts/train_sft.py --max-steps 20 +``` + +Details: [ray-llm-posttrain README](https://github.com/feast-dev/feast/tree/master/examples/ray-llm-posttrain). + +## Takeaways + +1. **Keep conversation features in Feast** — this example’s `web_documents`. +2. **Stream with Ray** — `to_ray_dataset()`, then preprocess training text yourself. +3. **Want ODFVs** — `.to_df()` / `.to_arrow()` to materialize, then train. +4. **Bring your own trainer** — GPT-2 in the example is optional. + +## References + +**This example** + +- [ray-llm-posttrain example](https://github.com/feast-dev/feast/tree/master/examples/ray-llm-posttrain) +- [feature_definitions.py](https://github.com/feast-dev/feast/blob/master/examples/ray-llm-posttrain/feature_repo/feature_definitions.py) +- [train_sft.py](https://github.com/feast-dev/feast/blob/master/examples/ray-llm-posttrain/scripts/train_sft.py) + +**Docs** + +- [Ray offline store](https://docs.feast.dev/reference/offline-stores/ray) +- [Ray data source](https://docs.feast.dev/reference/data-sources/ray) +- [Ray compute engine](https://docs.feast.dev/reference/compute-engine/ray) +- [On demand feature views](https://docs.feast.dev/reference/beta-on-demand-feature-view) +- [Feature retrieval](https://docs.feast.dev/getting-started/concepts/feature-retrieval) +- [FAQ: historical features without entity dataframe](https://docs.feast.dev/getting-started/faq#how-do-i-run-get_historical_features-without-providing-an-entity-dataframe) + +**Related blogs & tutorials** + +- [Historical Features Without Entity IDs](/blog/entity-less-historical-features-retrieval) +- [Scaling ML with Feast and Ray](/blog/feast-ray-distributed-processing) +- [Validating historical features](https://docs.feast.dev/tutorials/validating-historical-features) diff --git a/infra/website/docs/blog/feast-unity-catalog-integration.md b/infra/website/docs/blog/feast-unity-catalog-integration.md new file mode 100644 index 00000000000..1ee977a71eb --- /dev/null +++ b/infra/website/docs/blog/feast-unity-catalog-integration.md @@ -0,0 +1,342 @@ +--- +title: "Feast Gets Native Apache Iceberg Support" +description: "Feast now reads features from any Iceberg catalog — REST, SQL, Hive, Glue, DynamoDB. Connect to Unity Catalog, Apache Polaris, Nessie, or your own PyIceberg catalog. Full support for get_historical_features, materialize, and online serving." +date: 2026-07-18 +authors: ["Nikhil Kathole"] +--- + +# Feast Gets Native Apache Iceberg Support + +Apache Iceberg has become the open table format. Your data lake is probably already on it — whether through Databricks, Snowflake, AWS, or self-managed infrastructure. But until now, connecting Feast to Iceberg tables meant either going through Spark (heavyweight, slow to start) or copying data into Feast-managed Parquet files (data duplication, governance gap). + +Feast now ships a native `IcebergSource` that reads directly from any Iceberg catalog. No data copies. Your feature tables live where they already live — in your Iceberg catalog — and Feast reads from them via PyIceberg. With the DuckDB offline store, you don't even need a Spark cluster — reads happen entirely in-process. + +## Why This Matters + +Before this, the path from "data in Iceberg" to "features in Feast" looked like this: + +1. Data engineers build Iceberg tables in their catalog (UC, Glue, Hive) +2. ML engineers copy data to Feast-managed Parquet files, or configure a SparkSource that couples them to a specific compute engine +3. Two copies of the data. Two metadata systems. No connection between them. + +Now the path is: + +1. Data engineers build Iceberg tables in their catalog +2. ML engineers point `IcebergSource` at the table +3. Done. Feast reads directly from the catalog via PyIceberg. One copy. One source of truth. Choose DuckDB for lightweight local reads or Spark when you need distributed compute — the data source definition stays the same either way. + +## What You Get + +### Any Iceberg Catalog + +`IcebergSource` supports every catalog backend that PyIceberg supports: + +| `catalog_type` | Backend | Example Use Case | +|---|---|---| +| `"rest"` | Iceberg REST Catalog | Databricks Unity Catalog, Apache Polaris, Project Nessie, Snowflake Open Catalog | +| `"sql"` | SQL-backed catalog | Local dev with SQLite, CI/CD, PostgreSQL-backed catalogs | +| `"hive"` | Hive Metastore | On-premise Hadoop, EMR | +| `"glue"` | AWS Glue Data Catalog | AWS-native lakehouse | +| `"dynamodb"` | DynamoDB catalog | Serverless AWS | + +### Both Offline Stores + +| Operation | DuckDB | Spark | +|---|---|---| +| `feast apply` | Yes | Yes | +| `get_historical_features` | Yes | Yes | +| `materialize` / `materialize-incremental` | Yes | Yes | +| `get_online_features` | Yes | Yes | + +Both offline stores use PyIceberg for the actual Iceberg table scan — the difference is what happens after. DuckDB processes the Arrow table in-process (no JVM, no cluster), making it ideal for local development and moderate-scale workloads. Spark is there when you need distributed compute over large datasets. The same `IcebergSource` definition works with either offline store — just change `offline_store.type` in your YAML. + +### Full Iceberg Semantics + +Every read goes through PyIceberg's `table.scan().to_arrow()`. This means you get proper Iceberg semantics: schema evolution, partition pruning, and snapshot isolation — not just raw Parquet file reads. + +## Quick Start + +### Install + +```bash +pip install "feast[iceberg]" +``` + +### Define a Source + +```python +from feast.infra.data_sources.contrib.iceberg_catalog import IcebergSource + +driver_stats = IcebergSource( + warehouse="my_catalog", + namespace="ml_features", + table="driver_hourly_stats", + catalog_type="rest", + endpoint="https://my-iceberg-catalog.example.com", + token_env_var="CATALOG_TOKEN", + timestamp_field="event_timestamp", +) +``` + +### Use It + +```python +from datetime import timedelta +from feast import Entity, FeatureView, Field +from feast.types import Float64, Int64 + +driver = Entity(name="driver", join_keys=["driver_id"]) + +driver_stats_fv = FeatureView( + name="driver_hourly_stats", + entities=[driver], + ttl=timedelta(days=365), + schema=[ + Field(name="driver_id", dtype=Int64), + Field(name="conv_rate", dtype=Float64), + Field(name="acc_rate", dtype=Float64), + Field(name="avg_daily_trips", dtype=Int64), + ], + source=driver_stats, + online=True, +) +``` + +```yaml +# feature_store.yaml +project: my_project +registry: data/registry.db +provider: local +online_store: + type: sqlite + path: data/online_store.db +offline_store: + type: duckdb +``` + +```bash +feast apply +``` + +Then use it like any other Feast source: + +```python +from feast import FeatureStore +import pandas as pd +from datetime import datetime, timezone + +store = FeatureStore(repo_path=".") + +# Training data +training_df = store.get_historical_features( + entity_df=pd.DataFrame({ + "driver_id": [1001, 1002, 1003], + "event_timestamp": [datetime(2026, 7, 1, tzinfo=timezone.utc)] * 3, + }), + features=["driver_hourly_stats:conv_rate", "driver_hourly_stats:acc_rate"], +).to_df() + +# Materialize to online store +store.materialize_incremental(end_date=datetime.now(tz=timezone.utc)) + +# Online serving +online = store.get_online_features( + features=["driver_hourly_stats:conv_rate", "driver_hourly_stats:acc_rate"], + entity_rows=[{"driver_id": 1001}], +).to_dict() +``` + +## Catalog Examples + +### AWS Glue + +```python +glue_source = IcebergSource( + warehouse="my_glue_database", + namespace="ml_features", + table="driver_stats", + catalog_type="glue", + catalog_properties={"region_name": "us-east-1"}, + timestamp_field="event_timestamp", +) +``` + +### Hive Metastore + +```python +hive_source = IcebergSource( + endpoint="thrift://hive-metastore:9083", + warehouse="warehouse", + namespace="features", + table="driver_stats", + catalog_type="hive", + timestamp_field="event_timestamp", +) +``` + +### Apache Polaris / Nessie + +```python +polaris_source = IcebergSource( + endpoint="https://polaris.example.com", + warehouse="my_catalog", + namespace="ml", + table="features", + catalog_type="rest", + token_env_var="POLARIS_TOKEN", + timestamp_field="event_timestamp", +) +``` + +### Local Development (SQLite-backed) + +For development and CI/CD, use a local PyIceberg SQL catalog — no external service required: + +```python +local_source = IcebergSource( + warehouse="dev_warehouse", + namespace="default", + table="driver_stats", + catalog_type="sql", + catalog_name="dev_catalog", + catalog_properties={ + "uri": "sqlite:////tmp/iceberg_catalog.db", + "warehouse": "file:///tmp/iceberg_warehouse", + }, + timestamp_field="event_timestamp", +) +``` + +## Use Case: Unity Catalog Integration + +Unity Catalog users get everything above with simpler configuration. `UnityCatalogSource` extends `IcebergSource` with UC-specific defaults: + +- Default connection via `DATABRICKS_HOST` and `DATABRICKS_TOKEN` environment variables — no manual endpoint or token setup +- Three-level naming (`warehouse.namespace.table`) maps directly to UC's catalog/schema/table hierarchy + +### Databricks Setup + +```bash +export DATABRICKS_HOST="https://your-workspace.cloud.databricks.com" +export DATABRICKS_TOKEN="dapi_your_token_here" +``` + +```python +from feast.infra.data_sources.contrib.iceberg_catalog import UnityCatalogSource + +driver_stats_source = UnityCatalogSource( + warehouse="ml_catalog", + namespace="driver_features", + table="driver_hourly_stats", + timestamp_field="event_timestamp", + created_timestamp_column="created", + description="Hourly aggregated driver statistics", +) +``` + +That's it. No endpoint or token parameters needed — they come from the environment variables. + +### Optional: Governance Metadata Sync + +If you want `feast apply` to annotate your UC tables with `feast.*` properties (project name, feature view, primary keys, owner), you can use the `UnityCatalogProvider`: + +```yaml +# feature_store.yaml +provider: unity_catalog +``` + +This is optional. For most users, `provider: local` is sufficient. Reads, materialization, and online serving work the same regardless of which provider you use. + +### OSS Unity Catalog + +The integration also works with the open-source Unity Catalog, with some differences: + +| Capability | Databricks UC | OSS UC | +|---|---|---| +| Read via Iceberg REST | Built-in | Requires `uniform_iceberg_metadata_location` in H2 DB | +| Read via SQL catalog | Yes | Yes | +| Credential vending | Yes | Not available | + +For OSS UC, set `credential_vending=False` and `token_env_var=None`: + +```python +source = UnityCatalogSource( + warehouse="unity", + namespace="default", + table="driver_hourly_stats", + endpoint="http://localhost:8080/api/2.1/unity-catalog/iceberg", + token_env_var=None, + credential_vending=False, + catalog_type="sql", # recommended for OSS UC + catalog_name="my_catalog", + catalog_properties={ + "uri": "sqlite:////tmp/pyiceberg_catalog.db", + "warehouse": "file:///tmp/warehouse", + }, + timestamp_field="event_timestamp", + register_as_feature_table=False, +) +``` + +## How It Works + +### Read Path + +All data reads go through PyIceberg, regardless of catalog type: + +``` +IcebergSource.get_catalog_client() + → PyIceberg catalog (REST / SQL / Hive / Glue / DynamoDB) + → table.scan().to_arrow() + → Arrow table + → DuckDB ibis memtable or Spark DataFrame +``` + +If the catalog is misconfigured, you get a clear error — consistent with how every other Feast data source works. + +### Catalog Name Isolation + +Each source has a `catalog_name` parameter (default: `"feast_iceberg"`). This is the instance name PyIceberg uses when loading the catalog. If you have multiple sources pointing at different catalogs, use different names to avoid collisions: + +```python +production = IcebergSource(catalog_name="prod_catalog", ...) +staging = IcebergSource(catalog_name="staging_catalog", ...) +``` + +## What's Not Supported + +- **Write-back to Iceberg/UC tables.** Feast reads from existing tables; it doesn't write feature data back. The `write_to_offline_store` API only supports `FileSource` (DuckDB) and `SparkSource` (Spark). Your data engineering pipelines create and populate the tables. +- **Table creation.** Tables must exist before Feast can read from them. `feast apply` registers the feature view in Feast's registry, not the table in the catalog. + +## Configuration Reference + +### IcebergSource + +| Parameter | Default | Description | +|---|---|---| +| `warehouse` | *required* | Catalog or warehouse name | +| `namespace` | *required* | Schema or namespace | +| `table` | *required* | Table name | +| `catalog_type` | `"rest"` | Backend: `"rest"`, `"sql"`, `"hive"`, `"glue"`, `"dynamodb"` | +| `catalog_name` | `"feast_iceberg"` | PyIceberg instance name (unique per catalog to avoid collisions) | +| `endpoint` | `None` | Catalog endpoint URL | +| `catalog_properties` | `{}` | Additional catalog config (e.g., `{"uri": "sqlite:///..."}`) | +| `token_env_var` | `None` | Env var containing auth token | +| `credential_vending` | `True` | Request scoped storage credentials | +| `timestamp_field` | `None` | Event timestamp column | +| `created_timestamp_column` | `None` | Creation timestamp for deduplication | + +### UnityCatalogSource (extends IcebergSource) + +All `IcebergSource` parameters plus: + +| Parameter | Default | Description | +|---|---|---| +| `endpoint` | From `DATABRICKS_HOST` | Defaults to `{DATABRICKS_HOST}/api/2.1/unity-catalog/iceberg` | +| `token_env_var` | `"DATABRICKS_TOKEN"` | Defaults to Databricks token env var | +| `register_as_feature_table` | `True` | Sync `feast.*` properties to UC on `feast apply` | +| `sync_lineage` | `True` | Record lineage in UC (Databricks only) | + +--- + +*Native Iceberg support is available in Feast 0.64+. Install with `pip install "feast[iceberg]"` and check the [Iceberg data source documentation](/reference/data-sources/iceberg) for the full API reference.* diff --git a/infra/website/public/images/blog/feast-openai-compat-flow.png b/infra/website/public/images/blog/feast-openai-compat-flow.png new file mode 100644 index 00000000000..c7dcac8a5ed Binary files /dev/null and b/infra/website/public/images/blog/feast-openai-compat-flow.png differ diff --git a/infra/website/src/components/Navigation.astro b/infra/website/src/components/Navigation.astro index a5987bf6348..143fcbc047f 100644 --- a/infra/website/src/components/Navigation.astro +++ b/infra/website/src/components/Navigation.astro @@ -14,11 +14,29 @@ COMMUNITY - +
@@ -38,7 +56,7 @@ top: 0; left: 0; right: 0; - background-color: white; + background-color: var(--color-nav-bg); z-index: 1000; } @@ -80,6 +98,35 @@ margin-left: 32px; } + .nav-right { + display: flex; + align-items: center; + gap: 4px; + padding-right: var(--content-padding); + } + + .theme-toggle { + background: none; + border: none; + padding: 8px; + cursor: pointer; + color: var(--color-text); + display: flex; + align-items: center; + justify-content: center; + opacity: 0.7; + transition: opacity 0.2s ease; + } + + .theme-toggle:hover { + opacity: 1; + } + + .icon-sun { display: none; } + .icon-moon { display: block; } + :global([data-theme="dark"]) .icon-sun { display: block; } + :global([data-theme="dark"]) .icon-moon { display: none; } + .mobile-menu-button { display: block; background: none; @@ -87,7 +134,6 @@ padding: 8px; cursor: pointer; color: var(--color-text); - margin-right: var(--content-padding); } @media (min-width: 1024px) { @@ -95,7 +141,7 @@ display: flex; align-items: center; } - + .mobile-menu-button { display: none; } @@ -104,9 +150,9 @@ .mobile-menu { display: none; width: 100%; - background: white; + background: var(--color-nav-bg); padding: 16px 0; - border-top: 1px solid #eee; + border-top: 1px solid var(--color-border); position: absolute; top: 52px; left: 0; @@ -122,17 +168,27 @@ } .mobile-menu a:hover { - background-color: #f5f5f5; + background-color: var(--color-nav-hover); } \ No newline at end of file diff --git a/infra/website/src/layouts/BaseLayout.astro b/infra/website/src/layouts/BaseLayout.astro index 05d940005ba..96e30a6aee2 100644 --- a/infra/website/src/layouts/BaseLayout.astro +++ b/infra/website/src/layouts/BaseLayout.astro @@ -52,7 +52,17 @@ const { - + +
diff --git a/infra/website/src/layouts/BlogLayout.astro b/infra/website/src/layouts/BlogLayout.astro index c5f8379fc80..9fe68a9daa7 100644 --- a/infra/website/src/layouts/BlogLayout.astro +++ b/infra/website/src/layouts/BlogLayout.astro @@ -14,7 +14,7 @@ const { frontmatter } = Astro.props;

{frontmatter.title}

{frontmatter.date && ( -
+ + ) : ( + + setShowTree(true)} + /> + + )} + + )} + + {/* Content */} +
+ + + {renderContent()} +
+ + ); +}; + +export default DatasetCatalogBrowser; diff --git a/ui/src/pages/saved-data-sets/DatasetInstance.tsx b/ui/src/pages/saved-data-sets/DatasetInstance.tsx index ce13cb26e3c..2856af5f342 100644 --- a/ui/src/pages/saved-data-sets/DatasetInstance.tsx +++ b/ui/src/pages/saved-data-sets/DatasetInstance.tsx @@ -95,7 +95,6 @@ const DatasetInstance = () => { { setEditError(null); setShowEditModal(true); @@ -106,7 +105,6 @@ const DatasetInstance = () => { setShowDeleteConfirm(true)} > diff --git a/ui/src/pages/saved-data-sets/DatasetOverviewTab.tsx b/ui/src/pages/saved-data-sets/DatasetOverviewTab.tsx index b765da70241..65e3cc2732b 100644 --- a/ui/src/pages/saved-data-sets/DatasetOverviewTab.tsx +++ b/ui/src/pages/saved-data-sets/DatasetOverviewTab.tsx @@ -15,6 +15,7 @@ import { EuiCallOut, } from "@elastic/eui"; import { useParams } from "react-router-dom"; +import EuiCustomLink from "../../components/EuiCustomLink"; import DatasetFeaturesTable from "./DatasetFeaturesTable"; import DatasetJoinKeysTable from "./DatasetJoinKeysTable"; import useLoadDataset from "./useLoadDataset"; @@ -56,7 +57,7 @@ function formatTimestamp(ts: any): string { } const DatasetOverviewTab = () => { - let { datasetName } = useParams(); + const { datasetName, projectName } = useParams(); if (!datasetName) { throw new Error( @@ -96,6 +97,10 @@ const DatasetOverviewTab = () => { const tags = data.spec?.tags || {}; const featureServiceName = data.spec?.featureServiceName || data.spec?.feature_service_name; + const namespace = data.spec?.namespace || ""; + const collection = data.spec?.collection || ""; + const description = data.spec?.description || ""; + const dataSources: string[] = data.spec?.dataSources || []; const createdTs = data.meta?.createdTimestamp || data.meta?.created_timestamp; const minEventTs = data.meta?.minEventTimestamp || data.meta?.min_event_timestamp; @@ -168,6 +173,33 @@ const DatasetOverviewTab = () => { + {description && ( + <> + Description + + {description} + + + )} + + {namespace && ( + <> + Namespace + + {namespace} + + + )} + + {collection && ( + <> + Collection + + {collection} + + + )} + Storage Type {storageInfo.type} @@ -180,6 +212,26 @@ const DatasetOverviewTab = () => { + {dataSources.length > 0 && ( + <> + + Data Source{dataSources.length > 1 ? "s" : ""} + + + {dataSources.map((dsName, idx) => ( + + {idx > 0 && ", "} + + {dsName} + + + ))} + + + )} + {featureServiceName && ( <> diff --git a/ui/src/pages/saved-data-sets/DatasetsCardGrid.tsx b/ui/src/pages/saved-data-sets/DatasetsCardGrid.tsx index 2c96a0ee109..d5c69082978 100644 --- a/ui/src/pages/saved-data-sets/DatasetsCardGrid.tsx +++ b/ui/src/pages/saved-data-sets/DatasetsCardGrid.tsx @@ -11,6 +11,7 @@ import { EuiToolTip, EuiIcon, EuiCopy, + EuiLink, } from "@elastic/eui"; import { useNavigate, useParams } from "react-router-dom"; @@ -103,6 +104,9 @@ const DatasetCard: React.FC = ({ const tags = spec.tags || {}; const featureServiceName = spec.featureServiceName || spec.feature_service_name; + const namespace = spec.namespace || ""; + const collection = spec.collection || ""; + const description = spec.description || ""; const createdTimestamp = meta.createdTimestamp || meta.created_timestamp; const storagePath = extractStoragePath(dataset); const storageType = detectStorageType(dataset); @@ -169,6 +173,32 @@ const DatasetCard: React.FC = ({ + {/* Description */} + {description && ( + +

{description}

+
+ )} + + {/* Namespace / Collection badges */} + {(namespace || collection) && ( + <> + + + {namespace && ( + + {namespace} + + )} + {collection && ( + + {collection} + + )} + + + )} + {/* Storage path */} @@ -213,6 +243,28 @@ const DatasetCard: React.FC = ({ )} + {spec.dataSources && spec.dataSources.length > 0 && ( + + + Source + + + {spec.dataSources.map((dsName: string, idx: number) => ( + + {idx > 0 && ", "} + { + e.stopPropagation(); + navigate(`/p/${datasetProject}/data-source/${dsName}`); + }} + > + {dsName} + + + ))} + + + )} {/* Spacer to push footer */} diff --git a/ui/src/pages/saved-data-sets/DatasetsListingTable.tsx b/ui/src/pages/saved-data-sets/DatasetsListingTable.tsx index 823f42176f3..0ce44b35dc0 100644 --- a/ui/src/pages/saved-data-sets/DatasetsListingTable.tsx +++ b/ui/src/pages/saved-data-sets/DatasetsListingTable.tsx @@ -15,6 +15,9 @@ function detectStorageType(dataset: any): string { if (storage.snowflakeStorage) return "Snowflake"; if (storage.redshiftStorage) return "Redshift"; if (storage.sparkStorage) return "Spark"; + if (storage.trinoStorage) return "Trino"; + if (storage.athenaStorage) return "Athena"; + if (storage.customStorage) return "Custom"; return "—"; } @@ -35,6 +38,44 @@ const DatasetsListingTable = ({ datasets }: DatasetsListingTableProps) => { ); }, }, + { + name: "Namespace", + render: (item: any) => { + const ns = item.spec?.namespace; + return ns ? ( + {ns} + ) : ( + + ); + }, + width: "140px", + }, + { + name: "Collection", + render: (item: any) => { + const col = item.spec?.collection; + return col ? ( + {col} + ) : ( + + ); + }, + width: "140px", + }, + { + name: "Description", + render: (item: any) => { + const desc = item.spec?.description; + return desc ? ( + + {desc.length > 50 ? desc.slice(0, 50) + "…" : desc} + + ) : ( + + ); + }, + width: "200px", + }, { name: "Features", render: (item: any) => (item.spec?.features || []).length, @@ -53,6 +94,26 @@ const DatasetsListingTable = ({ datasets }: DatasetsListingTableProps) => { ), width: "120px", }, + { + name: "Data Source", + render: (item: any) => { + const dsList: string[] = item.spec?.dataSources || []; + if (dsList.length === 0) return "—"; + const itemProject = item.project || item.spec?.project || projectName; + return ( + <> + {dsList.map((dsName, idx) => ( + + {idx > 0 && ", "} + + {dsName} + + + ))} + + ); + }, + }, { name: "Feature Service", render: (item: any) => diff --git a/ui/src/pages/saved-data-sets/EditDatasetModal.tsx b/ui/src/pages/saved-data-sets/EditDatasetModal.tsx index db567a15703..507827e6dd0 100644 --- a/ui/src/pages/saved-data-sets/EditDatasetModal.tsx +++ b/ui/src/pages/saved-data-sets/EditDatasetModal.tsx @@ -99,6 +99,33 @@ const ALL_STORAGE_TYPES: StorageTypeDef[] = [ helpText: "Athena table reference.", sourceTypeMatch: ["BATCH_ATHENA"], }, + { + value: "postgres", + label: "PostgreSQL", + description: "PostgreSQL table reference", + placeholder: "schema.table_name", + helpText: + "PostgreSQL table reference. Data is read via the PostgreSQL offline store.", + sourceTypeMatch: ["CUSTOM_SOURCE"], + }, + { + value: "clickhouse", + label: "ClickHouse", + description: "ClickHouse table reference", + placeholder: "database.table_name", + helpText: + "ClickHouse table reference. Data is read via the ClickHouse offline store.", + sourceTypeMatch: ["CUSTOM_SOURCE"], + }, + { + value: "couchbase", + label: "Couchbase Columnar", + description: "Couchbase Columnar collection reference", + placeholder: "database.scope.collection", + helpText: + "Couchbase Columnar reference in format: database.scope.collection", + sourceTypeMatch: ["CUSTOM_SOURCE"], + }, { value: "custom", label: "Custom", @@ -128,6 +155,11 @@ function detectDataSourceTypes(dataSources: any[]): Set { if (ds.spec?.trinoOptions || ds.trinoOptions) types.add("BATCH_TRINO"); if (ds.spec?.athenaOptions || ds.athenaOptions) types.add("BATCH_ATHENA"); if (ds.spec?.customOptions || ds.customOptions) types.add("CUSTOM_SOURCE"); + const classType = + ds.spec?.dataSourceClassType || ds.dataSourceClassType || ""; + if (classType.includes("postgres")) types.add("CUSTOM_SOURCE"); + if (classType.includes("clickhouse")) types.add("CUSTOM_SOURCE"); + if (classType.includes("couchbase")) types.add("CUSTOM_SOURCE"); } return types; } @@ -160,7 +192,25 @@ function detectStorageType(dataset: any): string { if (storage.sparkStorage) return "spark"; if (storage.trinoStorage) return "trino"; if (storage.athenaStorage) return "athena"; - if (storage.customStorage) return "custom"; + if (storage.customStorage) { + try { + const config = storage.customStorage.configuration || ""; + const parsed = typeof config === "string" ? JSON.parse(config) : config; + if (parsed.database && parsed.scope && parsed.collection) + return "couchbase"; + if (parsed.table) { + const classType = + dataset?.spec?.dataSourceClassType || + dataset?.dataSourceClassType || + ""; + if (classType.includes("postgres")) return "postgres"; + if (classType.includes("clickhouse")) return "clickhouse"; + } + } catch { + // fall through + } + return "custom"; + } return "file"; } @@ -180,6 +230,14 @@ function extractStoragePath(dataset: any): string { return ""; } +function extractStorageFileFormat(dataset: any): string { + const storage = dataset?.spec?.storage; + if (storage?.sparkStorage?.fileFormat) return storage.sparkStorage.fileFormat; + if (storage?.sparkStorage?.file_format) + return storage.sparkStorage.file_format; + return "parquet"; +} + const EditDatasetModal = ({ dataset, onClose, @@ -307,6 +365,12 @@ const EditDatasetModal = ({ // Form state const [storagePath, setStoragePath] = useState(extractStoragePath(dataset)); const [storageType, setStorageType] = useState(detectStorageType(dataset)); + const [namespace, setNamespace] = useState(spec.namespace || ""); + const [collection, setCollection] = useState(spec.collection || ""); + const [description, setDescription] = useState(spec.description || ""); + const [storageFileFormat, setStorageFileFormat] = useState( + extractStorageFileFormat(dataset), + ); const [featuresInput, setFeaturesInput] = useState( (spec.features || []).map((f: string) => ({ label: f })), ); @@ -365,9 +429,14 @@ const EditDatasetModal = ({ join_keys: joinKeysInput.map((o) => o.label), storage_path: storagePath.trim(), storage_type: storageType, + storage_file_format: + storageType === "spark" ? storageFileFormat : undefined, tags: tagsObj, full_feature_names: fullFeatureNames, feature_service_name: featureServiceName || undefined, + namespace: namespace.trim() || undefined, + collection: collection.trim() || undefined, + description: description.trim() || undefined, allow_override: true, }; await onSubmit(payload); @@ -421,6 +490,20 @@ const EditDatasetModal = ({ + + + setDescription(e.target.value)} + placeholder="e.g. Training data for driver fraud model" + /> + + + + + + + {/* Organization */} + +

Organization (optional)

+
+ + + + + + setNamespace(e.target.value)} + placeholder="e.g. fraud" + /> + + + + + setCollection(e.target.value)} + placeholder="e.g. training" + /> + + + + @@ -490,6 +608,41 @@ const EditDatasetModal = ({ /> + {storageType === "spark" && ( + + Parquet, + }, + { + value: "avro", + inputDisplay: "Avro", + dropdownDisplay: Avro, + }, + { + value: "csv", + inputDisplay: "CSV", + dropdownDisplay: CSV, + }, + { + value: "json", + inputDisplay: "JSON", + dropdownDisplay: JSON, + }, + ]} + valueOfSelected={storageFileFormat} + onChange={setStorageFileFormat} + fullWidth + /> + + )} + diff --git a/ui/src/pages/saved-data-sets/Index.tsx b/ui/src/pages/saved-data-sets/Index.tsx index 752863aadad..1c587d83883 100644 --- a/ui/src/pages/saved-data-sets/Index.tsx +++ b/ui/src/pages/saved-data-sets/Index.tsx @@ -16,7 +16,6 @@ import { EuiFlexGroup, EuiFlexItem, EuiFieldSearch, - EuiStat, EuiPanel, EuiSelect, EuiText, @@ -32,6 +31,7 @@ import { DatasetIcon } from "../../graphics/DatasetIcon"; import { useDocumentTitle } from "../../hooks/useDocumentTitle"; import DatasetsCardGrid from "./DatasetsCardGrid"; import DatasetsListingTable from "./DatasetsListingTable"; +import DatasetCatalogBrowser from "./DatasetCatalogBrowser"; import DatasetsIndexEmptyState from "./DatasetsIndexEmptyState"; import AddToCatalogModal from "./AddToCatalogModal"; import type { RegisterDatasetPayload } from "./RegisterDatasetModal"; @@ -62,6 +62,7 @@ const SORT_OPTIONS = [ ]; const VIEW_TOGGLE_BUTTONS = [ + { id: "catalog", label: "Catalog", iconType: "folderClosed" }, { id: "cards", label: "Cards", iconType: "grid" }, { id: "table", label: "Table", iconType: "list" }, ]; @@ -78,12 +79,14 @@ function getDatasetSortValue(dataset: any, key: string): any { const Index = () => { const { projectName } = useParams(); - const { isLoading, isSuccess, isError, data } = useLoadSavedDataSets(); + const { isLoading, isSuccess, isError, isPermissionDenied, data } = + useLoadSavedDataSets(); const [showRegisterModal, setShowRegisterModal] = useState(false); const [deleteTarget, setDeleteTarget] = useState(null); const [searchQuery, setSearchQuery] = useState(""); const [sortBy, setSortBy] = useState("created_desc"); - const [viewMode, setViewMode] = useState("cards"); + const [viewMode, setViewMode] = useState("catalog"); + const [namespaceFilter, setNamespaceFilter] = useState("all"); const [submitError, setSubmitError] = useState(null); const [successMessage, setSuccessMessage] = useState(null); @@ -196,12 +199,18 @@ const Index = () => { // Compute summary stats const stats = useMemo(() => { if (!data) - return { total: 0, totalFeatures: 0, storageTypes: new Set() }; + return { + total: 0, + totalFeatures: 0, + storageTypes: new Set(), + namespaces: [] as string[], + }; const totalFeatures = data.reduce( (acc: number, ds: any) => acc + (ds.spec?.features?.length || 0), 0, ); const storageTypes = new Set(); + const namespacesSet = new Set(); data.forEach((ds: any) => { const storage = ds.spec?.storage; if (storage?.fileStorage) storageTypes.add("File"); @@ -212,8 +221,11 @@ const Index = () => { else if (storage?.trinoStorage) storageTypes.add("Trino"); else if (storage?.athenaStorage) storageTypes.add("Athena"); else if (storage?.customStorage) storageTypes.add("Custom"); + const ns = ds.spec?.namespace; + if (ns) namespacesSet.add(ns); }); - return { total: data.length, totalFeatures, storageTypes }; + const namespaces = Array.from(namespacesSet).sort(); + return { total: data.length, totalFeatures, storageTypes, namespaces }; }, [data]); // Filter and sort @@ -221,9 +233,20 @@ const Index = () => { if (!data) return []; let filtered = data; + // Namespace filter + if (namespaceFilter !== "all") { + if (namespaceFilter === "_none") { + filtered = filtered.filter((ds: any) => !ds.spec?.namespace); + } else { + filtered = filtered.filter( + (ds: any) => ds.spec?.namespace === namespaceFilter, + ); + } + } + if (searchQuery.trim()) { const q = searchQuery.toLowerCase(); - filtered = data.filter((ds: any) => { + filtered = filtered.filter((ds: any) => { const name = (ds.spec?.name || "").toLowerCase(); const tags = JSON.stringify(ds.spec?.tags || {}).toLowerCase(); const features = (ds.spec?.features || []).join(" ").toLowerCase(); @@ -232,11 +255,17 @@ const Index = () => { ds.spec?.feature_service_name || "" ).toLowerCase(); + const ns = (ds.spec?.namespace || "").toLowerCase(); + const col = (ds.spec?.collection || "").toLowerCase(); + const desc = (ds.spec?.description || "").toLowerCase(); return ( name.includes(q) || tags.includes(q) || features.includes(q) || - service.includes(q) + service.includes(q) || + ns.includes(q) || + col.includes(q) || + desc.includes(q) ); }); } @@ -253,7 +282,7 @@ const Index = () => { }); return filtered; - }, [data, searchQuery, sortBy]); + }, [data, searchQuery, sortBy, namespaceFilter]); const hasData = data && data.length > 0; @@ -413,7 +442,12 @@ const Index = () => { )} - {isError && ( + {isPermissionDenied && ( + +

You do not have permission to view saved datasets.

+
+ )} + {isError && !isPermissionDenied && ( { {isSuccess && hasData && ( <> - {/* Summary Stats */} - - - - - - - - - - + {/* View mode toggle — always visible */} + + + + + + {stats.total} datasets + {stats.namespaces.length > 0 && ( + <> + {" "} + across {stats.namespaces.length}{" "} + namespaces + + )} + + + - - - - + + setViewMode(id)} + isIconOnly + buttonSize="compressed" + /> - + - {/* Search + Sort + View Toggle */} + {/* Search + Namespace Filter + Sort — shared toolbar */} setSearchQuery(e.target.value)} isClearable fullWidth /> + {stats.namespaces.length > 0 && ( + + ({ + value: ns, + text: ns, + })), + ]} + value={namespaceFilter} + onChange={(e) => setNamespaceFilter(e.target.value)} + compressed + prepend="Namespace" + /> + + )} { prepend="Sort" /> - - setViewMode(id)} - isIconOnly - buttonSize="compressed" - /> - - + - {/* Results count when searching */} - {searchQuery.trim() && ( + {/* Results count when filtering */} + {(searchQuery.trim() || namespaceFilter !== "all") && ( <> Showing {processedData.length} of {data.length} datasets + {namespaceFilter !== "all" && namespaceFilter !== "_none" && ( + <> + {" "} + in namespace {namespaceFilter} + + )} + {namespaceFilter === "_none" && <> with no namespace} )} - {/* Content */} - {viewMode === "cards" ? ( + {/* Catalog (hierarchical) view */} + {viewMode === "catalog" && ( + setDeleteTarget(name)} + /> + )} + + {/* Flat views (cards / table) */} + {viewMode === "cards" && ( setDeleteTarget(name)} /> - ) : ( + )} + {viewMode === "table" && ( )} diff --git a/ui/src/pages/saved-data-sets/RegisterDatasetModal.tsx b/ui/src/pages/saved-data-sets/RegisterDatasetModal.tsx index 90f93b8ca31..e7543ddadf1 100644 --- a/ui/src/pages/saved-data-sets/RegisterDatasetModal.tsx +++ b/ui/src/pages/saved-data-sets/RegisterDatasetModal.tsx @@ -34,9 +34,13 @@ export interface RegisterDatasetPayload { join_keys: string[]; storage_path: string; storage_type: string; + storage_file_format?: string; tags: Record; full_feature_names: boolean; feature_service_name?: string; + namespace?: string; + collection?: string; + description?: string; } interface RegisterDatasetModalProps { @@ -115,6 +119,33 @@ const ALL_STORAGE_TYPES: StorageTypeDefinition[] = [ helpText: "Athena table reference. Data is queried via Athena.", sourceTypeMatch: ["BATCH_ATHENA"], }, + { + value: "postgres", + label: "PostgreSQL", + description: "PostgreSQL table reference", + placeholder: "schema.table_name", + helpText: + "PostgreSQL table reference. Data is read via the PostgreSQL offline store.", + sourceTypeMatch: ["CUSTOM_SOURCE"], + }, + { + value: "clickhouse", + label: "ClickHouse", + description: "ClickHouse table reference", + placeholder: "database.table_name", + helpText: + "ClickHouse table reference. Data is read via the ClickHouse offline store.", + sourceTypeMatch: ["CUSTOM_SOURCE"], + }, + { + value: "couchbase", + label: "Couchbase Columnar", + description: "Couchbase Columnar collection reference", + placeholder: "database.scope.collection", + helpText: + "Couchbase Columnar reference in format: database.scope.collection", + sourceTypeMatch: ["CUSTOM_SOURCE"], + }, { value: "custom", label: "Custom", @@ -144,6 +175,11 @@ function detectDataSourceTypes(dataSources: any[]): Set { if (ds.spec?.trinoOptions || ds.trinoOptions) types.add("BATCH_TRINO"); if (ds.spec?.athenaOptions || ds.athenaOptions) types.add("BATCH_ATHENA"); if (ds.spec?.customOptions || ds.customOptions) types.add("CUSTOM_SOURCE"); + const classType = + ds.spec?.dataSourceClassType || ds.dataSourceClassType || ""; + if (classType.includes("postgres")) types.add("CUSTOM_SOURCE"); + if (classType.includes("clickhouse")) types.add("CUSTOM_SOURCE"); + if (classType.includes("couchbase")) types.add("CUSTOM_SOURCE"); } return types; } @@ -241,8 +277,12 @@ const RegisterDatasetModal = ({ // Form state const [name, setName] = useState(""); + const [namespace, setNamespace] = useState(""); + const [collection, setCollection] = useState(""); + const [description, setDescription] = useState(""); const [storagePath, setStoragePath] = useState(""); const [storageType, setStorageType] = useState("file"); + const [storageFileFormat, setStorageFileFormat] = useState("parquet"); const [featuresInput, setFeaturesInput] = useState( [], ); @@ -336,9 +376,14 @@ const RegisterDatasetModal = ({ join_keys: joinKeysInput.map((o) => o.label), storage_path: storagePath.trim(), storage_type: storageType, + storage_file_format: + storageType === "spark" ? storageFileFormat : undefined, tags: tagsObj, full_feature_names: fullFeatureNames, feature_service_name: featureServiceName || undefined, + namespace: namespace.trim() || undefined, + collection: collection.trim() || undefined, + description: description.trim() || undefined, }; await onSubmit(payload); }; @@ -395,6 +440,18 @@ const RegisterDatasetModal = ({ /> + + + setDescription(e.target.value)} + placeholder="e.g. Training data for driver fraud model" + /> + + + + + {/* Section: Organization */} + +

Organization (optional)

+
+ + + + + Group datasets into namespaces and collections for hierarchical + organization. Leave empty to keep the dataset at the top level. + + + + + + + + setNamespace(e.target.value)} + placeholder="e.g. fraud" + /> + + + + + setCollection(e.target.value)} + placeholder="e.g. training" + /> + + + + @@ -464,6 +564,41 @@ const RegisterDatasetModal = ({ /> + {storageType === "spark" && ( + + Parquet, + }, + { + value: "avro", + inputDisplay: "Avro", + dropdownDisplay: Avro, + }, + { + value: "csv", + inputDisplay: "CSV", + dropdownDisplay: CSV, + }, + { + value: "json", + inputDisplay: "JSON", + dropdownDisplay: JSON, + }, + ]} + valueOfSelected={storageFileFormat} + onChange={setStorageFileFormat} + fullWidth + /> + + )} + diff --git a/ui/src/queries/mutations/useDataSourceMutations.ts b/ui/src/queries/mutations/useDataSourceMutations.ts index 737ced0dbfd..a59313bf7a7 100644 --- a/ui/src/queries/mutations/useDataSourceMutations.ts +++ b/ui/src/queries/mutations/useDataSourceMutations.ts @@ -15,6 +15,12 @@ interface ApplyDataSourcePayload { redshift_options?: { table: string; database: string; schema_: string }; kafka_options?: { kafka_bootstrap_servers: string; topic: string }; spark_options?: { table: string; path: string }; + custom_options?: { + configuration?: string; + class_name?: string; + config?: string; + }; + data_source_class_type?: string; } interface DeleteDataSourcePayload { diff --git a/ui/src/queries/mutations/useFeatureServiceMutations.ts b/ui/src/queries/mutations/useFeatureServiceMutations.ts new file mode 100644 index 00000000000..85bd3f4fd44 --- /dev/null +++ b/ui/src/queries/mutations/useFeatureServiceMutations.ts @@ -0,0 +1,100 @@ +import { useMutation, useQueryClient } from "react-query"; + +interface FeatureViewProjectionPayload { + feature_view_name: string; + feature_names?: string[]; +} + +interface ApplyFeatureServicePayload { + name: string; + project: string; + features: FeatureViewProjectionPayload[]; + description?: string; + tags?: Record; + owner?: string; +} + +interface DeleteFeatureServicePayload { + name: string; + project: string; +} + +interface MutationResult { + name: string; + project: string; + status: string; +} + +const API_BASE = "/api/v1"; + +const applyFeatureService = async ( + payload: ApplyFeatureServicePayload, +): Promise => { + const response = await fetch(`${API_BASE}/feature_services`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify(payload), + }); + + if (!response.ok) { + const error = await response + .json() + .catch(() => ({ detail: response.statusText })); + throw new Error( + error.detail || `Failed to apply feature service: ${response.status}`, + ); + } + + return response.json(); +}; + +const deleteFeatureService = async ( + payload: DeleteFeatureServicePayload, +): Promise => { + const response = await fetch( + `${API_BASE}/feature_services/${encodeURIComponent(payload.name)}?project=${encodeURIComponent(payload.project)}`, + { method: "DELETE" }, + ); + + if (!response.ok) { + const error = await response + .json() + .catch(() => ({ detail: response.statusText })); + throw new Error( + error.detail || `Failed to delete feature service: ${response.status}`, + ); + } + + return response.json(); +}; + +const useApplyFeatureService = () => { + const queryClient = useQueryClient(); + + return useMutation(applyFeatureService, { + onSuccess: () => { + queryClient.invalidateQueries(["rest"]); + queryClient.invalidateQueries(["feature-services-rest"]); + queryClient.invalidateQueries(["feature-service-rest"]); + }, + }); +}; + +const useDeleteFeatureService = () => { + const queryClient = useQueryClient(); + + return useMutation(deleteFeatureService, { + onSuccess: () => { + queryClient.invalidateQueries(["rest"]); + queryClient.invalidateQueries(["feature-services-rest"]); + queryClient.invalidateQueries(["feature-service-rest"]); + }, + }); +}; + +export { useApplyFeatureService, useDeleteFeatureService }; +export type { + ApplyFeatureServicePayload, + DeleteFeatureServicePayload, + FeatureViewProjectionPayload, +}; diff --git a/ui/src/queries/mutations/usePermissionMutations.ts b/ui/src/queries/mutations/usePermissionMutations.ts new file mode 100644 index 00000000000..aa3961c8ff9 --- /dev/null +++ b/ui/src/queries/mutations/usePermissionMutations.ts @@ -0,0 +1,74 @@ +import { useMutation, useQueryClient } from "react-query"; +import { restPost, restDelete } from "../restApiClient"; + +interface PolicyPayload { + role_based_policy?: { roles: string[] }; + group_based_policy?: { groups: string[] }; + namespace_based_policy?: { namespaces: string[] }; + combined_group_namespace_policy?: { + groups: string[]; + namespaces: string[]; + }; +} + +interface ApplyPermissionPayload { + name: string; + project: string; + types: string[]; + name_patterns: string[]; + actions: string[]; + policy: PolicyPayload; + tags?: Record; + required_tags?: Record; +} + +interface DeletePermissionPayload { + name: string; + project: string; +} + +interface MutationResult { + name: string; + project: string; + status: string; +} + +const API_BASE = "/api/v1"; + +const useApplyPermission = () => { + const queryClient = useQueryClient(); + + return useMutation( + (payload: ApplyPermissionPayload) => + restPost(API_BASE, "/permissions", payload), + { + onSuccess: () => { + queryClient.invalidateQueries(["rest"]); + queryClient.invalidateQueries(["permissions-rest"]); + queryClient.invalidateQueries(["registry-rest-bulk"]); + }, + }, + ); +}; + +const useDeletePermission = () => { + const queryClient = useQueryClient(); + + return useMutation( + (payload: DeletePermissionPayload) => + restDelete( + API_BASE, + `/permissions/${encodeURIComponent(payload.name)}?project=${encodeURIComponent(payload.project)}`, + ), + { + onSuccess: () => { + queryClient.invalidateQueries(["rest"]); + queryClient.invalidateQueries(["permissions-rest"]); + queryClient.invalidateQueries(["registry-rest-bulk"]); + }, + }, + ); +}; + +export { useApplyPermission, useDeletePermission }; +export type { ApplyPermissionPayload, DeletePermissionPayload, PolicyPayload }; diff --git a/ui/src/queries/useLoadComputeEngine.ts b/ui/src/queries/useLoadComputeEngine.ts index f9077297e21..dc9dfae73bd 100644 --- a/ui/src/queries/useLoadComputeEngine.ts +++ b/ui/src/queries/useLoadComputeEngine.ts @@ -2,7 +2,7 @@ import { useContext } from "react"; import { useQuery } from "react-query"; import RegistryPathContext from "../contexts/RegistryPathContext"; import { useDataMode } from "../contexts/DataModeContext"; -import restFetch from "./restApiClient"; +import restFetch, { RestApiError } from "./restApiClient"; export interface ComputeEngineConfig { type: string; @@ -68,9 +68,18 @@ export function useLoadComputeEngine(projectName?: string) { { enabled: !!registryUrl, staleTime: 30_000, + retry: (failureCount, error) => { + if (error instanceof RestApiError && error.status === 403) return false; + return failureCount < 3; + }, }, ); + const isPermissionDenied = + engineQuery.isError && + engineQuery.error instanceof RestApiError && + engineQuery.error.status === 403; + let engineInfo: ComputeEngineInfo | null = null; let featureViewInfos: FeatureViewEngineInfo[] = []; @@ -105,6 +114,7 @@ export function useLoadComputeEngine(projectName?: string) { isLoading: engineQuery.isLoading, isSuccess: engineQuery.isSuccess, isError: engineQuery.isError, + isPermissionDenied, engineInfo, featureViewInfos, }; diff --git a/ui/src/queries/useLoadRegistry.ts b/ui/src/queries/useLoadRegistry.ts index 477f9be3db8..9e7cf2db2d5 100644 --- a/ui/src/queries/useLoadRegistry.ts +++ b/ui/src/queries/useLoadRegistry.ts @@ -100,6 +100,22 @@ const assembleFeatureStoreData = ( // REST fetch strategy // --------------------------------------------------------------------------- +const permissionSafeFetch = async ( + apiBaseUrl: string, + path: string, + fallback: T, + fetchOptions?: FetchOptions, +): Promise => { + try { + return await restFetch(apiBaseUrl, path, fetchOptions); + } catch (err: any) { + if (err?.status === 403 || err?.status === 401) { + return fallback; + } + throw err; + } +}; + const fetchREST = async ( apiBaseUrl: string, projectName?: string, @@ -111,6 +127,8 @@ const fetchREST = async ( : ""; const useAllEndpoint = !projectParam; + const emptyList = (key: string) => ({ [key]: [] }); + const [ entitiesResp, featureViewsResp, @@ -120,49 +138,60 @@ const fetchREST = async ( savedDatasetsResp, projectsResp, ] = await Promise.all([ - restFetch( + permissionSafeFetch( apiBaseUrl, useAllEndpoint ? "/entities/all?include_relationships=true" : `/entities${projectParam}&include_relationships=true`, + emptyList("entities"), fetchOptions, ), - restFetch( + permissionSafeFetch( apiBaseUrl, useAllEndpoint ? "/feature_views/all?include_relationships=true" : `/feature_views${projectParam}&include_relationships=true`, + emptyList("featureViews"), fetchOptions, ), - restFetch( + permissionSafeFetch( apiBaseUrl, useAllEndpoint ? "/label_views/all?include_relationships=true" : `/label_views${projectParam}&include_relationships=true`, + emptyList("featureViews"), fetchOptions, ), - restFetch( + permissionSafeFetch( apiBaseUrl, useAllEndpoint ? "/feature_services/all?include_relationships=true" : `/feature_services${projectParam}&include_relationships=true`, + emptyList("featureServices"), fetchOptions, ), - restFetch( + permissionSafeFetch( apiBaseUrl, useAllEndpoint ? "/data_sources/all?include_relationships=true" : `/data_sources${projectParam}&include_relationships=true`, + emptyList("dataSources"), fetchOptions, ), - restFetch( + permissionSafeFetch( apiBaseUrl, useAllEndpoint ? "/saved_datasets/all?include_relationships=true" : `/saved_datasets${projectParam}&include_relationships=true`, + emptyList("savedDatasets"), + fetchOptions, + ), + permissionSafeFetch( + apiBaseUrl, + "/projects", + emptyList("projects"), fetchOptions, ), - restFetch(apiBaseUrl, "/projects", fetchOptions), ]); const entities = entitiesResp.entities || []; diff --git a/ui/src/queries/useResourceQuery.ts b/ui/src/queries/useResourceQuery.ts index 857eeefc131..b099d04fc4d 100644 --- a/ui/src/queries/useResourceQuery.ts +++ b/ui/src/queries/useResourceQuery.ts @@ -1,8 +1,9 @@ import { useContext } from "react"; -import { useQuery, UseQueryResult } from "react-query"; +import { useQuery } from "react-query"; import RegistryPathContext from "../contexts/RegistryPathContext"; import { useDataMode } from "../contexts/DataModeContext"; import restFetch from "./restApiClient"; +import { RestApiError } from "./restApiClient"; import { FEAST_FV_TYPES, genericFVType } from "../parsers/mergedFVTypes"; interface ResourceQueryOptions { @@ -25,19 +26,30 @@ function useResourceQuery({ restPath, restSelect, enabled = true, -}: ResourceQueryOptions): UseQueryResult { +}: ResourceQueryOptions) { const registryUrl = useContext(RegistryPathContext); const { fetchOptions } = useDataMode(); - return useQuery( + const query = useQuery( ["rest", resourceType, registryUrl, project || "all"], () => restFetch(registryUrl, restPath, fetchOptions), { enabled: !!registryUrl && enabled, staleTime: 30_000, select: restSelect, + retry: (failureCount, error) => { + if (error instanceof RestApiError && error.status === 403) return false; + return failureCount < 3; + }, }, ); + + const isPermissionDenied = + query.isError && + query.error instanceof RestApiError && + query.error.status === 403; + + return { ...query, isPermissionDenied }; } // --------------------------------------------------------------------------- @@ -110,6 +122,13 @@ function labelViewDetailPath(name: string, project: string): string { return `/label_views/${encodeURIComponent(name)}?project=${encodeURIComponent(project)}&include_relationships=true`; } +function permissionListPath(project?: string): string { + if (project && project !== "all") { + return `/permissions?project=${encodeURIComponent(project)}`; + } + return `/permissions?project=default`; +} + function featuresListPath(project?: string): string { if (project && project !== "all") { return `/features?project=${encodeURIComponent(project)}`; @@ -214,6 +233,7 @@ export { savedDatasetDetailPath, labelViewListPath, labelViewDetailPath, + permissionListPath, featuresListPath, featureDetailPath, restFeatureViewsToMergedList, diff --git a/ui/src/utils/permissionUtils.ts b/ui/src/utils/permissionUtils.ts index c4c4cef032e..2f1f8e06628 100644 --- a/ui/src/utils/permissionUtils.ts +++ b/ui/src/utils/permissionUtils.ts @@ -1,5 +1,16 @@ import { FEAST_FCO_TYPES } from "../parsers/types"; +/** + * Test if a regex pattern is potentially vulnerable to catastrophic backtracking. + * Rejects patterns with nested quantifiers like (a+)+ or (a*)* + */ +const isSafePattern = (pattern: string): boolean => { + if (pattern.length > 1000) return false; + // Reject nested quantifiers: a quantifier applied to a group containing a quantifier + if (/(\([^)]*[+*][^)]*\))[+*{]/.test(pattern)) return false; + return true; +}; + /** * Get permissions for a specific entity * @param permissions List of all permissions @@ -42,6 +53,9 @@ export const getEntityPermissions = ( matchesName = true; // If no name patterns, matches all names } else { matchesName = permission.spec?.name_patterns?.some((pattern: string) => { + if (!pattern || !isSafePattern(pattern)) { + return pattern === entityName; + } try { const regex = new RegExp(pattern); return regex.test(entityName); diff --git a/ui/yarn.lock b/ui/yarn.lock index cca5da3763e..cb5589a81f9 100644 --- a/ui/yarn.lock +++ b/ui/yarn.lock @@ -7562,6 +7562,11 @@ jsonpointer@^5.0.0: object.assign "^4.1.4" object.values "^1.1.6" +keycloak-js@^26.2.4: + version "26.2.4" + resolved "https://registry.npmjs.org/keycloak-js/-/keycloak-js-26.2.4.tgz" + integrity sha512-PnXpR3ubETGOt0B/Qt2lxmPbkZr5bc3vlQsOqDoTPPQsZRp7JjhTKxlJ187uWh8qJhvBab6Gsjb06a8ayOPfuw== + keyv@^4.5.3: version "4.5.4" resolved "https://registry.npmjs.org/keyv/-/keyv-4.5.4.tgz"