diff --git a/.github/workflows/disconnected-readiness.yml b/.github/workflows/disconnected-readiness.yml new file mode 100644 index 00000000000..07f941a5663 --- /dev/null +++ b/.github/workflows/disconnected-readiness.yml @@ -0,0 +1,12 @@ +name: Disconnected Readiness +on: + pull_request: + branches: + - main + - master + +jobs: + check: + uses: opendatahub-io/disconnected-readiness-scorer/.github/workflows/disconnected-readiness-check.yml@v1 + with: + rules: "" diff --git a/.github/workflows/sync_stable_branch.yml b/.github/workflows/sync_stable_branch.yml new file mode 100644 index 00000000000..47cb5f40f55 --- /dev/null +++ b/.github/workflows/sync_stable_branch.yml @@ -0,0 +1,106 @@ +name: Sync to Stable Branch + +on: + pull_request: + types: + - closed + branches: + - master + +jobs: + cherry-pick-to-stable: + permissions: + contents: write + pull-requests: write + if: >- + github.event.pull_request.merged == true && + contains(github.event.pull_request.labels.*.name, 'sync-to-stable') + runs-on: ubuntu-latest + + env: + GH_TOKEN: ${{ secrets.SYNC_STABLE_PAT }} + PR_NUMBER: ${{ github.event.pull_request.number }} + MERGE_SHA: ${{ github.event.pull_request.merge_commit_sha }} + + steps: + - name: Checkout repository + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 + with: + fetch-depth: 0 + token: ${{ secrets.SYNC_STABLE_PAT }} + + - name: Configure Git + run: | + git config user.name "github-actions[bot]" + git config user.email "github-actions[bot]@users.noreply.github.com" + + - name: Create cherry-pick branch + id: cherry-pick + run: | + BRANCH="auto-sync/stable/${PR_NUMBER}" + echo "branch=${BRANCH}" >> "$GITHUB_OUTPUT" + + git checkout -b "$BRANCH" origin/stable + + # -m 1 handles merge commits by picking the first parent + if git cherry-pick -m 1 "$MERGE_SHA"; then + echo "conflict=false" >> "$GITHUB_OUTPUT" + else + git cherry-pick --abort || true + git commit --allow-empty -m "cherry-pick of #${PR_NUMBER} failed — manual resolution needed" + echo "conflict=true" >> "$GITHUB_OUTPUT" + fi + + git push origin "$BRANCH" --force + + - name: Create Pull Request + id: create-pr + env: + PR_TITLE: ${{ github.event.pull_request.title }} + SOURCE_PR_URL: ${{ github.event.pull_request.html_url }} + run: | + BRANCH="${{ steps.cherry-pick.outputs.branch }}" + CONFLICT="${{ steps.cherry-pick.outputs.conflict }}" + + if [ "$CONFLICT" = "true" ]; then + PR_URL=$(gh pr create \ + --base stable \ + --head "$BRANCH" \ + --title "sync(stable): ${PR_TITLE} (#${PR_NUMBER}) [CONFLICT]" \ + --body "## Auto-sync from master (PR #${PR_NUMBER}) + + Cherry-pick of ${SOURCE_PR_URL} **failed due to merge conflicts**. + + ### Manual resolution required + \`\`\`bash + git fetch origin + git checkout -B auto-sync/stable/${PR_NUMBER} origin/stable + git cherry-pick -m 1 ${MERGE_SHA} + # resolve conflicts, then: + git add . + git cherry-pick --continue + git push origin auto-sync/stable/${PR_NUMBER} --force + \`\`\`" \ + --label "sync-to-stable" \ + --label "needs-manual-resolution") + else + PR_URL=$(gh pr create \ + --base stable \ + --head "$BRANCH" \ + --title "sync(stable): ${PR_TITLE} (#${PR_NUMBER})" \ + --body "## Auto-sync from master (PR #${PR_NUMBER}) + + Cherry-pick of ${SOURCE_PR_URL} to \`stable\` branch. + + This PR was automatically created and will auto-merge after CI passes." \ + --label "sync-to-stable") + fi + + echo "pr_url=${PR_URL}" >> "$GITHUB_OUTPUT" + echo "Created PR: ${PR_URL}" + + - name: Enable auto-merge + if: steps.cherry-pick.outputs.conflict == 'false' + run: | + PR_URL="${{ steps.create-pr.outputs.pr_url }}" + gh pr merge "$PR_URL" --auto --squash diff --git a/.github/workflows/trigger-konflux-nightly.yaml b/.github/workflows/trigger-konflux-nightly.yaml new file mode 100644 index 00000000000..d577097976e --- /dev/null +++ b/.github/workflows/trigger-konflux-nightly.yaml @@ -0,0 +1,96 @@ +name: Nightly Test Status Monitor + +# The Feast nightly integration test is scheduled and executed entirely within +# Konflux via the PaC cron trigger in .tekton/feast-nightly-test.yaml (daily +# at 8 AM UTC). The Konflux pipeline posts the result back to GitHub as a +# commit status (context: konflux/nightly-feast) on the master HEAD. +# +# This workflow runs after the Konflux pipeline should have completed and +# reports the nightly status in the GitHub Actions summary. +# +# No Konflux API secrets are required. + +on: + schedule: + - cron: '0 11 * * *' # 11 AM UTC — Konflux nightly starts 8 AM, runs ~2h + workflow_dispatch: + inputs: + branch: + description: 'Branch to check nightly status for' + required: false + default: 'master' + type: string + +env: + STATUS_CONTEXT: 'konflux/nightly-feast' + +jobs: + check-nightly-status: + runs-on: ubuntu-latest + permissions: + statuses: read + steps: + - name: Get branch HEAD SHA + id: sha + env: + GH_TOKEN: ${{ github.token }} + BRANCH: ${{ inputs.branch || 'master' }} + run: | + SHA=$(gh api "repos/${{ github.repository }}/git/refs/heads/${BRANCH}" \ + --jq '.object.sha') + echo "sha=${SHA}" >> "$GITHUB_OUTPUT" + echo "branch=${BRANCH}" >> "$GITHUB_OUTPUT" + echo "Branch: ${BRANCH} SHA: ${SHA}" + + - name: Read Konflux nightly commit status + id: nightly + env: + GH_TOKEN: ${{ github.token }} + run: | + # The Konflux nightly pipeline posts commit status via report-nightly-status task. + # Statuses are ordered newest-first; pick the first matching context. + PAYLOAD=$(gh api \ + "repos/${{ github.repository }}/commits/${{ steps.sha.outputs.sha }}/statuses" \ + --jq "[.[] | select(.context == \"${STATUS_CONTEXT}\")] | first // {}") + + STATE=$(echo "$PAYLOAD" | jq -r '.state // "not_found"') + DESCRIPTION=$(echo "$PAYLOAD" | jq -r '.description // ""') + TARGET_URL=$(echo "$PAYLOAD" | jq -r '.target_url // ""') + + echo "state=${STATE}" >> "$GITHUB_OUTPUT" + echo "description=${DESCRIPTION}" >> "$GITHUB_OUTPUT" + echo "target_url=${TARGET_URL}" >> "$GITHUB_OUTPUT" + + echo "Konflux nightly status: ${STATE}" + echo "Description: ${DESCRIPTION}" + echo "Details URL: ${TARGET_URL}" + + if [ "${STATE}" = "pending" ]; then + echo "::warning::Konflux nightly pipeline is still pending — results not available yet." + elif [ "${STATE}" = "not_found" ]; then + echo "::warning::No commit status found for context '${STATUS_CONTEXT}' on ${BRANCH} HEAD." + echo "::warning::The Konflux pipeline may not have run yet or status was not posted." + elif [ "${STATE}" = "failure" ]; then + echo "::error::Konflux nightly test failed — see Details URL for pipeline run logs." + fi + + - name: Summary + if: always() + env: + STATE: ${{ steps.nightly.outputs.state }} + DESCRIPTION: ${{ steps.nightly.outputs.description }} + TARGET_URL: ${{ steps.nightly.outputs.target_url }} + run: | + { + echo "## Feast Nightly Test Status" + echo "" + echo "| | |" + echo "|---|---|" + echo "| **Branch** | \`${{ steps.sha.outputs.branch }}\` |" + echo "| **Commit** | \`${{ steps.sha.outputs.sha }}\` |" + echo "| **Konflux status** | \`${STATE}\` |" + echo "| **Description** | ${DESCRIPTION} |" + if [ -n "${TARGET_URL}" ]; then + echo "| **Details** | [View pipeline run](${TARGET_URL}) |" + fi + } >> "$GITHUB_STEP_SUMMARY" diff --git a/.secrets.baseline b/.secrets.baseline index f6a5bbb4b0a..c90de9dbeab 100644 --- a/.secrets.baseline +++ b/.secrets.baseline @@ -1172,7 +1172,7 @@ "filename": "infra/feast-operator/internal/controller/services/services.go", "hashed_secret": "36dc326eb15c7bdd8d91a6b87905bcea20b637d1", "is_verified": false, - "line_number": 180 + "line_number": 185 } ], "infra/feast-operator/internal/controller/services/tls_test.go": [ diff --git a/.tekton/README.md b/.tekton/README.md new file mode 100644 index 00000000000..a991e7035d9 --- /dev/null +++ b/.tekton/README.md @@ -0,0 +1,221 @@ +# Tekton Pipelines (Pipelines-as-Code) + +This directory contains Tekton `PipelineRun` definitions used for building and +testing the Open Data Hub (ODH) Feast components on the Konflux ITS OpenShift +environment. Pipelines are triggered by +[Pipelines-as-Code](https://pipelinesascode.com/) based on repository events +and comments. + +For Tekton concepts (Tasks, Pipelines, PipelineRuns), see the +[Tekton Getting Started](http://tekton.dev/docs/getting-started/) documentation. + +--- + +## Overview + +``` +PR opened / updated + ├─ odh-feast-operator-pull-request → quay.io/opendatahub/feast-operator:odh-pr- + └─ odh-feature-server-pull-request → quay.io/opendatahub/feature-server:odh-pr- + │ + ├─ /group-test or group-test event auto triggered → feast-group-test (tests PR images) + └─ /pr-e2etest comment for retest → feast-pr-test (tests PR images) + +Merge to master + ├─ odh-feast-operator-push → quay.io/opendatahub/feast-operator:odh-master + └─ odh-feature-server-push → quay.io/opendatahub/feature-server:odh-master + │ + └─ Daily 8 AM UTC cron / → feast-nightly-test (tests odh-master images) +``` + +All integration test pipelines provision an ephemeral HyperShift cluster +(m5.2xlarge, latest OCP 4.x via EaaS), deploy the Feast operator and feature +server, run the full test suite, collect must-gather artifacts, and post +results back to GitHub. + +--- + +## Build pipelines + +### `odh-feast-operator-pull-request.yaml` — Operator image build (PR) + +Builds the Feast operator image from a pull request and pushes it to quay.io. +Required before running `/group-test` or `/pr-e2etest` on a PR. + +| | | +|---|---| +| **Trigger** | Pull request opened or updated targeting `master` | +| **Source** | PR head branch at `{{revision}}` | +| **Build context** | `infra/feast-operator` | +| **Dockerfile** | `infra/feast-operator/Dockerfile` | +| **Output image** | `quay.io/opendatahub/feast-operator:odh-pr-` and `odh-pr-` | +| **Service account** | `build-pipeline-odh-feast-operator` | + +--- + +### `odh-feature-server-pull-request.yaml` — Feature server image build (PR) + +Builds the feature server image from a pull request and pushes it to quay.io. +Required before running `/group-test` or `/pr-e2etest` on a PR. + +| | | +|---|---| +| **Trigger** | Pull request opened or updated targeting `master` | +| **Source** | PR head branch at `{{revision}}` | +| **Build context** | `sdk/python/feast/infra/feature_servers/multicloud` | +| **Dockerfile** | `sdk/python/feast/infra/feature_servers/multicloud/Dockerfile` | +| **Output image** | `quay.io/opendatahub/feature-server:odh-pr-` and `odh-pr-` | +| **Service account** | `build-pipeline-odh-feature-server` | + +--- + +### `odh-feast-operator-push.yaml` — Operator image build (merge to master) + +Builds and pushes the Feast operator image on every merge to `master`. This is +the image the nightly test pipeline resolves via the `odh-master` tag. + +| | | +|---|---| +| **Trigger** | Push to `master` | +| **Source** | `master` branch | +| **Build context** | `infra/feast-operator` | +| **Dockerfile** | `infra/feast-operator/Dockerfile` | +| **Output image** | `quay.io/opendatahub/feast-operator:odh-master` | +| **Service account** | `build-pipeline-odh-feast-operator` | +| **Pipeline** | [`multi-arch-container-build.yaml`](https://github.com/opendatahub-io/odh-konflux-central/blob/main/pipeline/multi-arch-container-build.yaml) | + +--- + +### `odh-feature-server-push.yaml` — Feature server image build (merge to master) + +Builds and pushes the feature server image on every merge to `master`. This is +the image the nightly test pipeline resolves via the `odh-master` tag. + +| | | +|---|---| +| **Trigger** | Push to `master` | +| **Source** | `master` branch | +| **Build context** | `sdk/python/feast/infra/feature_servers/multicloud` | +| **Dockerfile** | `sdk/python/feast/infra/feature_servers/multicloud/Dockerfile` | +| **Output image** | `quay.io/opendatahub/feature-server:odh-master` | +| **Service account** | `build-pipeline-odh-feature-server` | +| **Pipeline** | [`multi-arch-container-build.yaml`](https://github.com/opendatahub-io/odh-konflux-central/blob/main/pipeline/multi-arch-container-build.yaml) | + +--- + +## Integration test pipelines + +All three test pipelines run the same test suite on an ephemeral HyperShift +cluster. They differ only in what images they test and how they are triggered. + +**Tests that run:** + +| Test | Command | Description | +|------|---------|-------------| +| Feature Store Operator E2E | `make install && make deploy && make test-e2e` | Full operator + feature server e2e after deployment | +| Registry REST API | `uv run pytest ...test_registry_rest_api.py --integration` | Integration tests for the registry REST API | +| Previous-version compatibility | `make test-previous-version` | Validates compatibility with the previous operator version | +| Upgrade test | `make test-upgrade` | Validates upgrading the operator to the tested version | + +After the test step, the pipeline runs must-gather (Feast + OpenShift), +pushes artifacts to the CI artifacts repo and OCI storage, and posts +results back to GitHub. + +The test runner image is defined in +[`Dockerfile.go-its`](https://github.com/opendatahub-io/odh-konflux-central/blob/main/integration-tests/feast/Dockerfile.go-its) +and includes Go, Python, `oc`/kubectl, `uv`, `skopeo`, and other tools. + +--- + +### `feast-nightly-test.yaml` — Nightly integration test + +Runs the full integration test suite against the latest `odh-master` images +built by the push pipelines. Tests master branch quality on a daily cadence. + +| | | +|---|---| +| **Trigger** | Daily cron at **8 AM UTC** on `master` | +| **Images tested** | `quay.io/opendatahub/feast-operator:odh-master` and `quay.io/opendatahub/feature-server:odh-master` — resolved by digest at run time | +| **Source cloned** | `opendatahub-io/feast` at the commit embedded in the `odh-master` image label | +| **Pipeline** | [`nightly-testing-pipeline.yaml`](https://github.com/opendatahub-io/odh-konflux-central/blob/main/integration-tests/feast/nightly-testing-pipeline.yaml) | +| **Result reporting** | Commit status `konflux/nightly-feast` posted to master HEAD; GitHub Actions monitors and opens issues on failure | + +**How images are resolved** + +The nightly pipeline does **not** build images inline. A `resolve-images` task +runs `skopeo inspect` on the `odh-master` tags and reads the +`io.openshift.build.commit.id` label to determine the exact source commit. +This ensures tests always run against properly Konflux-built images (feast +installed from PyPI, all assets present). + +--- + +### `feast-group-test.yaml` — Group integration test + +Runs integration tests using images built from the **current pull request**. +Use this when changes may affect both the Feast operator and the feature server +(e.g. API changes, shared library updates). + +| | | +|---|---| +| **Trigger** | Automatic — fires when both `odh-feast-operator-pull-request` and `odh-feature-server-pull-request` complete successfully on a PR. +| **Trigger (manual)** | Comment `/group-test` on a pull request | +| **Images tested** | PR-built images resolved by the `generate-snapshot` task (`odh-pr-`) | +| **Source cloned** | `opendatahub-io/feast` at the PR commit | +| **Pipeline** | [`pr-group-testing-pipeline.yaml`](https://github.com/opendatahub-io/odh-konflux-central/blob/main/integration-tests/feast/pr-group-testing-pipeline.yaml) | +| **Result reporting** | GitHub check run `Red Hat Konflux / feast-group-test` posted to the PR by Pipelines-as-Code; PR comment with artifact browser link posted on completion | + +**How images get into the group test** + +1. When a PR is opened or updated, `odh-feast-operator-pull-request` and + `odh-feature-server-pull-request` build and push images tagged + `odh-pr-` and `odh-pr-`. +2. Once both build pipelines succeed, `feast-group-test` is automatically + triggered. The `generate-snapshot` task finds those PR images by component + name, assembles a snapshot JSON with their digests + and git metadata, and passes it to `deploy-and-test`. + +Both the images and the source checkout use the same PR commit, so the code +under test is always consistent. + +--- + +### `feast-pr-test.yaml` — PR integration test (manual) + +Identical test flow to `feast-group-test` but triggered manually by a PR +comment. Use this to run a full integration test on demand without waiting for +the group-test event. + +| | | +|---|---| +| **Trigger (manual)** | Comment `/pr-e2etest` on a pull request | +| **Images tested** | PR-built images resolved by the `generate-snapshot` task (`odh-pr-`) | +| **Source cloned** | `opendatahub-io/feast` at the PR commit | +| **Pipeline** | [`pr-group-testing-pipeline.yaml`](https://github.com/opendatahub-io/odh-konflux-central/blob/main/integration-tests/feast/pr-group-testing-pipeline.yaml) | +| **Max keep runs** | 5 (higher than group-test to retain more history for manual runs) | + +--- + +## Trigger reference + +| Comment / event | Pipeline | Images tested | When to use | +|-----------------|----------|---------------|-------------| +| PR opened/updated | `odh-feast-operator-pull-request` + `odh-feature-server-pull-request` | — (build only) | Automatic on every PR | +| `group-test` event or `/group-test` | `feast-group-test` | PR images | Multi-component changes | +| `/pr-e2etest` | `feast-pr-test` | PR images | On-demand full e2e on a PR | +| Daily 8 AM UTC `feast-nightly-test` | `odh-master` images | Master branch quality gate | +| Merge to master | `odh-feast-operator-push` + `odh-feature-server-push` | — (build only) | Automatic on every merge | + +--- + +## Pipeline definitions + +All integration test pipeline definitions live in +[`opendatahub-io/odh-konflux-central`](https://github.com/opendatahub-io/odh-konflux-central/tree/main/integration-tests/feast): + +| File | Used by | +|------|---------| +| [`nightly-testing-pipeline.yaml`](https://github.com/opendatahub-io/odh-konflux-central/blob/main/integration-tests/feast/nightly-testing-pipeline.yaml) | `feast-nightly-test` | +| [`pr-group-testing-pipeline.yaml`](https://github.com/opendatahub-io/odh-konflux-central/blob/main/integration-tests/feast/pr-group-testing-pipeline.yaml) | `feast-group-test`, `feast-pr-test` | +| [`Dockerfile.go-its`](https://github.com/opendatahub-io/odh-konflux-central/blob/main/integration-tests/feast/Dockerfile.go-its) | Test runner image for all test pipelines | +| [`multi-arch-container-build.yaml`](https://github.com/opendatahub-io/odh-konflux-central/blob/main/pipeline/multi-arch-container-build.yaml) | All build pipelines | diff --git a/.tekton/early-gate-ci-build.yaml b/.tekton/early-gate-ci-build.yaml new file mode 100644 index 00000000000..41cc4330005 --- /dev/null +++ b/.tekton/early-gate-ci-build.yaml @@ -0,0 +1,37 @@ +apiVersion: tekton.dev/v1 +kind: PipelineRun +metadata: + annotations: + build.appstudio.redhat.com/commit_sha: '{{revision}}' + build.appstudio.redhat.com/target_branch: '{{target_branch}}' + pipelinesascode.tekton.dev/max-keep-runs: "3" + pipelinesascode.tekton.dev/on-event: "[issue_comment]" + pipelinesascode.tekton.dev/on-comment: "^/early-gate(-build)?$" + labels: + appstudio.openshift.io/application: early-gate + appstudio.openshift.io/component: early-gate-ci + pipelines.appstudio.openshift.io/type: build + name: early-gate-ci-build + namespace: open-data-hub-tenant +spec: + params: + - name: git-url + value: '{{source_url}}' + - name: revision + value: '{{revision}}' + pipelineRef: + resolver: git + params: + - name: url + value: https://github.com/opendatahub-io/odh-konflux-central.git + - name: revision + value: 'earlygate2' + - name: pathInRepo + value: early-gate/early-gate-component-pipeline.yaml + taskRunTemplate: + serviceAccountName: build-pipeline-early-gate-ci + workspaces: + - name: git-auth + secret: + secretName: '{{ git_auth_secret }}' +status: {} diff --git a/.tekton/early-gate-ci-test.yaml b/.tekton/early-gate-ci-test.yaml new file mode 100644 index 00000000000..648f4246abb --- /dev/null +++ b/.tekton/early-gate-ci-test.yaml @@ -0,0 +1,40 @@ +apiVersion: tekton.dev/v1 +kind: PipelineRun +metadata: + annotations: + build.appstudio.redhat.com/commit_sha: '{{revision}}' + build.appstudio.redhat.com/target_branch: '{{target_branch}}' + pipelinesascode.tekton.dev/max-keep-runs: "3" + pipelinesascode.tekton.dev/on-event: "[issue_comment]" + pipelinesascode.tekton.dev/on-comment: "^/early-gate-test$" + labels: + appstudio.openshift.io/application: early-gate + appstudio.openshift.io/component: early-gate-ci + pipelines.appstudio.openshift.io/type: test + name: early-gate-ci-test + namespace: open-data-hub-tenant +spec: + params: + - name: git-url + value: '{{source_url}}' + - name: revision + value: '{{revision}}' + timeouts: + pipeline: 6h + tasks: 5h + pipelineRef: + resolver: git + params: + - name: url + value: https://github.com/opendatahub-io/odh-konflux-central.git + - name: revision + value: 'main' + - name: pathInRepo + value: early-gate/early-gate-test-pipeline.yaml + taskRunTemplate: + serviceAccountName: build-pipeline-early-gate-ci + workspaces: + - name: git-auth + secret: + secretName: '{{ git_auth_secret }}' +status: {} diff --git a/.tekton/feast-group-test.yaml b/.tekton/feast-group-test.yaml new file mode 100644 index 00000000000..67c8ad45639 --- /dev/null +++ b/.tekton/feast-group-test.yaml @@ -0,0 +1,67 @@ +--- +# Feast Group Test Pipeline (Pipelines-as-Code) +# +# This PipelineRun triggers integration tests across multiple Feast components +# (feast-operator and feature-server) in a single run. It is used for +# group testing when changes may affect more than one component. +# +# Triggers: +# - Event: group-test (e.g. from Konflux/App Studio group-test workflow) +# - Comment: /group-test on a pull request +# +# Pipeline definition: odh-konflux-central/integration-tests/feast/pr-group-testing-pipeline.yaml +# Components tested: odh-feast-operator-ci, odh-feature-server-ci +# +# Images used in the test are built from the PR by the per-component PR pipelines +# (odh-feast-operator-pull-request, odh-feature-server-pull-request). generate-snapshot +# assembles a snapshot of those PR-built image refs; deploy-and-test uses them and the +# PR commit for e2e and REST API tests. See .tekton/README.md for details. +# +# Required template variables (supplied by Pipelines-as-Code): +# - revision, target_branch, pull_request_number, git_auth_secret +# +apiVersion: tekton.dev/v1 +kind: PipelineRun +metadata: + annotations: + build.appstudio.openshift.io/repo: https://github.com/opendatahub-io/feast?rev={{revision}} + build.appstudio.redhat.com/commit_sha: '{{revision}}' + build.appstudio.redhat.com/target_branch: '{{target_branch}}' + build.appstudio.redhat.com/pull_request_number: '{{pull_request_number}}' + pipelinesascode.tekton.dev/cancel-in-progress: "false" + pipelinesascode.tekton.dev/max-keep-runs: "3" + pipelinesascode.tekton.dev/on-cel-expression: event == "group-test" + pipelinesascode.tekton.dev/on-comment: "^/group-test" + name: feast-group-test + namespace: open-data-hub-tenant + labels: + appstudio.openshift.io/application: group-testing + appstudio.openshift.io/component: feast-group + pipelines.appstudio.openshift.io/type: test +spec: + params: + - name: group-components + value: '{ "odh-feast-operator-ci": "opendatahub/feast-operator", "odh-feature-server-ci": "opendatahub/feature-server" }' + pipelineRef: + resolver: git + params: + - name: url + value: https://github.com/opendatahub-io/odh-konflux-central.git + - name: revision + value: main + - name: pathInRepo + value: integration-tests/feast/pr-group-testing-pipeline.yaml + taskRunTemplate: + podTemplate: + nodeSelector: + konflux-ci.dev/workload: konflux-tenants + tolerations: + - effect: NoSchedule + key: konflux-ci.dev/workload + operator: Equal + value: konflux-tenants + serviceAccountName: konflux-integration-runner + workspaces: + - name: git-auth + secret: + secretName: '{{ git_auth_secret }}' diff --git a/.tekton/feast-nightly-test.yaml b/.tekton/feast-nightly-test.yaml new file mode 100644 index 00000000000..4b30e9476ab --- /dev/null +++ b/.tekton/feast-nightly-test.yaml @@ -0,0 +1,60 @@ +--- +# Feast Nightly Test Pipeline (Pipelines-as-Code) +# +# Triggers the nightly integration test pipeline which resolves the latest +# Konflux-built odh-master images (pushed on every merge to master by the +# odh-feast-operator-push and odh-feature-server-push pipelines), provisions +# an ephemeral HyperShift cluster, deploys the operator and feature-server, +# and runs the full e2e + REST API + previous-version + upgrade test suite. +# +# Triggers: +# - Cron: daily at 8 AM UTC +# +# Pipeline definition: odh-konflux-central/integration-tests/feast/nightly-testing-pipeline.yaml +# +apiVersion: tekton.dev/v1 +kind: PipelineRun +metadata: + annotations: + build.appstudio.openshift.io/repo: https://github.com/opendatahub-io/feast?rev={{revision}} + build.appstudio.redhat.com/commit_sha: '{{revision}}' + build.appstudio.redhat.com/target_branch: '{{target_branch}}' + pipelinesascode.tekton.dev/cancel-in-progress: "false" + pipelinesascode.tekton.dev/max-keep-runs: "3" + pipelinesascode.tekton.dev/on-cron: "0 8 * * *" + pipelinesascode.tekton.dev/on-target-branch: "master" + name: feast-nightly-test + namespace: open-data-hub-tenant + labels: + appstudio.openshift.io/application: group-testing + appstudio.openshift.io/component: feast-nightly + pipelines.appstudio.openshift.io/type: test +spec: + params: + - name: git-repo + value: opendatahub-io/feast + - name: git-branch + value: master + pipelineRef: + resolver: git + params: + - name: url + value: https://github.com/opendatahub-io/odh-konflux-central.git + - name: revision + value: main + - name: pathInRepo + value: integration-tests/feast/nightly-testing-pipeline.yaml + taskRunTemplate: + podTemplate: + nodeSelector: + konflux-ci.dev/workload: konflux-tenants + tolerations: + - effect: NoSchedule + key: konflux-ci.dev/workload + operator: Equal + value: konflux-tenants + serviceAccountName: konflux-integration-runner + workspaces: + - name: git-auth + secret: + secretName: '{{ git_auth_secret }}' diff --git a/.tekton/feast-pr-test.yaml b/.tekton/feast-pr-test.yaml new file mode 100644 index 00000000000..47ed5407d48 --- /dev/null +++ b/.tekton/feast-pr-test.yaml @@ -0,0 +1,59 @@ +--- +# Feast PR integration test (Pipelines-as-Code) +# +# Runs the same Hypershift + deploy + e2e flow as /group-test: uses Konflux +# snapshot images built from the pull request (odh-feast-operator-ci, +# odh-feature-server-ci), not freshly built nightly images. +# +# Triggers: +# - Comment: /pr-e2etest on a pull request (manual trigger) +# +# Pipeline definition: opendatahub-io/odh-konflux-central +# integration-tests/feast/pr-group-testing-pipeline.yaml +# +# Components tested: odh-feast-operator-ci, odh-feature-server-ci +# +apiVersion: tekton.dev/v1 +kind: PipelineRun +metadata: + annotations: + build.appstudio.openshift.io/repo: https://github.com/opendatahub-io/feast?rev={{revision}} + build.appstudio.redhat.com/commit_sha: '{{revision}}' + build.appstudio.redhat.com/target_branch: '{{target_branch}}' + build.appstudio.redhat.com/pull_request_number: '{{pull_request_number}}' + pipelinesascode.tekton.dev/cancel-in-progress: "false" + pipelinesascode.tekton.dev/max-keep-runs: "5" + pipelinesascode.tekton.dev/on-comment: "^/pr-e2etest" + name: feast-pr-test + namespace: open-data-hub-tenant + labels: + appstudio.openshift.io/application: group-testing + appstudio.openshift.io/component: feast-group + pipelines.appstudio.openshift.io/type: test +spec: + params: + - name: group-components + value: '{ "odh-feast-operator-ci": "opendatahub/feast-operator", "odh-feature-server-ci": "opendatahub/feature-server" }' + pipelineRef: + resolver: git + params: + - name: url + value: https://github.com/opendatahub-io/odh-konflux-central.git + - name: revision + value: main + - name: pathInRepo + value: integration-tests/feast/pr-group-testing-pipeline.yaml + taskRunTemplate: + podTemplate: + nodeSelector: + konflux-ci.dev/workload: konflux-tenants + tolerations: + - effect: NoSchedule + key: konflux-ci.dev/workload + operator: Equal + value: konflux-tenants + serviceAccountName: konflux-integration-runner + workspaces: + - name: git-auth + secret: + secretName: '{{ git_auth_secret }}' diff --git a/.tekton/odh-feast-operator-pull-request.yaml b/.tekton/odh-feast-operator-pull-request.yaml new file mode 100644 index 00000000000..02381d0b920 --- /dev/null +++ b/.tekton/odh-feast-operator-pull-request.yaml @@ -0,0 +1,55 @@ +apiVersion: tekton.dev/v1 +kind: PipelineRun +metadata: + annotations: + build.appstudio.openshift.io/repo: https://github.com/opendatahub-io/feast?rev={{revision}} + build.appstudio.redhat.com/commit_sha: '{{revision}}' + build.appstudio.redhat.com/target_branch: '{{target_branch}}' + build.appstudio.redhat.com/pull_request_number: '{{pull_request_number}}' + pipelinesascode.tekton.dev/cancel-in-progress: "false" + pipelinesascode.tekton.dev/max-keep-runs: "3" + pipelinesascode.tekton.dev/on-cel-expression: event == "pull_request" && target_branch + == "master" + creationTimestamp: null + labels: + appstudio.openshift.io/application: opendatahub-builds + appstudio.openshift.io/component: odh-feast-operator-ci + pipelines.appstudio.openshift.io/type: build + name: odh-feast-operator-on-pull-request + namespace: open-data-hub-tenant +spec: + params: + - name: git-url + value: '{{source_url}}' + - name: revision + value: '{{revision}}' + - name: output-image + value: quay.io/opendatahub/feast-operator:odh-pr + - name: dockerfile + value: Dockerfile + - name: path-context + value: infra/feast-operator + - name: additional-tags + value: + - 'odh-pr-{{revision}}' + - 'odh-pr-{{pull_request_number}}' + - name: pipeline-type + value: pull-request + - name: enable-group-testing + value: "true" + pipelineRef: + resolver: git + params: + - name: url + value: https://github.com/opendatahub-io/odh-konflux-central.git + - name: revision + value: main + - name: pathInRepo + value: pipeline/multi-arch-container-build.yaml + taskRunTemplate: + serviceAccountName: build-pipeline-odh-feast-operator + workspaces: + - name: git-auth + secret: + secretName: '{{ git_auth_secret }}' +status: {} diff --git a/.tekton/odh-feast-operator-push.yaml b/.tekton/odh-feast-operator-push.yaml new file mode 100644 index 00000000000..e1759e31479 --- /dev/null +++ b/.tekton/odh-feast-operator-push.yaml @@ -0,0 +1,46 @@ +apiVersion: tekton.dev/v1 +kind: PipelineRun +metadata: + annotations: + build.appstudio.openshift.io/repo: https://github.com/opendatahub-io/feast?rev={{revision}} + build.appstudio.redhat.com/commit_sha: '{{revision}}' + build.appstudio.redhat.com/target_branch: '{{target_branch}}' + pipelinesascode.tekton.dev/cancel-in-progress: "false" + pipelinesascode.tekton.dev/max-keep-runs: "3" + pipelinesascode.tekton.dev/on-cel-expression: event == "push" && target_branch + == "master" + creationTimestamp: null + labels: + appstudio.openshift.io/application: opendatahub-builds + appstudio.openshift.io/component: odh-feast-operator-ci + pipelines.appstudio.openshift.io/type: build + name: odh-feast-operator-on-push + namespace: open-data-hub-tenant +spec: + params: + - name: git-url + value: '{{source_url}}' + - name: revision + value: '{{revision}}' + - name: output-image + value: quay.io/opendatahub/feast-operator:odh-master + - name: dockerfile + value: Dockerfile + - name: path-context + value: infra/feast-operator + pipelineRef: + resolver: git + params: + - name: url + value: https://github.com/opendatahub-io/odh-konflux-central.git + - name: revision + value: main + - name: pathInRepo + value: pipeline/multi-arch-container-build.yaml + taskRunTemplate: + serviceAccountName: build-pipeline-odh-feast-operator + workspaces: + - name: git-auth + secret: + secretName: '{{ git_auth_secret }}' +status: {} diff --git a/.tekton/odh-feature-server-pull-request.yaml b/.tekton/odh-feature-server-pull-request.yaml new file mode 100644 index 00000000000..1aa1edd97bc --- /dev/null +++ b/.tekton/odh-feature-server-pull-request.yaml @@ -0,0 +1,55 @@ +apiVersion: tekton.dev/v1 +kind: PipelineRun +metadata: + annotations: + build.appstudio.openshift.io/repo: https://github.com/opendatahub-io/feast?rev={{revision}} + build.appstudio.redhat.com/commit_sha: '{{revision}}' + build.appstudio.redhat.com/target_branch: '{{target_branch}}' + build.appstudio.redhat.com/pull_request_number: '{{pull_request_number}}' + pipelinesascode.tekton.dev/cancel-in-progress: "false" + pipelinesascode.tekton.dev/max-keep-runs: "3" + pipelinesascode.tekton.dev/on-cel-expression: event == "pull_request" && target_branch + == "master" + creationTimestamp: null + labels: + appstudio.openshift.io/application: opendatahub-builds + appstudio.openshift.io/component: odh-feature-server-ci + pipelines.appstudio.openshift.io/type: build + name: odh-feature-server-on-pull-request + namespace: open-data-hub-tenant +spec: + params: + - name: git-url + value: '{{source_url}}' + - name: revision + value: '{{revision}}' + - name: output-image + value: quay.io/opendatahub/feature-server:odh-pr + - name: dockerfile + value: Dockerfile + - name: path-context + value: sdk/python/feast/infra/feature_servers/multicloud + - name: additional-tags + value: + - 'odh-pr-{{revision}}' + - 'odh-pr-{{pull_request_number}}' + - name: pipeline-type + value: pull-request + - name: enable-group-testing + value: "true" + pipelineRef: + resolver: git + params: + - name: url + value: https://github.com/opendatahub-io/odh-konflux-central.git + - name: revision + value: main + - name: pathInRepo + value: pipeline/multi-arch-container-build.yaml + taskRunTemplate: + serviceAccountName: build-pipeline-odh-feature-server + workspaces: + - name: git-auth + secret: + secretName: '{{ git_auth_secret }}' +status: {} diff --git a/.tekton/odh-feature-server-push.yaml b/.tekton/odh-feature-server-push.yaml new file mode 100644 index 00000000000..7e9ee947fc5 --- /dev/null +++ b/.tekton/odh-feature-server-push.yaml @@ -0,0 +1,46 @@ +apiVersion: tekton.dev/v1 +kind: PipelineRun +metadata: + annotations: + build.appstudio.openshift.io/repo: https://github.com/opendatahub-io/feast?rev={{revision}} + build.appstudio.redhat.com/commit_sha: '{{revision}}' + build.appstudio.redhat.com/target_branch: '{{target_branch}}' + pipelinesascode.tekton.dev/cancel-in-progress: "false" + pipelinesascode.tekton.dev/max-keep-runs: "3" + pipelinesascode.tekton.dev/on-cel-expression: event == "push" && target_branch + == "master" + creationTimestamp: null + labels: + appstudio.openshift.io/application: opendatahub-builds + appstudio.openshift.io/component: odh-feature-server-ci + pipelines.appstudio.openshift.io/type: build + name: odh-feature-server-on-push + namespace: open-data-hub-tenant +spec: + params: + - name: git-url + value: '{{source_url}}' + - name: revision + value: '{{revision}}' + - name: output-image + value: quay.io/opendatahub/feature-server:odh-master + - name: dockerfile + value: Dockerfile + - name: path-context + value: sdk/python/feast/infra/feature_servers/multicloud + pipelineRef: + resolver: git + params: + - name: url + value: https://github.com/opendatahub-io/odh-konflux-central.git + - name: revision + value: main + - name: pathInRepo + value: pipeline/multi-arch-container-build.yaml + taskRunTemplate: + serviceAccountName: build-pipeline-odh-feature-server + workspaces: + - name: git-auth + secret: + secretName: '{{ git_auth_secret }}' +status: {} diff --git a/infra/feast-operator/cmd/main.go b/infra/feast-operator/cmd/main.go index 0d833f1469b..03768b67805 100644 --- a/infra/feast-operator/cmd/main.go +++ b/infra/feast-operator/cmd/main.go @@ -19,8 +19,12 @@ package main import ( "context" "crypto/tls" + "errors" "flag" + "fmt" "os" + "strings" + "time" // Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.) // to ensure that exec-entrypoint and run can make use of them. @@ -34,17 +38,22 @@ import ( corev1 "k8s.io/api/core/v1" policyv1 "k8s.io/api/policy/v1" rbacv1 "k8s.io/api/rbac/v1" + apiextensionsclient "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" apierrors "k8s.io/apimachinery/pkg/api/errors" apimeta "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" utilruntime "k8s.io/apimachinery/pkg/util/runtime" clientgoscheme "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/rest" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/cache" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/healthz" "sigs.k8s.io/controller-runtime/pkg/log/zap" + "sigs.k8s.io/controller-runtime/pkg/manager" "sigs.k8s.io/controller-runtime/pkg/metrics/filters" metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" "sigs.k8s.io/controller-runtime/pkg/webhook" @@ -102,7 +111,7 @@ func main() { var probeAddr string var secureMetrics bool var featureStoreMetrics bool - var tlsOpts []func(*tls.Config) + tlsOpts := make([]func(*tls.Config), 0, 2) flag.StringVar(&metricsAddr, "metrics-bind-address", "0", "The address the metrics endpoint binds to. "+ "Use :8443 for HTTPS or :8080 for HTTP, or leave as 0 to disable the metrics service.") flag.StringVar(&probeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.") @@ -131,38 +140,42 @@ func main() { } tlsProfileFetched := false - tlsProfile, err := tlspkg.FetchAPIServerTLSProfile(context.Background(), bootstrapClient) + profileCtx, cancelProfile := context.WithTimeout(context.Background(), 10*time.Second) + defer cancelProfile() + tlsProfile, err := tlspkg.FetchAPIServerTLSProfile(profileCtx, bootstrapClient) if err != nil { switch { case apimeta.IsNoMatchError(err): - setupLog.Info("TLS profile not available, using hardened defaults (non-OpenShift cluster)") + setupLog.Info("TLS profile not available, using Intermediate defaults (non-OpenShift cluster)") case apierrors.IsNotFound(err): - setupLog.Info("APIServer resource not found, using hardened defaults") + setupLog.Info("APIServer resource not found, using Intermediate defaults") + case apierrors.IsServiceUnavailable(err), + apierrors.IsTimeout(err), + apierrors.IsServerTimeout(err), + apierrors.IsTooManyRequests(err), + errors.Is(err, context.DeadlineExceeded): + setupLog.Info("Transient API error reading TLS profile, using Intermediate defaults", "error", err) + tlsProfileFetched = true default: setupLog.Error(err, "unable to read APIServer TLS profile, refusing to start with unknown TLS posture") os.Exit(1) } + tlsProfile = *configv1.TLSProfiles[configv1.TLSProfileIntermediateType] } else { tlsProfileFetched = true - tlsConfigFn, unsupported := tlspkg.NewTLSConfigFromProfile(tlsProfile) - if len(unsupported) > 0 { - setupLog.Info("TLS profile contains ciphers unsupported by Go", "unsupported", unsupported) - } - tlsOpts = append(tlsOpts, tlsConfigFn) } + tlsConfigFn, unsupported := tlspkg.NewTLSConfigFromProfile(tlsProfile) + if len(unsupported) > 0 { + setupLog.Info("TLS profile contains ciphers unsupported by Go", "unsupported", unsupported) + } + tlsOpts = append(tlsOpts, tlsConfigFn) tlsAdherenceFetched := false - tlsAdherence, err := tlspkg.FetchAPIServerTLSAdherencePolicy(context.Background(), bootstrapClient) + adherenceCtx, cancelAdherence := context.WithTimeout(context.Background(), 10*time.Second) + defer cancelAdherence() + tlsAdherence, err := tlspkg.FetchAPIServerTLSAdherencePolicy(adherenceCtx, bootstrapClient) if err != nil { - switch { - case apimeta.IsNoMatchError(err): - setupLog.Info("TLS adherence policy not available (non-OpenShift cluster)") - case apierrors.IsNotFound(err): - setupLog.Info("APIServer resource not found, skipping adherence policy") - default: - setupLog.Error(err, "unable to read APIServer TLS adherence policy, refusing to start") - os.Exit(1) - } + setupLog.Info("unable to fetch TLS adherence policy, watcher will retry", "error", err) } else { tlsAdherenceFetched = true } @@ -265,6 +278,43 @@ func main() { setupLog.Error(err, "unable to create controller", "controller", "FeatureStore") os.Exit(1) } + + // Setup Notebook ConfigMap controller for OpenDataHub integration + // Default to kubeflow.org/v1 Notebook CRD (used by OpenDataHub) + notebookGVK := schema.GroupVersionKind{ + Group: "kubeflow.org", + Version: "v1", + Kind: "Notebook", + } + // Validate Notebook CRD availability and skip controller setup if CRD is missing + crdExists, err := validateNotebookCRD(context.Background(), mgr.GetConfig(), notebookGVK) + if err != nil { + setupLog.Error(err, "Notebook CRD validation failed for feast specific configmap", "GVK", notebookGVK) + if !crdExists { + setupLog.Info("Skipping Notebook ConfigMap controller setup - Notebook CRD not found") + } else { + setupLog.Info("Notebook ConfigMap controller will be set up anyway (could not verify CRD)") + } + } + if crdExists { + if err = setupNotebookController(mgr, notebookGVK); err != nil { + setupLog.Error(err, "unable to create controller", "controller", "NotebookConfigMap") + os.Exit(1) + } + } else { + // CRD not found at startup (not an error, just not installed yet). + // During DSC deployment the Notebook CRD may be installed after the Feast + // operator starts. Poll in the background and trigger a process restart + // when the CRD appears so the controller can register during normal startup. + // Dynamic controller registration on a running manager is not supported by + // controller-runtime v0.18 (cache sync fails for late-added informers). + setupLog.Info("Notebook CRD not found at startup, will poll for CRD availability", "GVK", notebookGVK) + if addErr := mgr.Add(manager.RunnableFunc(func(ctx context.Context) error { + return waitForCRDAndRestart(ctx, mgr.GetConfig(), notebookGVK) + })); addErr != nil { + setupLog.Error(addErr, "Failed to add CRD watcher runnable") + } + } // +kubebuilder:scaffold:builder // Register SecurityProfileWatcher to restart on TLS profile changes @@ -308,3 +358,92 @@ func main() { os.Exit(1) } } + +// validateNotebookCRD checks if the Notebook CRD exists in the cluster +// Returns (crdExists bool, error) where crdExists is true if CRD exists, false if not found or unknown +func validateNotebookCRD(ctx context.Context, config *rest.Config, gvk schema.GroupVersionKind) (bool, error) { + apiextensionsClientset, err := apiextensionsclient.NewForConfig(config) + if err != nil { + return false, fmt.Errorf("failed to create apiextensions client: %w", err) + } + + // Construct CRD name from GVK (format: plural.group) + // For kubeflow.org/v1 Notebook, the CRD name is "notebooks.kubeflow.org" + // Simple heuristic: convert Kind to lowercase and add 's' for plural + plural := strings.ToLower(gvk.Kind) + "s" + crdName := plural + "." + gvk.Group + + _, err = apiextensionsClientset.ApiextensionsV1().CustomResourceDefinitions().Get( + ctx, + crdName, + metav1.GetOptions{}, + ) + if err != nil { + if apierrors.IsNotFound(err) { + // Not an error: integration is optional and controller setup will be skipped. + setupLog.Info( + "Notebook CRD not found; skipping Notebook ConfigMap controller setup", + "GVK", gvk, + "expectedCRD", crdName, + ) + return false, nil + } + if apierrors.IsForbidden(err) { + // Can't verify - assume it might exist and let controller try + setupLog.Info( + "Unable to validate Notebook CRD (insufficient permissions), will attempt setup", + "CRD", crdName, + "GVK", gvk, + ) + return true, nil // Return true to allow controller setup + } + return false, fmt.Errorf("failed to check Notebook CRD availability: %w", err) + } + + setupLog.Info("Notebook CRD validated successfully", "CRD", crdName, "GVK", gvk) + return true, nil +} + +const crdPollInterval = 5 * time.Second + +func setupNotebookController(mgr ctrl.Manager, notebookGVK schema.GroupVersionKind) error { + if err := (&controller.NotebookConfigMapReconciler{ + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + NotebookGVK: notebookGVK, + }).SetupWithManager(mgr); err != nil { + return err + } + setupLog.Info("Notebook ConfigMap controller setup completed successfully") + return nil +} + +// waitForCRDAndRestart polls for the Notebook CRD and exits the process when +// it becomes available. Kubernetes will restart the pod, and on the next startup +// the controller registers normally via the standard code path. This avoids +// dynamically adding controllers to a running manager, which causes cache sync +// failures in controller-runtime v0.18. +func waitForCRDAndRestart(ctx context.Context, config *rest.Config, gvk schema.GroupVersionKind) error { + ticker := time.NewTicker(crdPollInterval) + defer ticker.Stop() + + setupLog.Info("Waiting for Notebook CRD to become available...", "GVK", gvk) + + for { + select { + case <-ctx.Done(): + setupLog.Info("Context cancelled, stopping CRD watch", "GVK", gvk) + return nil + case <-ticker.C: + crdExists, err := validateNotebookCRD(ctx, config, gvk) + if err != nil { + setupLog.Error(err, "Failed to check Notebook CRD availability, will retry") + continue + } + if crdExists { + setupLog.Info("Notebook CRD detected, restarting operator to initialize Notebook ConfigMap controller", "GVK", gvk) + os.Exit(0) + } + } + } +} diff --git a/infra/feast-operator/config/rbac/role.yaml b/infra/feast-operator/config/rbac/role.yaml index f6e6801dfa8..1c94d289697 100644 --- a/infra/feast-operator/config/rbac/role.yaml +++ b/infra/feast-operator/config/rbac/role.yaml @@ -8,6 +8,27 @@ rules: - "" resources: - configmaps + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - "" + resources: + - namespaces + verbs: + - get + - list + - patch + - update + - watch +- apiGroups: + - "" + resources: - persistentvolumeclaims - serviceaccounts - services @@ -21,7 +42,6 @@ rules: - apiGroups: - "" resources: - - namespaces - pods - secrets verbs: @@ -34,6 +54,13 @@ rules: - pods/exec verbs: - create +- apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions + verbs: + - get + - list - apiGroups: - apps resources: @@ -109,6 +136,14 @@ rules: - get - patch - update +- apiGroups: + - kubeflow.org + resources: + - notebooks + verbs: + - get + - list + - watch - apiGroups: - monitoring.coreos.com resources: diff --git a/infra/feast-operator/dist/install.yaml b/infra/feast-operator/dist/install.yaml index 44aab6c8af5..955c54358bd 100644 --- a/infra/feast-operator/dist/install.yaml +++ b/infra/feast-operator/dist/install.yaml @@ -22036,6 +22036,27 @@ rules: - "" resources: - configmaps + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - "" + resources: + - namespaces + verbs: + - get + - list + - patch + - update + - watch +- apiGroups: + - "" + resources: - persistentvolumeclaims - serviceaccounts - services @@ -22049,7 +22070,6 @@ rules: - apiGroups: - "" resources: - - namespaces - pods - secrets verbs: @@ -22062,6 +22082,13 @@ rules: - pods/exec verbs: - create +- apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions + verbs: + - get + - list - apiGroups: - apps resources: @@ -22137,6 +22164,14 @@ rules: - get - patch - update +- apiGroups: + - kubeflow.org + resources: + - notebooks + verbs: + - get + - list + - watch - apiGroups: - monitoring.coreos.com resources: diff --git a/infra/feast-operator/docs/auto-access.md b/infra/feast-operator/docs/auto-access.md new file mode 100644 index 00000000000..be36d0f11a6 --- /dev/null +++ b/infra/feast-operator/docs/auto-access.md @@ -0,0 +1,47 @@ +# Feast Auto Access Enablement + +## Overview + +The Feast Operator automatically enables discovery and view access for users and groups defined in Feast permissions. Dashboard applications use user credentials to list Feast namespaces and client ConfigMaps, with no dependency on a central registry or service account. + +## Namespace Labeling + +The operator adds the label `opendatahub.io/feast: "true"` to namespaces that contain a deployed FeatureStore. This enables dashboards to discover Feast namespaces cluster-wide by listing namespaces with this label selector. + +- **Label**: `opendatahub.io/feast=true` +- **When added**: After a FeatureStore is successfully deployed +- **When removed**: When the last FeatureStore in the namespace is deleted + +## Auto-Access RBAC + +When the registry REST API is enabled and permissions are defined in `permissions.py`, the operator creates RBAC so users and groups from Feast permissions get view access: + +1. **GroupBasedPolicy**: All users in the listed groups +2. **NamespaceBasedPolicy**: All users/groups with any RoleBinding in the Data Science Project (K8s namespace) +3. **CombinedGroupNamespacePolicy**: Both groups and namespace-based subjects + +### RBAC Resources Created + +- **ClusterRole** `feast-discover-namespaces`: Allows `get`, `list`, `watch` on namespaces (for cluster-wide discovery) +- **ClusterRoleBinding** (per FeatureStore): Binds the ClusterRole to subjects from permissions +- **Role** (namespace-scoped): Allows `get`, `list`, `watch` on the client ConfigMap +- **RoleBinding** (namespace-scoped): Binds the Role to the same subjects + +### View Access + +View access grants: +- List namespaces cluster-wide (with label `opendatahub.io/feast=true`) +- Get/list/watch the Feature Store client ConfigMap in each accessible namespace + +## Dashboard Contract + +Dashboards should: + +1. List namespaces with label selector `opendatahub.io/feast=true` using the user's token +2. For each namespace the user can access, list ConfigMaps to find client ConfigMaps (e.g. `feast--client`) + +## Prerequisites + +- Registry REST API must be enabled (`spec.services.registry.local.server.restAPI: true`) +- Permissions must be applied via `feast apply` (from `permissions.py` in the feature repo) +- The operator reconciles permissions periodically (every 30 seconds) to pick up changes diff --git a/infra/feast-operator/docs/notebook-configmap-integration.md b/infra/feast-operator/docs/notebook-configmap-integration.md new file mode 100644 index 00000000000..ebf9359fec3 --- /dev/null +++ b/infra/feast-operator/docs/notebook-configmap-integration.md @@ -0,0 +1,219 @@ +# Notebook ConfigMap Integration + +## Overview + +The Feast operator can watch OpenDataHub Notebook custom resources and automatically create/update a `notebook-feast-config` ConfigMap in each notebook's namespace. This ConfigMap contains Feast client configuration for all projects referenced by the notebook's annotation. + +## Architecture + +The integration works bidirectionally: + +1. **Notebook → ConfigMap**: When a Notebook's annotation changes, the operator creates/updates the `notebook-feast-config` with Feast client configs for all referenced projects. + +2. **FeatureStore → ConfigMap**: When a FeatureStore's client ConfigMap changes, the operator updates all `notebook-feast-config` that reference that project. + +## Label Format + +Notebooks must have the following label: + +```yaml +labels: + opendatahub.io/feast-integration: "true" +``` +And the following annotation: + +```yaml +annotations: + opendatahub.io/feast-config: "project1,project2,project3" +``` + +The annotation value is a comma-separated list of Feast project names. The operator will: +- Fetch the client ConfigMap for each project from the FeatureStore +- Create/update `notebook-feast-config` in the notebook's namespace +- Use project names as keys and client ConfigMap YAML content as values + +## ConfigMap Structure + +The `notebook-feast-config` ConfigMap is created in the same namespace as the Notebook and has the following structure: + +```yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: -feast-config + namespace: + labels: + managed-by: feast-operator + ownerReferences: + - apiVersion: kubeflow.org/v1 + kind: Notebook + name: + uid: +data: + : | + +``` + +Each key in `data` is a Feast project name, and the value is the YAML content from that project's client ConfigMap. The ConfigMap is owned by the Notebook and will be deleted when the Notebook is deleted. + +## How It Works + +### 1. Notebook Reconciliation + +When a Notebook is created or updated: + +1. Operator checks for `opendatahub.io/feast-integration` label is set to `true` +2. Parses comma-separated project names from `opendatahub.io/feast-config` annotation +3. Creates/updates `notebook-feast-config` ConfigMap with project names as keys and YAML content from the project's client ConfigMap as values in the notebook's namespace + +### 2. FeatureStore Reconciliation + +When a FeatureStore's client ConfigMap changes: + +1. Operator identifies the project name from the FeatureStore's `spec.feastProject` +2. Lists all Notebooks across all namespaces +3. For each Notebook with the project in its `opendatahub.io/feast-config` annotation: + - Triggers reconciliation of that Notebook's `notebook-feast-config` ConfigMap +4. Updates the `notebook-feast-config` ConfigMap with the new client config + +### 3. Cleanup + +When a Notebook is deleted: +- The `notebook-feast-config` ConfigMap is automatically deleted (via owner reference) + +When a project is removed from a Notebook's `opendatahub.io/feast-config` annotation: +- The corresponding entry is removed from the ConfigMap + +## Example Usage + +### 1. Create a FeatureStore + +```yaml +apiVersion: feast.dev/v1alpha1 +kind: FeatureStore +metadata: + name: my-feast-store + namespace: feast-system +spec: + feastProject: my-project + services: + # ... service configuration +``` + +This creates a client ConfigMap (e.g., `my-feast-store-client`) with the Feast configuration. + +### 2. Create a Notebook with Feast Projects + +```yaml +apiVersion: kubeflow.org/v1 +kind: Notebook +metadata: + name: my-notebook + namespace: user-namespace + labels: + opendatahub.io/feast-integration: "true" + annotations: + opendatahub.io/feast-config: "my-project,another-project" +spec: + # ... notebook configuration +``` + +The operator automatically creates `my-notebook-feast-config` ConfigMap in `user-namespace`: + +```yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: my-notebook-feast-config + namespace: user-namespace + labels: + managed-by: feast-operator + ownerReferences: + - apiVersion: kubeflow.org/v1 + kind: Notebook + name: my-notebook + uid: +data: + my-project: | + # YAML content from my-project's client ConfigMap + another-project: | + # YAML content from another-project's client ConfigMap +``` + +### 3. Update Notebook Annotation + +If you update the Notebook annotation: + +```yaml +annotations: + opendatahub.io/feast-config: "my-project" # Removed another-project +``` + +The operator automatically removes `another-project` from the ConfigMap. + +### 4. Update FeatureStore + +If you update the FeatureStore configuration, the operator automatically updates all `my-notebook-feast-config` ConfigMaps that reference that project. + +## Configuration + +The Notebook GVK is configured in `cmd/main.go`: + +```go +notebookGVK := schema.GroupVersionKind{ + Group: "kubeflow.org", + Version: "v1", + Kind: "Notebook", +} +``` + +To use a different Notebook CRD, modify this GVK accordingly. + +## RBAC Permissions + +The operator requires the following permissions: + +```yaml +- apiGroups: + - kubeflow.org + resources: + - notebooks + verbs: + - get + - list + - watch +- apiGroups: + - feast.dev + resources: + - featurestores + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - configmaps + verbs: + - get + - list + - watch + - create + - update + - patch + - delete +``` + +## Troubleshooting + +### ConfigMap Not Created + +- Verify the Notebook has the `opendatahub.io/feast-integration` label set to `true` +- Check that the annotation `opendatahub.io/feast-config` contains valid project names +- Ensure FeatureStores exist for all referenced projects +- Check operator logs for errors +### Project Not Found +- Ensure a FeatureStore exists with `spec.feastProject` matching the project name +- Verify the FeatureStore is in Ready state +- Check that the FeatureStore has a client ConfigMap created + diff --git a/infra/feast-operator/internal/controller/access/access.go b/infra/feast-operator/internal/controller/access/access.go new file mode 100644 index 00000000000..8b0b0ab17cd --- /dev/null +++ b/infra/feast-operator/internal/controller/access/access.go @@ -0,0 +1,80 @@ +/* +Copyright 2024 Feast Community. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package access + +import ( + "context" + "fmt" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/log" +) + +const ( + FeastNamespaceLabelKey = "opendatahub.io/feast" + FeastNamespaceLabelValue = "true" +) + +// EnsureNamespaceLabel adds the Feast discovery label to the namespace. +// Call when a FeatureStore is successfully deployed. +func EnsureNamespaceLabel(ctx context.Context, c client.Client, namespace string) error { + ns := &corev1.Namespace{} + if err := c.Get(ctx, client.ObjectKey{Name: namespace}, ns); err != nil { + if apierrors.IsNotFound(err) { + return nil + } + return fmt.Errorf("failed to get namespace %s: %w", namespace, err) + } + if ns.Labels == nil { + ns.Labels = make(map[string]string) + } + if ns.Labels[FeastNamespaceLabelKey] == FeastNamespaceLabelValue { + return nil + } + ns.Labels[FeastNamespaceLabelKey] = FeastNamespaceLabelValue + if err := c.Update(ctx, ns); err != nil { + return fmt.Errorf("failed to patch namespace %s with Feast label: %w", namespace, err) + } + log.FromContext(ctx).Info("Added Feast discovery label to namespace", "namespace", namespace) + return nil +} + +// RemoveNamespaceLabelIfLast removes the Feast label from the namespace when +// otherFeatureStoreCount is 0. Call when a FeatureStore is being deleted. +func RemoveNamespaceLabelIfLast(ctx context.Context, c client.Client, namespace string, otherFeatureStoreCount int) error { + if otherFeatureStoreCount > 0 { + return nil + } + ns := &corev1.Namespace{} + if err := c.Get(ctx, client.ObjectKey{Name: namespace}, ns); err != nil { + if apierrors.IsNotFound(err) { + return nil + } + return fmt.Errorf("failed to get namespace %s: %w", namespace, err) + } + if ns.Labels == nil || ns.Labels[FeastNamespaceLabelKey] == "" { + return nil + } + delete(ns.Labels, FeastNamespaceLabelKey) + if err := c.Update(ctx, ns); err != nil { + return fmt.Errorf("failed to remove Feast label from namespace %s: %w", namespace, err) + } + log.FromContext(ctx).Info("Removed Feast discovery label from namespace", "namespace", namespace) + return nil +} diff --git a/infra/feast-operator/internal/controller/access/access_test.go b/infra/feast-operator/internal/controller/access/access_test.go new file mode 100644 index 00000000000..e6336fe6cf3 --- /dev/null +++ b/infra/feast-operator/internal/controller/access/access_test.go @@ -0,0 +1,145 @@ +/* +Copyright 2024 Feast Community. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package access + +import ( + "context" + "testing" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" +) + +func newScheme() *runtime.Scheme { + s := runtime.NewScheme() + _ = corev1.AddToScheme(s) + return s +} + +func newNamespace(name string, labels map[string]string) *corev1.Namespace { + return &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Labels: labels, + }, + } +} + +func getNamespace(t *testing.T, c client.Client) *corev1.Namespace { + t.Helper() + ns := &corev1.Namespace{} + if err := c.Get(context.Background(), client.ObjectKey{Name: "test-ns"}, ns); err != nil { + t.Fatalf("Failed to get namespace test-ns: %v", err) + } + return ns +} + +func TestEnsureNamespaceLabel_AddsLabel(t *testing.T) { + ns := newNamespace("test-ns", nil) + c := fake.NewClientBuilder().WithScheme(newScheme()).WithObjects(ns).Build() + + if err := EnsureNamespaceLabel(context.Background(), c, "test-ns"); err != nil { + t.Fatalf("unexpected error: %v", err) + } + updated := getNamespace(t, c) + if updated.Labels[FeastNamespaceLabelKey] != FeastNamespaceLabelValue { + t.Fatalf("expected label %s=%s, got %v", FeastNamespaceLabelKey, FeastNamespaceLabelValue, updated.Labels) + } +} + +func TestEnsureNamespaceLabel_AlreadyLabeled(t *testing.T) { + ns := newNamespace("test-ns", map[string]string{FeastNamespaceLabelKey: FeastNamespaceLabelValue}) + c := fake.NewClientBuilder().WithScheme(newScheme()).WithObjects(ns).Build() + + if err := EnsureNamespaceLabel(context.Background(), c, "test-ns"); err != nil { + t.Fatalf("unexpected error: %v", err) + } + updated := getNamespace(t, c) + if updated.Labels[FeastNamespaceLabelKey] != FeastNamespaceLabelValue { + t.Fatalf("label should still be present") + } +} + +func TestEnsureNamespaceLabel_NamespaceNotFound(t *testing.T) { + c := fake.NewClientBuilder().WithScheme(newScheme()).Build() + + if err := EnsureNamespaceLabel(context.Background(), c, "missing-ns"); err != nil { + t.Fatalf("expected nil error for missing namespace, got: %v", err) + } +} + +func TestEnsureNamespaceLabel_PreservesExistingLabels(t *testing.T) { + ns := newNamespace("test-ns", map[string]string{"existing": "label"}) + c := fake.NewClientBuilder().WithScheme(newScheme()).WithObjects(ns).Build() + + if err := EnsureNamespaceLabel(context.Background(), c, "test-ns"); err != nil { + t.Fatalf("unexpected error: %v", err) + } + updated := getNamespace(t, c) + if updated.Labels["existing"] != "label" { + t.Fatal("existing label was removed") + } + if updated.Labels[FeastNamespaceLabelKey] != FeastNamespaceLabelValue { + t.Fatal("feast label was not added") + } +} + +func TestRemoveNamespaceLabelIfLast_RemovesWhenZero(t *testing.T) { + ns := newNamespace("test-ns", map[string]string{FeastNamespaceLabelKey: FeastNamespaceLabelValue}) + c := fake.NewClientBuilder().WithScheme(newScheme()).WithObjects(ns).Build() + + if err := RemoveNamespaceLabelIfLast(context.Background(), c, "test-ns", 0); err != nil { + t.Fatalf("unexpected error: %v", err) + } + updated := getNamespace(t, c) + if _, ok := updated.Labels[FeastNamespaceLabelKey]; ok { + t.Fatal("label should have been removed") + } +} + +func TestRemoveNamespaceLabelIfLast_KeepsWhenOthersExist(t *testing.T) { + ns := newNamespace("test-ns", map[string]string{FeastNamespaceLabelKey: FeastNamespaceLabelValue}) + c := fake.NewClientBuilder().WithScheme(newScheme()).WithObjects(ns).Build() + + if err := RemoveNamespaceLabelIfLast(context.Background(), c, "test-ns", 1); err != nil { + t.Fatalf("unexpected error: %v", err) + } + updated := getNamespace(t, c) + if updated.Labels[FeastNamespaceLabelKey] != FeastNamespaceLabelValue { + t.Fatal("label should not have been removed when other FeatureStores exist") + } +} + +func TestRemoveNamespaceLabelIfLast_NoLabelNoOp(t *testing.T) { + ns := newNamespace("test-ns", nil) + c := fake.NewClientBuilder().WithScheme(newScheme()).WithObjects(ns).Build() + + if err := RemoveNamespaceLabelIfLast(context.Background(), c, "test-ns", 0); err != nil { + t.Fatalf("unexpected error: %v", err) + } +} + +func TestRemoveNamespaceLabelIfLast_NamespaceNotFound(t *testing.T) { + c := fake.NewClientBuilder().WithScheme(newScheme()).Build() + + if err := RemoveNamespaceLabelIfLast(context.Background(), c, "missing-ns", 0); err != nil { + t.Fatalf("expected nil error for missing namespace, got: %v", err) + } +} diff --git a/infra/feast-operator/internal/controller/access/rbac.go b/infra/feast-operator/internal/controller/access/rbac.go new file mode 100644 index 00000000000..ce83e2e51b8 --- /dev/null +++ b/infra/feast-operator/internal/controller/access/rbac.go @@ -0,0 +1,268 @@ +/* +Copyright 2024 Feast Community. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package access + +import ( + "context" + "fmt" + "strings" + + rbacv1 "k8s.io/api/rbac/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/log" + + "github.com/feast-dev/feast/infra/feast-operator/internal/controller/registry" +) + +const ( + FeastDiscoverClusterRoleName = "feast-discover-namespaces" +) + +// ReconcileAutoAccessRBAC creates or updates RBAC so subjects from registry permissions +// get view access to the Feature Store namespace and client ConfigMap. +func ReconcileAutoAccessRBAC(ctx context.Context, c client.Client, scheme *runtime.Scheme, owner client.Object, namespace, name, clientConfigMapName string, policies []registry.PermissionPolicy) error { + subjects := buildSubjects(ctx, c, policies) + if len(subjects) == 0 { + return CleanupAutoAccessRBAC(ctx, c, namespace, name) + } + if err := ensureFeastDiscoverClusterRole(ctx, c); err != nil { + return err + } + clusterBindingName := "feast-" + namespace + "--" + name + "-discover" + if err := reconcileClusterRoleBinding(ctx, c, scheme, owner, clusterBindingName, subjects); err != nil { + return err + } + roleName := "feast-" + namespace + "--" + name + "-viewer" + if err := reconcileViewerRole(ctx, c, scheme, owner, namespace, roleName, clientConfigMapName); err != nil { + return err + } + if err := reconcileViewerRoleBinding(ctx, c, scheme, owner, namespace, roleName, subjects); err != nil { + return err + } + return nil +} + +func buildSubjects(ctx context.Context, c client.Client, policies []registry.PermissionPolicy) []rbacv1.Subject { + seen := make(map[string]struct{}) + var subjects []rbacv1.Subject + for _, p := range policies { + for _, g := range p.Groups { + key := "Group:" + g + if _, ok := seen[key]; ok { + continue + } + seen[key] = struct{}{} + subjects = append(subjects, rbacv1.Subject{ + APIGroup: rbacv1.GroupName, + Kind: "Group", + Name: g, + }) + } + for _, ns := range p.Namespaces { + ns = strings.TrimSpace(ns) + if ns == "" { + log.FromContext(ctx).V(1).Info("Skipping empty namespace in NamespaceBasedPolicy for auto-access RBAC") + continue + } + subs, err := listSubjectsInNamespace(ctx, c, ns) + if err != nil { + log.FromContext(ctx).V(1).Info("Failed to list RoleBindings in namespace for NamespaceBasedPolicy", "namespace", ns, "error", err) + continue + } + for _, s := range subs { + key := subjectKey(s) + if _, ok := seen[key]; ok { + continue + } + seen[key] = struct{}{} + subjects = append(subjects, s) + } + } + } + return subjects +} + +func subjectKey(s rbacv1.Subject) string { + apiGroup := s.APIGroup + if apiGroup == "" { + apiGroup = "rbac.authorization.k8s.io" + } + return s.Kind + ":" + apiGroup + ":" + s.Name + ":" + s.Namespace +} + +func listSubjectsInNamespace(ctx context.Context, c client.Client, namespace string) ([]rbacv1.Subject, error) { + var list rbacv1.RoleBindingList + if err := c.List(ctx, &list, client.InNamespace(namespace)); err != nil { + return nil, err + } + var out []rbacv1.Subject + seen := make(map[string]struct{}) + for i := range list.Items { + for _, s := range list.Items[i].Subjects { + if s.Kind != "User" && s.Kind != "Group" { + continue + } + if s.Name == "" { + continue + } + key := subjectKey(s) + if _, ok := seen[key]; ok { + continue + } + seen[key] = struct{}{} + out = append(out, s) + } + } + return out, nil +} + +func ensureFeastDiscoverClusterRole(ctx context.Context, c client.Client) error { + cr := &rbacv1.ClusterRole{ + ObjectMeta: metav1.ObjectMeta{Name: FeastDiscoverClusterRoleName}, + } + _, err := controllerutil.CreateOrUpdate(ctx, c, cr, func() error { + cr.Rules = []rbacv1.PolicyRule{ + { + APIGroups: []string{""}, + Resources: []string{"namespaces"}, + Verbs: []string{"get", "list", "watch"}, + }, + } + return nil + }) + return err +} + +func reconcileClusterRoleBinding(ctx context.Context, c client.Client, _ *runtime.Scheme, owner client.Object, name string, subjects []rbacv1.Subject) error { + crb := &rbacv1.ClusterRoleBinding{ + ObjectMeta: metav1.ObjectMeta{Name: name}, + } + _, err := controllerutil.CreateOrUpdate(ctx, c, crb, func() error { + crb.RoleRef = rbacv1.RoleRef{ + APIGroup: rbacv1.GroupName, + Kind: "ClusterRole", + Name: FeastDiscoverClusterRoleName, + } + crb.Subjects = subjects + // Cannot set owner reference: ClusterRoleBinding is cluster-scoped while + // FeatureStore is namespace-scoped. Cleanup is handled by CleanupAutoAccessRBAC. + if owner != nil { + if crb.Labels == nil { + crb.Labels = make(map[string]string) + } + crb.Labels["feast.dev/owner-name"] = owner.GetName() + crb.Labels["feast.dev/owner-namespace"] = owner.GetNamespace() + } + return nil + }) + return err +} + +func reconcileViewerRole(ctx context.Context, c client.Client, scheme *runtime.Scheme, owner client.Object, namespace, roleName, configMapName string) error { + role := &rbacv1.Role{ + ObjectMeta: metav1.ObjectMeta{Name: roleName, Namespace: namespace}, + } + _, err := controllerutil.CreateOrUpdate(ctx, c, role, func() error { + role.Rules = []rbacv1.PolicyRule{ + { + APIGroups: []string{""}, + Resources: []string{"configmaps"}, + ResourceNames: []string{configMapName}, + Verbs: []string{"get", "list", "watch"}, + }, + } + if owner != nil && scheme != nil { + return controllerutil.SetControllerReference(owner, role, scheme) + } + return nil + }) + return err +} + +func reconcileViewerRoleBinding(ctx context.Context, c client.Client, scheme *runtime.Scheme, owner client.Object, namespace, roleName string, subjects []rbacv1.Subject) error { + rb := &rbacv1.RoleBinding{ + ObjectMeta: metav1.ObjectMeta{Name: roleName, Namespace: namespace}, + } + _, err := controllerutil.CreateOrUpdate(ctx, c, rb, func() error { + rb.RoleRef = rbacv1.RoleRef{ + APIGroup: rbacv1.GroupName, + Kind: "Role", + Name: roleName, + } + rb.Subjects = subjects + if owner != nil && scheme != nil { + return controllerutil.SetControllerReference(owner, rb, scheme) + } + return nil + }) + return err +} + +// CleanupDiscoverClusterRoleIfLast deletes the shared feast-discover-namespaces +// ClusterRole when otherFeatureStoreCount is 0 (i.e. no other FeatureStore in +// the cluster still needs it). +func CleanupDiscoverClusterRoleIfLast(ctx context.Context, c client.Client, otherFeatureStoreCount int) error { + if otherFeatureStoreCount > 0 { + return nil + } + cr := &rbacv1.ClusterRole{} + if err := c.Get(ctx, client.ObjectKey{Name: FeastDiscoverClusterRoleName}, cr); err != nil { + if apierrors.IsNotFound(err) { + return nil + } + return err + } + if err := c.Delete(ctx, cr); err != nil && !apierrors.IsNotFound(err) { + return fmt.Errorf("failed to delete ClusterRole %s: %w", FeastDiscoverClusterRoleName, err) + } + return nil +} + +// CleanupAutoAccessRBAC removes the auto-access ClusterRoleBinding, Role, and RoleBinding. +func CleanupAutoAccessRBAC(ctx context.Context, c client.Client, namespace, name string) error { + clusterBindingName := "feast-" + namespace + "--" + name + "-discover" + crb := &rbacv1.ClusterRoleBinding{} + if err := c.Get(ctx, client.ObjectKey{Name: clusterBindingName}, crb); err != nil { + if !apierrors.IsNotFound(err) { + return err + } + } else if err := c.Delete(ctx, crb); err != nil && !apierrors.IsNotFound(err) { + return fmt.Errorf("failed to delete ClusterRoleBinding %s: %w", clusterBindingName, err) + } + roleName := "feast-" + namespace + "--" + name + "-viewer" + role := &rbacv1.Role{} + if err := c.Get(ctx, client.ObjectKey{Namespace: namespace, Name: roleName}, role); err != nil { + if !apierrors.IsNotFound(err) { + return err + } + } else if err := c.Delete(ctx, role); err != nil && !apierrors.IsNotFound(err) { + return fmt.Errorf("failed to delete Role %s: %w", roleName, err) + } + rb := &rbacv1.RoleBinding{} + if err := c.Get(ctx, client.ObjectKey{Namespace: namespace, Name: roleName}, rb); err != nil { + if !apierrors.IsNotFound(err) { + return err + } + } else if err := c.Delete(ctx, rb); err != nil && !apierrors.IsNotFound(err) { + return fmt.Errorf("failed to delete RoleBinding %s: %w", roleName, err) + } + return nil +} diff --git a/infra/feast-operator/internal/controller/access/rbac_test.go b/infra/feast-operator/internal/controller/access/rbac_test.go new file mode 100644 index 00000000000..d5d9ebb2554 --- /dev/null +++ b/infra/feast-operator/internal/controller/access/rbac_test.go @@ -0,0 +1,363 @@ +/* +Copyright 2024 Feast Community. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package access + +import ( + "context" + "testing" + + corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + "github.com/feast-dev/feast/infra/feast-operator/internal/controller/registry" +) + +func newRBACScheme() *runtime.Scheme { + s := runtime.NewScheme() + _ = corev1.AddToScheme(s) + _ = rbacv1.AddToScheme(s) + return s +} + +func TestBuildSubjects_GroupBasedPolicy(t *testing.T) { + c := fake.NewClientBuilder().WithScheme(newRBACScheme()).Build() + policies := []registry.PermissionPolicy{ + {Groups: []string{"admins", "data-team"}}, + } + subjects := buildSubjects(context.Background(), c, policies) + if len(subjects) != 2 { + t.Fatalf("expected 2 subjects, got %d", len(subjects)) + } + assertSubject(t, subjects[0], "Group", "admins") + assertSubject(t, subjects[1], "Group", "data-team") +} + +func TestBuildSubjects_NamespaceBasedPolicy(t *testing.T) { + rb := &rbacv1.RoleBinding{ + ObjectMeta: metav1.ObjectMeta{Name: "rb1", Namespace: "ds-project"}, + Subjects: []rbacv1.Subject{ + {Kind: "User", Name: "alice", APIGroup: rbacv1.GroupName}, + {Kind: "Group", Name: "team-x", APIGroup: rbacv1.GroupName}, + }, + } + c := fake.NewClientBuilder().WithScheme(newRBACScheme()).WithObjects(rb).Build() + policies := []registry.PermissionPolicy{ + {Namespaces: []string{"ds-project"}}, + } + subjects := buildSubjects(context.Background(), c, policies) + if len(subjects) != 2 { + t.Fatalf("expected 2 subjects, got %d", len(subjects)) + } +} + +func TestBuildSubjects_CombinedPolicy(t *testing.T) { + rb := &rbacv1.RoleBinding{ + ObjectMeta: metav1.ObjectMeta{Name: "rb1", Namespace: "ns1"}, + Subjects: []rbacv1.Subject{ + {Kind: "User", Name: "bob", APIGroup: rbacv1.GroupName}, + }, + } + c := fake.NewClientBuilder().WithScheme(newRBACScheme()).WithObjects(rb).Build() + policies := []registry.PermissionPolicy{ + {Groups: []string{"engineers"}, Namespaces: []string{"ns1"}}, + } + subjects := buildSubjects(context.Background(), c, policies) + if len(subjects) != 2 { + t.Fatalf("expected 2 subjects (1 group + 1 user), got %d", len(subjects)) + } +} + +func TestBuildSubjects_DeduplicatesGroups(t *testing.T) { + c := fake.NewClientBuilder().WithScheme(newRBACScheme()).Build() + policies := []registry.PermissionPolicy{ + {Groups: []string{"admins"}}, + {Groups: []string{"admins", "other"}}, + } + subjects := buildSubjects(context.Background(), c, policies) + if len(subjects) != 2 { + t.Fatalf("expected 2 unique subjects, got %d: %v", len(subjects), subjects) + } +} + +func TestBuildSubjects_DeduplicatesNamespaceSubjects(t *testing.T) { + rb1 := &rbacv1.RoleBinding{ + ObjectMeta: metav1.ObjectMeta{Name: "rb1", Namespace: "ns1"}, + Subjects: []rbacv1.Subject{{Kind: "User", Name: "alice", APIGroup: rbacv1.GroupName}}, + } + rb2 := &rbacv1.RoleBinding{ + ObjectMeta: metav1.ObjectMeta{Name: "rb2", Namespace: "ns1"}, + Subjects: []rbacv1.Subject{{Kind: "User", Name: "alice", APIGroup: rbacv1.GroupName}}, + } + c := fake.NewClientBuilder().WithScheme(newRBACScheme()).WithObjects(rb1, rb2).Build() + policies := []registry.PermissionPolicy{ + {Namespaces: []string{"ns1"}}, + } + subjects := buildSubjects(context.Background(), c, policies) + if len(subjects) != 1 { + t.Fatalf("expected 1 deduplicated subject, got %d", len(subjects)) + } +} + +func TestBuildSubjects_EmptyPolicies(t *testing.T) { + c := fake.NewClientBuilder().WithScheme(newRBACScheme()).Build() + subjects := buildSubjects(context.Background(), c, nil) + if len(subjects) != 0 { + t.Fatalf("expected 0 subjects for nil policies, got %d", len(subjects)) + } +} + +func TestBuildSubjects_NonexistentNamespace(t *testing.T) { + c := fake.NewClientBuilder().WithScheme(newRBACScheme()).Build() + policies := []registry.PermissionPolicy{ + {Namespaces: []string{"nonexistent"}}, + } + // Should not error, just return no subjects from that namespace + subjects := buildSubjects(context.Background(), c, policies) + if len(subjects) != 0 { + t.Fatalf("expected 0 subjects for nonexistent namespace, got %d", len(subjects)) + } +} + +func TestReconcileAutoAccessRBAC_CreatesAllResources(t *testing.T) { + scheme := newRBACScheme() + ns := newNamespace("feast-ns", nil) + c := fake.NewClientBuilder().WithScheme(scheme).WithObjects(ns).Build() + + policies := []registry.PermissionPolicy{ + {Groups: []string{"data-scientists"}}, + } + err := ReconcileAutoAccessRBAC(context.Background(), c, scheme, nil, "feast-ns", "my-feast", "feast-client-config", policies) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + // Verify ClusterRole + cr := &rbacv1.ClusterRole{} + if err := c.Get(context.Background(), client.ObjectKey{Name: FeastDiscoverClusterRoleName}, cr); err != nil { + t.Fatalf("Failed to get ClusterRole: %v", err) + } + if len(cr.Rules) != 1 || cr.Rules[0].Resources[0] != "namespaces" { + t.Fatalf("unexpected ClusterRole rules: %v", cr.Rules) + } + + // Verify ClusterRoleBinding + crb := &rbacv1.ClusterRoleBinding{} + if err := c.Get(context.Background(), client.ObjectKey{Name: "feast-feast-ns--my-feast-discover"}, crb); err != nil { + t.Fatalf("Failed to get ClusterRoleBinding: %v", err) + } + if len(crb.Subjects) != 1 || crb.Subjects[0].Name != "data-scientists" { + t.Fatalf("unexpected ClusterRoleBinding subjects: %v", crb.Subjects) + } + + // Verify Role + role := &rbacv1.Role{} + if err := c.Get(context.Background(), client.ObjectKey{Namespace: "feast-ns", Name: "feast-feast-ns--my-feast-viewer"}, role); err != nil { + t.Fatalf("Failed to get Role: %v", err) + } + if len(role.Rules) != 1 || role.Rules[0].ResourceNames[0] != "feast-client-config" { + t.Fatalf("unexpected Role rules: %v", role.Rules) + } + + // Verify RoleBinding + rb := &rbacv1.RoleBinding{} + if err := c.Get(context.Background(), client.ObjectKey{Namespace: "feast-ns", Name: "feast-feast-ns--my-feast-viewer"}, rb); err != nil { + t.Fatalf("Failed to get RoleBinding: %v", err) + } + if len(rb.Subjects) != 1 || rb.Subjects[0].Name != "data-scientists" { + t.Fatalf("unexpected RoleBinding subjects: %v", rb.Subjects) + } +} + +func TestReconcileAutoAccessRBAC_NoSubjects(t *testing.T) { + scheme := newRBACScheme() + c := fake.NewClientBuilder().WithScheme(scheme).Build() + + // Empty policies -> no subjects -> should be a no-op + err := ReconcileAutoAccessRBAC(context.Background(), c, scheme, nil, "ns", "name", "cm", nil) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + crb := &rbacv1.ClusterRoleBinding{} + if err := c.Get(context.Background(), client.ObjectKey{Name: "feast-ns-name-discover"}, crb); err == nil { + t.Fatal("ClusterRoleBinding should not exist when there are no subjects") + } +} + +func TestReconcileAutoAccessRBAC_UpdatesExistingResources(t *testing.T) { + scheme := newRBACScheme() + ns := newNamespace("feast-ns", nil) + c := fake.NewClientBuilder().WithScheme(scheme).WithObjects(ns).Build() + + // First reconcile with one group + policies := []registry.PermissionPolicy{{Groups: []string{"group-a"}}} + if err := ReconcileAutoAccessRBAC(context.Background(), c, scheme, nil, "feast-ns", "fs", "cm", policies); err != nil { + t.Fatalf("unexpected error on first reconcile: %v", err) + } + + // Second reconcile with different group + policies = []registry.PermissionPolicy{{Groups: []string{"group-b"}}} + if err := ReconcileAutoAccessRBAC(context.Background(), c, scheme, nil, "feast-ns", "fs", "cm", policies); err != nil { + t.Fatalf("unexpected error on second reconcile: %v", err) + } + + crb := &rbacv1.ClusterRoleBinding{} + if err := c.Get(context.Background(), client.ObjectKey{Name: "feast-feast-ns--fs-discover"}, crb); err != nil { + t.Fatalf("Failed to get ClusterRoleBinding: %v", err) + } + if len(crb.Subjects) != 1 || crb.Subjects[0].Name != "group-b" { + t.Fatalf("expected updated subject group-b, got: %v", crb.Subjects) + } +} + +func TestCleanupAutoAccessRBAC(t *testing.T) { + scheme := newRBACScheme() + c := fake.NewClientBuilder().WithScheme(scheme).WithObjects( + &rbacv1.ClusterRoleBinding{ + ObjectMeta: metav1.ObjectMeta{Name: "feast-ns--my-feast-discover"}, + }, + &rbacv1.Role{ + ObjectMeta: metav1.ObjectMeta{Name: "feast-ns--my-feast-viewer", Namespace: "ns"}, + }, + &rbacv1.RoleBinding{ + ObjectMeta: metav1.ObjectMeta{Name: "feast-ns--my-feast-viewer", Namespace: "ns"}, + }, + ).Build() + + if err := CleanupAutoAccessRBAC(context.Background(), c, "ns", "my-feast"); err != nil { + t.Fatalf("unexpected error: %v", err) + } + + crb := &rbacv1.ClusterRoleBinding{} + if err := c.Get(context.Background(), client.ObjectKey{Name: "feast-ns--my-feast-discover"}, crb); err == nil { + t.Fatal("ClusterRoleBinding should have been deleted") + } + role := &rbacv1.Role{} + if err := c.Get(context.Background(), client.ObjectKey{Namespace: "ns", Name: "feast-ns--my-feast-viewer"}, role); err == nil { + t.Fatal("Role should have been deleted") + } + rb := &rbacv1.RoleBinding{} + if err := c.Get(context.Background(), client.ObjectKey{Namespace: "ns", Name: "feast-ns--my-feast-viewer"}, rb); err == nil { + t.Fatal("RoleBinding should have been deleted") + } +} + +func TestCleanupAutoAccessRBAC_AlreadyGone(t *testing.T) { + scheme := newRBACScheme() + c := fake.NewClientBuilder().WithScheme(scheme).Build() + + // Should not error when resources don't exist + if err := CleanupAutoAccessRBAC(context.Background(), c, "ns", "name"); err != nil { + t.Fatalf("unexpected error: %v", err) + } +} + +func TestEnsureFeastDiscoverClusterRole_Idempotent(t *testing.T) { + scheme := newRBACScheme() + c := fake.NewClientBuilder().WithScheme(scheme).Build() + + if err := ensureFeastDiscoverClusterRole(context.Background(), c); err != nil { + t.Fatalf("first call failed: %v", err) + } + if err := ensureFeastDiscoverClusterRole(context.Background(), c); err != nil { + t.Fatalf("second call (idempotent) failed: %v", err) + } + cr := &rbacv1.ClusterRole{} + if err := c.Get(context.Background(), client.ObjectKey{Name: FeastDiscoverClusterRoleName}, cr); err != nil { + t.Fatalf("Failed to get ClusterRole: %v", err) + } +} + +func TestListSubjectsInNamespace(t *testing.T) { + rb1 := &rbacv1.RoleBinding{ + ObjectMeta: metav1.ObjectMeta{Name: "rb1", Namespace: "ns1"}, + Subjects: []rbacv1.Subject{ + {Kind: "User", Name: "alice", APIGroup: rbacv1.GroupName}, + {Kind: "Group", Name: "team-a", APIGroup: rbacv1.GroupName}, + }, + } + rb2 := &rbacv1.RoleBinding{ + ObjectMeta: metav1.ObjectMeta{Name: "rb2", Namespace: "ns1"}, + Subjects: []rbacv1.Subject{ + {Kind: "User", Name: "alice", APIGroup: rbacv1.GroupName}, + {Kind: "User", Name: "bob", APIGroup: rbacv1.GroupName}, + }, + } + c := fake.NewClientBuilder().WithScheme(newRBACScheme()).WithObjects(rb1, rb2).Build() + + subjects, err := listSubjectsInNamespace(context.Background(), c, "ns1") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + // alice (deduplicated), team-a, bob = 3 unique subjects + if len(subjects) != 3 { + t.Fatalf("expected 3 unique subjects, got %d: %v", len(subjects), subjects) + } +} + +func TestCleanupDiscoverClusterRoleIfLast_DeletesWhenZero(t *testing.T) { + scheme := newRBACScheme() + cr := &rbacv1.ClusterRole{ + ObjectMeta: metav1.ObjectMeta{Name: FeastDiscoverClusterRoleName}, + } + c := fake.NewClientBuilder().WithScheme(scheme).WithObjects(cr).Build() + + if err := CleanupDiscoverClusterRoleIfLast(context.Background(), c, 0); err != nil { + t.Fatalf("unexpected error: %v", err) + } + + got := &rbacv1.ClusterRole{} + if err := c.Get(context.Background(), client.ObjectKey{Name: FeastDiscoverClusterRoleName}, got); err == nil { + t.Fatal("ClusterRole should have been deleted") + } +} + +func TestCleanupDiscoverClusterRoleIfLast_KeepsWhenOthersExist(t *testing.T) { + scheme := newRBACScheme() + cr := &rbacv1.ClusterRole{ + ObjectMeta: metav1.ObjectMeta{Name: FeastDiscoverClusterRoleName}, + } + c := fake.NewClientBuilder().WithScheme(scheme).WithObjects(cr).Build() + + if err := CleanupDiscoverClusterRoleIfLast(context.Background(), c, 1); err != nil { + t.Fatalf("unexpected error: %v", err) + } + + got := &rbacv1.ClusterRole{} + if err := c.Get(context.Background(), client.ObjectKey{Name: FeastDiscoverClusterRoleName}, got); err != nil { + t.Fatal("ClusterRole should still exist when other FeatureStores remain") + } +} + +func TestCleanupDiscoverClusterRoleIfLast_AlreadyGone(t *testing.T) { + scheme := newRBACScheme() + c := fake.NewClientBuilder().WithScheme(scheme).Build() + + if err := CleanupDiscoverClusterRoleIfLast(context.Background(), c, 0); err != nil { + t.Fatalf("expected nil error when ClusterRole already absent, got: %v", err) + } +} + +func assertSubject(t *testing.T, s rbacv1.Subject, expectedKind, expectedName string) { + t.Helper() + if s.Kind != expectedKind || s.Name != expectedName { + t.Fatalf("expected subject %s/%s, got %s/%s", expectedKind, expectedName, s.Kind, s.Name) + } +} diff --git a/infra/feast-operator/internal/controller/featurestore_controller.go b/infra/feast-operator/internal/controller/featurestore_controller.go index 1980fb1f089..15575d040fb 100644 --- a/infra/feast-operator/internal/controller/featurestore_controller.go +++ b/infra/feast-operator/internal/controller/featurestore_controller.go @@ -41,16 +41,19 @@ import ( "sigs.k8s.io/controller-runtime/pkg/reconcile" feastdevv1 "github.com/feast-dev/feast/infra/feast-operator/api/v1" + "github.com/feast-dev/feast/infra/feast-operator/internal/controller/access" "github.com/feast-dev/feast/infra/feast-operator/internal/controller/authz" feasthandler "github.com/feast-dev/feast/infra/feast-operator/internal/controller/handler" feastmetrics "github.com/feast-dev/feast/infra/feast-operator/internal/controller/metrics" + "github.com/feast-dev/feast/infra/feast-operator/internal/controller/registry" "github.com/feast-dev/feast/infra/feast-operator/internal/controller/services" routev1 "github.com/openshift/api/route/v1" ) // Constants for requeue const ( - RequeueDelayError = 5 * time.Second + RequeueDelayError = 5 * time.Second + RequeuePeriodicInterval = 30 * time.Second ) // FeatureStoreReconciler reconciles a FeatureStore object @@ -67,7 +70,8 @@ type FeatureStoreReconciler struct { // +kubebuilder:rbac:groups=apps,resources=deployments,verbs=get;list;create;update;watch;delete // +kubebuilder:rbac:groups=core,resources=services;configmaps;persistentvolumeclaims;serviceaccounts,verbs=get;list;create;update;watch;delete // +kubebuilder:rbac:groups=rbac.authorization.k8s.io,resources=roles;rolebindings;clusterroles;clusterrolebindings;subjectaccessreviews,verbs=get;list;create;update;watch;delete -// +kubebuilder:rbac:groups=core,resources=secrets;pods;namespaces,verbs=get;list;watch +// +kubebuilder:rbac:groups=core,resources=namespaces,verbs=get;list;watch;patch;update +// +kubebuilder:rbac:groups=core,resources=secrets;pods,verbs=get;list;watch // +kubebuilder:rbac:groups=core,resources=pods/exec,verbs=create // +kubebuilder:rbac:groups=authentication.k8s.io,resources=tokenreviews,verbs=create // +kubebuilder:rbac:groups=route.openshift.io,resources=routes,verbs=get;list;create;update;watch;delete @@ -88,21 +92,11 @@ func (r *FeatureStoreReconciler) Reconcile(ctx context.Context, req ctrl.Request err := r.Get(ctx, req.NamespacedName, cr) if err != nil { if apierrors.IsNotFound(err) { - // CR deleted since request queued, child objects getting GC'd, no requeue logger.V(1).Info("FeatureStore CR not found, has been deleted") if r.Metrics != nil { r.Metrics.DeleteFeatureStore(req.NamespacedName.Namespace, req.NamespacedName.Name) } - // Clean up namespace registry entry even if the CR is not found - if err := r.cleanupNamespaceRegistry(ctx, &feastdevv1.FeatureStore{ - ObjectMeta: metav1.ObjectMeta{ - Name: req.NamespacedName.Name, - Namespace: req.NamespacedName.Namespace, - }, - }); err != nil { - logger.Error(err, "Failed to clean up namespace registry entry for deleted FeatureStore") - // Don't return error here as the CR is already deleted - } + r.cleanupOnDeletion(ctx, req.NamespacedName.Namespace, req.NamespacedName.Name) return ctrl.Result{}, nil } logger.Error(err, "Unable to get FeatureStore CR") @@ -110,16 +104,11 @@ func (r *FeatureStoreReconciler) Reconcile(ctx context.Context, req ctrl.Request } currentStatus := cr.Status.DeepCopy() - // Handle deletion - clean up namespace registry entry if cr.DeletionTimestamp != nil { - logger.Info("FeatureStore is being deleted, cleaning up namespace registry entry") if r.Metrics != nil { r.Metrics.DeleteFeatureStore(cr.Namespace, cr.Name) } - if err := r.cleanupNamespaceRegistry(ctx, cr); err != nil { - logger.Error(err, "Failed to clean up namespace registry entry") - return ctrl.Result{}, err - } + r.cleanupOnDeletion(ctx, cr.Namespace, cr.Name) return ctrl.Result{}, nil } @@ -142,25 +131,152 @@ func (r *FeatureStoreReconciler) Reconcile(ctx context.Context, req ctrl.Request } } - // Add to namespace registry if deployment was successful and not being deleted - if recErr == nil && cr.DeletionTimestamp == nil { - feast := services.FeastServices{ - Handler: feasthandler.FeastHandler{ - Client: r.Client, - Context: ctx, - FeatureStore: cr, - Scheme: r.Scheme, - }, + if recErr == nil && cr.DeletionTimestamp == nil && apimeta.IsStatusConditionTrue(cr.Status.Conditions, feastdevv1.ReadyType) { + if err := access.EnsureNamespaceLabel(ctx, r.Client, cr.Namespace); err != nil { + logger.Error(err, "Failed to add Feast label to namespace") } - if err := feast.AddToNamespaceRegistry(); err != nil { - logger.Error(err, "Failed to add FeatureStore to namespace registry") - // Don't return error here as the FeatureStore is already deployed successfully + r.addToNamespaceRegistry(ctx, cr) + policies, err := r.fetchPermissionsFromRegistry(ctx, cr) + if err != nil { + logger.Error(err, "Failed to fetch permissions from registry") + } + if err != nil || len(policies) == 0 || cr.Status.ClientConfigMap == "" { + logger.V(1).Info("Auto-access prerequisites missing or registry unreachable; cleaning up stale auto-access RBAC", + "policies", len(policies), + "clientConfigMapSet", cr.Status.ClientConfigMap != "", + "fetchError", err != nil, + ) + if err := access.CleanupAutoAccessRBAC(ctx, r.Client, cr.Namespace, cr.Name); err != nil { + logger.Error(err, "Failed to cleanup stale auto-access RBAC") + } + } else { + if err := access.ReconcileAutoAccessRBAC(ctx, r.Client, r.Scheme, cr, cr.Namespace, cr.Name, cr.Status.ClientConfigMap, policies); err != nil { + logger.Error(err, "Failed to reconcile auto-access RBAC") + } } } + if recErr == nil && result.RequeueAfter == 0 { + result.RequeueAfter = RequeuePeriodicInterval + } return result, recErr } +func (r *FeatureStoreReconciler) fetchPermissionsFromRegistry(ctx context.Context, cr *feastdevv1.FeatureStore) ([]registry.PermissionPolicy, error) { + logger := log.FromContext(ctx) + registryRest := cr.Status.ServiceHostnames.RegistryRest + if registryRest == "" { + logger.V(1).Info("Skipping permission fetch: registry REST API hostname is not set (ensure RestAPI is enabled)") + return nil, nil + } + project := cr.Status.Applied.FeastProject + if project == "" { + project = cr.Spec.FeastProject + } + if project == "" { + logger.Info("Skipping permission fetch: feast project name is not set") + return nil, nil + } + intraCommToken, err := r.readIntraCommunicationToken(ctx, cr) + if err != nil { + return nil, err + } + useTLS := cr.Status.Applied.Services.Registry != nil && + cr.Status.Applied.Services.Registry.Local != nil && + cr.Status.Applied.Services.Registry.Local.Server != nil && + cr.Status.Applied.Services.Registry.Local.Server.TLS.IsTLS() + return registry.ListPermissions(ctx, registryRest, project, intraCommToken, useTLS) +} + +func (r *FeatureStoreReconciler) readIntraCommunicationToken(ctx context.Context, cr *feastdevv1.FeatureStore) (string, error) { + cmName := services.GetIntraCommunicationConfigMapName(cr.Name) + cm := &corev1.ConfigMap{} + if err := r.Get(ctx, types.NamespacedName{Name: cmName, Namespace: cr.Namespace}, cm); err != nil { + return "", err + } + return cm.Data["token"], nil +} + +func (r *FeatureStoreReconciler) addToNamespaceRegistry(ctx context.Context, cr *feastdevv1.FeatureStore) { + logger := log.FromContext(ctx) + feast := services.FeastServices{ + Handler: feasthandler.FeastHandler{ + Client: r.Client, + Context: ctx, + FeatureStore: cr, + Scheme: r.Scheme, + }, + } + if err := feast.AddToNamespaceRegistry(); err != nil { + logger.Error(err, "Failed to add feature store to namespace registry") + } +} + +func (r *FeatureStoreReconciler) cleanupOnDeletion(ctx context.Context, namespace, name string) { + logger := log.FromContext(ctx) + otherCount := r.countOtherFeatureStoresInNamespace(ctx, namespace, name) + if err := access.RemoveNamespaceLabelIfLast(ctx, r.Client, namespace, otherCount); err != nil { + logger.Error(err, "Failed to remove Feast label from namespace") + } + if err := access.CleanupAutoAccessRBAC(ctx, r.Client, namespace, name); err != nil { + logger.Error(err, "Failed to cleanup auto-access RBAC") + } + clusterCount := r.countOtherFeatureStoresInCluster(ctx, namespace, name) + if err := access.CleanupDiscoverClusterRoleIfLast(ctx, r.Client, clusterCount); err != nil { + logger.Error(err, "Failed to cleanup discover ClusterRole") + } + r.cleanupNamespaceRegistry(ctx, &feastdevv1.FeatureStore{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + }) +} + +func (r *FeatureStoreReconciler) cleanupNamespaceRegistry(ctx context.Context, cr *feastdevv1.FeatureStore) { + logger := log.FromContext(ctx) + feast := services.FeastServices{ + Handler: feasthandler.FeastHandler{ + Client: r.Client, + Context: ctx, + FeatureStore: cr, + Scheme: r.Scheme, + }, + } + if err := feast.RemoveFromNamespaceRegistry(); err != nil { + logger.Error(err, "Failed to remove feature store from namespace registry") + } +} + +func (r *FeatureStoreReconciler) countOtherFeatureStoresInNamespace(ctx context.Context, namespace, excludeName string) int { + var list feastdevv1.FeatureStoreList + if err := r.List(ctx, &list, client.InNamespace(namespace)); err != nil { + return -1 + } + count := 0 + for i := range list.Items { + if list.Items[i].Name != excludeName && list.Items[i].DeletionTimestamp == nil { + count++ + } + } + return count +} + +func (r *FeatureStoreReconciler) countOtherFeatureStoresInCluster(ctx context.Context, namespace, excludeName string) int { + var list feastdevv1.FeatureStoreList + if err := r.List(ctx, &list); err != nil { + return -1 + } + count := 0 + for i := range list.Items { + item := &list.Items[i] + if (item.Name != excludeName || item.Namespace != namespace) && item.DeletionTimestamp == nil { + count++ + } + } + return count +} + func (r *FeatureStoreReconciler) deployFeast(ctx context.Context, cr *feastdevv1.FeatureStore) (result ctrl.Result, err error) { logger := log.FromContext(ctx) condition := metav1.Condition{ @@ -273,20 +389,6 @@ func (r *FeatureStoreReconciler) SetupWithManager(mgr ctrl.Manager) error { } -// cleanupNamespaceRegistry removes the feature store instance from the namespace registry -func (r *FeatureStoreReconciler) cleanupNamespaceRegistry(ctx context.Context, cr *feastdevv1.FeatureStore) error { - feast := services.FeastServices{ - Handler: feasthandler.FeastHandler{ - Client: r.Client, - Context: ctx, - FeatureStore: cr, - Scheme: r.Scheme, - }, - } - - return feast.RemoveFromNamespaceRegistry() -} - // if a remotely referenced FeatureStore is changed, reconcile any FeatureStores that reference it. func (r *FeatureStoreReconciler) mapFeastRefsToFeastRequests(ctx context.Context, object client.Object) []reconcile.Request { logger := log.FromContext(ctx) diff --git a/infra/feast-operator/internal/controller/featurestore_controller_db_store_test.go b/infra/feast-operator/internal/controller/featurestore_controller_db_store_test.go index 08276293032..905d29cf245 100644 --- a/infra/feast-operator/internal/controller/featurestore_controller_db_store_test.go +++ b/infra/feast-operator/internal/controller/featurestore_controller_db_store_test.go @@ -538,7 +538,7 @@ var _ = Describe("FeatureStore Controller - db storage services", func() { cmList := corev1.ConfigMapList{} err = k8sClient.List(ctx, &cmList, listOpts) Expect(err).NotTo(HaveOccurred()) - Expect(cmList.Items).To(HaveLen(1)) + Expect(cmList.Items).To(HaveLen(2)) feast := services.FeastServices{ Handler: handler.FeastHandler{ @@ -558,7 +558,7 @@ var _ = Describe("FeatureStore Controller - db storage services", func() { }, deploy) Expect(err).NotTo(HaveOccurred()) registryContainer := services.GetRegistryContainer(*deploy) - Expect(registryContainer.Env).To(HaveLen(1)) + Expect(registryContainer.Env).To(HaveLen(2)) env := getFeatureStoreYamlEnvVar(registryContainer.Env) Expect(env).NotTo(BeNil()) @@ -602,7 +602,7 @@ var _ = Describe("FeatureStore Controller - db storage services", func() { Expect(repoConfig).To(Equal(testConfig)) offlineContainer := services.GetOfflineContainer(*deploy) - Expect(offlineContainer.Env).To(HaveLen(1)) + Expect(offlineContainer.Env).To(HaveLen(2)) assertEnvFrom(*offlineContainer) env = getFeatureStoreYamlEnvVar(offlineContainer.Env) Expect(env).NotTo(BeNil()) @@ -620,7 +620,7 @@ var _ = Describe("FeatureStore Controller - db storage services", func() { onlineContainer := services.GetOnlineContainer(*deploy) Expect(onlineContainer.VolumeMounts).To(HaveLen(1)) - Expect(onlineContainer.Env).To(HaveLen(1)) + Expect(onlineContainer.Env).To(HaveLen(2)) assertEnvFrom(*onlineContainer) Expect(onlineContainer.ImagePullPolicy).To(Equal(corev1.PullAlways)) env = getFeatureStoreYamlEnvVar(onlineContainer.Env) @@ -637,7 +637,7 @@ var _ = Describe("FeatureStore Controller - db storage services", func() { Expect(err).NotTo(HaveOccurred()) Expect(repoConfigOnline).To(Equal(testConfig)) onlineContainer = services.GetOnlineContainer(*deploy) - Expect(onlineContainer.Env).To(HaveLen(1)) + Expect(onlineContainer.Env).To(HaveLen(2)) // check client config cm := &corev1.ConfigMap{} diff --git a/infra/feast-operator/internal/controller/featurestore_controller_ephemeral_test.go b/infra/feast-operator/internal/controller/featurestore_controller_ephemeral_test.go index 212fa80228b..bbaca588388 100644 --- a/infra/feast-operator/internal/controller/featurestore_controller_ephemeral_test.go +++ b/infra/feast-operator/internal/controller/featurestore_controller_ephemeral_test.go @@ -253,7 +253,7 @@ var _ = Describe("FeatureStore Controller-Ephemeral services", func() { cmList := corev1.ConfigMapList{} err = k8sClient.List(ctx, &cmList, listOpts) Expect(err).NotTo(HaveOccurred()) - Expect(cmList.Items).To(HaveLen(1)) + Expect(cmList.Items).To(HaveLen(2)) feast := services.FeastServices{ Handler: handler.FeastHandler{ @@ -274,7 +274,7 @@ var _ = Describe("FeatureStore Controller-Ephemeral services", func() { Expect(err).NotTo(HaveOccurred()) Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(4)) registryContainer := services.GetRegistryContainer(*deploy) - Expect(registryContainer.Env).To(HaveLen(1)) + Expect(registryContainer.Env).To(HaveLen(2)) env := getFeatureStoreYamlEnvVar(registryContainer.Env) Expect(env).NotTo(BeNil()) @@ -307,7 +307,7 @@ var _ = Describe("FeatureStore Controller-Ephemeral services", func() { Expect(repoConfig).To(Equal(testConfig)) offlineContainer := services.GetOfflineContainer(*deploy) - Expect(offlineContainer.Env).To(HaveLen(1)) + Expect(offlineContainer.Env).To(HaveLen(2)) assertEnvFrom(*offlineContainer) env = getFeatureStoreYamlEnvVar(offlineContainer.Env) Expect(env).NotTo(BeNil()) @@ -327,7 +327,7 @@ var _ = Describe("FeatureStore Controller-Ephemeral services", func() { Expect(repoConfigOffline).To(Equal(testConfig)) onlineContainer := services.GetOnlineContainer(*deploy) - Expect(onlineContainer.Env).To(HaveLen(3)) + Expect(onlineContainer.Env).To(HaveLen(4)) Expect(onlineContainer.ImagePullPolicy).To(Equal(corev1.PullAlways)) env = getFeatureStoreYamlEnvVar(onlineContainer.Env) Expect(env).NotTo(BeNil()) diff --git a/infra/feast-operator/internal/controller/featurestore_controller_kubernetes_auth_test.go b/infra/feast-operator/internal/controller/featurestore_controller_kubernetes_auth_test.go index 3bfab485e85..fd82456428c 100644 --- a/infra/feast-operator/internal/controller/featurestore_controller_kubernetes_auth_test.go +++ b/infra/feast-operator/internal/controller/featurestore_controller_kubernetes_auth_test.go @@ -378,7 +378,7 @@ var _ = Describe("FeatureStore Controller-Kubernetes authorization", func() { cmList := corev1.ConfigMapList{} err = k8sClient.List(ctx, &cmList, listOpts) Expect(err).NotTo(HaveOccurred()) - Expect(cmList.Items).To(HaveLen(1)) + Expect(cmList.Items).To(HaveLen(2)) feast := services.FeastServices{ Handler: handler.FeastHandler{ diff --git a/infra/feast-operator/internal/controller/featurestore_controller_namespace_registry_test.go b/infra/feast-operator/internal/controller/featurestore_controller_namespace_registry_test.go deleted file mode 100644 index b92d5cec50e..00000000000 --- a/infra/feast-operator/internal/controller/featurestore_controller_namespace_registry_test.go +++ /dev/null @@ -1,453 +0,0 @@ -/* -Copyright 2025 Feast Community. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package controller - -import ( - "context" - "encoding/json" - "fmt" - "time" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - corev1 "k8s.io/api/core/v1" - rbacv1 "k8s.io/api/rbac/v1" - "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/types" - "sigs.k8s.io/controller-runtime/pkg/reconcile" - - feastdevv1 "github.com/feast-dev/feast/infra/feast-operator/api/v1" - "github.com/feast-dev/feast/infra/feast-operator/internal/controller/services" -) - -const DefaultNamespace = "default" -const FeastControllerNamespace = "feast-operator-system" - -var ctx = context.Background() - -var _ = Describe("FeatureStore Controller - Namespace Registry", func() { - - Context("When deploying a FeatureStore with namespace registry", func() { - const resourceName = "namespace-registry-test" - var pullPolicy = corev1.PullAlways - var image = "feastdev/feast:latest" - - typeNamespacedName := types.NamespacedName{ - Name: resourceName, - Namespace: DefaultNamespace, - } - featurestore := &feastdevv1.FeatureStore{} - - BeforeEach(func() { - By("Ensuring manager namespace exists") - namespace := &corev1.Namespace{ - ObjectMeta: metav1.ObjectMeta{ - Name: FeastControllerNamespace, - }, - } - // Try to create, ignore if already exists - err := k8sClient.Create(ctx, namespace) - if err != nil && !errors.IsAlreadyExists(err) { - Expect(err).NotTo(HaveOccurred()) - } - - By("Creating a FeatureStore resource") - featurestore = createFeatureStoreResource(resourceName, image, pullPolicy, nil, nil) - Expect(k8sClient.Create(ctx, featurestore)).Should(Succeed()) - - // Wait for the resource to be created - Eventually(func() error { - return k8sClient.Get(ctx, typeNamespacedName, featurestore) - }, time.Second*10, time.Millisecond*250).Should(Succeed()) - }) - - AfterEach(func() { - By("Cleaning up the FeatureStore resource") - // Only delete if the resource still exists - err := k8sClient.Get(ctx, typeNamespacedName, featurestore) - if err == nil { - Expect(k8sClient.Delete(ctx, featurestore)).Should(Succeed()) - - // Wait for the resource to be deleted - Eventually(func() bool { - err := k8sClient.Get(ctx, typeNamespacedName, featurestore) - return errors.IsNotFound(err) - }, time.Second*10, time.Millisecond*250).Should(BeTrue()) - } - }) - - It("should create namespace registry ConfigMap", func() { - By("Reconciling the FeatureStore") - reconciler := &FeatureStoreReconciler{ - Client: k8sClient, - Scheme: k8sClient.Scheme(), - } - - _, err := reconciler.Reconcile(ctx, reconcile.Request{ - NamespacedName: typeNamespacedName, - }) - Expect(err).NotTo(HaveOccurred()) - - By("Checking that namespace registry ConfigMap is created") - Eventually(func() error { - cm := &corev1.ConfigMap{} - return k8sClient.Get(ctx, types.NamespacedName{ - Name: services.NamespaceRegistryConfigMapName, - Namespace: services.DefaultKubernetesNamespace, // Assuming Kubernetes environment - }, cm) - }, time.Second*30, time.Millisecond*500).Should(Succeed()) - }) - - It("should create namespace registry Role and RoleBinding", func() { - By("Reconciling the FeatureStore") - reconciler := &FeatureStoreReconciler{ - Client: k8sClient, - Scheme: k8sClient.Scheme(), - } - - _, err := reconciler.Reconcile(ctx, reconcile.Request{ - NamespacedName: typeNamespacedName, - }) - Expect(err).NotTo(HaveOccurred()) - - By("Checking that namespace registry Role is created") - Eventually(func() error { - role := &rbacv1.Role{} - return k8sClient.Get(ctx, types.NamespacedName{ - Name: services.NamespaceRegistryConfigMapName + "-reader", - Namespace: services.DefaultKubernetesNamespace, - }, role) - }, time.Second*30, time.Millisecond*500).Should(Succeed()) - - By("Checking that namespace registry RoleBinding is created") - Eventually(func() error { - roleBinding := &rbacv1.RoleBinding{} - return k8sClient.Get(ctx, types.NamespacedName{ - Name: services.NamespaceRegistryConfigMapName + "-reader", - Namespace: services.DefaultKubernetesNamespace, - }, roleBinding) - }, time.Second*30, time.Millisecond*500).Should(Succeed()) - }) - - It("should register feature store in namespace registry", func() { - By("Reconciling the FeatureStore") - reconciler := &FeatureStoreReconciler{ - Client: k8sClient, - Scheme: k8sClient.Scheme(), - } - - _, err := reconciler.Reconcile(ctx, reconcile.Request{ - NamespacedName: typeNamespacedName, - }) - Expect(err).NotTo(HaveOccurred()) - - By("Checking that feature store is registered in namespace registry") - Eventually(func() error { - cm := &corev1.ConfigMap{} - err := k8sClient.Get(ctx, types.NamespacedName{ - Name: services.NamespaceRegistryConfigMapName, - Namespace: services.DefaultKubernetesNamespace, - }, cm) - if err != nil { - return err - } - - // Check if the ConfigMap contains the expected data - if cm.Data == nil || cm.Data[services.NamespaceRegistryDataKey] == "" { - return fmt.Errorf("namespace registry data is empty") - } - - // Parse the JSON data - var registryData services.NamespaceRegistryData - err = json.Unmarshal([]byte(cm.Data[services.NamespaceRegistryDataKey]), ®istryData) - if err != nil { - return err - } - - // Check if the feature store namespace is registered - if registryData.Namespaces == nil { - return fmt.Errorf("namespaces map is nil") - } - - // The feature store should be registered in its namespace - featureStoreNamespace := featurestore.Namespace - if featureStoreNamespace == "" { - featureStoreNamespace = DefaultNamespace - } - - configs, exists := registryData.Namespaces[featureStoreNamespace] - if !exists { - return fmt.Errorf("feature store namespace %s not found in registry", featureStoreNamespace) - } - - // Check if the client config is registered - expectedConfigName := featurestore.Status.ClientConfigMap - if expectedConfigName == "" { - // If no client config name is set, we expect at least one config - if len(configs) == 0 { - return fmt.Errorf("no client configs found for namespace %s", featureStoreNamespace) - } - } else { - // Check if the specific config is registered - found := false - for _, config := range configs { - if config == expectedConfigName { - found = true - break - } - } - if !found { - return fmt.Errorf("expected client config %s not found in registry", expectedConfigName) - } - } - - return nil - }, time.Second*30, time.Millisecond*500).Should(Succeed()) - }) - - It("should clean up namespace registry entry when FeatureStore is deleted", func() { - By("Reconciling the FeatureStore to create registry entry") - reconciler := &FeatureStoreReconciler{ - Client: k8sClient, - Scheme: k8sClient.Scheme(), - } - - _, err := reconciler.Reconcile(ctx, reconcile.Request{ - NamespacedName: typeNamespacedName, - }) - Expect(err).NotTo(HaveOccurred()) - - By("Verifying feature store is registered") - Eventually(func() error { - cm := &corev1.ConfigMap{} - err := k8sClient.Get(ctx, types.NamespacedName{ - Name: services.NamespaceRegistryConfigMapName, - Namespace: services.DefaultKubernetesNamespace, - }, cm) - if err != nil { - return err - } - - if cm.Data == nil || cm.Data[services.NamespaceRegistryDataKey] == "" { - return fmt.Errorf("namespace registry data is empty") - } - - var registryData services.NamespaceRegistryData - err = json.Unmarshal([]byte(cm.Data[services.NamespaceRegistryDataKey]), ®istryData) - if err != nil { - return err - } - - featureStoreNamespace := featurestore.Namespace - if featureStoreNamespace == "" { - featureStoreNamespace = DefaultNamespace - } - - _, exists := registryData.Namespaces[featureStoreNamespace] - if !exists { - return fmt.Errorf("feature store not registered") - } - - return nil - }, time.Second*30, time.Millisecond*500).Should(Succeed()) - - By("Deleting the FeatureStore") - Expect(k8sClient.Delete(ctx, featurestore)).Should(Succeed()) - - By("Reconciling the deletion") - _, err = reconciler.Reconcile(ctx, reconcile.Request{ - NamespacedName: typeNamespacedName, - }) - Expect(err).NotTo(HaveOccurred()) - - By("Verifying namespace registry entry is cleaned up") - Eventually(func() error { - cm := &corev1.ConfigMap{} - err := k8sClient.Get(ctx, types.NamespacedName{ - Name: services.NamespaceRegistryConfigMapName, - Namespace: services.DefaultKubernetesNamespace, - }, cm) - if err != nil { - return err - } - - if cm.Data == nil || cm.Data[services.NamespaceRegistryDataKey] == "" { - // Empty registry is acceptable after cleanup - return nil - } - - var registryData services.NamespaceRegistryData - err = json.Unmarshal([]byte(cm.Data[services.NamespaceRegistryDataKey]), ®istryData) - if err != nil { - return err - } - - featureStoreNamespace := featurestore.Namespace - if featureStoreNamespace == "" { - featureStoreNamespace = DefaultNamespace - } - - // Check that the specific FeatureStore's config is removed - configs, exists := registryData.Namespaces[featureStoreNamespace] - if exists { - expectedClientConfigName := "feast-" + featurestore.Name + "-client" - for _, config := range configs { - if config == expectedClientConfigName { - return fmt.Errorf("feature store config %s still exists after deletion", expectedClientConfigName) - } - } - } - - return nil - }, time.Second*30, time.Millisecond*500).Should(Succeed()) - }) - }) - - Context("When testing namespace registry data operations", func() { - It("should correctly serialize and deserialize namespace registry data", func() { - By("Creating test data") - originalData := &services.NamespaceRegistryData{ - Namespaces: map[string][]string{ - "test-namespace-1": {"client-config-1", "client-config-2"}, - "test-namespace-2": {"client-config-3"}, - }, - } - - By("Marshaling to JSON") - jsonData, err := json.Marshal(originalData) - Expect(err).NotTo(HaveOccurred()) - - By("Unmarshaling back") - var unmarshaledData services.NamespaceRegistryData - err = json.Unmarshal(jsonData, &unmarshaledData) - Expect(err).NotTo(HaveOccurred()) - - By("Verifying data integrity") - Expect(unmarshaledData.Namespaces).To(Equal(originalData.Namespaces)) - }) - - It("should handle empty namespace registry data", func() { - By("Creating empty data") - originalData := &services.NamespaceRegistryData{ - Namespaces: make(map[string][]string), - } - - By("Marshaling to JSON") - jsonData, err := json.Marshal(originalData) - Expect(err).NotTo(HaveOccurred()) - - By("Unmarshaling back") - var unmarshaledData services.NamespaceRegistryData - err = json.Unmarshal(jsonData, &unmarshaledData) - Expect(err).NotTo(HaveOccurred()) - - By("Verifying empty data") - Expect(unmarshaledData.Namespaces).To(Equal(originalData.Namespaces)) - Expect(unmarshaledData.Namespaces).To(BeEmpty()) - }) - - It("should correctly remove entries from namespace registry data", func() { - By("Creating test data with multiple entries") - originalData := &services.NamespaceRegistryData{ - Namespaces: map[string][]string{ - "namespace-1": {"config-1", "config-2", "config-3"}, - "namespace-2": {"config-4"}, - }, - } - - By("Marshaling to JSON") - jsonData, err := json.Marshal(originalData) - Expect(err).NotTo(HaveOccurred()) - - By("Unmarshaling back") - var data services.NamespaceRegistryData - err = json.Unmarshal(jsonData, &data) - Expect(err).NotTo(HaveOccurred()) - - By("Simulating removal of specific config") - namespace := "namespace-1" - configToRemove := "config-2" - - if configs, exists := data.Namespaces[namespace]; exists { - var updatedConfigs []string - for _, config := range configs { - if config != configToRemove { - updatedConfigs = append(updatedConfigs, config) - } - } - data.Namespaces[namespace] = updatedConfigs - } - - By("Verifying removal worked") - expectedConfigs := []string{"config-1", "config-3"} - Expect(data.Namespaces[namespace]).To(Equal(expectedConfigs)) - - By("Verifying other namespace is unchanged") - Expect(data.Namespaces["namespace-2"]).To(Equal([]string{"config-4"})) - }) - - It("should remove entire namespace when last config is removed", func() { - By("Creating test data with single config per namespace") - originalData := &services.NamespaceRegistryData{ - Namespaces: map[string][]string{ - "namespace-1": {"config-1"}, - "namespace-2": {"config-2"}, - }, - } - - By("Marshaling to JSON") - jsonData, err := json.Marshal(originalData) - Expect(err).NotTo(HaveOccurred()) - - By("Unmarshaling back") - var data services.NamespaceRegistryData - err = json.Unmarshal(jsonData, &data) - Expect(err).NotTo(HaveOccurred()) - - By("Simulating removal of the only config from namespace-1") - namespace := "namespace-1" - configToRemove := "config-1" - - if configs, exists := data.Namespaces[namespace]; exists { - var updatedConfigs []string - for _, config := range configs { - if config != configToRemove { - updatedConfigs = append(updatedConfigs, config) - } - } - - // If no configs left, remove the namespace entry - if len(updatedConfigs) == 0 { - delete(data.Namespaces, namespace) - } else { - data.Namespaces[namespace] = updatedConfigs - } - } - - By("Verifying namespace was removed") - _, exists := data.Namespaces[namespace] - Expect(exists).To(BeFalse()) - - By("Verifying other namespace is unchanged") - Expect(data.Namespaces["namespace-2"]).To(Equal([]string{"config-2"})) - - By("Verifying total namespace count") - Expect(data.Namespaces).To(HaveLen(1)) - }) - }) -}) diff --git a/infra/feast-operator/internal/controller/featurestore_controller_objectstore_test.go b/infra/feast-operator/internal/controller/featurestore_controller_objectstore_test.go index a326752a78f..5aabdef737a 100644 --- a/infra/feast-operator/internal/controller/featurestore_controller_objectstore_test.go +++ b/infra/feast-operator/internal/controller/featurestore_controller_objectstore_test.go @@ -260,7 +260,7 @@ var _ = Describe("FeatureStore Controller-Ephemeral services", func() { cmList := corev1.ConfigMapList{} err = k8sClient.List(ctx, &cmList, listOpts) Expect(err).NotTo(HaveOccurred()) - Expect(cmList.Items).To(HaveLen(1)) + Expect(cmList.Items).To(HaveLen(2)) feast := services.FeastServices{ Handler: handler.FeastHandler{ @@ -287,7 +287,7 @@ var _ = Describe("FeatureStore Controller-Ephemeral services", func() { Expect(services.GetOfflineContainer(*deploy)).To(BeNil()) Expect(deploy.Spec.Template.Spec.Volumes).To(HaveLen(1)) Expect(deploy.Spec.Template.Spec.Containers[0].VolumeMounts).To(HaveLen(1)) - Expect(deploy.Spec.Template.Spec.Containers[0].Env).To(HaveLen(1)) + Expect(deploy.Spec.Template.Spec.Containers[0].Env).To(HaveLen(2)) env := getFeatureStoreYamlEnvVar(deploy.Spec.Template.Spec.Containers[0].Env) Expect(env).NotTo(BeNil()) diff --git a/infra/feast-operator/internal/controller/featurestore_controller_oidc_auth_test.go b/infra/feast-operator/internal/controller/featurestore_controller_oidc_auth_test.go index e15f8ecfa8a..105367bedf3 100644 --- a/infra/feast-operator/internal/controller/featurestore_controller_oidc_auth_test.go +++ b/infra/feast-operator/internal/controller/featurestore_controller_oidc_auth_test.go @@ -322,7 +322,7 @@ var _ = Describe("FeatureStore Controller-OIDC authorization", func() { cmList := corev1.ConfigMapList{} err = k8sClient.List(ctx, &cmList, listOpts) Expect(err).NotTo(HaveOccurred()) - Expect(cmList.Items).To(HaveLen(1)) + Expect(cmList.Items).To(HaveLen(2)) feast := services.FeastServices{ Handler: handler.FeastHandler{ diff --git a/infra/feast-operator/internal/controller/featurestore_controller_pvc_test.go b/infra/feast-operator/internal/controller/featurestore_controller_pvc_test.go index 8e7303cee34..645013526e6 100644 --- a/infra/feast-operator/internal/controller/featurestore_controller_pvc_test.go +++ b/infra/feast-operator/internal/controller/featurestore_controller_pvc_test.go @@ -459,7 +459,7 @@ var _ = Describe("FeatureStore Controller-Ephemeral services", func() { cmList := corev1.ConfigMapList{} err = k8sClient.List(ctx, &cmList, listOpts) Expect(err).NotTo(HaveOccurred()) - Expect(cmList.Items).To(HaveLen(1)) + Expect(cmList.Items).To(HaveLen(2)) feast := services.FeastServices{ Handler: handler.FeastHandler{ @@ -481,7 +481,7 @@ var _ = Describe("FeatureStore Controller-Ephemeral services", func() { Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(3)) Expect(deploy.Spec.Template.Spec.SecurityContext).To(Equal(securityContext)) registryContainer := services.GetRegistryContainer(*deploy) - Expect(registryContainer.Env).To(HaveLen(1)) + Expect(registryContainer.Env).To(HaveLen(2)) env := getFeatureStoreYamlEnvVar(registryContainer.Env) Expect(env).NotTo(BeNil()) @@ -515,7 +515,7 @@ var _ = Describe("FeatureStore Controller-Ephemeral services", func() { Expect(repoConfig).To(Equal(testConfig)) offlineContainer := services.GetOfflineContainer(*deploy) - Expect(offlineContainer.Env).To(HaveLen(1)) + Expect(offlineContainer.Env).To(HaveLen(2)) env = getFeatureStoreYamlEnvVar(offlineContainer.Env) Expect(env).NotTo(BeNil()) @@ -533,7 +533,7 @@ var _ = Describe("FeatureStore Controller-Ephemeral services", func() { // check online config onlineContainer := services.GetOnlineContainer(*deploy) - Expect(onlineContainer.Env).To(HaveLen(3)) + Expect(onlineContainer.Env).To(HaveLen(4)) Expect(onlineContainer.ImagePullPolicy).To(Equal(corev1.PullAlways)) env = getFeatureStoreYamlEnvVar(onlineContainer.Env) Expect(env).NotTo(BeNil()) diff --git a/infra/feast-operator/internal/controller/featurestore_controller_test.go b/infra/feast-operator/internal/controller/featurestore_controller_test.go index a9ac5235eda..9da8171dfe0 100644 --- a/infra/feast-operator/internal/controller/featurestore_controller_test.go +++ b/infra/feast-operator/internal/controller/featurestore_controller_test.go @@ -124,7 +124,7 @@ var _ = Describe("FeatureStore Controller", func() { cmList := corev1.ConfigMapList{} err = k8sClient.List(ctx, &cmList, listOpts) Expect(err).NotTo(HaveOccurred()) - Expect(cmList.Items).To(HaveLen(1)) + Expect(cmList.Items).To(HaveLen(2)) feast := services.FeastServices{ Handler: handler.FeastHandler{ @@ -336,7 +336,7 @@ var _ = Describe("FeatureStore Controller", func() { Expect(deploy.Spec.Template.Spec.ServiceAccountName).To(Equal(deploy.Name)) Expect(deploy.Spec.Strategy.Type).To(Equal(appsv1.RecreateDeploymentStrategyType)) Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(1)) - Expect(deploy.Spec.Template.Spec.Containers[0].Env).To(HaveLen(1)) + Expect(deploy.Spec.Template.Spec.Containers[0].Env).To(HaveLen(2)) env := getFeatureStoreYamlEnvVar(deploy.Spec.Template.Spec.Containers[0].Env) Expect(env).NotTo(BeNil()) @@ -401,7 +401,7 @@ var _ = Describe("FeatureStore Controller", func() { testConfig.Project = resourceNew.Spec.FeastProject Expect(deploy.Spec.Strategy.Type).To(Equal(appsv1.RollingUpdateDeploymentStrategyType)) - Expect(deploy.Spec.Template.Spec.Containers[0].Env).To(HaveLen(1)) + Expect(deploy.Spec.Template.Spec.Containers[0].Env).To(HaveLen(2)) env = getFeatureStoreYamlEnvVar(deploy.Spec.Template.Spec.Containers[0].Env) Expect(env).NotTo(BeNil()) @@ -720,7 +720,7 @@ var _ = Describe("FeatureStore Controller", func() { cmList := corev1.ConfigMapList{} err = k8sClient.List(ctx, &cmList, listOpts) Expect(err).NotTo(HaveOccurred()) - Expect(cmList.Items).To(HaveLen(1)) + Expect(cmList.Items).To(HaveLen(2)) feast := services.FeastServices{ Handler: handler.FeastHandler{ @@ -748,7 +748,7 @@ var _ = Describe("FeatureStore Controller", func() { Expect(deploy.Spec.Template.Spec.InitContainers[1].EnvFrom).NotTo(BeEmpty()) Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(4)) registryContainer := services.GetRegistryContainer(*deploy) - Expect(registryContainer.Env).To(HaveLen(1)) + Expect(registryContainer.Env).To(HaveLen(2)) env := getFeatureStoreYamlEnvVar(registryContainer.Env) Expect(env).NotTo(BeNil()) @@ -770,7 +770,7 @@ var _ = Describe("FeatureStore Controller", func() { // check offline config Expect(deploy.Spec.Template.Spec.ServiceAccountName).To(Equal(deploy.Name)) offlineContainer := services.GetOfflineContainer(*deploy) - Expect(offlineContainer.Env).To(HaveLen(1)) + Expect(offlineContainer.Env).To(HaveLen(2)) env = getFeatureStoreYamlEnvVar(offlineContainer.Env) Expect(env).NotTo(BeNil()) @@ -789,7 +789,7 @@ var _ = Describe("FeatureStore Controller", func() { // check online config onlineContainer := services.GetOnlineContainer(*deploy) - Expect(onlineContainer.Env).To(HaveLen(3)) + Expect(onlineContainer.Env).To(HaveLen(4)) Expect(onlineContainer.ImagePullPolicy).To(Equal(corev1.PullAlways)) env = getFeatureStoreYamlEnvVar(onlineContainer.Env) Expect(env).NotTo(BeNil()) @@ -866,7 +866,7 @@ var _ = Describe("FeatureStore Controller", func() { feast.Handler.FeatureStore = resource testConfig.Project = resourceNew.Spec.FeastProject - Expect(deploy.Spec.Template.Spec.Containers[0].Env).To(HaveLen(1)) + Expect(deploy.Spec.Template.Spec.Containers[0].Env).To(HaveLen(2)) env = getFeatureStoreYamlEnvVar(deploy.Spec.Template.Spec.Containers[0].Env) Expect(env).NotTo(BeNil()) @@ -914,7 +914,7 @@ var _ = Describe("FeatureStore Controller", func() { cmList := corev1.ConfigMapList{} err = k8sClient.List(ctx, &cmList, listOpts) Expect(err).NotTo(HaveOccurred()) - Expect(cmList.Items).To(HaveLen(1)) + Expect(cmList.Items).To(HaveLen(2)) feast := services.FeastServices{ Handler: handler.FeastHandler{ @@ -940,7 +940,7 @@ var _ = Describe("FeatureStore Controller", func() { Expect(deploy.Spec.Template.Spec.ServiceAccountName).To(Equal(deploy.Name)) Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(4)) onlineContainer := services.GetOnlineContainer(*deploy) - Expect(onlineContainer.Env).To(HaveLen(3)) + Expect(onlineContainer.Env).To(HaveLen(4)) Expect(areEnvVarArraysEqual(onlineContainer.Env, []corev1.EnvVar{{Name: testEnvVarName, Value: testEnvVarValue}, {Name: services.TmpFeatureStoreYamlEnvVar, Value: fsYamlStr}, {Name: "fieldRefName", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{APIVersion: "v1", FieldPath: "metadata.namespace"}}}})).To(BeTrue()) Expect(onlineContainer.ImagePullPolicy).To(Equal(corev1.PullAlways)) @@ -964,7 +964,7 @@ var _ = Describe("FeatureStore Controller", func() { Expect(err).NotTo(HaveOccurred()) onlineContainer = services.GetOnlineContainer(*deploy) - Expect(onlineContainer.Env).To(HaveLen(3)) + Expect(onlineContainer.Env).To(HaveLen(4)) Expect(areEnvVarArraysEqual(onlineContainer.Env, []corev1.EnvVar{{Name: testEnvVarName, Value: testEnvVarValue + "1"}, {Name: services.TmpFeatureStoreYamlEnvVar, Value: fsYamlStr}, {Name: "fieldRefName", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{APIVersion: "v1", FieldPath: "metadata.name"}}}})).To(BeTrue()) }) @@ -1813,18 +1813,23 @@ func noAuthzConfig() services.AuthzConfig { } func areEnvVarArraysEqual(arr1 []corev1.EnvVar, arr2 []corev1.EnvVar) bool { - if len(arr1) != len(arr2) { + // Filter out operator-injected env vars that aren't part of user config + var filtered []corev1.EnvVar + for _, env := range arr1 { + if env.Name != services.IntraCommunicationBase64EnvVar { + filtered = append(filtered, env) + } + } + + if len(filtered) != len(arr2) { return false } - // Create a map to count occurrences of EnvVars in the first array. envMap := make(map[string]corev1.EnvVar) - - for _, env := range arr1 { + for _, env := range filtered { envMap[env.Name] = env } - // Check the second array against the map. for _, env := range arr2 { if _, exists := envMap[env.Name]; !exists || !reflect.DeepEqual(envMap[env.Name], env) { return false diff --git a/infra/feast-operator/internal/controller/featurestore_controller_tls_test.go b/infra/feast-operator/internal/controller/featurestore_controller_tls_test.go index bc60aed4374..8e3d8032cbe 100644 --- a/infra/feast-operator/internal/controller/featurestore_controller_tls_test.go +++ b/infra/feast-operator/internal/controller/featurestore_controller_tls_test.go @@ -255,7 +255,7 @@ var _ = Describe("FeatureStore Controller - Feast service TLS", func() { cmList := corev1.ConfigMapList{} err = k8sClient.List(ctx, &cmList, listOpts) Expect(err).NotTo(HaveOccurred()) - Expect(cmList.Items).To(HaveLen(1)) + Expect(cmList.Items).To(HaveLen(2)) // check deployment deploy := &appsv1.Deployment{} @@ -267,7 +267,7 @@ var _ = Describe("FeatureStore Controller - Feast service TLS", func() { Expect(err).NotTo(HaveOccurred()) Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(4)) registryContainer := services.GetRegistryContainer(*deploy) - Expect(registryContainer.Env).To(HaveLen(1)) + Expect(registryContainer.Env).To(HaveLen(2)) env := getFeatureStoreYamlEnvVar(registryContainer.Env) Expect(env).NotTo(BeNil()) @@ -288,7 +288,7 @@ var _ = Describe("FeatureStore Controller - Feast service TLS", func() { // check offline config offlineContainer := services.GetOfflineContainer(*deploy) - Expect(offlineContainer.Env).To(HaveLen(1)) + Expect(offlineContainer.Env).To(HaveLen(2)) env = getFeatureStoreYamlEnvVar(offlineContainer.Env) Expect(env).NotTo(BeNil()) @@ -305,7 +305,7 @@ var _ = Describe("FeatureStore Controller - Feast service TLS", func() { // check online config onlineContainer := services.GetOnlineContainer(*deploy) - Expect(onlineContainer.Env).To(HaveLen(1)) + Expect(onlineContainer.Env).To(HaveLen(2)) env = getFeatureStoreYamlEnvVar(onlineContainer.Env) Expect(env).NotTo(BeNil()) @@ -319,7 +319,7 @@ var _ = Describe("FeatureStore Controller - Feast service TLS", func() { err = yaml.Unmarshal(envByte, repoConfigOnline) Expect(err).NotTo(HaveOccurred()) Expect(repoConfigOnline).To(Equal(&testConfig)) - Expect(deploy.Spec.Template.Spec.Containers[0].Env).To(HaveLen(1)) + Expect(deploy.Spec.Template.Spec.Containers[0].Env).To(HaveLen(2)) // check client config cm := &corev1.ConfigMap{} diff --git a/infra/feast-operator/internal/controller/notebook_configmap_controller.go b/infra/feast-operator/internal/controller/notebook_configmap_controller.go new file mode 100644 index 00000000000..eae295e3b90 --- /dev/null +++ b/infra/feast-operator/internal/controller/notebook_configmap_controller.go @@ -0,0 +1,509 @@ +/* +Copyright 2024 Feast Community. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller + +import ( + "context" + "fmt" + "strings" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + handler "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + feastdevv1 "github.com/feast-dev/feast/infra/feast-operator/api/v1" + "github.com/feast-dev/feast/infra/feast-operator/internal/controller/services" +) + +const ( + // NotebookFeastIntegrationLabel is the label that must be set to 'true' to enable feast integration + NotebookFeastIntegrationLabel = "opendatahub.io/feast-integration" + // NotebookFeastConfigAnnotation is the annotation key on Notebooks that contains comma-separated feast project names + NotebookFeastConfigAnnotation = "opendatahub.io/feast-config" + // NotebookConfigMapNameSuffix is the suffix for ConfigMap names created for each notebook + NotebookConfigMapNameSuffix = "-feast-config" + // TrueValue is the value that indicates feast integration is enabled + TrueValue = "true" +) + +// NotebookConfigMapReconciler reconciles Notebooks to create/update notebook-feast-config ConfigMaps +// based on feast project annotations +type NotebookConfigMapReconciler struct { + client.Client + Scheme *runtime.Scheme + // NotebookGVK is the GroupVersionKind of the Notebook custom resource + NotebookGVK schema.GroupVersionKind +} + +// +kubebuilder:rbac:groups=kubeflow.org,resources=notebooks,verbs=get;list;watch +// +kubebuilder:rbac:groups=core,resources=configmaps,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=feast.dev,resources=featurestores,verbs=get;list;watch +// +kubebuilder:rbac:groups=apiextensions.k8s.io,resources=customresourcedefinitions,verbs=get;list + +// Reconcile handles Notebook reconciliation +func (r *NotebookConfigMapReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + logger := log.FromContext(ctx) + // Fetch the Notebook + notebook := &unstructured.Unstructured{} + notebook.SetGroupVersionKind(r.NotebookGVK) + if err := r.Get(ctx, req.NamespacedName, notebook); err != nil { + if apierrors.IsNotFound(err) { + // Notebook deleted - ConfigMap will be automatically deleted via owner reference + logger.V(1).Info("Notebook not found, ConfigMap will be cleaned up by owner reference") + return ctrl.Result{}, nil + } + logger.Error(err, "Unable to fetch Notebook") + return ctrl.Result{}, err + } + + // If Notebook is being deleted, owner reference will handle ConfigMap cleanup + if notebook.GetDeletionTimestamp() != nil { + logger.V(1).Info("Notebook is being deleted, ConfigMap will be cleaned up by owner reference") + return ctrl.Result{}, nil + } + + // Check if feast integration is enabled via label + labels := notebook.GetLabels() + if labels == nil { + logger.V(1).Info("No integration label found on Notebook, skipping reconciliation") + return r.cleanupNotebookConfigMap(ctx, req.Namespace, req.Name) + } + + feastIntegrationEnabled, exists := labels[NotebookFeastIntegrationLabel] + if !exists || feastIntegrationEnabled != TrueValue { + logger.V(1).Info("Feast integration not enabled, skipping reconciliation", "label", NotebookFeastIntegrationLabel, "value", feastIntegrationEnabled) + return r.cleanupNotebookConfigMap(ctx, req.Namespace, req.Name) + } + + // Extract feast project names from annotation + annotations := notebook.GetAnnotations() + + feastConfigAnnotation, exists := annotations[NotebookFeastConfigAnnotation] + if feastIntegrationEnabled == TrueValue && (!exists || feastConfigAnnotation == "") { + logger.V(1).Info("No feast config annotation found on Notebook, still keeping the ConfigMap for the notebook") + } + // Parse comma-separated project names from annotation + projectNames := r.parseProjectNames(feastConfigAnnotation) + + logger.Info("Reconciling notebook ConfigMap", "projects", projectNames, "namespace", req.Namespace) + + // Create or update the notebook-feast-config ConfigMap + if err := r.reconcileNotebookConfigMap(ctx, req.Namespace, notebook, projectNames); err != nil { + logger.Error(err, "Failed to reconcile notebook ConfigMap") + return ctrl.Result{}, err + } + + return ctrl.Result{}, nil +} + +// parseProjectNames parses comma-separated project names from the label value +func (r *NotebookConfigMapReconciler) parseProjectNames(labelValue string) []string { + projects := strings.Split(labelValue, ",") + var validProjects []string + for _, project := range projects { + project = strings.TrimSpace(project) + if project != "" { + validProjects = append(validProjects, project) + } + } + return validProjects +} + +// reconcileNotebookConfigMap creates or updates the notebook-feast-config ConfigMap for this notebook +func (r *NotebookConfigMapReconciler) reconcileNotebookConfigMap( + ctx context.Context, + namespace string, + notebook client.Object, + projectNames []string, +) error { + logger := log.FromContext(ctx) + + // Each notebook gets its own ConfigMap with a unique name + configMapName := fmt.Sprintf("%s%s", notebook.GetName(), NotebookConfigMapNameSuffix) + configMap := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: configMapName, + Namespace: namespace, + }, + } + + if op, err := controllerutil.CreateOrUpdate(ctx, r.Client, configMap, func() error { + return r.setNotebookConfigMapData(ctx, configMap, notebook, projectNames) + }); err != nil { + return fmt.Errorf("failed to create or update notebook ConfigMap: %w", err) + } else if op == controllerutil.OperationResultCreated || op == controllerutil.OperationResultUpdated { + logger.Info("Successfully reconciled notebook ConfigMap", "ConfigMap", configMap.Name, "operation", op) + } + + return nil +} + +// setNotebookConfigMapData sets the ConfigMap data with feast project configs for this specific notebook +func (r *NotebookConfigMapReconciler) setNotebookConfigMapData( + ctx context.Context, + cm *corev1.ConfigMap, + notebook client.Object, + projectNames []string, +) error { + logger := log.FromContext(ctx) + + if cm.Data == nil { + cm.Data = make(map[string]string) + } + + // Track which projects should exist + expectedProjects := make(map[string]bool) + for _, projectName := range projectNames { + expectedProjects[projectName] = true + } + + // Add or update projects that are in the annotation + // This always fetches fresh data, so any updates to FeatureStore client ConfigMaps will be picked up + for _, projectName := range projectNames { + clientConfigYAML, err := r.getClientConfigYAMLForProject(ctx, projectName) + if err != nil { + // Log error and remove this project from ConfigMap if it exists + logger.Error(err, "Failed to get client config for project, removing from ConfigMap", "project", projectName, "notebook", notebook.GetName()) + delete(cm.Data, projectName) + continue + } + + // Check if this is an update + existingYAML, exists := cm.Data[projectName] + if exists && existingYAML != clientConfigYAML { + logger.Info("Updating project config in ConfigMap (FeatureStore client ConfigMap changed)", "project", projectName, "notebook", notebook.GetName()) + } + + // Set or update the project config + cm.Data[projectName] = clientConfigYAML + } + + // Remove projects that are no longer in the annotation + for key := range cm.Data { + if !expectedProjects[key] { + logger.Info("Removing project from ConfigMap (no longer in annotation)", "project", key, "notebook", notebook.GetName()) + delete(cm.Data, key) + } + } + + // Set labels + if cm.Labels == nil { + cm.Labels = make(map[string]string) + } + cm.Labels[services.ManagedByLabelKey] = services.ManagedByLabelValue + cm.Labels["source-resource"] = notebook.GetName() + cm.Labels["source-kind"] = notebook.GetObjectKind().GroupVersionKind().Kind + + // Set owner reference to the Notebook with blockOwnerDeletion: false + // This is required because Notebook CRD doesn't support finalizers + // The owner reference still allows Kubernetes to track the relationship + // and we handle cleanup manually in Reconcile() when Notebook is deleted + gvk := notebook.GetObjectKind().GroupVersionKind() + controller := false + blockOwnerDeletion := false + cm.OwnerReferences = []metav1.OwnerReference{ + { + APIVersion: gvk.GroupVersion().String(), + Kind: gvk.Kind, + Name: notebook.GetName(), + UID: notebook.GetUID(), + Controller: &controller, + BlockOwnerDeletion: &blockOwnerDeletion, + }, + } + + return nil +} + +// getClientConfigYAMLForProject finds a FeatureStore with the given project name and returns its client config YAML +func (r *NotebookConfigMapReconciler) getClientConfigYAMLForProject(ctx context.Context, projectName string) (string, error) { + logger := log.FromContext(ctx) + + // List all FeatureStores to find one with matching feastProject + var featureStoreList feastdevv1.FeatureStoreList + if err := r.List(ctx, &featureStoreList, client.InNamespace("")); err != nil { + return "", fmt.Errorf("failed to list FeatureStores: %w", err) + } + + // Find FeatureStore with matching project name + // Try to match by spec.feastProject first, then by metadata.name as fallback + var matchingFeatureStore *feastdevv1.FeatureStore + + for i := range featureStoreList.Items { + fs := &featureStoreList.Items[i] + var matches bool + + // First, try to match by spec.feastProject + project := fs.Spec.FeastProject + if fs.Status.Applied.FeastProject != "" { + project = fs.Status.Applied.FeastProject + } + if project == projectName { + matches = true + } else if fs.GetName() == projectName { + // Fallback: match by FeatureStore metadata name + matches = true + } + + if matches { + if fs.Status.ClientConfigMap != "" { + matchingFeatureStore = fs + break + } + } + } + + if matchingFeatureStore == nil { + // Provide helpful error message with available projects + var availableProjects []string + for i := range featureStoreList.Items { + fs := &featureStoreList.Items[i] + project := fs.Spec.FeastProject + if fs.Status.Applied.FeastProject != "" { + project = fs.Status.Applied.FeastProject + } + if project != "" { + availableProjects = append(availableProjects, fmt.Sprintf("%s (FeatureStore: %s/%s)", project, fs.Namespace, fs.Name)) + } + } + + if len(availableProjects) > 0 { + return "", fmt.Errorf("no FeatureStore found with feastProject: %s. Available projects are: %v", projectName, availableProjects) + } + return "", fmt.Errorf("no FeatureStore found with feastProject: %s. No FeatureStores found in the cluster", projectName) + } + + // Get the client configmap name from status + clientConfigMapName := matchingFeatureStore.Status.ClientConfigMap + if clientConfigMapName == "" { + featureStoreKey := fmt.Sprintf("%s/%s", matchingFeatureStore.Namespace, matchingFeatureStore.Name) + logger.Error(nil, "FeatureStore found but has no ClientConfigMap", "featurestore", featureStoreKey) + return "", fmt.Errorf("FeatureStore %s/%s does not have a client ConfigMap (may still be deploying)", matchingFeatureStore.Namespace, matchingFeatureStore.Name) + } + + // Fetch the client configmap + clientConfigMap := &corev1.ConfigMap{} + clientConfigMapKey := types.NamespacedName{ + Name: clientConfigMapName, + Namespace: matchingFeatureStore.Namespace, + } + + if err := r.Get(ctx, clientConfigMapKey, clientConfigMap); err != nil { + return "", fmt.Errorf("failed to get client ConfigMap %s: %w", clientConfigMapKey, err) + } + + // Extract the YAML content + yamlContent, exists := clientConfigMap.Data[services.FeatureStoreYamlCmKey] + if !exists { + return "", fmt.Errorf("client ConfigMap %s does not contain key %s", clientConfigMapName, services.FeatureStoreYamlCmKey) + } + + return yamlContent, nil +} + +// cleanupNotebookConfigMap removes the notebook-feast-config ConfigMap when integration label is completely removed and config annotation is empty +func (r *NotebookConfigMapReconciler) cleanupNotebookConfigMap(ctx context.Context, namespace, notebookName string) (ctrl.Result, error) { + logger := log.FromContext(ctx) + + configMapName := fmt.Sprintf("%s%s", notebookName, NotebookConfigMapNameSuffix) + configMap := &corev1.ConfigMap{} + if err := r.Get(ctx, types.NamespacedName{Name: configMapName, Namespace: namespace}, configMap); err != nil { + if apierrors.IsNotFound(err) { + return ctrl.Result{}, nil + } + return ctrl.Result{}, err + } + + if err := r.Delete(ctx, configMap); err != nil { + if apierrors.IsNotFound(err) { + return ctrl.Result{}, nil + } + logger.Error(err, "Failed to delete notebook ConfigMap", "ConfigMap", configMapName) + return ctrl.Result{}, err + } + + logger.Info("Deleted notebook ConfigMap (label removed)", "ConfigMap", configMapName, "namespace", namespace) + return ctrl.Result{}, nil +} + +// mapFeatureStoreToNotebookRequests maps a FeatureStore change to all Notebooks that reference its project +// This is called when a FeatureStore CR is created, updated, or deleted +func (r *NotebookConfigMapReconciler) mapFeatureStoreToNotebookRequests(ctx context.Context, obj client.Object) []reconcile.Request { + logger := log.FromContext(ctx) + + featureStore, ok := obj.(*feastdevv1.FeatureStore) + if !ok { + return nil + } + + // Get the project name from the FeatureStore + projectName := featureStore.Spec.FeastProject + if featureStore.Status.Applied.FeastProject != "" { + projectName = featureStore.Status.Applied.FeastProject + } + + if projectName == "" { + return nil + } + + logger.V(1).Info("FeatureStore changed, finding affected Notebooks", "project", projectName, "featurestore", fmt.Sprintf("%s/%s", featureStore.Namespace, featureStore.Name)) + + // List all Notebooks across all namespaces + notebookList := &unstructured.UnstructuredList{} + notebookList.SetGroupVersionKind(schema.GroupVersionKind{ + Group: r.NotebookGVK.Group, + Version: r.NotebookGVK.Version, + Kind: r.NotebookGVK.Kind + "List", + }) + + if err := r.List(ctx, notebookList, client.InNamespace("")); err != nil { + logger.Error(err, "Failed to list Notebooks") + return nil + } + + var requests []reconcile.Request + for i := range notebookList.Items { + notebook := ¬ebookList.Items[i] + labels := notebook.GetLabels() + if labels == nil { + continue + } + + // Check if feast integration is enabled + feastIntegrationEnabled, exists := labels[NotebookFeastIntegrationLabel] + if !exists || feastIntegrationEnabled != TrueValue { + continue + } + + // Read projects from annotation + annotations := notebook.GetAnnotations() + if annotations == nil { + continue + } + + feastConfigAnnotation, exists := annotations[NotebookFeastConfigAnnotation] + if !exists || feastConfigAnnotation == "" { + continue + } + + // Check if this notebook references the project + projectNames := r.parseProjectNames(feastConfigAnnotation) + for _, name := range projectNames { + if name == projectName { + requests = append(requests, reconcile.Request{ + NamespacedName: types.NamespacedName{ + Name: notebook.GetName(), + Namespace: notebook.GetNamespace(), + }, + }) + break + } + } + } + + if len(requests) > 0 { + logger.Info("FeatureStore change triggers Notebook reconciliation", "project", projectName, "notebooks", len(requests)) + } + + return requests +} + +// mapConfigMapToNotebookRequest maps a ConfigMap change back to its owning Notebook +// This ensures that if a user deletes or modifies the ConfigMap, it gets recreated/restored +func (r *NotebookConfigMapReconciler) mapConfigMapToNotebookRequest(ctx context.Context, obj client.Object) []reconcile.Request { + configMap, ok := obj.(*corev1.ConfigMap) + if !ok { + return nil + } + + // Check if this is a notebook ConfigMap by checking the name suffix + if !strings.HasSuffix(configMap.Name, NotebookConfigMapNameSuffix) { + return nil + } + + // Extract notebook name from ConfigMap name + // ConfigMap name format: {notebook-name}-feast-config + notebookName := strings.TrimSuffix(configMap.Name, NotebookConfigMapNameSuffix) + if notebookName == "" { + return nil + } + + // Check if ConfigMap has owner reference to a Notebook + // If it does, use that namespace, otherwise use ConfigMap's namespace + namespace := configMap.Namespace + for _, ownerRef := range configMap.OwnerReferences { + if ownerRef.Kind == r.NotebookGVK.Kind { + // Found Notebook owner reference + return []reconcile.Request{ + { + NamespacedName: types.NamespacedName{ + Name: notebookName, + Namespace: namespace, + }, + }, + } + } + } + + // If no owner reference, still try to reconcile the notebook + // (in case owner reference was removed or ConfigMap was recreated) + return []reconcile.Request{ + { + NamespacedName: types.NamespacedName{ + Name: notebookName, + Namespace: namespace, + }, + }, + } +} + +// SetupWithManager sets up the controller with the Manager +func (r *NotebookConfigMapReconciler) SetupWithManager(mgr ctrl.Manager) error { + // Create a source for unstructured Notebook resources + notebookSource := &unstructured.Unstructured{} + notebookSource.SetGroupVersionKind(r.NotebookGVK) + + bldr := ctrl.NewControllerManagedBy(mgr). + // For() watches Notebooks - any Notebook CRUD triggers Reconcile() + // Reconcile() filters to only process Notebooks with opendatahub.io/feast-integration label set to "true" + For(notebookSource). + // Watch ConfigMaps - when a notebook ConfigMap is modified or deleted, + // trigger reconciliation of the owning Notebook to restore it + Watches( + &corev1.ConfigMap{}, + handler.EnqueueRequestsFromMapFunc(r.mapConfigMapToNotebookRequest), + ). + // Watch FeatureStores - when a FeatureStore changes (CRUD), find all Notebooks + // that reference its project and trigger their reconciliation + // This ensures notebook ConfigMaps are updated when FeatureStore client configs change + Watches( + &feastdevv1.FeatureStore{}, + handler.EnqueueRequestsFromMapFunc(r.mapFeatureStoreToNotebookRequests), + ) + + return bldr.Complete(r) +} diff --git a/infra/feast-operator/internal/controller/notebook_configmap_controller_test.go b/infra/feast-operator/internal/controller/notebook_configmap_controller_test.go new file mode 100644 index 00000000000..88f325f2f91 --- /dev/null +++ b/infra/feast-operator/internal/controller/notebook_configmap_controller_test.go @@ -0,0 +1,633 @@ +/* +Copyright 2024 Feast Community. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller + +import ( + "context" + "fmt" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + apiextensionsclient "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + corev1 "k8s.io/api/core/v1" + + feastdevv1 "github.com/feast-dev/feast/infra/feast-operator/api/v1" + "github.com/feast-dev/feast/infra/feast-operator/internal/controller/services" +) + +var _ = Describe("NotebookConfigMap Controller", func() { + const ( + notebookName = "test-notebook" + namespace = "default" + projectName = "test-project" + featureStoreName = "test-featurestore" + clientConfigMapName = "test-featurestore-client" + ) + + var ( + ctx context.Context + reconciler *NotebookConfigMapReconciler + notebookGVK schema.GroupVersionKind + testYAMLContent string + ) + + BeforeEach(func() { + ctx = context.Background() + notebookGVK = schema.GroupVersionKind{ + Group: "kubeflow.org", + Version: "v1", + Kind: "Notebook", + } + + // Create Notebook CRD using apiextensions client + cfg := testEnv.Config + apiextensionsClientset, err := apiextensionsclient.NewForConfig(cfg) + Expect(err).NotTo(HaveOccurred()) + + crd := &apiextensionsv1.CustomResourceDefinition{ + ObjectMeta: metav1.ObjectMeta{ + Name: "notebooks.kubeflow.org", + }, + Spec: apiextensionsv1.CustomResourceDefinitionSpec{ + Group: "kubeflow.org", + Versions: []apiextensionsv1.CustomResourceDefinitionVersion{ + { + Name: "v1", + Served: true, + Storage: true, + Schema: &apiextensionsv1.CustomResourceValidation{ + OpenAPIV3Schema: &apiextensionsv1.JSONSchemaProps{ + Type: "object", + Properties: map[string]apiextensionsv1.JSONSchemaProps{ + "spec": { + Type: "object", + }, + "status": { + Type: "object", + }, + }, + }, + }, + }, + }, + Scope: apiextensionsv1.NamespaceScoped, + Names: apiextensionsv1.CustomResourceDefinitionNames{ + Plural: "notebooks", + Singular: "notebook", + Kind: "Notebook", + }, + }, + } + + _, err = apiextensionsClientset.ApiextensionsV1().CustomResourceDefinitions().Create(ctx, crd, metav1.CreateOptions{}) + if err != nil && !errors.IsAlreadyExists(err) { + Expect(err).NotTo(HaveOccurred()) + } + + // Create and delete a dummy notebook to force the REST mapper to discover the CRD + dummyNotebook := &unstructured.Unstructured{} + dummyNotebook.SetGroupVersionKind(notebookGVK) + dummyNotebook.SetName("dummy-notebook-for-crd-discovery") + dummyNotebook.SetNamespace(namespace) + err = k8sClient.Create(ctx, dummyNotebook) + if err == nil { + _ = k8sClient.Delete(ctx, dummyNotebook) + } + + reconciler = &NotebookConfigMapReconciler{ + Client: k8sClient, + Scheme: k8sClient.Scheme(), + NotebookGVK: notebookGVK, + } + testYAMLContent = "project: test-project\nprovider: local" + }) + + AfterEach(func() { + // Clean up any test notebooks + notebookList := &unstructured.UnstructuredList{} + notebookList.SetGroupVersionKind(schema.GroupVersionKind{ + Group: notebookGVK.Group, + Version: notebookGVK.Version, + Kind: notebookGVK.Kind + "List", + }) + _ = k8sClient.List(ctx, notebookList, client.InNamespace(namespace)) + for i := range notebookList.Items { + _ = k8sClient.Delete(ctx, ¬ebookList.Items[i]) + } + + // Clean up all configmaps in the namespace (test configmaps) + configMapList := &corev1.ConfigMapList{} + _ = k8sClient.List(ctx, configMapList, client.InNamespace(namespace)) + for i := range configMapList.Items { + cm := &configMapList.Items[i] + // Delete all test-related configmaps + _ = k8sClient.Delete(ctx, cm) + } + + // Clean up feature stores + fsList := &feastdevv1.FeatureStoreList{} + _ = k8sClient.List(ctx, fsList, client.InNamespace(namespace)) + for i := range fsList.Items { + _ = k8sClient.Delete(ctx, &fsList.Items[i]) + } + }) + + createUnstructuredNotebook := func(name, ns string, labels, annotations map[string]string) *unstructured.Unstructured { + notebook := &unstructured.Unstructured{} + notebook.SetGroupVersionKind(notebookGVK) + notebook.SetName(name) + notebook.SetNamespace(ns) + if labels != nil { + notebook.SetLabels(labels) + } + if annotations != nil { + notebook.SetAnnotations(annotations) + } + return notebook + } + + createFeatureStore := func(name, ns, project string) *feastdevv1.FeatureStore { + fs := &feastdevv1.FeatureStore{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: ns, + }, + Spec: feastdevv1.FeatureStoreSpec{ + FeastProject: project, + }, + } + return fs + } + + // Helper to set FeatureStore status after creation + setFeatureStoreStatus := func(fs *feastdevv1.FeatureStore, project string, configMapName string) { + fs.Status.ClientConfigMap = configMapName + fs.Status.Applied.FeastProject = project + } + + createClientConfigMap := func(name, ns string) *corev1.ConfigMap { + return &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: ns, + }, + Data: map[string]string{ + services.FeatureStoreYamlCmKey: testYAMLContent, + }, + } + } + + Context("Reconcile", func() { + It("should return nil when Notebook is not found", func() { + // Create and delete a notebook first to force the REST mapper to discover the CRD + // Use Eventually to wait for CRD to be recognized + tempNotebook := createUnstructuredNotebook("temp-notebook", namespace, nil, nil) + Eventually(func() error { + return k8sClient.Create(ctx, tempNotebook) + }, "10s", "1s").Should(Succeed()) + Expect(k8sClient.Delete(ctx, tempNotebook)).To(Succeed()) + + req := reconcile.Request{ + NamespacedName: types.NamespacedName{ + Name: "non-existent", + Namespace: namespace, + }, + } + result, err := reconciler.Reconcile(ctx, req) + Expect(err).NotTo(HaveOccurred()) + Expect(result).To(Equal(reconcile.Result{})) + }) + + It("should return nil when Notebook is being deleted", func() { + notebook := createUnstructuredNotebook(notebookName, namespace, map[string]string{ + NotebookFeastIntegrationLabel: TrueValue, + }, map[string]string{ + NotebookFeastConfigAnnotation: projectName, + }) + now := metav1.Now() + notebook.SetDeletionTimestamp(&now) + Expect(k8sClient.Create(ctx, notebook)).To(Succeed()) + + req := reconcile.Request{ + NamespacedName: types.NamespacedName{ + Name: notebookName, + Namespace: namespace, + }, + } + result, err := reconciler.Reconcile(ctx, req) + Expect(err).NotTo(HaveOccurred()) + Expect(result).To(Equal(reconcile.Result{})) + + Expect(k8sClient.Delete(ctx, notebook)).To(Succeed()) + }) + + It("should cleanup ConfigMap when integration label is not set", func() { + notebook := createUnstructuredNotebook(notebookName, namespace, nil, nil) + Expect(k8sClient.Create(ctx, notebook)).To(Succeed()) + + configMap := createClientConfigMap(notebookName+NotebookConfigMapNameSuffix, namespace) + Expect(k8sClient.Create(ctx, configMap)).To(Succeed()) + + req := reconcile.Request{ + NamespacedName: types.NamespacedName{ + Name: notebookName, + Namespace: namespace, + }, + } + result, err := reconciler.Reconcile(ctx, req) + Expect(err).NotTo(HaveOccurred()) + Expect(result).To(Equal(reconcile.Result{})) + + cm := &corev1.ConfigMap{} + err = k8sClient.Get(ctx, types.NamespacedName{Name: configMap.Name, Namespace: namespace}, cm) + Expect(err).To(HaveOccurred()) + + Expect(k8sClient.Delete(ctx, notebook)).To(Succeed()) + }) + + It("should cleanup ConfigMap when integration label is not 'true'", func() { + notebook := createUnstructuredNotebook(notebookName, namespace, map[string]string{ + NotebookFeastIntegrationLabel: "false", + }, nil) + Expect(k8sClient.Create(ctx, notebook)).To(Succeed()) + + configMap := createClientConfigMap(notebookName+NotebookConfigMapNameSuffix, namespace) + Expect(k8sClient.Create(ctx, configMap)).To(Succeed()) + + req := reconcile.Request{ + NamespacedName: types.NamespacedName{ + Name: notebookName, + Namespace: namespace, + }, + } + result, err := reconciler.Reconcile(ctx, req) + Expect(err).NotTo(HaveOccurred()) + Expect(result).To(Equal(reconcile.Result{})) + + cm := &corev1.ConfigMap{} + err = k8sClient.Get(ctx, types.NamespacedName{Name: configMap.Name, Namespace: namespace}, cm) + Expect(err).To(HaveOccurred()) + + Expect(k8sClient.Delete(ctx, notebook)).To(Succeed()) + }) + + It("should create ConfigMap when integration is enabled but annotation is empty", func() { + fs := createFeatureStore(featureStoreName, namespace, projectName) + Expect(k8sClient.Create(ctx, fs)).To(Succeed()) + // Set and update status separately + setFeatureStoreStatus(fs, projectName, clientConfigMapName) + Expect(k8sClient.Status().Update(ctx, fs)).To(Succeed()) + defer func() { + _ = k8sClient.Delete(ctx, fs) + }() + + clientCM := createClientConfigMap(clientConfigMapName, namespace) + Expect(k8sClient.Create(ctx, clientCM)).To(Succeed()) + defer func() { + _ = k8sClient.Delete(ctx, clientCM) + }() + + notebook := createUnstructuredNotebook(notebookName, namespace, map[string]string{ + NotebookFeastIntegrationLabel: TrueValue, + }, nil) + Expect(k8sClient.Create(ctx, notebook)).To(Succeed()) + defer func() { + _ = k8sClient.Delete(ctx, notebook) + }() + + req := reconcile.Request{ + NamespacedName: types.NamespacedName{ + Name: notebookName, + Namespace: namespace, + }, + } + result, err := reconciler.Reconcile(ctx, req) + Expect(err).NotTo(HaveOccurred()) + Expect(result).To(Equal(reconcile.Result{})) + + cm := &corev1.ConfigMap{} + err = k8sClient.Get(ctx, types.NamespacedName{ + Name: notebookName + NotebookConfigMapNameSuffix, + Namespace: namespace, + }, cm) + Expect(err).NotTo(HaveOccurred()) + Expect(cm.Data).To(BeEmpty()) + }) + + It("should create ConfigMap with project config when integration is enabled with annotation", func() { + fs := createFeatureStore(featureStoreName, namespace, projectName) + Expect(k8sClient.Create(ctx, fs)).To(Succeed()) + // Set and update status separately + setFeatureStoreStatus(fs, projectName, clientConfigMapName) + Expect(k8sClient.Status().Update(ctx, fs)).To(Succeed()) + defer func() { + _ = k8sClient.Delete(ctx, fs) + }() + + clientCM := createClientConfigMap(clientConfigMapName, namespace) + Expect(k8sClient.Create(ctx, clientCM)).To(Succeed()) + defer func() { + _ = k8sClient.Delete(ctx, clientCM) + }() + + notebook := createUnstructuredNotebook(notebookName, namespace, map[string]string{ + NotebookFeastIntegrationLabel: TrueValue, + }, map[string]string{ + NotebookFeastConfigAnnotation: projectName, + }) + Expect(k8sClient.Create(ctx, notebook)).To(Succeed()) + defer func() { + _ = k8sClient.Delete(ctx, notebook) + }() + + req := reconcile.Request{ + NamespacedName: types.NamespacedName{ + Name: notebookName, + Namespace: namespace, + }, + } + result, err := reconciler.Reconcile(ctx, req) + Expect(err).NotTo(HaveOccurred()) + Expect(result).To(Equal(reconcile.Result{})) + + cm := &corev1.ConfigMap{} + err = k8sClient.Get(ctx, types.NamespacedName{ + Name: notebookName + NotebookConfigMapNameSuffix, + Namespace: namespace, + }, cm) + Expect(err).NotTo(HaveOccurred()) + Expect(cm.Data).To(HaveKey(projectName)) + Expect(cm.Data[projectName]).To(Equal(testYAMLContent)) + Expect(cm.Labels[services.ManagedByLabelKey]).To(Equal(services.ManagedByLabelValue)) + Expect(cm.Labels["source-resource"]).To(Equal(notebookName)) + Expect(cm.Labels["source-kind"]).To(Equal("Notebook")) + Expect(cm.OwnerReferences).To(HaveLen(1)) + Expect(cm.OwnerReferences[0].Name).To(Equal(notebookName)) + }) + + It("should handle multiple projects in annotation", func() { + projectName2 := "test-project-2" + featureStoreName2 := "test-featurestore-2" + clientConfigMapName2 := "test-featurestore-2-client" + + fs1 := createFeatureStore(featureStoreName, namespace, projectName) + Expect(k8sClient.Create(ctx, fs1)).To(Succeed()) + // Set and update status separately + setFeatureStoreStatus(fs1, projectName, clientConfigMapName) + Expect(k8sClient.Status().Update(ctx, fs1)).To(Succeed()) + defer func() { + _ = k8sClient.Delete(ctx, fs1) + }() + + fs2 := createFeatureStore(featureStoreName2, namespace, projectName2) + Expect(k8sClient.Create(ctx, fs2)).To(Succeed()) + // Set and update status separately + setFeatureStoreStatus(fs2, projectName2, clientConfigMapName2) + Expect(k8sClient.Status().Update(ctx, fs2)).To(Succeed()) + defer func() { + _ = k8sClient.Delete(ctx, fs2) + }() + + clientCM1 := createClientConfigMap(clientConfigMapName, namespace) + Expect(k8sClient.Create(ctx, clientCM1)).To(Succeed()) + defer func() { + _ = k8sClient.Delete(ctx, clientCM1) + }() + + testYAMLContent2 := "project: test-project-2\nprovider: local" + clientCM2 := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: clientConfigMapName2, + Namespace: namespace, + }, + Data: map[string]string{ + services.FeatureStoreYamlCmKey: testYAMLContent2, + }, + } + Expect(k8sClient.Create(ctx, clientCM2)).To(Succeed()) + defer func() { + _ = k8sClient.Delete(ctx, clientCM2) + }() + + notebook := createUnstructuredNotebook(notebookName, namespace, map[string]string{ + NotebookFeastIntegrationLabel: TrueValue, + }, map[string]string{ + NotebookFeastConfigAnnotation: fmt.Sprintf("%s, %s", projectName, projectName2), + }) + Expect(k8sClient.Create(ctx, notebook)).To(Succeed()) + defer func() { + _ = k8sClient.Delete(ctx, notebook) + }() + + req := reconcile.Request{ + NamespacedName: types.NamespacedName{ + Name: notebookName, + Namespace: namespace, + }, + } + result, err := reconciler.Reconcile(ctx, req) + Expect(err).NotTo(HaveOccurred()) + Expect(result).To(Equal(reconcile.Result{})) + + cm := &corev1.ConfigMap{} + err = k8sClient.Get(ctx, types.NamespacedName{ + Name: notebookName + NotebookConfigMapNameSuffix, + Namespace: namespace, + }, cm) + Expect(err).NotTo(HaveOccurred()) + Expect(cm.Data).To(HaveKey(projectName)) + Expect(cm.Data).To(HaveKey(projectName2)) + Expect(cm.Data[projectName]).To(Equal(testYAMLContent)) + Expect(cm.Data[projectName2]).To(Equal(testYAMLContent2)) + }) + + It("should handle project not found in FeatureStore", func() { + notebook := createUnstructuredNotebook(notebookName, namespace, map[string]string{ + NotebookFeastIntegrationLabel: TrueValue, + }, map[string]string{ + NotebookFeastConfigAnnotation: "non-existent-project", + }) + Expect(k8sClient.Create(ctx, notebook)).To(Succeed()) + defer func() { + _ = k8sClient.Delete(ctx, notebook) + }() + + req := reconcile.Request{ + NamespacedName: types.NamespacedName{ + Name: notebookName, + Namespace: namespace, + }, + } + result, err := reconciler.Reconcile(ctx, req) + Expect(err).NotTo(HaveOccurred()) + Expect(result).To(Equal(reconcile.Result{})) + + cm := &corev1.ConfigMap{} + err = k8sClient.Get(ctx, types.NamespacedName{ + Name: notebookName + NotebookConfigMapNameSuffix, + Namespace: namespace, + }, cm) + Expect(err).NotTo(HaveOccurred()) + Expect(cm.Data).NotTo(HaveKey("non-existent-project")) + }) + + It("should update ConfigMap when project annotation changes", func() { + fs1 := createFeatureStore(featureStoreName, namespace, projectName) + Expect(k8sClient.Create(ctx, fs1)).To(Succeed()) + // Set and update status separately + setFeatureStoreStatus(fs1, projectName, clientConfigMapName) + Expect(k8sClient.Status().Update(ctx, fs1)).To(Succeed()) + defer func() { + _ = k8sClient.Delete(ctx, fs1) + }() + + projectName2 := "test-project-2" + featureStoreName2 := "test-featurestore-2" + clientConfigMapName2 := "test-featurestore-2-client" + + fs2 := createFeatureStore(featureStoreName2, namespace, projectName2) + Expect(k8sClient.Create(ctx, fs2)).To(Succeed()) + // Set and update status separately + setFeatureStoreStatus(fs2, projectName2, clientConfigMapName2) + Expect(k8sClient.Status().Update(ctx, fs2)).To(Succeed()) + defer func() { + _ = k8sClient.Delete(ctx, fs2) + }() + + clientCM1 := createClientConfigMap(clientConfigMapName, namespace) + Expect(k8sClient.Create(ctx, clientCM1)).To(Succeed()) + defer func() { + _ = k8sClient.Delete(ctx, clientCM1) + }() + + testYAMLContent2 := "project: test-project-2\nprovider: local" + clientCM2 := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: clientConfigMapName2, + Namespace: namespace, + }, + Data: map[string]string{ + services.FeatureStoreYamlCmKey: testYAMLContent2, + }, + } + Expect(k8sClient.Create(ctx, clientCM2)).To(Succeed()) + defer func() { + _ = k8sClient.Delete(ctx, clientCM2) + }() + + notebook := createUnstructuredNotebook(notebookName, namespace, map[string]string{ + NotebookFeastIntegrationLabel: TrueValue, + }, map[string]string{ + NotebookFeastConfigAnnotation: projectName, + }) + Expect(k8sClient.Create(ctx, notebook)).To(Succeed()) + defer func() { + _ = k8sClient.Delete(ctx, notebook) + }() + + req := reconcile.Request{ + NamespacedName: types.NamespacedName{ + Name: notebookName, + Namespace: namespace, + }, + } + result, err := reconciler.Reconcile(ctx, req) + Expect(err).NotTo(HaveOccurred()) + Expect(result).To(Equal(reconcile.Result{})) + + cm := &corev1.ConfigMap{} + err = k8sClient.Get(ctx, types.NamespacedName{ + Name: notebookName + NotebookConfigMapNameSuffix, + Namespace: namespace, + }, cm) + Expect(err).NotTo(HaveOccurred()) + Expect(cm.Data).To(HaveKey(projectName)) + Expect(cm.Data).NotTo(HaveKey(projectName2)) + + notebook.SetAnnotations(map[string]string{ + NotebookFeastConfigAnnotation: projectName2, + }) + Expect(k8sClient.Update(ctx, notebook)).To(Succeed()) + + result, err = reconciler.Reconcile(ctx, req) + Expect(err).NotTo(HaveOccurred()) + Expect(result).To(Equal(reconcile.Result{})) + + err = k8sClient.Get(ctx, types.NamespacedName{ + Name: notebookName + NotebookConfigMapNameSuffix, + Namespace: namespace, + }, cm) + Expect(err).NotTo(HaveOccurred()) + Expect(cm.Data).NotTo(HaveKey(projectName)) + Expect(cm.Data).To(HaveKey(projectName2)) + Expect(cm.Data[projectName2]).To(Equal(testYAMLContent2)) + }) + }) + + Context("getClientConfigYAMLForProject", func() { + It("should return YAML content for matching project", func() { + fs := createFeatureStore(featureStoreName, namespace, projectName) + Expect(k8sClient.Create(ctx, fs)).To(Succeed()) + // Set and update status separately + setFeatureStoreStatus(fs, projectName, clientConfigMapName) + Expect(k8sClient.Status().Update(ctx, fs)).To(Succeed()) + defer func() { + _ = k8sClient.Delete(ctx, fs) + }() + + clientCM := createClientConfigMap(clientConfigMapName, namespace) + Expect(k8sClient.Create(ctx, clientCM)).To(Succeed()) + defer func() { + _ = k8sClient.Delete(ctx, clientCM) + }() + + yaml, err := reconciler.getClientConfigYAMLForProject(ctx, projectName) + Expect(err).NotTo(HaveOccurred()) + Expect(yaml).To(Equal(testYAMLContent)) + }) + + It("should return error when project not found", func() { + _, err := reconciler.getClientConfigYAMLForProject(ctx, "non-existent-project") + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("no FeatureStore found")) + }) + + It("should return error when ClientConfigMap doesn't exist", func() { + fs := createFeatureStore(featureStoreName, namespace, projectName) + Expect(k8sClient.Create(ctx, fs)).To(Succeed()) + // Set status with non-existent configmap + setFeatureStoreStatus(fs, projectName, "non-existent-configmap") + Expect(k8sClient.Status().Update(ctx, fs)).To(Succeed()) + defer func() { + _ = k8sClient.Delete(ctx, fs) + }() + + _, err := reconciler.getClientConfigYAMLForProject(ctx, projectName) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("failed to get client ConfigMap")) + }) + }) +}) diff --git a/infra/feast-operator/internal/controller/registry/client.go b/infra/feast-operator/internal/controller/registry/client.go new file mode 100644 index 00000000000..878511d961b --- /dev/null +++ b/infra/feast-operator/internal/controller/registry/client.go @@ -0,0 +1,172 @@ +/* +Copyright 2024 Feast Community. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package registry + +import ( + "context" + "crypto/hmac" + "crypto/sha256" + "crypto/tls" + "encoding/base64" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "strings" + "time" + + "sigs.k8s.io/controller-runtime/pkg/log" +) + +const ( + permissionsPath = "/api/v1/permissions" + requestTimeout = 10 * time.Second +) + +// PermissionPolicy holds the extracted policy data from a Feast permission. +type PermissionPolicy struct { + Groups []string + Namespaces []string +} + +// ListPermissions fetches permissions from the registry REST API for the given project. +// The intraCommToken is the per-instance secret used to bypass per-user auth on the registry. +// When useTLS is true, uses https with InsecureSkipVerify for cluster-internal service certs. +// Returns policies from GroupBasedPolicy, NamespaceBasedPolicy, and CombinedGroupNamespacePolicy only. +func ListPermissions(ctx context.Context, registryRestURL, project, intraCommToken string, useTLS bool) ([]PermissionPolicy, error) { + if registryRestURL == "" || project == "" { + return nil, nil + } + baseURL := registryRestURL + if !strings.HasPrefix(baseURL, "http://") && !strings.HasPrefix(baseURL, "https://") { + scheme := "http://" + if useTLS { + scheme = "https://" + } + baseURL = scheme + baseURL + } + u, err := url.Parse(baseURL) + if err != nil { + return nil, fmt.Errorf("failed to parse registry URL: %w", err) + } + u.Path = strings.TrimSuffix(u.Path, "/") + permissionsPath + q := u.Query() + q.Set("project", project) + u.RawQuery = q.Encode() + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.String(), nil) + if err != nil { + return nil, fmt.Errorf("failed to create request: %w", err) + } + if intraCommToken != "" { + req.Header.Set("Authorization", "Bearer "+BuildIntraCommunicationJWT(intraCommToken)) + } + client := &http.Client{ + Timeout: requestTimeout, + Transport: &http.Transport{ + TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, + }, + } + resp, err := client.Do(req) + if err != nil { + return nil, fmt.Errorf("failed to fetch permissions: %w", err) + } + defer func() { _ = resp.Body.Close() }() + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("registry returned status %d", resp.StatusCode) + } + logger := log.FromContext(ctx) + body, err := io.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("failed to read permissions response body: %w", err) + } + logger.V(1).Info("Registry permissions response", "body", string(body)) + var result struct { + Permissions []struct { + Spec *struct { + Policy map[string]json.RawMessage `json:"policy,omitempty"` + } `json:"spec,omitempty"` + } `json:"permissions,omitempty"` + } + if err := json.Unmarshal(body, &result); err != nil { + return nil, fmt.Errorf("failed to decode permissions response: %w", err) + } + var policies []PermissionPolicy + for _, p := range result.Permissions { + if p.Spec == nil || p.Spec.Policy == nil { + continue + } + pol := extractPolicy(p.Spec.Policy) + if pol != nil { + policies = append(policies, *pol) + } + } + logger.V(1).Info("Extracted permission policies", "permissionsCount", len(result.Permissions), "policiesCount", len(policies)) + return policies, nil +} + +// BuildIntraCommunicationJWT creates a JWT matching the Python SDK's +// intra-service communication format: HS256 signed with an empty key, +// sub=":::". The registry recognises this and bypasses per-user +// permission checks. +func BuildIntraCommunicationJWT(intraCommToken string) string { + b64 := base64.RawURLEncoding.EncodeToString + header := b64([]byte(`{"alg":"HS256","typ":"JWT"}`)) + payload := b64([]byte(`{"sub":":::` + intraCommToken + `"}`)) + signingInput := header + "." + payload + mac := hmac.New(sha256.New, []byte("")) + mac.Write([]byte(signingInput)) + sig := b64(mac.Sum(nil)) + return signingInput + "." + sig +} + +func extractPolicy(policy map[string]json.RawMessage) *PermissionPolicy { + var groups, namespaces []string + for k, v := range policy { + norm := strings.ToLower(strings.ReplaceAll(k, "_", "")) + switch norm { + case "groupbasedpolicy": + var g struct { + Groups []string `json:"groups,omitempty"` + } + if err := json.Unmarshal(v, &g); err == nil { + groups = append(groups, g.Groups...) + } + case "namespacebasedpolicy": + var n struct { + Namespaces []string `json:"namespaces,omitempty"` + } + if err := json.Unmarshal(v, &n); err == nil { + namespaces = append(namespaces, n.Namespaces...) + } + case "combinedgroupnamespacepolicy": + var c struct { + Groups []string `json:"groups,omitempty"` + Namespaces []string `json:"namespaces,omitempty"` + } + if err := json.Unmarshal(v, &c); err == nil { + groups = append(groups, c.Groups...) + namespaces = append(namespaces, c.Namespaces...) + } + } + } + if len(groups) == 0 && len(namespaces) == 0 { + return nil + } + return &PermissionPolicy{Groups: groups, Namespaces: namespaces} +} diff --git a/infra/feast-operator/internal/controller/registry/client_test.go b/infra/feast-operator/internal/controller/registry/client_test.go new file mode 100644 index 00000000000..ce2710f310a --- /dev/null +++ b/infra/feast-operator/internal/controller/registry/client_test.go @@ -0,0 +1,295 @@ +/* +Copyright 2024 Feast Community. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package registry + +import ( + "context" + "encoding/base64" + "encoding/json" + "net/http" + "net/http/httptest" + "strings" + "testing" +) + +func TestExtractPolicy_GroupBasedPolicy(t *testing.T) { + policy := map[string]json.RawMessage{ + "groupBasedPolicy": json.RawMessage(`{"groups":["admin-group","data-team"]}`), + } + result := extractPolicy(policy) + if result == nil { + t.Fatal("expected non-nil result") + } + assertStrSlice(t, result.Groups, []string{"admin-group", "data-team"}) + assertStrSlice(t, result.Namespaces, nil) +} + +func TestExtractPolicy_NamespaceBasedPolicy(t *testing.T) { + policy := map[string]json.RawMessage{ + "namespaceBasedPolicy": json.RawMessage(`{"namespaces":["ds-project-1","ds-project-2"]}`), + } + result := extractPolicy(policy) + if result == nil { + t.Fatal("expected non-nil result") + } + assertStrSlice(t, result.Groups, nil) + assertStrSlice(t, result.Namespaces, []string{"ds-project-1", "ds-project-2"}) +} + +func TestExtractPolicy_CombinedGroupNamespacePolicy(t *testing.T) { + policy := map[string]json.RawMessage{ + "combinedGroupNamespacePolicy": json.RawMessage(`{"groups":["ml-engineers"],"namespaces":["prod-ns"]}`), + } + result := extractPolicy(policy) + if result == nil { + t.Fatal("expected non-nil result") + } + assertStrSlice(t, result.Groups, []string{"ml-engineers"}) + assertStrSlice(t, result.Namespaces, []string{"prod-ns"}) +} + +func TestExtractPolicy_RoleBasedPolicyIgnored(t *testing.T) { + policy := map[string]json.RawMessage{ + "roleBasedPolicy": json.RawMessage(`{"roles":["admin"]}`), + } + result := extractPolicy(policy) + if result != nil { + t.Fatalf("expected nil for RoleBasedPolicy, got %+v", result) + } +} + +func TestExtractPolicy_EmptyPolicy(t *testing.T) { + result := extractPolicy(map[string]json.RawMessage{}) + if result != nil { + t.Fatalf("expected nil for empty policy, got %+v", result) + } +} + +func TestExtractPolicy_SnakeCaseKeys(t *testing.T) { + // Verify normalization handles snake_case (e.g. from alternative serializers) + policy := map[string]json.RawMessage{ + "group_based_policy": json.RawMessage(`{"groups":["team-a"]}`), + } + result := extractPolicy(policy) + if result == nil { + t.Fatal("expected non-nil result for snake_case key") + } + assertStrSlice(t, result.Groups, []string{"team-a"}) +} + +func TestListPermissions_EmptyInputs(t *testing.T) { + policies, err := ListPermissions(context.Background(), "", "project", "tok", false) + if err != nil || policies != nil { + t.Fatalf("expected nil,nil for empty URL; got %v, %v", policies, err) + } + policies, err = ListPermissions(context.Background(), "http://localhost", "", "tok", false) + if err != nil || policies != nil { + t.Fatalf("expected nil,nil for empty project; got %v, %v", policies, err) + } +} + +func TestListPermissions_Success(t *testing.T) { + responseBody := `{ + "permissions": [ + { + "spec": { + "name": "perm1", + "project": "my_project", + "policy": { + "groupBasedPolicy": {"groups": ["group-a", "group-b"]} + } + } + }, + { + "spec": { + "name": "perm2", + "project": "my_project", + "policy": { + "namespaceBasedPolicy": {"namespaces": ["ns-1"]} + } + } + }, + { + "spec": { + "name": "perm3", + "project": "my_project", + "policy": { + "roleBasedPolicy": {"roles": ["admin"]} + } + } + } + ] + }` + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != permissionsPath { + t.Errorf("unexpected path: %s", r.URL.Path) + } + if r.URL.Query().Get("project") != "my_project" { + t.Errorf("unexpected project param: %s", r.URL.Query().Get("project")) + } + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte(responseBody)) + })) + defer server.Close() + + policies, err := ListPermissions(context.Background(), server.URL, "my_project", "test-token", false) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + // RoleBasedPolicy should be filtered out, leaving 2 policies + if len(policies) != 2 { + t.Fatalf("expected 2 policies, got %d", len(policies)) + } + assertStrSlice(t, policies[0].Groups, []string{"group-a", "group-b"}) + assertStrSlice(t, policies[1].Namespaces, []string{"ns-1"}) +} + +func TestListPermissions_NonOKStatus(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusInternalServerError) + })) + defer server.Close() + + _, err := ListPermissions(context.Background(), server.URL, "p", "tok", false) + if err == nil { + t.Fatal("expected error for non-200 status") + } +} + +func TestListPermissions_EmptyPermissions(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"permissions":[]}`)) + })) + defer server.Close() + + policies, err := ListPermissions(context.Background(), server.URL, "p", "tok", false) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(policies) != 0 { + t.Fatalf("expected 0 policies, got %d", len(policies)) + } +} + +func TestListPermissions_NoSpecPermission(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"permissions":[{}]}`)) + })) + defer server.Close() + + policies, err := ListPermissions(context.Background(), server.URL, "p", "tok", false) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(policies) != 0 { + t.Fatalf("expected 0 policies for nil spec, got %d", len(policies)) + } +} + +func TestListPermissions_URLWithoutScheme_TLS(t *testing.T) { + _, err := ListPermissions(context.Background(), "127.0.0.1:8080", "p", "tok", true) + if err == nil { + t.Fatal("expected error for unreachable host") + } +} + +func TestListPermissions_URLWithoutScheme_NoTLS(t *testing.T) { + _, err := ListPermissions(context.Background(), "127.0.0.1:8080", "p", "tok", false) + if err == nil { + t.Fatal("expected error for unreachable host") + } +} + +func TestBuildIntraCommunicationJWT(t *testing.T) { + secretToken := "my-random-secret-value" // pragma: allowlist secret + jwt := BuildIntraCommunicationJWT(secretToken) + parts := strings.Split(jwt, ".") + if len(parts) != 3 { + t.Fatalf("expected 3 JWT parts, got %d", len(parts)) + } + + payloadBytes, err := base64.RawURLEncoding.DecodeString(parts[1]) + if err != nil { + t.Fatalf("failed to decode payload: %v", err) + } + var payload struct { + Sub string `json:"sub"` + } + if err := json.Unmarshal(payloadBytes, &payload); err != nil { + t.Fatalf("failed to unmarshal payload: %v", err) + } + expectedSub := ":::" + secretToken + if payload.Sub != expectedSub { + t.Fatalf("expected sub %q, got %q", expectedSub, payload.Sub) + } +} + +func TestListPermissions_SendsIntraCommunicationJWT(t *testing.T) { + var receivedAuth string + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + receivedAuth = r.Header.Get("Authorization") + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"permissions":[]}`)) + })) + defer server.Close() + + _, err := ListPermissions(context.Background(), server.URL, "p", "my-secret", false) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if !strings.HasPrefix(receivedAuth, "Bearer ") { + t.Fatalf("expected Bearer token, got %q", receivedAuth) + } + token := strings.TrimPrefix(receivedAuth, "Bearer ") + parts := strings.Split(token, ".") + if len(parts) != 3 { + t.Fatalf("expected valid JWT with 3 parts, got %d", len(parts)) + } +} + +func TestListPermissions_NoTokenNoAuthHeader(t *testing.T) { + var receivedAuth string + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + receivedAuth = r.Header.Get("Authorization") + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"permissions":[]}`)) + })) + defer server.Close() + + _, err := ListPermissions(context.Background(), server.URL, "p", "", false) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if receivedAuth != "" { + t.Fatalf("expected no Authorization header when token is empty, got %q", receivedAuth) + } +} + +func assertStrSlice(t *testing.T, got, want []string) { + t.Helper() + if len(got) != len(want) { + t.Fatalf("slice length mismatch: got %v, want %v", got, want) + } + for i := range got { + if got[i] != want[i] { + t.Fatalf("slice[%d] mismatch: got %q, want %q", i, got[i], want[i]) + } + } +} diff --git a/infra/feast-operator/internal/controller/services/cronjob.go b/infra/feast-operator/internal/controller/services/cronjob.go index f3b978928f7..c0eeb7567c2 100644 --- a/infra/feast-operator/internal/controller/services/cronjob.go +++ b/infra/feast-operator/internal/controller/services/cronjob.go @@ -167,6 +167,7 @@ func (feast *FeastServices) setCronJobContainers(podSpec *corev1.PodSpec) { } func (feast *FeastServices) getCronJobContainer(containerName, cronJobCmd string) corev1.Container { + intraCommCMName := GetIntraCommunicationConfigMapName(feast.Handler.FeatureStore.Name) return *getContainer( containerName, "", @@ -177,6 +178,7 @@ func (feast *FeastServices) getCronJobContainer(containerName, cronJobCmd string }, feast.Handler.FeatureStore.Status.Applied.CronJob.ContainerConfigs.ContainerConfigs, "", + intraCommCMName, ) } diff --git a/infra/feast-operator/internal/controller/services/services.go b/infra/feast-operator/internal/controller/services/services.go index c6271c28d3a..ca33d117bd5 100644 --- a/infra/feast-operator/internal/controller/services/services.go +++ b/infra/feast-operator/internal/controller/services/services.go @@ -17,6 +17,8 @@ limitations under the License. package services import ( + "crypto/rand" + "encoding/base64" "errors" "strconv" "strings" @@ -79,6 +81,9 @@ func (feast *FeastServices) Deploy() error { if err := feast.createServiceAccount(); err != nil { return err } + if err := feast.createIntraCommunicationConfigMap(); err != nil { + return err + } if err := feast.createDeployment(); err != nil { return err } @@ -343,6 +348,38 @@ func (feast *FeastServices) createServiceAccount() error { return nil } +// GetIntraCommunicationConfigMapName returns the name of the ConfigMap holding the intra-communication token. +func GetIntraCommunicationConfigMapName(featureStoreName string) string { + return handler.FeastPrefix + featureStoreName + "-intra-comm" +} + +func (feast *FeastServices) createIntraCommunicationConfigMap() error { + logger := log.FromContext(feast.Handler.Context) + cr := feast.Handler.FeatureStore + cmName := GetIntraCommunicationConfigMapName(cr.Name) + cm := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{Name: cmName, Namespace: cr.Namespace}, + } + if op, err := controllerutil.CreateOrUpdate(feast.Handler.Context, feast.Handler.Client, cm, controllerutil.MutateFn(func() error { + if cm.Data == nil || cm.Data[intraCommunicationTokenKey] == "" { + b := make([]byte, 32) + if _, err := rand.Read(b); err != nil { + return err + } + cm.Data = map[string]string{ + intraCommunicationTokenKey: base64.StdEncoding.EncodeToString(b), + } + } + cm.Labels = feast.getLabels() + return controllerutil.SetControllerReference(cr, cm, feast.Handler.Scheme) + })); err != nil { + return err + } else if op == controllerutil.OperationResultCreated || op == controllerutil.OperationResultUpdated { + logger.Info("Successfully reconciled", "ConfigMap", cmName, "operation", op) + } + return nil +} + func (feast *FeastServices) createDeployment() error { logger := log.FromContext(feast.Handler.Context) deploy := feast.initFeastDeploy() @@ -477,7 +514,8 @@ func (feast *FeastServices) setContainer(containers *[]corev1.Container, feastTy name := string(feastType) workingDir := feast.getFeatureRepoDir() cmd := feast.getContainerCommand(feastType) - container := getContainer(name, workingDir, cmd, serverConfigs.ContainerConfigs, fsYamlB64) + intraCommCMName := GetIntraCommunicationConfigMapName(feast.Handler.FeatureStore.Name) + container := getContainer(name, workingDir, cmd, serverConfigs.ContainerConfigs, fsYamlB64, intraCommCMName) tls := feast.getTlsConfigs(feastType) probeHandler := feast.getProbeHandler(feastType, tls) container.Ports = []corev1.ContainerPort{} @@ -534,7 +572,7 @@ func (feast *FeastServices) setContainer(containers *[]corev1.Container, feastTy } } -func getContainer(name, workingDir string, cmd []string, containerConfigs feastdevv1.ContainerConfigs, fsYamlB64 string) *corev1.Container { +func getContainer(name, workingDir string, cmd []string, containerConfigs feastdevv1.ContainerConfigs, fsYamlB64, intraCommCMName string) *corev1.Container { container := &corev1.Container{ Name: name, Command: cmd, @@ -542,13 +580,22 @@ func getContainer(name, workingDir string, cmd []string, containerConfigs feastd if len(workingDir) > 0 { container.WorkingDir = workingDir } - if len(fsYamlB64) > 0 { - container.Env = []corev1.EnvVar{ - { - Name: TmpFeatureStoreYamlEnvVar, - Value: fsYamlB64, + container.Env = []corev1.EnvVar{ + { + Name: IntraCommunicationBase64EnvVar, + ValueFrom: &corev1.EnvVarSource{ + ConfigMapKeyRef: &corev1.ConfigMapKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{Name: intraCommCMName}, + Key: intraCommunicationTokenKey, + }, }, - } + }, + } + if len(fsYamlB64) > 0 { + container.Env = append(container.Env, corev1.EnvVar{ + Name: TmpFeatureStoreYamlEnvVar, + Value: fsYamlB64, + }) } applyCtrConfigs(container, containerConfigs) return container diff --git a/infra/feast-operator/internal/controller/services/services_types.go b/infra/feast-operator/internal/controller/services/services_types.go index 098362af96b..0e7694e52ae 100644 --- a/infra/feast-operator/internal/controller/services/services_types.go +++ b/infra/feast-operator/internal/controller/services/services_types.go @@ -25,15 +25,17 @@ import ( ) const ( - TmpFeatureStoreYamlEnvVar = "TMP_FEATURE_STORE_YAML_BASE64" - feastServerImageVar = "RELATED_IMAGE_FEATURE_SERVER" - cronJobImageVar = "RELATED_IMAGE_CRON_JOB" - FeatureStoreYamlCmKey = "feature_store.yaml" - EphemeralPath = "/feast-data" - FeatureRepoDir = "feature_repo" - DefaultRegistryPath = "registry.db" - DefaultOnlineStorePath = "online_store.db" - svcDomain = ".svc.cluster.local" + TmpFeatureStoreYamlEnvVar = "TMP_FEATURE_STORE_YAML_BASE64" + IntraCommunicationBase64EnvVar = "INTRA_COMMUNICATION_BASE64" + intraCommunicationTokenKey = "token" + feastServerImageVar = "RELATED_IMAGE_FEATURE_SERVER" + cronJobImageVar = "RELATED_IMAGE_CRON_JOB" + FeatureStoreYamlCmKey = "feature_store.yaml" + EphemeralPath = "/feast-data" + FeatureRepoDir = "feature_repo" + DefaultRegistryPath = "registry.db" + DefaultOnlineStorePath = "online_store.db" + svcDomain = ".svc.cluster.local" // Namespace registry ConfigMap constants NamespaceRegistryConfigMapName = "feast-configs-registry" diff --git a/infra/feast-operator/test/e2e_rhoai/e2e_suite_test.go b/infra/feast-operator/test/e2e_rhoai/e2e_suite_test.go new file mode 100644 index 00000000000..14b1dae2c59 --- /dev/null +++ b/infra/feast-operator/test/e2e_rhoai/e2e_suite_test.go @@ -0,0 +1,32 @@ +/* +Copyright 2025 Feast Community. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2erhoai + +import ( + "fmt" + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +// Run e2e RHOAI feast tests using the Ginkgo runner. +func TestFeastE2ETestSuite(t *testing.T) { + RegisterFailHandler(Fail) + _, _ = fmt.Fprintf(GinkgoWriter, "Feast E2E Test suite\n") + RunSpecs(t, "e2erhoai Feast E2E test suite") +} diff --git a/infra/feast-operator/test/e2e_rhoai/feast_oidc_test.go b/infra/feast-operator/test/e2e_rhoai/feast_oidc_test.go new file mode 100644 index 00000000000..6cd6e6cc504 --- /dev/null +++ b/infra/feast-operator/test/e2e_rhoai/feast_oidc_test.go @@ -0,0 +1,131 @@ +/* +Copyright 2026 Feast Community. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package e2erhoai provides end-to-end (E2E) test coverage for Feast integration with +// Red Hat OpenShift AI (RHOAI) environments. +// This test validates Feast workbench integration with OIDC authentication, +// verifying that OIDC token-based auth works correctly for Feast operations. +package e2erhoai + +import ( + "fmt" + "time" + + . "github.com/feast-dev/feast/infra/feast-operator/test/e2e_rhoai/utils" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("Feast OIDC Authentication Integration Testing", Ordered, func() { + const ( + namespace = "test-ns-feast-oidc" + configMapName = "feast-oidc-wb-cm" + rolebindingName = "rb-feast-oidc-test" + notebookFile = "test/e2e_rhoai/resources/feast-oidc-auth-test.ipynb" + pvcFile = "test/e2e_rhoai/resources/pvc.yaml" + permissionFile = "test/e2e_rhoai/resources/permissions_oidc.py" + notebookPVC = "jupyterhub-nb-kube-3aadmin-pvc" + testDir = "/test/e2e_rhoai" + notebookName = "feast-oidc-auth-test.ipynb" + feastDeploymentName = FeastPrefix + "test-feast-oidc" + feastCRName = "test-feast-oidc" + feastProject = "driver_ranking_oidc" + feastOidcYaml = "test/e2e_rhoai/resources/feast_oidc.yaml" + feastConfigMapName = "jupyter-nb-kube-3aadmin-feast-config" + feastClientConfigMapKey = "driver_ranking_oidc" + ) + + // Verify feast ConfigMap contains OIDC auth type + verifyFeastOidcConfigMap := func() { + By(fmt.Sprintf("Listing ConfigMaps and verifying %s exists with OIDC auth", feastConfigMapName)) + + expectedContent := []string{ + "project: driver_ranking_oidc", + "type: oidc", + } + + const maxRetries = 5 + const retryInterval = 5 * time.Second + var configMapExists bool + var err error + + for i := 0; i < maxRetries; i++ { + exists, listErr := VerifyConfigMapExistsInList(namespace, feastConfigMapName) + if listErr != nil { + err = listErr + if i < maxRetries-1 { + fmt.Printf("Failed to list ConfigMaps, retrying in %v... (attempt %d/%d)\n", retryInterval, i+1, maxRetries) + time.Sleep(retryInterval) + continue + } + } else if exists { + configMapExists = true + fmt.Printf("ConfigMap %s found in ConfigMap list\n", feastConfigMapName) + break + } + + if i < maxRetries-1 { + fmt.Printf("ConfigMap %s not found in list yet, retrying in %v... (attempt %d/%d)\n", feastConfigMapName, retryInterval, i+1, maxRetries) + time.Sleep(retryInterval) + } + } + + if !configMapExists { + Expect(err).ToNot(HaveOccurred(), fmt.Sprintf("Failed to find ConfigMap %s in ConfigMap list after %d attempts: %v", feastConfigMapName, maxRetries, err)) + } + + err = VerifyFeastConfigMapContent(namespace, feastConfigMapName, feastClientConfigMapKey, expectedContent) + Expect(err).ToNot(HaveOccurred(), fmt.Sprintf("Failed to verify Feast ConfigMap %s content: %v", feastConfigMapName, err)) + fmt.Printf("Feast ConfigMap %s verified successfully with OIDC auth type\n", feastConfigMapName) + } + + BeforeAll(func() { + By(fmt.Sprintf("Creating test namespace: %s", namespace)) + Expect(CreateNamespace(namespace, testDir)).To(Succeed()) + fmt.Printf("Namespace %s created successfully\n", namespace) + }) + + AfterAll(func() { + By(fmt.Sprintf("Deleting test namespace: %s", namespace)) + Expect(DeleteNamespace(namespace, testDir)).To(Succeed()) + fmt.Printf("Namespace %s deleted successfully\n", namespace) + }) + + Context("Feast OIDC Authentication Tests", func() { + BeforeEach(func() { + By("Applying and validating the OIDC FeatureStore CR") + ApplyFeastOidcYamlAndVerify(namespace, testDir, feastDeploymentName, feastCRName, feastOidcYaml) + + By("Verify Feature Store CR is in Ready state") + ValidateFeatureStoreCRStatus(namespace, feastCRName) + }) + + It("Should apply OIDC permissions and verify Feast methods with OIDC auth", func() { + By("Applying Feast OIDC permissions") + ApplyFeastOidcPermissions(permissionFile, "/feast-data/driver_ranking_oidc/feature_repo/permissions.py", namespace, feastDeploymentName) + + By("Creating notebook with OIDC token env var") + CreateOidcNotebookTest(namespace, configMapName, notebookFile, "test/e2e_rhoai/resources/feature_repo", + pvcFile, rolebindingName, notebookPVC, notebookName, testDir, feastProject) + + By("Verifying Feast ConfigMap was created with OIDC auth type") + verifyFeastOidcConfigMap() + + By("Monitoring notebook execution") + MonitorNotebookTest(namespace, notebookName) + }) + }) +}) diff --git a/infra/feast-operator/test/e2e_rhoai/feast_postupgrade_test.go b/infra/feast-operator/test/e2e_rhoai/feast_postupgrade_test.go new file mode 100644 index 00000000000..1649705f25f --- /dev/null +++ b/infra/feast-operator/test/e2e_rhoai/feast_postupgrade_test.go @@ -0,0 +1,73 @@ +/* +Copyright 2025 Feast Community. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2erhoai + +import ( + "fmt" + + . "github.com/feast-dev/feast/infra/feast-operator/test/e2e_rhoai/utils" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("Feast PostUpgrade scenario Testing", Ordered, func() { + const ( + namespace = "test-ns-feast-upgrade" + testDir = "/test/e2e_rhoai" + feastDeploymentName = FeastPrefix + "test-s3" + feastCRName = "test-s3" + ) + + AfterAll(func() { + By(fmt.Sprintf("Deleting test namespace: %s", namespace)) + Expect(DeleteNamespace(namespace, testDir)).To(Succeed()) + fmt.Printf("Namespace %s deleted successfully\n", namespace) + }) + runPostUpgradeTest := func() { + + By("Checking if the Feast deployment is available after upgrade") + CheckDeployment(namespace, feastDeploymentName) + + By("Verify Feature Store CR is in Ready state") + ValidateFeatureStoreCRStatus(namespace, feastCRName) + + By("Validating feature_store.yaml contains S3 registry path and driver_ranking project") + ValidateFeatureStoreYamlS3(namespace, feastDeploymentName) + + By("Validating pre-upgrade registry objects are intact in S3 (before feast apply)") + ValidateRegistryIntact(namespace, feastDeploymentName, testDir) + + By("Validating materialization intervals (last_updated_timestamp) are preserved post-upgrade") + ValidateMaterializationIntervals(namespace, feastDeploymentName, testDir) + + By("Running `feast apply` and `feast materialize-incremental` to validate registry definitions (S3 / driver_ranking)") + VerifyApplyFeatureStoreDefinitionsS3(namespace, feastCRName, feastDeploymentName) + + By("Validating Feast project list for driver_ranking") + VerifyFeastMethodsForDriverRanking(namespace, feastDeploymentName, testDir) + + By("Verifying online feature serving is queryable post-upgrade and post-materialization") + VerifyOnlineFeatureServing(namespace, feastDeploymentName, testDir) + + } + + // This context verifies that a pre-created Feast FeatureStore CR continues to function as expected + // after an upgrade. It validates `feast apply`, registry sync, feature retrieval, and model execution. + Context("Feast post Upgrade Test", func() { + It("Should run a feastPostUpgrade test scenario for S3 test-s3 FeatureStore apply and materialize successfully", runPostUpgradeTest) + }) +}) diff --git a/infra/feast-operator/test/e2e_rhoai/feast_preupgrade_test.go b/infra/feast-operator/test/e2e_rhoai/feast_preupgrade_test.go new file mode 100644 index 00000000000..16e9796c820 --- /dev/null +++ b/infra/feast-operator/test/e2e_rhoai/feast_preupgrade_test.go @@ -0,0 +1,57 @@ +/* +Copyright 2025 Feast Community. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2erhoai + +import ( + "fmt" + + . "github.com/feast-dev/feast/infra/feast-operator/test/e2e_rhoai/utils" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("Feast PreUpgrade scenario Testing", Ordered, func() { + const ( + namespace = "test-ns-feast-upgrade" + testDir = "/test/e2e_rhoai" + ) + + BeforeAll(func() { + By(fmt.Sprintf("Creating test namespace: %s", namespace)) + Expect(CreateNamespace(namespace, testDir)).To(Succeed()) + fmt.Printf("Namespace %s created successfully\n", namespace) + }) + + AfterAll(func() { + // Only delete namespace on failure; successful runs preserve resources for post-upgrade verification + if CurrentSpecReport().Failed() { + By(fmt.Sprintf("Deleting test namespace: %s", namespace)) + Expect(DeleteNamespace(namespace, testDir)).To(Succeed()) + fmt.Printf("Namespace %s deleted successfully\n", namespace) + } + }) + + runPreUpgradeTest := func() { + By("Applying Feast S3 manifest (no postgres/redis) and verifying setup") + ApplyFeastS3YamlAndVerify(namespace, testDir, "test/e2e_rhoai/resources/feast_s3.yaml", "test-s3") + } + + // This context ensures the Feast CR setup is functional prior to any upgrade + Context("Feast Pre Upgrade Test", func() { + It("Should create and run a feastPreUpgrade test scenario feast S3 (test-s3) FeatureStore setup successfully", runPreUpgradeTest) + }) +}) diff --git a/infra/feast-operator/test/e2e_rhoai/feast_wb_connection_integration_test.go b/infra/feast-operator/test/e2e_rhoai/feast_wb_connection_integration_test.go new file mode 100644 index 00000000000..98208884c88 --- /dev/null +++ b/infra/feast-operator/test/e2e_rhoai/feast_wb_connection_integration_test.go @@ -0,0 +1,168 @@ +/* +Copyright 2025 Feast Community. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package e2erhoai provides end-to-end (E2E) test coverage for Feast integration with +// Red Hat OpenShift AI (RHOAI) environments. +// This specific test validates the functionality +// of executing a Feast workbench integration connection with kubernetes auth and without auth successfully +package e2erhoai + +import ( + "fmt" + "time" + + . "github.com/feast-dev/feast/infra/feast-operator/test/e2e_rhoai/utils" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("Feast Workbench Integration Connection Testing", Ordered, func() { + const ( + namespace = "test-ns-feast" + configMapName = "feast-wb-cm" + rolebindingName = "rb-feast-test" + notebookFile = "test/e2e_rhoai/resources/feast-wb-connection-credit-scoring.ipynb" + pvcFile = "test/e2e_rhoai/resources/pvc.yaml" + permissionFile = "test/e2e_rhoai/resources/permissions.py" + notebookPVC = "jupyterhub-nb-kube-3aadmin-pvc" + testDir = "/test/e2e_rhoai" + notebookName = "feast-wb-connection-credit-scoring.ipynb" + feastDeploymentName = FeastPrefix + "credit-scoring" + feastCRName = "credit-scoring" + ) + + // Verify feast ConfigMap + verifyFeastConfigMap := func(authEnabled bool) { + feastConfigMapName := "jupyter-nb-kube-3aadmin-feast-config" + configMapKey := "credit_scoring_local" + By(fmt.Sprintf("Listing ConfigMaps and verifying %s exists with correct content", feastConfigMapName)) + + // Build expected content based on auth type + expectedContent := []string{ + "project: credit_scoring_local", + } + if authEnabled { + expectedContent = append(expectedContent, "type: kubernetes") + } else { + expectedContent = append(expectedContent, "type: no_auth") + } + + // First, list ConfigMaps and check if target ConfigMap exists + // Retry with polling since the ConfigMap may be created asynchronously + const maxRetries = 5 + const retryInterval = 5 * time.Second + var configMapExists bool + var err error + + for i := 0; i < maxRetries; i++ { + exists, listErr := VerifyConfigMapExistsInList(namespace, feastConfigMapName) + if listErr != nil { + err = listErr + if i < maxRetries-1 { + fmt.Printf("Failed to list ConfigMaps, retrying in %v... (attempt %d/%d)\n", retryInterval, i+1, maxRetries) + time.Sleep(retryInterval) + continue + } + } else if exists { + configMapExists = true + fmt.Printf("ConfigMap %s found in ConfigMap list\n", feastConfigMapName) + break + } + + if i < maxRetries-1 { + fmt.Printf("ConfigMap %s not found in list yet, retrying in %v... (attempt %d/%d)\n", feastConfigMapName, retryInterval, i+1, maxRetries) + time.Sleep(retryInterval) + } + } + + if !configMapExists { + Expect(err).ToNot(HaveOccurred(), fmt.Sprintf("Failed to find ConfigMap %s in ConfigMap list after %d attempts: %v", feastConfigMapName, maxRetries, err)) + } + + // Once ConfigMap exists in list, verify content (project name and auth type) + err = VerifyFeastConfigMapContent(namespace, feastConfigMapName, configMapKey, expectedContent) + Expect(err).ToNot(HaveOccurred(), fmt.Sprintf("Failed to verify Feast ConfigMap %s content: %v", feastConfigMapName, err)) + fmt.Printf("Feast ConfigMap %s verified successfully with project and auth type\n", feastConfigMapName) + } + + // Parameterized test function that handles both auth and non-auth scenarios + runFeastWorkbenchIntegration := func(authEnabled bool) { + // Apply permissions only if auth is enabled + if authEnabled { + By("Applying Feast permissions for kubernetes authenticated scenario") + ApplyFeastPermissions(permissionFile, "/feast-data/credit_scoring_local/feature_repo/permissions.py", namespace, feastDeploymentName) + } + + // Create notebook with all setup steps + // Pass feastProject parameter to set the opendatahub.io/feast-config annotation + CreateNotebookTest(namespace, configMapName, notebookFile, "test/e2e_rhoai/resources/feature_repo", pvcFile, rolebindingName, notebookPVC, notebookName, testDir, "credit_scoring_local") + + // Verify Feast ConfigMap was created with correct auth type + verifyFeastConfigMap(authEnabled) + + // Monitor notebook execution + MonitorNotebookTest(namespace, notebookName) + } + + BeforeAll(func() { + By(fmt.Sprintf("Creating test namespace: %s", namespace)) + Expect(CreateNamespace(namespace, testDir)).To(Succeed()) + fmt.Printf("Namespace %s created successfully\n", namespace) + + By("Applying Feast infra manifests and verifying setup") + ApplyFeastInfraManifestsAndVerify(namespace, testDir) + }) + + AfterAll(func() { + By(fmt.Sprintf("Deleting test namespace: %s", namespace)) + Expect(DeleteNamespace(namespace, testDir)).To(Succeed()) + fmt.Printf("Namespace %s deleted successfully\n", namespace) + }) + + Context("Feast Workbench Integration Tests - Without Auth", func() { + BeforeEach(func() { + By("Applying and validating the credit-scoring FeatureStore CR without auth") + ApplyFeastYamlAndVerify(namespace, testDir, feastDeploymentName, feastCRName, "test/testdata/feast_integration_test_crs/feast.yaml") + + By("Verify Feature Store CR is in Ready state") + ValidateFeatureStoreCRStatus(namespace, feastCRName) + + By("Running `feast apply` and `feast materialize-incremental` to validate registry definitions") + VerifyApplyFeatureStoreDefinitions(namespace, feastCRName, feastDeploymentName) + }) + + It("Should create and run a FeastWorkbenchIntegrationWithoutAuth scenario successfully", func() { + runFeastWorkbenchIntegration(false) + }) + }) + + Context("Feast Workbench Integration Tests - With Auth", func() { + BeforeEach(func() { + By("Applying and validating the credit-scoring FeatureStore CR (with auth)") + ApplyFeastYamlAndVerify(namespace, testDir, feastDeploymentName, feastCRName, "test/e2e_rhoai/resources/feast_kube_auth.yaml") + + By("Verify Feature Store CR is in Ready state") + ValidateFeatureStoreCRStatus(namespace, feastCRName) + + By("Running `feast apply` and `feast materialize-incremental` to validate registry definitions") + VerifyApplyFeatureStoreDefinitions(namespace, feastCRName, feastDeploymentName) + }) + + It("Should create and run a FeastWorkbenchIntegrationWithAuth scenario successfully", func() { + runFeastWorkbenchIntegration(true) + }) + }) +}) diff --git a/infra/feast-operator/test/e2e_rhoai/feast_wb_milvus_test.go b/infra/feast-operator/test/e2e_rhoai/feast_wb_milvus_test.go new file mode 100644 index 00000000000..288f10285d7 --- /dev/null +++ b/infra/feast-operator/test/e2e_rhoai/feast_wb_milvus_test.go @@ -0,0 +1,65 @@ +/* +Copyright 2025 Feast Community. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package e2erhoai provides end-to-end (E2E) test coverage for Feast integration with +// Red Hat OpenShift AI (RHOAI) environments. This specific test validates the functionality +// of executing a Feast Jupyter notebook within a fully configured OpenShift namespace +package e2erhoai + +import ( + "fmt" + + . "github.com/feast-dev/feast/infra/feast-operator/test/e2e_rhoai/utils" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("Feast Jupyter Notebook Testing", Ordered, func() { + const ( + namespace = "test-ns-feast-wb" + configMapName = "feast-wb-cm" + rolebindingName = "rb-feast-test" + notebookFile = "test/e2e_rhoai/resources/feast-wb-milvus-test.ipynb" + pvcFile = "test/e2e_rhoai/resources/pvc.yaml" + notebookPVC = "jupyterhub-nb-kube-3aadmin-pvc" + testDir = "/test/e2e_rhoai" + notebookName = "feast-wb-milvus-test.ipynb" + feastMilvusTest = "TestFeastMilvusNotebook" + ) + + BeforeAll(func() { + By(fmt.Sprintf("Creating test namespace: %s", namespace)) + Expect(CreateNamespace(namespace, testDir)).To(Succeed()) + fmt.Printf("Namespace %s created successfully\n", namespace) + }) + + AfterAll(func() { + By(fmt.Sprintf("Deleting test namespace: %s", namespace)) + Expect(DeleteNamespace(namespace, testDir)).To(Succeed()) + fmt.Printf("Namespace %s deleted successfully\n", namespace) + }) + + Context("Feast Jupyter Notebook Test", func() { + It("Should create and run a "+feastMilvusTest+" successfully", func() { + // Create notebook with all setup steps + // Pass empty string for feastProject to keep annotation empty + CreateNotebookTest(namespace, configMapName, notebookFile, "test/e2e_rhoai/resources/feature_repo", pvcFile, rolebindingName, notebookPVC, notebookName, testDir, "") + + // Monitor notebook execution + MonitorNotebookTest(namespace, notebookName) + }) + }) +}) diff --git a/infra/feast-operator/test/e2e_rhoai/feast_wb_ray_offline_store_test.go b/infra/feast-operator/test/e2e_rhoai/feast_wb_ray_offline_store_test.go new file mode 100644 index 00000000000..a936901e08b --- /dev/null +++ b/infra/feast-operator/test/e2e_rhoai/feast_wb_ray_offline_store_test.go @@ -0,0 +1,83 @@ +/* +Copyright 2025 Feast Community. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package e2erhoai provides end-to-end (E2E) test coverage for Feast integration with +// Red Hat OpenShift AI (RHOAI) environments. This specific test validates the functionality +// of executing a Feast Jupyter notebook with Ray offline store within a fully configured OpenShift namespace +package e2erhoai + +import ( + "fmt" + "os/exec" + + . "github.com/feast-dev/feast/infra/feast-operator/test/e2e_rhoai/utils" + testutils "github.com/feast-dev/feast/infra/feast-operator/test/utils" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("Feast Jupyter Notebook Testing with Ray Offline Store", Ordered, func() { + const ( + namespace = "test-ns-feast-wb-ray" + configMapName = "feast-wb-ray-cm" + rolebindingName = "rb-feast-ray-test" + notebookFile = "test/e2e_rhoai/resources/feast-wb-ray-test.ipynb" + pvcFile = "test/e2e_rhoai/resources/pvc.yaml" + kueueResourcesFile = "test/e2e_rhoai/resources/kueue_resources_setup.yaml" + notebookPVC = "jupyterhub-nb-kube-3aadmin-pvc" + testDir = "/test/e2e_rhoai" + notebookName = "feast-wb-ray-test.ipynb" + feastRayTest = "TestFeastRayOfflineStoreNotebook" + ) + + BeforeAll(func() { + By(fmt.Sprintf("Creating test namespace: %s", namespace)) + Expect(CreateNamespace(namespace, testDir)).To(Succeed()) + fmt.Printf("Namespace %s created successfully\n", namespace) + + By("Applying Kueue resources setup") + // Apply with namespace flag - cluster-scoped resources (ResourceFlavor, ClusterQueue) will be applied at cluster level, + // and namespace-scoped resources (LocalQueue) will be applied in the specified namespace + cmd := exec.Command("kubectl", "apply", "-f", kueueResourcesFile, "-n", namespace) + output, err := testutils.Run(cmd, testDir) + Expect(err).ToNot(HaveOccurred(), fmt.Sprintf("Failed to apply Kueue resources: %v\nOutput: %s", err, output)) + fmt.Printf("Kueue resources applied successfully\n") + }) + + AfterAll(func() { + By("Deleting Kueue resources") + // Delete with namespace flag - will delete namespace-scoped resources from the namespace + // and cluster-scoped resources from the cluster + cmd := exec.Command("kubectl", "delete", "-f", kueueResourcesFile, "-n", namespace, "--ignore-not-found=true", "--timeout=60s") + _, _ = testutils.Run(cmd, testDir) + fmt.Printf("Kueue resources cleanup completed\n") + + By(fmt.Sprintf("Deleting test namespace: %s", namespace)) + Expect(DeleteNamespace(namespace, testDir)).To(Succeed()) + fmt.Printf("Namespace %s deleted successfully\n", namespace) + }) + + Context("Feast Jupyter Notebook Test with Ray Offline store", func() { + It("Should create and run a "+feastRayTest+" successfully", func() { + // Create notebook with all setup steps + // Pass empty string for feastProject to keep annotation empty + CreateNotebookTest(namespace, configMapName, notebookFile, "test/e2e_rhoai/resources/feature_repo", pvcFile, rolebindingName, notebookPVC, notebookName, testDir, "") + + // Monitor notebook execution + MonitorNotebookTest(namespace, notebookName) + }) + }) +}) diff --git a/infra/feast-operator/test/e2e_rhoai/resources/custom-nb.yaml b/infra/feast-operator/test/e2e_rhoai/resources/custom-nb.yaml new file mode 100644 index 00000000000..18473fb0d7f --- /dev/null +++ b/infra/feast-operator/test/e2e_rhoai/resources/custom-nb.yaml @@ -0,0 +1,96 @@ +# This template maybe used to spin up a custom notebook image +# i.e.: sed s/{{.IngressDomain}}/$(oc get ingresses.config/cluster -o jsonpath={.spec.domain})/g tests/resources/custom-nb.template | oc apply -f - +# resources generated: +# pod/jupyter-nb-kube-3aadmin-0 +# service/jupyter-nb-kube-3aadmin +# route.route.openshift.io/jupyter-nb-kube-3aadmin (jupyter-nb-kube-3aadmin-opendatahub.apps.tedbig412.cp.fyre.ibm.com) +# service/jupyter-nb-kube-3aadmin-tls +apiVersion: kubeflow.org/v1 +kind: Notebook +metadata: + annotations: + notebooks.opendatahub.io/inject-auth: "true" + notebooks.opendatahub.io/last-size-selection: Small + opendatahub.io/link: https://jupyter-nb-kube-3aadmin-{{.Namespace}}.{{.IngressDomain}}/notebook/{{.Namespace}}/jupyter-nb-kube-3aadmin + opendatahub.io/username: {{.Username}} + opendatahub.io/feast-config: {{.FeastProject}} + generation: 1 + labels: + app: jupyter-nb-kube-3aadmin + opendatahub.io/dashboard: "true" + opendatahub.io/odh-managed: "true" + opendatahub.io/user: {{.Username}} + opendatahub.io/feast-integration: 'true' + name: jupyter-nb-kube-3aadmin + namespace: {{.Namespace}} +spec: + template: + spec: + affinity: + nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - preference: + matchExpressions: + - key: nvidia.com/gpu.present + operator: NotIn + values: + - "true" + weight: 1 + containers: + - env: + - name: NOTEBOOK_ARGS + value: |- + --ServerApp.port=8888 + --ServerApp.token='' + --ServerApp.password='' + --ServerApp.base_url=/notebook/{{.Namespace}}/jupyter-nb-kube-3aadmin + --ServerApp.quit_button=False + --ServerApp.tornado_settings={"user":"{{.Username}}","hub_host":"https://odh-dashboard-{{.OpenDataHubNamespace}}.{{.IngressDomain}}","hub_prefix":"/notebookController/{{.Username}}"} + - name: JUPYTER_IMAGE + value: {{.NotebookImage}} + - name: JUPYTER_NOTEBOOK_PORT + value: "8888" + - name: PIP_INDEX_URL + value: {{.PipIndexUrl}} + - name: PIP_TRUSTED_HOST + value: {{.PipTrustedHost}} + - name: FEAST_VERSION + value: {{.FeastVersion}} + - name: OPENAI_API_KEY + value: {{.OpenAIAPIKey}} + - name: NAMESPACE + value: {{.Namespace}} +{{- range $name, $value := .AdditionalEnv }} + - name: {{$name}} + value: {{$value}} +{{- end }} + image: {{.NotebookImage}} + command: {{.Command}} + imagePullPolicy: Always + name: jupyter-nb-kube-3aadmin + ports: + - containerPort: 8888 + name: notebook-port + protocol: TCP + resources: + limits: + cpu: "2" + memory: 3Gi + requests: + cpu: "1" + memory: 3Gi + volumeMounts: + - mountPath: /opt/app-root/src + name: jupyterhub-nb-kube-3aadmin-pvc + - mountPath: /opt/app-root/notebooks + name: {{.NotebookConfigMapName}} + workingDir: /opt/app-root/src + enableServiceLinks: false + serviceAccountName: default + volumes: + - name: jupyterhub-nb-kube-3aadmin-pvc + persistentVolumeClaim: + claimName: {{.NotebookPVC}} + - name: {{.NotebookConfigMapName}} + configMap: + name: {{.NotebookConfigMapName}} diff --git a/infra/feast-operator/test/e2e_rhoai/resources/feast-oidc-auth-test.ipynb b/infra/feast-operator/test/e2e_rhoai/resources/feast-oidc-auth-test.ipynb new file mode 100644 index 00000000000..ecc32adbdc0 --- /dev/null +++ b/infra/feast-operator/test/e2e_rhoai/resources/feast-oidc-auth-test.ipynb @@ -0,0 +1,504 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "id": "cell-4", + "metadata": {}, + "outputs": [], + "source": [ + "from feast import FeatureStore\n", + "fs_oidc = FeatureStore(fs_yaml_file='/opt/app-root/src/feast-config/driver_ranking_oidc')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c4c03bc6", + "metadata": {}, + "outputs": [], + "source": [ + "auth_config = fs_oidc.config.auth_config\n", + "auth_type = getattr(auth_config, 'auth_type', None) or getattr(auth_config, 'type', None)\n", + "assert auth_type == 'oidc', f\"Expected auth type 'oidc', got: '{auth_type}'\"\n", + "print(f\"✅ Auth type verified from FeatureStore config: {auth_type}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cell-7", + "metadata": {}, + "outputs": [], + "source": [ + "# Verify project\n", + "project_name = \"driver_ranking_oidc\"\n", + "project = fs_oidc.get_project(project_name)\n", + "\n", + "assert project is not None, f\"get_project('{project_name}') returned None\"\n", + "\n", + "if isinstance(project, dict):\n", + " returned_name = project.get(\"spec\", {}).get(\"name\")\n", + "else:\n", + " returned_name = getattr(project, \"name\", None)\n", + " if not returned_name and hasattr(project, \"spec\") and hasattr(project.spec, \"name\"):\n", + " returned_name = project.spec.name\n", + "\n", + "assert returned_name, f\"Returned project does not contain a valid name: {project}\"\n", + "assert returned_name == project_name, (\n", + " f\"Expected project '{project_name}', but got '{returned_name}'\"\n", + ")\n", + "\n", + "print(f\"get_project('{project_name}') validation passed!\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e26f3c0a", + "metadata": {}, + "outputs": [], + "source": [ + "# Validate get_entity\n", + "entity = fs_oidc.get_entity(\"driver\")\n", + "assert entity is not None, \"Entity 'driver' not found in registry\"\n", + "assert entity.name == \"driver\", f\"Entity name mismatch: {entity.name}\"\n", + "\n", + "join_keys = getattr(entity, \"join_keys\", None) or [getattr(entity, \"join_key\", \"\")]\n", + "assert \"driver_id\" in join_keys, f\"Expected join_key 'driver_id', got: {join_keys}\"\n", + "print(f\"✅ get_entity('driver'): name={entity.name}, join_keys={join_keys}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b7f470b2", + "metadata": {}, + "outputs": [], + "source": [ + "# driver_hourly_stats_fresh is the registered view (PushSource-backed)\n", + "fv = fs_oidc.get_feature_view(\"driver_hourly_stats_fresh\")\n", + "assert fv is not None, \"FeatureView 'driver_hourly_stats_fresh' not found in registry\"\n", + "assert fv.name == \"driver_hourly_stats_fresh\", f\"FeatureView name mismatch: {fv.name}\"\n", + "feature_names = [f.name for f in fv.features]\n", + "for expected in [\"conv_rate\", \"acc_rate\", \"avg_daily_trips\",\n", + " \"driver_metadata\", \"driver_config\", \"driver_profile\"]:\n", + " assert expected in feature_names, f\"'{expected}' not in features: {feature_names}\"\n", + "print(f\"✅ get_feature_view('driver_hourly_stats_fresh'): {len(fv.features)} features — {feature_names}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "80e6449f", + "metadata": {}, + "outputs": [], + "source": [ + "# Validate get_data_source\n", + "ds = fs_oidc.get_data_source(\"driver_hourly_stats_source\")\n", + "assert ds is not None, \"DataSource 'driver_hourly_stats_source' not found in registry\"\n", + "assert ds.name == \"driver_hourly_stats_source\", f\"DataSource name mismatch: {ds.name}\"\n", + "print(f\"✅ get_data_source('driver_hourly_stats_source'): {ds.name}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "0e4faeb0", + "metadata": {}, + "outputs": [], + "source": [ + "# Validate get_feature_service\n", + "for fs_name in [\"driver_activity_v1\", \"driver_activity_v2\", \"driver_activity_v3\"]:\n", + " svc = fs_oidc.get_feature_service(fs_name)\n", + " assert svc is not None, f\"FeatureService '{fs_name}' not found in registry\"\n", + " assert svc.name == fs_name, f\"FeatureService name mismatch: {svc.name}\"\n", + " print(f\"✅ get_feature_service('{fs_name}'): {len(svc.feature_view_projections)} projections\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8b2ccacd", + "metadata": {}, + "outputs": [], + "source": [ + "# Validate get_on_demand_feature_view\n", + "for odfv_name in [\"transformed_conv_rate\", \"transformed_conv_rate_fresh\"]:\n", + " odfv = fs_oidc.get_on_demand_feature_view(odfv_name)\n", + " assert odfv is not None, f\"OnDemandFeatureView '{odfv_name}' not found\"\n", + " if isinstance(odfv, dict):\n", + " odfv_returned_name = odfv.get(\"spec\", {}).get(\"name\")\n", + " else:\n", + " odfv_returned_name = getattr(odfv, \"name\", None)\n", + " assert odfv_returned_name == odfv_name, (\n", + " f\"OnDemandFeatureView name mismatch: {odfv_returned_name}\"\n", + " )\n", + " odfv_features = [f.name for f in getattr(odfv, \"features\", [])]\n", + " for expected in [\"conv_rate_plus_val1\", \"conv_rate_plus_val2\"]:\n", + " assert expected in odfv_features, (\n", + " f\"'{expected}' not in {odfv_name} features: {odfv_features}\"\n", + " )\n", + " print(f\"✅ get_on_demand_feature_view('{odfv_name}'): features={odfv_features}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "02256c21", + "metadata": {}, + "outputs": [], + "source": [ + "# Validate all list_* methods with OIDC auth.\n", + "non_empty_required = {\n", + " \"list_projects\",\n", + " \"list_entities\",\n", + " \"list_feature_views\",\n", + " \"list_all_feature_views\",\n", + " \"list_batch_feature_views\",\n", + " \"list_on_demand_feature_views\",\n", + " \"list_feature_services\",\n", + " \"list_data_sources\",\n", + "}\n", + "\n", + "feast_list_functions = [\n", + " \"list_projects\",\n", + " \"list_entities\",\n", + " \"list_feature_views\",\n", + " \"list_all_feature_views\",\n", + " \"list_batch_feature_views\",\n", + " \"list_on_demand_feature_views\",\n", + " \"list_feature_services\",\n", + " \"list_data_sources\",\n", + " \"list_saved_datasets\",\n", + "]\n", + "\n", + "def validate_list_method(fs_obj, method_name, require_non_empty=False):\n", + " assert hasattr(fs_obj, method_name), f\"Method not found: {method_name}\"\n", + " result = getattr(fs_obj, method_name)()\n", + " assert isinstance(result, list), (\n", + " f\"{method_name}() must return a list, got {type(result)}\"\n", + " )\n", + " if require_non_empty:\n", + " assert len(result) > 0, (\n", + " f\"{method_name}() returned an empty list — expected pre-applied data\"\n", + " )\n", + " print(f\"✅ {method_name}() returned {len(result)} items\")\n", + "\n", + "for m in feast_list_functions:\n", + " validate_list_method(fs_oidc, m, require_non_empty=(m in non_empty_required))\n", + "\n", + "print(\"\\nAll list_* methods validated successfully with OIDC auth\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "60a56715", + "metadata": {}, + "outputs": [], + "source": [ + "import time\n", + "import pandas as pd\n", + "\n", + "ts = pd.Timestamp.now(tz=\"UTC\")\n", + "online_write_df = pd.DataFrame({\n", + " \"driver_id\": [1001],\n", + " \"conv_rate\": [0.75],\n", + " \"acc_rate\": [0.91],\n", + " \"avg_daily_trips\": [42],\n", + " \"driver_metadata\": [{\"region\": \"west\"}],\n", + " \"driver_config\": [{\"level\": \"senior\"}],\n", + " \"driver_profile\": [{\"name\": \"John\", \"age\": \"30\"}],\n", + " \"event_timestamp\": [ts],\n", + " \"created\": [ts],\n", + "})\n", + "\n", + "fs_oidc.write_to_online_store(\n", + " feature_view_name=\"driver_hourly_stats_fresh\",\n", + " df=online_write_df,\n", + " allow_registry_cache=False,\n", + ")\n", + "print(\"✅ write_to_online_store('driver_hourly_stats_fresh') succeeded\")\n", + "\n", + "response = None\n", + "last_error = None\n", + "for attempt in range(1, 4):\n", + " try:\n", + " response = fs_oidc.get_online_features(\n", + " features=[\n", + " \"driver_hourly_stats_fresh:conv_rate\",\n", + " \"driver_hourly_stats_fresh:acc_rate\",\n", + " \"driver_hourly_stats_fresh:avg_daily_trips\",\n", + " ],\n", + " entity_rows=[{\"driver_id\": 1001}],\n", + " ).to_dict()\n", + " break\n", + " except Exception as exc:\n", + " last_error = exc\n", + " if attempt < 3:\n", + " print(f\"⚠️ Online read attempt {attempt}/3: {exc}\")\n", + " time.sleep(5)\n", + " else:\n", + " raise\n", + "\n", + "assert response is not None, f\"Online feature read failed: {last_error}\"\n", + "assert \"conv_rate\" in response, f\"'conv_rate' missing: {list(response.keys())}\"\n", + "assert \"acc_rate\" in response, f\"'acc_rate' missing: {list(response.keys())}\"\n", + "assert response[\"avg_daily_trips\"] == [42], (\n", + " f\"avg_daily_trips mismatch: {response['avg_daily_trips']}\"\n", + ")\n", + "print(f\"✅ get_online_features(): conv_rate={response['conv_rate']}, \"\n", + " f\"acc_rate={response['acc_rate']}, avg_daily_trips={response['avg_daily_trips']}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d3160e1e", + "metadata": {}, + "outputs": [], + "source": [ + "# Validate list_permissions returns the OIDC admin group permission\n", + "permissions = fs_oidc.list_permissions()\n", + "assert isinstance(permissions, list), (\n", + " f\"list_permissions() must return a list, got {type(permissions)}\"\n", + ")\n", + "assert len(permissions) > 0, \"list_permissions() returned empty — expected admin_group_permission\"\n", + "\n", + "permission_names = [p.name for p in permissions]\n", + "print(f\"Permissions found: {permission_names}\")\n", + "\n", + "assert \"admin_group_permission\" in permission_names, (\n", + " f\"Expected 'admin_group_permission' in permissions, but got: {permission_names}\"\n", + ")\n", + "print(\"✅ admin_group_permission exists in Feast OIDC permissions\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "090ac5ef", + "metadata": {}, + "outputs": [], + "source": [ + "# Validate get_historical_features with OIDC admin auth.\n", + "from datetime import datetime, timezone\n", + "import pandas as pd\n", + "\n", + "entity_df = pd.DataFrame({\n", + " \"driver_id\": [1001, 1002, 1003],\n", + " \"event_timestamp\": [datetime.now(tz=timezone.utc)] * 3,\n", + "})\n", + "\n", + "job = fs_oidc.get_historical_features(\n", + " entity_df=entity_df,\n", + " features=[\n", + " \"driver_hourly_stats_fresh:conv_rate\",\n", + " \"driver_hourly_stats_fresh:acc_rate\",\n", + " \"driver_hourly_stats_fresh:avg_daily_trips\",\n", + " ],\n", + ")\n", + "training_df = job.to_df()\n", + "\n", + "assert training_df is not None, \"get_historical_features() returned None\"\n", + "for col in [\"conv_rate\", \"acc_rate\", \"avg_daily_trips\"]:\n", + " assert col in training_df.columns, (\n", + " f\"'{col}' missing from historical features: {list(training_df.columns)}\"\n", + " )\n", + "print(f\"✅ get_historical_features(): {len(training_df)} rows\")\n", + "print(f\" Columns: {list(training_df.columns)}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9c8f1184", + "metadata": {}, + "outputs": [], + "source": [ + "# Validate materialize_incremental with OIDC admin auth.\n", + "\n", + "from datetime import datetime, timezone\n", + "\n", + "end_date = datetime.now(tz=timezone.utc)\n", + "\n", + "fs_oidc.materialize_incremental(\n", + " end_date=end_date,\n", + " feature_views=[\"driver_hourly_stats_fresh\"],\n", + ")\n", + "print(f\"✅ materialize_incremental(end_date={end_date.isoformat()}) succeeded with OIDC admin auth\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "79af11b8", + "metadata": {}, + "outputs": [], + "source": [ + "from feast.errors import (\n", + " FeatureServiceNotFoundException,\n", + " FeatureViewNotFoundException,\n", + " EntityNotFoundException,\n", + ")\n", + "\n", + "# delete_feature_service\n", + "for fs_name in [\"driver_activity_v1\", \"driver_activity_v2\", \"driver_activity_v3\"]:\n", + " fs_oidc.delete_feature_service(fs_name)\n", + " remaining = [s.name for s in fs_oidc.list_feature_services()]\n", + " assert fs_name not in remaining, (\n", + " f\"FeatureService '{fs_name}' still present after deletion\"\n", + " )\n", + " try:\n", + " fs_oidc.get_feature_service(fs_name)\n", + " raise AssertionError(f\"Expected get_feature_service('{fs_name}') to raise after deletion\")\n", + " except FeatureServiceNotFoundException:\n", + " print(f\"✅ delete_feature_service('{fs_name}'): confirmed FeatureServiceNotFoundException\")\n", + "\n", + "# delete_feature_view\n", + "for fv_name in [\"driver_hourly_stats_fresh\"]:\n", + " fs_oidc.delete_feature_view(fv_name)\n", + " remaining = [fv.name for fv in fs_oidc.list_feature_views()]\n", + " assert fv_name not in remaining, (\n", + " f\"FeatureView '{fv_name}' still present after deletion\"\n", + " )\n", + " try:\n", + " fs_oidc.get_feature_view(fv_name)\n", + " raise AssertionError(f\"Expected get_feature_view('{fv_name}') to raise after deletion\")\n", + " except FeatureViewNotFoundException:\n", + " print(f\"✅ delete_feature_view('{fv_name}'): confirmed FeatureViewNotFoundException\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "3f5964a3", + "metadata": {}, + "outputs": [], + "source": [ + "# Validate teardown with OIDC admin auth.\n", + "\n", + "fs_oidc.teardown()\n", + "print(\"✅ teardown() succeeded with OIDC admin auth\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "05940cf0", + "metadata": {}, + "outputs": [], + "source": [ + "# close() releases gRPC channels and registry connections.\n", + "fs_oidc.close()\n", + "print(\"✅ close() succeeded\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f4b3f65d", + "metadata": {}, + "outputs": [], + "source": [ + "import requests\n", + "\n", + "# Get feature server URL from the FeatureStore config already initialized\n", + "online_store_url = fs_oidc.config.online_store.path\n", + "assert online_store_url, \"Could not determine feature server URL from FeatureStore config\"\n", + "print(f\"Feature server URL: {online_store_url}\")\n", + "\n", + "# Send a request with an invalid token — OIDC must reject it with 401\n", + "response = requests.post(\n", + " f\"{online_store_url.rstrip('/')}/get-online-features\",\n", + " headers={\n", + " \"Authorization\": \"Bearer invalid-garbage-token\",\n", + " \"Content-Type\": \"application/json\",\n", + " },\n", + " json={\n", + " \"features\": [\"driver_hourly_stats_fresh:conv_rate\"],\n", + " \"entities\": {\"driver_id\": [1001]},\n", + " },\n", + " verify=False, # test cluster uses self-signed cert (verifySSL: false in CR)\n", + " timeout=10,\n", + ")\n", + "\n", + "assert response.status_code == 401, (\n", + " f\"Expected 401 Unauthorized for invalid OIDC token, \"\n", + " f\"got HTTP {response.status_code}: {response.text[:200]}\"\n", + ")\n", + "print(\"✅ Invalid token correctly rejected with HTTP 401\")\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e8d242a1", + "metadata": {}, + "outputs": [], + "source": [ + "response = requests.post(\n", + " f\"{online_store_url.rstrip('/')}/get-online-features\",\n", + " headers={\"Content-Type\": \"application/json\"}, # no Authorization header\n", + " json={\n", + " \"features\": [\"driver_hourly_stats_fresh:conv_rate\"],\n", + " \"entities\": {\"driver_id\": [1001]},\n", + " },\n", + " verify=False,\n", + " timeout=10,\n", + ")\n", + "assert response.status_code == 401, (\n", + " f\"Expected 401 for missing token, got HTTP {response.status_code}\"\n", + ")\n", + "print(\"✅ Missing Authorization header correctly rejected with HTTP 401\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d5fdf606", + "metadata": {}, + "outputs": [], + "source": [ + "response = requests.post(\n", + " f\"{online_store_url.rstrip('/')}/get-online-features\",\n", + " headers={\n", + " \"Authorization\": \"Basic dXNlcjpwYXNzd29yZA==\", # base64 user:password\n", + " \"Content-Type\": \"application/json\",\n", + " },\n", + " json={\n", + " \"features\": [\"driver_hourly_stats_fresh:conv_rate\"],\n", + " \"entities\": {\"driver_id\": [1001]},\n", + " },\n", + " verify=False,\n", + " timeout=10,\n", + ")\n", + "assert response.status_code == 401, (\n", + " f\"Expected 401 for wrong auth scheme, got HTTP {response.status_code}\"\n", + ")\n", + "print(\"✅ Wrong auth scheme (Basic) correctly rejected with HTTP 401\")" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbformat_minor": 5, + "version": "3.11.0" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/infra/feast-operator/test/e2e_rhoai/resources/feast-wb-connection-credit-scoring.ipynb b/infra/feast-operator/test/e2e_rhoai/resources/feast-wb-connection-credit-scoring.ipynb new file mode 100755 index 00000000000..7ea2116f69c --- /dev/null +++ b/infra/feast-operator/test/e2e_rhoai/resources/feast-wb-connection-credit-scoring.ipynb @@ -0,0 +1,633 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "import feast\n", + "\n", + "actual_version = feast.__version__\n", + "assert actual_version == os.environ.get(\"FEAST_VERSION\"), (\n", + " f\"❌ Feast version mismatch. Expected: {os.environ.get('FEAST_VERSION')}, Found: {actual_version}\"\n", + ")\n", + "print(f\"✅ Found Expected Feast version: {actual_version} in workbench\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# --- Configuration Variables ---\n", + "import os \n", + "\n", + "# Fetch token and server directly from oc CLI\n", + "import subprocess\n", + "\n", + "def oc(cmd_list):\n", + " \"\"\"Safely execute oc commands without shell injection risk.\"\"\"\n", + " return subprocess.check_output(cmd_list, shell=False).decode(\"utf-8\").strip()\n", + "\n", + "token = oc([\"oc\", \"whoami\", \"-t\"])\n", + "server = oc([\"oc\", \"whoami\", \"--show-server\"])\n", + "namespace = os.environ.get(\"NAMESPACE\")\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Safely execute oc login without shell injection risk \n", + "import subprocess\n", + "subprocess.check_output(\n", + " [\"oc\", \"login\", \"--token\", token, \"--server\", server],\n", + " shell=False\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Add user permission to namespace\n", + "import subprocess\n", + "\n", + "# Get current user safely\n", + "current_user = subprocess.check_output(\n", + " [\"oc\", \"whoami\"],\n", + " shell=False\n", + ").decode(\"utf-8\").strip()\n", + "\n", + "# Add role to user safely\n", + "subprocess.check_output(\n", + " [\"oc\", \"adm\", \"policy\", \"add-role-to-user\", \"admin\", current_user, \"-n\", namespace],\n", + " shell=False\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "import subprocess\n", + "\n", + "namespace = os.environ.get(\"NAMESPACE\") # read namespace from env\n", + "if not namespace:\n", + " raise ValueError(\"NAMESPACE environment variable is not set\")\n", + "\n", + "# Safely execute oc command and pipe through sed without shell injection risk\n", + "oc_process = subprocess.Popen(\n", + " [\"oc\", \"get\", \"configmap\", \"feast-credit-scoring-client\", \"-n\", namespace,\n", + " \"-o\", \"jsonpath={.data.feature_store\\\\.yaml}\"],\n", + " stdout=subprocess.PIPE,\n", + " stderr=subprocess.PIPE,\n", + " shell=False\n", + ")\n", + "sed_process = subprocess.Popen(\n", + " [\"sed\", \"s/\\\\\\\\n/\\\\n/g\"],\n", + " stdin=oc_process.stdout,\n", + " stdout=subprocess.PIPE,\n", + " stderr=subprocess.PIPE,\n", + " shell=False\n", + ")\n", + "oc_process.stdout.close()\n", + "yaml_content, _ = sed_process.communicate()\n", + "yaml_content = yaml_content.decode(\"utf-8\")\n", + "\n", + "# Save the configmap data into an environment variable (if needed)\n", + "os.environ[\"CONFIGMAP_DATA\"] = yaml_content" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from feast import FeatureStore\n", + "fs_credit_scoring_local = FeatureStore(fs_yaml_file='/opt/app-root/src/feast-config/credit_scoring_local')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "fs_credit_scoring_local.refresh_registry()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "project_name = \"credit_scoring_local\"\n", + "project = fs_credit_scoring_local.get_project(project_name)\n", + "\n", + "# 1. Assert object returned\n", + "assert project is not None, f\"❌ get_project('{project_name}') returned None\"\n", + "\n", + "# 2. Extract project name (works for dict or Feast object)\n", + "if isinstance(project, dict):\n", + " returned_name = project.get(\"spec\", {}).get(\"name\")\n", + "else:\n", + " # Feast Project object\n", + " returned_name = getattr(project, \"name\", None)\n", + " if not returned_name and hasattr(project, \"spec\") and hasattr(project.spec, \"name\"):\n", + " returned_name = project.spec.name\n", + "\n", + "# 3. Assert that name exists\n", + "assert returned_name, f\"❌ Returned project does not contain a valid name: {project}\"\n", + "\n", + "print(\"• Project Name Returned:\", returned_name)\n", + "\n", + "# 4. Assert the name matches expected\n", + "assert returned_name == project_name, (\n", + " f\"❌ Expected project '{project_name}', but got '{returned_name}'\"\n", + ")\n", + "\n", + "print(f\"\\n✓ get_project('{project_name}') validation passed!\")\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "feast_list_functions = [\n", + " \"list_projects\",\n", + " \"list_entities\",\n", + " \"list_feature_views\",\n", + " \"list_all_feature_views\",\n", + " \"list_batch_feature_views\",\n", + " \"list_on_demand_feature_views\",\n", + "]\n", + "\n", + "# validates feast list methods returns data and method type\n", + "def validate_list_method(fs_obj, method_name):\n", + " assert hasattr(fs_obj, method_name), f\"Method not found: {method_name}\"\n", + "\n", + " method = getattr(fs_obj, method_name)\n", + " result = method()\n", + "\n", + " assert isinstance(result, list), (\n", + " f\"{method_name}() must return a list, got {type(result)}\"\n", + " )\n", + " assert len(result) > 0, (\n", + " f\"{method_name}() returned an empty list — expected data\"\n", + " )\n", + "\n", + " print(f\"✓ {method_name}() returned {len(result)} items\")\n", + "\n", + "for m in feast_list_functions:\n", + " validate_list_method(fs_credit_scoring_local, m)\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "feast_list_functions = [\n", + " \"list_feature_services\",\n", + " # \"list_permissions\",\n", + " \"list_saved_datasets\",\n", + "]\n", + "\n", + "# validates feast methods exists and type is valid\n", + "def validate_list_func(fs_obj, method_name):\n", + " assert hasattr(fs_obj, method_name), f\"Method not found: {method_name}\"\n", + "\n", + " method = getattr(fs_obj, method_name)\n", + "\n", + " result = method()\n", + "\n", + " assert isinstance(result, list), (\n", + " f\"{method_name}() must return a list, got {type(result)}\"\n", + " )\n", + "\n", + "for m in feast_list_functions:\n", + " validate_list_func(fs_credit_scoring_local, m)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# validate_list_data_sources for with and without permissions \n", + "\n", + "import os\n", + "from feast.errors import FeastPermissionError\n", + "\n", + "def validate_list_data_sources(fs_obj):\n", + " \"\"\"\n", + " Validates list_data_sources() with special handling for Kubernetes auth mode.\n", + " If CONFIGMAP_DATA indicates auth=kubernetes, expect FeastPermissionError.\n", + " Otherwise validate output type normally.\n", + " \"\"\"\n", + " auth_mode = os.getenv(\"CONFIGMAP_DATA\")\n", + "\n", + " # Case 1: Kubernetes auth → expect permission error\n", + " if \"kubernetes\" in auth_mode.lower():\n", + " try:\n", + " fs_obj.list_data_sources()\n", + " raise AssertionError(\n", + " \"Expected FeastPermissionError due to Kubernetes auth, but the call succeeded.\"\n", + " )\n", + " except FeastPermissionError as e:\n", + " # Correct, this is expected\n", + " return\n", + " except Exception as e:\n", + " raise AssertionError(\n", + " f\"Expected FeastPermissionError, but got different exception: {type(e)} - {e}\"\n", + " )\n", + "\n", + " # Case 2: Non-Kubernetes auth → normal path\n", + " assert hasattr(fs_obj, \"list_data_sources\"), \"Method not found: list_data_sources\"\n", + " result = fs_obj.list_data_sources()\n", + " assert isinstance(result, list), (\n", + " f\"list_data_sources() must return a list, got {type(result)}\"\n", + " )\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "\n", + "entity = fs_credit_scoring_local.get_entity(\"dob_ssn\")\n", + "\n", + "assert entity is not None, \"❌ Entity 'dob_ssn' not found!\"\n", + "assert entity.name == \"dob_ssn\", f\"❌ Entity name mismatch: {entity.name}\"\n", + "\n", + "print(\"✓ Entity validation successful!\\n\", entity.name)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "from feast.errors import FeastPermissionError\n", + "\n", + "def validate_get_data_source(fs_obj, name: str):\n", + " auth_mode = os.getenv(\"CONFIGMAP_DATA\", \"\")\n", + "\n", + " print(\"📌 CONFIGMAP_DATA:\", auth_mode)\n", + "\n", + " # If Kubernetes auth is enabled → expect permission error\n", + " if \"auth\" in \"kubernetes\" in auth_mode.lower():\n", + " print(f\"🔒 Kubernetes auth detected, expecting permission error for get_data_source('{name}')\")\n", + "\n", + " try:\n", + " fs_obj.get_data_source(name)\n", + " raise AssertionError(\n", + " f\"❌ Expected FeastPermissionError when accessing data source '{name}', but call succeeded\"\n", + " )\n", + "\n", + " except FeastPermissionError as e:\n", + " print(f\"✅ Correctly blocked with FeastPermissionError: {e}\")\n", + " return\n", + "\n", + " except Exception as e:\n", + " raise AssertionError(\n", + " f\"❌ Expected FeastPermissionError but got {type(e)}: {e}\"\n", + " )\n", + "\n", + " # Otherwise → normal validation\n", + " print(f\"🔍 Fetching data source '{name}'...\")\n", + "\n", + " ds = fs_obj.get_data_source(name)\n", + "\n", + " print(\"\\n📌 Data Source Object:\")\n", + " print(ds)\n", + "\n", + " assert ds.name == name, (\n", + " f\"❌ Expected name '{name}', got '{ds.name}'\"\n", + " )\n", + "\n", + " print(f\"✅ Data source '{name}' exists and is correctly configured.\")\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import time\n", + "\n", + "feast_features = [\n", + " \"zipcode_features:city\",\n", + " \"zipcode_features:state\",\n", + "]\n", + "\n", + "entity_rows = [{\n", + " \"zipcode\": 1463,\n", + " \"dob_ssn\": \"19530219_5179\"\n", + "}]\n", + "\n", + "response = None\n", + "last_error = None\n", + "\n", + "for attempt in range(1, 4):\n", + " try:\n", + " response = fs_credit_scoring_local.get_online_features(\n", + " features=feast_features,\n", + " entity_rows=entity_rows,\n", + " ).to_dict()\n", + " break\n", + " except Exception as exc:\n", + " last_error = exc\n", + " if attempt < 3:\n", + " print(f\"⚠️ Online read failed (attempt {attempt}/3): {exc}\")\n", + " time.sleep(5)\n", + " else:\n", + " raise\n", + "\n", + "assert response is not None, f\"Online feature read failed: {last_error}\"\n", + "\n", + "print(\"Actual response:\", response)\n", + "\n", + "expected = {\n", + " 'zipcode': [1463],\n", + " 'dob_ssn': ['19530219_5179'],\n", + " 'city': ['PEPPERELL'],\n", + " 'state': ['MA'],\n", + "}\n", + "\n", + "assert response == expected" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Offline store write\n", + "import os\n", + "import pandas as pd\n", + "from feast.errors import FeastPermissionError\n", + "\n", + "write_feature_view = \"zipcode_features\"\n", + "write_location_type = \"PRIMARY_WRITE_TEST\"\n", + "write_total_wages = 310246739\n", + "\n", + "offline_base_write_data = {\n", + " \"zipcode\": [1463],\n", + " \"city\": [\"PEPPERELL\"],\n", + " \"state\": [\"MA\"],\n", + " \"location_type\": [write_location_type],\n", + " \"tax_returns_filed\": [5549],\n", + " \"population\": [10100],\n", + " \"total_wages\": [write_total_wages],\n", + " \"event_timestamp\": [pd.Timestamp(\"2017-01-01 12:00:00+00:00\")],\n", + " \"created_timestamp\": [pd.Timestamp(\"2017-01-01 12:00:00+00:00\")],\n", + "}\n", + "\n", + "# Offline store requires created_timestamp in this repo\n", + "offline_write_df = pd.DataFrame(offline_base_write_data)\n", + "\n", + "auth_mode = os.getenv(\"CONFIGMAP_DATA\", \"\")\n", + "if \"kubernetes\" in auth_mode.lower():\n", + " try:\n", + " fs_credit_scoring_local.write_to_offline_store(\n", + " feature_view_name=write_feature_view,\n", + " df=offline_write_df,\n", + " allow_registry_cache=False,\n", + " reorder_columns=True,\n", + " )\n", + " raise AssertionError(\"❌ Expected FeastPermissionError for offline write, but call succeeded\")\n", + " except FeastPermissionError as exc:\n", + " print(f\"✅ Correctly blocked offline write with FeastPermissionError: {exc}\")\n", + "else:\n", + " fs_credit_scoring_local.write_to_offline_store(\n", + " feature_view_name=write_feature_view,\n", + " df=offline_write_df,\n", + " allow_registry_cache=False,\n", + " reorder_columns=True,\n", + " )\n", + " print(f\"✅ write_to_offline_store() executed for '{write_feature_view}'\")\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "\n", + "import os\n", + "import pandas as pd\n", + "from datetime import timedelta\n", + "from feast import FeatureView\n", + "from feast.errors import FeastPermissionError\n", + "from feast.types import Int64\n", + "from feast.field import Field\n", + "from feast.entity import Entity\n", + "from feast.value_type import ValueType\n", + "from feast.infra.offline_stores.file_source import FileSource\n", + "from feast.data_format import ParquetFormat\n", + "\n", + "# ---- Define Entity (REQUIRED) ----\n", + "dob_ssn = Entity(\n", + " name=\"dob_ssn\",\n", + " value_type=ValueType.STRING,\n", + " description=\"Date of birth + last four digits of SSN\",\n", + ")\n", + "\n", + "# ---- FileSource WITHOUT created_timestamp ----\n", + "credit_history_source = FileSource(\n", + " name=\"Credit history\",\n", + " path=\"data/credit_history.parquet\",\n", + " file_format=ParquetFormat(),\n", + " timestamp_field=\"event_timestamp\",\n", + ")\n", + "\n", + "# ---- FeatureView (use Entity object) ----\n", + "credit_history = FeatureView(\n", + " name=\"credit_history\",\n", + " entities=[dob_ssn], # ✅ correct\n", + " ttl=timedelta(days=90),\n", + " schema=[\n", + " Field(name=\"credit_card_due\", dtype=Int64),\n", + " Field(name=\"mortgage_due\", dtype=Int64),\n", + " Field(name=\"student_loan_due\", dtype=Int64),\n", + " Field(name=\"vehicle_loan_due\", dtype=Int64),\n", + " Field(name=\"hard_pulls\", dtype=Int64),\n", + " Field(name=\"missed_payments_2y\", dtype=Int64),\n", + " Field(name=\"missed_payments_1y\", dtype=Int64),\n", + " Field(name=\"missed_payments_6m\", dtype=Int64),\n", + " Field(name=\"bankruptcies\", dtype=Int64),\n", + " ],\n", + " source=credit_history_source,\n", + ")\n", + "\n", + "auth_mode = os.getenv(\"CONFIGMAP_DATA\", \"\")\n", + "if \"kubernetes\" in auth_mode.lower():\n", + " try:\n", + " fs_credit_scoring_local.apply([dob_ssn, credit_history])\n", + " raise AssertionError(\"❌ Expected FeastPermissionError for apply, but call succeeded\")\n", + " except FeastPermissionError as exc:\n", + " print(f\"✅ Correctly blocked apply with FeastPermissionError: {exc}\")\n", + "else:\n", + " # ---- Apply FeatureView ----\n", + " fs_credit_scoring_local.apply([dob_ssn, credit_history])\n", + "\n", + " # ---- Online write (NO created_timestamp) ----\n", + " ts = pd.Timestamp(\"2020-04-26 18:01:04.746575\")\n", + "\n", + " online_write_df = pd.DataFrame({\n", + " \"dob_ssn\": [\"19530219_5179\"],\n", + " \"credit_card_due\": [8419],\n", + " \"mortgage_due\": [91803],\n", + " \"student_loan_due\": [22328],\n", + " \"vehicle_loan_due\": [15078],\n", + " \"hard_pulls\": [0],\n", + " \"missed_payments_2y\": [1],\n", + " \"missed_payments_1y\": [0],\n", + " \"missed_payments_6m\": [0],\n", + " \"bankruptcies\": [0],\n", + " \"event_timestamp\": [ts],\n", + " })\n", + "\n", + " fs_credit_scoring_local.write_to_online_store(\n", + " feature_view_name=\"credit_history\",\n", + " df=online_write_df,\n", + " allow_registry_cache=False,\n", + " )\n", + "\n", + " print(\"✅ Online store write succeeded\")\n", + "\n", + " # ---- Verify ----\n", + " online_df = fs_credit_scoring_local.get_online_features(\n", + " features=[\n", + " \"credit_history:credit_card_due\",\n", + " \"credit_history:mortgage_due\",\n", + " ],\n", + " entity_rows=[{\"dob_ssn\": \"19530219_5179\"}],\n", + " ).to_df()\n", + "\n", + " print(online_df)\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "from feast.errors import FeastPermissionError\n", + "\n", + "feature_view_name = \"credit_history\"\n", + "\n", + "auth_mode = os.getenv(\"CONFIGMAP_DATA\", \"\")\n", + "if \"kubernetes\" in auth_mode.lower():\n", + " try:\n", + " fs_credit_scoring_local.delete_feature_view(feature_view_name)\n", + " raise AssertionError(\"❌ Expected FeastPermissionError for delete_feature_view, but call succeeded\")\n", + " except FeastPermissionError as exc:\n", + " print(f\"✅ Correctly blocked delete_feature_view with FeastPermissionError: {exc}\")\n", + "else:\n", + " fs_credit_scoring_local.delete_feature_view(feature_view_name)\n", + "\n", + " # Verify deletion via list and get calls\n", + " remaining_fvs = [fv.name for fv in fs_credit_scoring_local.list_feature_views()]\n", + " assert feature_view_name not in remaining_fvs, (\n", + " f\"❌ FeatureView '{feature_view_name}' still present after deletion\"\n", + " )\n", + "\n", + " try:\n", + " fs_credit_scoring_local.get_feature_view(feature_view_name)\n", + " raise AssertionError(\n", + " f\"❌ Expected get_feature_view('{feature_view_name}') to fail after deletion\"\n", + " )\n", + " except Exception as exc:\n", + " print(f\"✅ Deletion verified for FeatureView '{feature_view_name}': {type(exc).__name__}\")\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "odfv_name = \"total_debt_calc\"\n", + "odfv = fs_credit_scoring_local.get_on_demand_feature_view(odfv_name)\n", + "\n", + "assert odfv is not None, f\"❌ get_on_demand_feature_view('{odfv_name}') returned None\"\n", + "\n", + "if isinstance(odfv, dict):\n", + " odfv_name_returned = odfv.get(\"spec\", {}).get(\"name\")\n", + " odfv_feature_names = [\n", + " f.get(\"spec\", {}).get(\"name\")\n", + " for f in odfv.get(\"spec\", {}).get(\"features\", [])\n", + " ]\n", + " odfv_write_to_online = odfv.get(\"spec\", {}).get(\"write_to_online_store\")\n", + "else:\n", + " odfv_name_returned = getattr(odfv, \"name\", None)\n", + " if not odfv_name_returned and hasattr(odfv, \"spec\") and hasattr(odfv.spec, \"name\"):\n", + " odfv_name_returned = odfv.spec.name\n", + " odfv_feature_names = [f.name for f in getattr(odfv, \"features\", [])]\n", + " odfv_write_to_online = getattr(odfv, \"write_to_online_store\", None)\n", + "\n", + "assert odfv_name_returned == odfv_name, (\n", + " f\"❌ Expected ODFV '{odfv_name}', but got '{odfv_name_returned}'\"\n", + ")\n", + "assert \"total_debt_due\" in odfv_feature_names, (\n", + " f\"❌ ODFV '{odfv_name}' missing 'total_debt_due' feature\"\n", + ")\n", + "assert odfv_write_to_online is False, (\n", + " f\"❌ ODFV '{odfv_name}' write_to_online_store expected False\"\n", + ")\n", + "\n", + "print(f\"✅ get_on_demand_feature_view('{odfv_name}')\")" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.5" + }, + "orig_nbformat": 4 + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/infra/feast-operator/test/e2e_rhoai/resources/feast-wb-milvus-test.ipynb b/infra/feast-operator/test/e2e_rhoai/resources/feast-wb-milvus-test.ipynb new file mode 100644 index 00000000000..e0505dac0ad --- /dev/null +++ b/infra/feast-operator/test/e2e_rhoai/resources/feast-wb-milvus-test.ipynb @@ -0,0 +1,462 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# ── Validate Feast version ───────────────────────────────────────────\n", + "import os\n", + "import feast\n", + "\n", + "actual_version = feast.__version__\n", + "expected_version = os.environ.get(\"FEAST_VERSION\")\n", + "\n", + "assert actual_version == expected_version, (\n", + " f\"❌ Feast version mismatch. Expected: {expected_version}, Found: {actual_version}\"\n", + ")\n", + "print(f\"✅ Feast version validated: {actual_version}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# ── Navigate to feature repo ─────────────────────────────────────────\n", + "%cd /opt/app-root/src/feature_repo\n", + "!ls -l" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# ── Validate feature_store.yaml ──────────────────────────────────────\n", + "import yaml\n", + "\n", + "with open(\"feature_store.yaml\") as f:\n", + " fs_config = yaml.safe_load(f)\n", + "\n", + "print(yaml.dump(fs_config, default_flow_style=False))\n", + "\n", + "assert fs_config.get(\"project\") == \"rag\", (\n", + " f\"❌ Expected project 'rag', got '{fs_config.get('project')}'\"\n", + ")\n", + "online_store = str(fs_config.get(\"online_store\", \"\")).lower()\n", + "assert \"milvus\" in online_store, (\n", + " f\"❌ Milvus not configured as online store. Found: {fs_config.get('online_store')}\"\n", + ")\n", + "print(\"✅ feature_store.yaml validated: project=rag, online_store=milvus\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# ── Download dataset ──────────────────────────────────────────────────\n", + "!mkdir -p data\n", + "!wget -q -O data/city_wikipedia_summaries_with_embeddings.parquet \\\n", + " https://raw.githubusercontent.com/opendatahub-io/feast/master/examples/rag/feature_repo/data/city_wikipedia_summaries_with_embeddings.parquet\n", + "print(\"✅ Dataset downloaded\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Load dataset and validate embeddings ──────────────────────────────\n", + "import pandas as pd\n", + "\n", + "df = pd.read_parquet(\"./data/city_wikipedia_summaries_with_embeddings.parquet\")\n", + "df[\"vector\"] = df[\"vector\"].apply(lambda x: x.tolist())\n", + "\n", + "assert not df.empty, \"❌ Loaded dataframe is empty\"\n", + "\n", + "required_columns = {\"item_id\", \"vector\", \"state\", \"sentence_chunks\", \"wiki_summary\"}\n", + "missing = required_columns - set(df.columns)\n", + "assert not missing, f\"❌ Missing columns in parquet: {missing}\"\n", + "\n", + "embedding_length = len(df[\"vector\"][0])\n", + "assert embedding_length == 384, f\"❌ Expected vector length 384, got {embedding_length}\"\n", + "\n", + "print(f\"✅ Dataset loaded: {len(df)} rows, embedding length={embedding_length}\")\n", + "df.head()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Install dependencies ──────────────────────────────────────────────\n", + "!pip install -q pymilvus[milvus_lite] transformers torch\n", + "print(\"✅ Dependencies installed\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Run feast apply and validate output ───────────────────────────────\n", + "import subprocess\n", + "\n", + "result = subprocess.run([\"feast\", \"apply\"], capture_output=True, text=True)\n", + "output = result.stdout + result.stderr\n", + "print(output)\n", + "\n", + "assert result.returncode == 0, (\n", + " f\"❌ feast apply exited with code {result.returncode}\\n{output}\"\n", + ")\n", + "\n", + "expected_messages = [\n", + " \"Applying changes for project rag\",\n", + " \"Deploying infrastructure for city_embeddings\",\n", + "]\n", + "for msg in expected_messages:\n", + " assert msg in output, f\"❌ Expected message not found in feast apply output: '{msg}'\"\n", + "\n", + "print(\"✅ feast apply succeeded with all expected messages\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Initialize FeatureStore ──────────────────────────────────────────\n", + "from feast import FeatureStore\n", + "\n", + "store = FeatureStore(repo_path=\".\")\n", + "print(f\"✅ FeatureStore initialized — project: {store.project}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "try:\n", + " store.write_to_online_store(feature_view_name=\"city_embeddings\", df=df)\n", + " print(\"✅ write_to_online_store executed successfully\")\n", + "except Exception as e:\n", + " raise AssertionError(f\"❌ write_to_online_store failed: {e}\")\n", + "\n", + "# Verify row count in Milvus matches source dataframe\n", + "milvus_client = store._provider._online_store._connect(store.config)\n", + "collections = milvus_client.list_collections()\n", + "assert len(collections) > 0, \"❌ No Milvus collections found after write_to_online_store\"\n", + "\n", + "actual_count = milvus_client.query(\n", + " collection_name=collections[0],\n", + " filter=\"\",\n", + " output_fields=[\"count(*)\"],\n", + ")[0].get(\"count(*)\", 0)\n", + "\n", + "assert actual_count == len(df), (\n", + " f\"❌ Row count mismatch: wrote {len(df)}, Milvus has {actual_count}\"\n", + ")\n", + "print(f\"✅ Milvus contains all {actual_count} written rows\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "batch_fvs = store.list_batch_feature_views()\n", + "\n", + "assert isinstance(batch_fvs, list), \"❌ list_batch_feature_views did not return a list\"\n", + "assert len(batch_fvs) == 1, (\n", + " f\"❌ Expected exactly 1 batch feature view, got {len(batch_fvs)}\"\n", + ")\n", + "print(f\"✅ Batch feature views count validated: {len(batch_fvs)}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "fv = store.get_feature_view(\"city_embeddings\")\n", + "\n", + "assert fv.name == \"city_embeddings\", (\n", + " f\"❌ Feature view name mismatch: expected 'city_embeddings', got '{fv.name}'\"\n", + ")\n", + "assert fv.entities == [\"item_id\"], (\n", + " f\"❌ Expected entities ['item_id'], got {fv.entities}\"\n", + ")\n", + "\n", + "feature_names = [f.name for f in fv.features]\n", + "for expected_field in [\"vector\", \"state\", \"sentence_chunks\", \"wiki_summary\"]:\n", + " assert expected_field in feature_names, (\n", + " f\"❌ Missing expected feature field: '{expected_field}'\"\n", + " )\n", + "\n", + "vector_feature = next(f for f in fv.features if f.name == \"vector\")\n", + "assert vector_feature.vector_index, \"❌ 'vector' feature is not configured as a vector index\"\n", + "assert vector_feature.vector_search_metric == \"COSINE\", (\n", + " f\"❌ Expected COSINE search metric, got '{vector_feature.vector_search_metric}'\"\n", + ")\n", + "\n", + "print(f\"✅ Feature view 'city_embeddings' schema validated\")\n", + "print(f\" Fields : {feature_names}\")\n", + "print(f\" Entities : {fv.entities}\")\n", + "print(f\" Metric : {vector_feature.vector_search_metric}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from feast.entity import Entity\n", + "from feast.types import ValueType\n", + "\n", + "test_entity = Entity(\n", + " name=\"item_id1\",\n", + " value_type=ValueType.INT64,\n", + " description=\"Temporary test entity\",\n", + " tags={\"team\": \"feast\"},\n", + ")\n", + "store.apply(test_entity)\n", + "\n", + "assert any(e.name == \"item_id1\" for e in store.list_entities()), (\n", + " \"❌ Entity 'item_id1' not found after apply\"\n", + ")\n", + "print(\"✅ Entity 'item_id1' added and verified\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "entity_to_delete = store.get_entity(\"item_id1\")\n", + "store.apply(objects=[], objects_to_delete=[entity_to_delete], partial=False)\n", + "\n", + "assert not any(e.name == \"item_id1\" for e in store.list_entities()), (\n", + " \"❌ Entity 'item_id1' still present after deletion\"\n", + ")\n", + "print(\"✅ Entity 'item_id1' deleted and verified\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "batch_fvs = store.list_batch_feature_views()\n", + "assert len(batch_fvs) == 1, (\n", + " f\"❌ Expected 1 feature view after entity delete, got {len(batch_fvs)}\"\n", + ")\n", + "print(f\"✅ Feature view count unchanged after entity delete: {len(batch_fvs)}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "\n", + "milvus_client = store._provider._online_store._connect(store.config)\n", + "\n", + "collections = milvus_client.list_collections()\n", + "assert len(collections) > 0, \"❌ No Milvus collections found\"\n", + "COLLECTION_NAME = collections[0]\n", + "print(f\"Collection: {COLLECTION_NAME}\")\n", + "\n", + "milvus_results = milvus_client.query(\n", + " collection_name=COLLECTION_NAME,\n", + " filter=\"item_id == '0'\",\n", + ")\n", + "assert len(milvus_results) > 0, \"❌ Milvus query for item_id=0 returned no results\"\n", + "\n", + "result_df = pd.DataFrame(milvus_results)\n", + "assert \"vector\" in result_df.columns, \"❌ 'vector' column missing from Milvus query result\"\n", + "\n", + "print(f\"✅ Milvus direct query validated: {len(result_df)} row(s) returned\")\n", + "result_df.head()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "\n", + "import torch\n", + "import torch.nn.functional as F\n", + "from transformers import AutoTokenizer, AutoModel\n", + "\n", + "TOKENIZER = \"sentence-transformers/all-MiniLM-L6-v2\"\n", + "MODEL = \"sentence-transformers/all-MiniLM-L6-v2\"\n", + "\n", + "def mean_pooling(model_output, attention_mask):\n", + " token_embeddings = model_output[0]\n", + " input_mask_expanded = (\n", + " attention_mask.unsqueeze(-1).expand(token_embeddings.size()).float()\n", + " )\n", + " return torch.sum(token_embeddings * input_mask_expanded, 1) / torch.clamp(\n", + " input_mask_expanded.sum(1), min=1e-9\n", + " )\n", + "\n", + "def run_model(sentences, tokenizer, model):\n", + " encoded_input = tokenizer(\n", + " sentences, padding=True, truncation=True, return_tensors=\"pt\"\n", + " )\n", + " with torch.no_grad():\n", + " model_output = model(**encoded_input)\n", + " embeddings = mean_pooling(model_output, encoded_input[\"attention_mask\"])\n", + " return F.normalize(embeddings, p=2, dim=1)\n", + "\n", + "tokenizer = AutoTokenizer.from_pretrained(TOKENIZER)\n", + "model = AutoModel.from_pretrained(MODEL)\n", + "print(\"✅ Embedding model loaded\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# ── Cell 17: Generate query embedding ─────────────────────────────────────────\n", + "question = \"Which city has the largest population in New York?\"\n", + "\n", + "query_embedding = run_model(question, tokenizer, model)\n", + "query = query_embedding.detach().cpu().numpy().tolist()[0]\n", + "\n", + "assert len(query) == 384, f\"❌ Query embedding length mismatch: expected 384, got {len(query)}\"\n", + "print(f\"✅ Query embedding generated: length={len(query)}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "\n", + "TOP_K = 3\n", + "\n", + "context_data = store.retrieve_online_documents_v2(\n", + " features=[\n", + " \"city_embeddings:vector\",\n", + " \"city_embeddings:item_id\",\n", + " \"city_embeddings:state\",\n", + " \"city_embeddings:sentence_chunks\",\n", + " \"city_embeddings:wiki_summary\",\n", + " ],\n", + " query=query,\n", + " top_k=TOP_K,\n", + " distance_metric=\"COSINE\",\n", + ").to_df()\n", + "\n", + "assert context_data is not None and not context_data.empty, (\n", + " \"❌ retrieve_online_documents_v2 returned empty results\"\n", + ")\n", + "assert len(context_data) == TOP_K, (\n", + " f\"❌ Expected top_k={TOP_K} results, got {len(context_data)}\"\n", + ")\n", + "\n", + "expected_columns = {\"state\", \"sentence_chunks\", \"wiki_summary\", \"item_id\", \"vector\"}\n", + "missing_cols = expected_columns - set(context_data.columns)\n", + "assert not missing_cols, f\"❌ Missing columns in RAG result: {missing_cols}\"\n", + "\n", + "if \"distance\" in context_data.columns:\n", + " assert context_data[\"distance\"].between(-1.0, 1.0).all(), (\n", + " \"❌ COSINE distance scores out of expected range [-1, 1]\"\n", + " )\n", + " print(f\" Distance range: [{context_data['distance'].min():.4f}, {context_data['distance'].max():.4f}]\")\n", + "\n", + "print(f\"✅ retrieve_online_documents_v2 returned {len(context_data)} results with correct schema\")\n", + "context_data" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "def format_documents(context_df):\n", + " output_context = \"\"\n", + " unique_documents = context_df.drop_duplicates(subset=[\"item_id\"]).apply(\n", + " lambda x: \"City & State = {\" + x[\"state\"] + \"}\\nSummary = {\" + x[\"wiki_summary\"].strip() + \"}\",\n", + " axis=1,\n", + " )\n", + " for i, document_text in enumerate(unique_documents):\n", + " output_context += f\"****START DOCUMENT {i}****\\n{document_text.strip()}\\n****END DOCUMENT {i}****\\n\"\n", + " return output_context\n", + "\n", + "# Pass item_id so drop_duplicates(subset=[\"item_id\"]) has the column available\n", + "RAG_CONTEXT = format_documents(context_data[[\"item_id\", \"state\", \"wiki_summary\"]])\n", + "\n", + "assert RAG_CONTEXT.strip(), \"❌ format_documents returned empty context\"\n", + "assert \"START DOCUMENT\" in RAG_CONTEXT, \"❌ START DOCUMENT marker missing from RAG context\"\n", + "assert \"END DOCUMENT\" in RAG_CONTEXT, \"❌ END DOCUMENT marker missing from RAG context\"\n", + "assert RAG_CONTEXT.count(\"START DOCUMENT\") == TOP_K, (\n", + " f\"❌ Expected {TOP_K} documents in context, found {RAG_CONTEXT.count('START DOCUMENT')}\"\n", + ")\n", + "\n", + "print(f\"✅ RAG context formatted correctly: {TOP_K} documents\")\n", + "print(RAG_CONTEXT)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "\n", + "FULL_PROMPT = f\"\"\"\n", + "You are an assistant for answering questions about states. You will be provided documentation\n", + "from Wikipedia. Provide a conversational answer. If you don't know the answer, just say\n", + "\"I do not know.\" Don't make up an answer.\n", + "\n", + "Here are document(s) you should use when answering the user's question:\n", + "{RAG_CONTEXT}\n", + "\"\"\"\n", + "\n", + "assert RAG_CONTEXT in FULL_PROMPT, \"❌ RAG context not embedded in prompt\"\n", + "print(\"✅ Full prompt constructed\")\n", + "print(FULL_PROMPT)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "3.11.0" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/infra/feast-operator/test/e2e_rhoai/resources/feast-wb-ray-test.ipynb b/infra/feast-operator/test/e2e_rhoai/resources/feast-wb-ray-test.ipynb new file mode 100644 index 00000000000..f11c43e1ffd --- /dev/null +++ b/infra/feast-operator/test/e2e_rhoai/resources/feast-wb-ray-test.ipynb @@ -0,0 +1,516 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# --- Configuration Variables ---\n", + "import os \n", + "\n", + "# Namespace where your resources exist\n", + "namespace = os.environ.get(\"NAMESPACE\")\n", + "\n", + "fsconfigmap = \"cm-fs-data\"\n", + "\n", + "# Fetch token and server directly from oc CLI\n", + "import subprocess\n", + "\n", + "def oc(cmd):\n", + " return subprocess.check_output(cmd, shell=True).decode(\"utf-8\").strip()\n", + "\n", + "token = oc(\"oc whoami -t\")\n", + "server = oc(\"oc whoami --show-server\")\n", + "\n", + "os.environ[\"CLUSTER_TOKEN\"] = token\n", + "os.environ[\"CLUSTER_SERVER\"] = server\n", + "\n", + "\n", + "# RayCluster name\n", + "raycluster = \"feastraytest\"\n", + "os.environ[\"RAY_CLUSTER\"] = raycluster\n", + "\n", + "# Show configured values\n", + "print(\"Configuration Variables:\")\n", + "print(f\" Namespace: {namespace}\")\n", + "print(f\" Server: {server}\")\n", + "print(f\" Token: {'*' * 20}\") # hide actual token\n", + "print(f\" Ray Cluster: {raycluster}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "! git clone https://github.com/Srihari1192/feast-rag-ray.git" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "%cd feast-rag-ray/feature_repo" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "!oc login --token=$token --server=$server" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "!oc create configmap $fsconfigmap --from-file=data/customer_daily_profile.parquet --from-file=data/driver_stats.parquet -n $namespace" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Import pieces from codeflare-sdk\n", + "from codeflare_sdk import Cluster, ClusterConfiguration, TokenAuthentication\n", + "\n", + "# Create authentication with token and server from oc\n", + "auth = TokenAuthentication(\n", + " token=token,\n", + " server=server,\n", + " skip_tls=True\n", + ")\n", + "auth.login()\n", + "print(\"✓ Authentication successful\")\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from kubernetes.client import (\n", + " V1Volume,\n", + " V1ConfigMapVolumeSource,\n", + " V1VolumeMount,\n", + ") \n", + "\n", + "data_volume = V1Volume(\n", + " name=\"data\",\n", + " config_map=V1ConfigMapVolumeSource(name=fsconfigmap)\n", + ")\n", + "\n", + "data_mount = V1VolumeMount(\n", + " name=\"data\",\n", + " mount_path=\"/opt/app-root/src/feast-rag-ray/feature_repo/data\",\n", + " read_only=True\n", + ")\n", + "\n", + "cluster = Cluster(ClusterConfiguration(\n", + " name=raycluster,\n", + " head_cpu_requests=1,\n", + " head_cpu_limits=1,\n", + " head_memory_requests=8,\n", + " head_memory_limits=8,\n", + " head_extended_resource_requests={'nvidia.com/gpu':0}, # For GPU enabled workloads set the head_extended_resource_requests and worker_extended_resource_requests\n", + " worker_extended_resource_requests={'nvidia.com/gpu':0},\n", + " num_workers=2,\n", + " worker_cpu_requests='500m',\n", + " worker_cpu_limits=1,\n", + " worker_memory_requests=8,\n", + " worker_memory_limits=8,\n", + " # image=\"\", # Optional Field \n", + " write_to_file=False, # When enabled Ray Cluster yaml files are written to /HOME/.codeflare/resources\n", + " local_queue=\"fs-user-queue\", # Specify the local queue manually\n", + " # ⭐ Best method: Use secretKeyRef to expose AWS credentials safely\n", + " volumes=[data_volume],\n", + " volume_mounts=[data_mount],\n", + " \n", + "))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "cluster.apply()\n", + "# cluster.wait_ready()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import time\n", + "\n", + "MAX_WAIT = 180 # 3 minutes\n", + "INTERVAL = 5 # check every 5 seconds\n", + "elapsed = 0\n", + "\n", + "print(\"⏳ Waiting up to 3 minutes for RayCluster to be READY...\\n\")\n", + "\n", + "while elapsed < MAX_WAIT:\n", + " details = cluster.details()\n", + " status = details.status.value\n", + "\n", + " print(details)\n", + " print(\"Cluster Status:\", status)\n", + "\n", + " if status == \"ready\":\n", + " print(\"✅ RayCluster is READY!\")\n", + " break\n", + " \n", + " print(f\"⏳ RayCluster is NOT ready yet: {status} ... checking again in {INTERVAL}s\\n\")\n", + " time.sleep(INTERVAL)\n", + " elapsed += INTERVAL\n", + "\n", + "else:\n", + " print(\"❌ Timeout: RayCluster did NOT become READY within 3 minutes.\")\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "! feast apply" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import sys\n", + "from pathlib import Path\n", + "from feast import FeatureStore\n", + "\n", + "# Add feature repo to PYTHONPATH\n", + "repo_path = Path(\".\")\n", + "sys.path.append(str(repo_path))\n", + "\n", + "# Initialize Feature Store\n", + "print(\"Initializing Feast with Ray configuration...\")\n", + "store = FeatureStore(repo_path=\".\")\n", + "\n", + "# Assertions: Verify store is initialized correctly\n", + "assert store is not None, \"FeatureStore should be initialized\"\n", + "assert store.config is not None, \"Store config should be available\"\n", + "assert store.config.offline_store is not None, \"Offline store should be configured\"\n", + "\n", + "print(f\"✓ Offline store: {store.config.offline_store.type}\")\n", + "if hasattr(store.config, \"batch_engine\") and store.config.batch_engine:\n", + " print(f\"✓ Compute engine: {store.config.batch_engine.type}\")\n", + " # Assertion: Verify batch engine is configured if present\n", + " assert store.config.batch_engine.type is not None, \"Batch engine type should be set\"\n", + "else:\n", + " print(\"⚠ No compute engine configured\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 2. Create Entity DataFrame\n", + "\n", + "Create an entity DataFrame for historical feature retrieval with point-in-time timestamps.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from datetime import datetime, timedelta\n", + "import pandas as pd\n", + "\n", + "# --- Create time window ---\n", + "end_date = datetime.now().replace(microsecond=0, second=0, minute=0)\n", + "start_date = end_date - timedelta(days=2)\n", + "\n", + "\n", + "entity_df = pd.DataFrame(\n", + " {\n", + " \"driver_id\": [1001, 1002, 1003],\n", + " \"customer_id\": [2001, 2002, 2003],\n", + " \"event_timestamp\": [\n", + " pd.Timestamp(end_date - timedelta(hours=24), tz=\"UTC\"),\n", + " pd.Timestamp(end_date - timedelta(hours=12), tz=\"UTC\"),\n", + " pd.Timestamp(end_date - timedelta(hours=6), tz=\"UTC\"),\n", + " ],\n", + " }\n", + ")\n", + "\n", + "# Assertions: Verify entity DataFrame is created correctly\n", + "assert len(entity_df) == 3, f\"Expected 3 rows, got {len(entity_df)}\"\n", + "assert \"driver_id\" in entity_df.columns, \"driver_id column should be present\"\n", + "assert \"customer_id\" in entity_df.columns, \"customer_id column should be present\"\n", + "assert \"event_timestamp\" in entity_df.columns, \"event_timestamp column should be present\"\n", + "assert all(entity_df[\"driver_id\"].isin([1001, 1002, 1003])), \"driver_id values should match expected\"\n", + "assert all(entity_df[\"customer_id\"].isin([2001, 2002, 2003])), \"customer_id values should match expected\"\n", + "assert entity_df[\"event_timestamp\"].notna().all(), \"All event_timestamp values should be non-null\"\n", + "\n", + "print(f\"✓ Created entity DataFrame with {len(entity_df)} rows\")\n", + "print(f\"✓ Time range: {start_date} to {end_date}\")\n", + "print(\"\\nEntity DataFrame:\")\n", + "print(entity_df)\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 3. Retrieve Historical Features\n", + "\n", + "Retrieve historical features using Ray compute engine for distributed point-in-time joins.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Cell 4: Retrieve Historical Features\n", + "print(\"Retrieving historical features with Ray compute engine...\")\n", + "print(\"(This demonstrates distributed point-in-time joins)\")\n", + "\n", + "try:\n", + " # Get historical features - this uses Ray compute engine for distributed processing\n", + " historical_features = store.get_historical_features(\n", + " entity_df=entity_df,\n", + " features=[\n", + " \"driver_hourly_stats:conv_rate\",\n", + " \"driver_hourly_stats:acc_rate\",\n", + " \"driver_hourly_stats:avg_daily_trips\",\n", + " \"customer_daily_profile:current_balance\",\n", + " \"customer_daily_profile:avg_passenger_count\",\n", + " \"customer_daily_profile:lifetime_trip_count\",\n", + " ],\n", + " )\n", + "\n", + " # Convert to DataFrame - Ray processes this efficiently\n", + " historical_df = historical_features.to_df()\n", + " \n", + " # Assertions: Verify historical features are retrieved correctly\n", + " assert historical_df is not None, \"Historical features DataFrame should not be None\"\n", + " assert len(historical_df) > 0, \"Should retrieve at least one row of historical features\"\n", + " assert \"driver_id\" in historical_df.columns, \"driver_id should be in the result\"\n", + " assert \"customer_id\" in historical_df.columns, \"customer_id should be in the result\"\n", + " \n", + " # Verify expected feature columns are present (some may be None if data doesn't exist)\n", + " expected_features = [\n", + " \"conv_rate\", \"acc_rate\", \"avg_daily_trips\",\n", + " \"current_balance\", \"avg_passenger_count\", \"lifetime_trip_count\"\n", + " ]\n", + " feature_columns = [col for col in historical_df.columns if col in expected_features]\n", + " assert len(feature_columns) > 0, f\"Should have at least one feature column, got: {historical_df.columns.tolist()}\"\n", + " \n", + " print(f\"✓ Retrieved {len(historical_df)} historical feature rows\")\n", + " print(f\"✓ Features: {list(historical_df.columns)}\")\n", + " \n", + " # Display the results\n", + " print(\"\\nHistorical Features DataFrame:\")\n", + " display(historical_df.head(10))\n", + "\n", + "except Exception as e:\n", + " print(f\"⚠ Historical features retrieval failed: {e}\")\n", + " print(\"This might be due to missing Ray dependencies or data\")\n", + " raise\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 4. Test On-Demand Feature Transformations\n", + "\n", + "Demonstrate on-demand feature transformations that are computed at request time.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Cell 5: Test On-Demand Features\n", + "print(\"Testing on-demand feature transformations...\")\n", + "\n", + "try:\n", + " # Get features including on-demand transformations\n", + " features_with_odfv = store.get_historical_features(\n", + " entity_df=entity_df.head(1),\n", + " features=[\n", + " \"driver_hourly_stats:conv_rate\",\n", + " \"driver_hourly_stats:acc_rate\",\n", + " \"driver_hourly_stats:avg_daily_trips\",\n", + " \"driver_activity_v2:conv_rate_plus_acc_rate\",\n", + " \"driver_activity_v2:trips_per_day_normalized\",\n", + " ],\n", + " )\n", + "\n", + " odfv_df = features_with_odfv.to_df()\n", + " \n", + " # Assertions: Verify on-demand features are computed correctly\n", + " assert odfv_df is not None, \"On-demand features DataFrame should not be None\"\n", + " assert len(odfv_df) > 0, \"Should retrieve at least one row with on-demand features\"\n", + " assert \"driver_id\" in odfv_df.columns, \"driver_id should be in the result\"\n", + " \n", + " # Verify on-demand feature columns if they exist\n", + " if \"conv_rate_plus_acc_rate\" in odfv_df.columns:\n", + " # Assertion: Verify the on-demand feature is computed\n", + " assert odfv_df[\"conv_rate_plus_acc_rate\"].notna().any(), \"conv_rate_plus_acc_rate should have non-null values\"\n", + " print(\"✓ On-demand feature 'conv_rate_plus_acc_rate' is computed\")\n", + " \n", + " if \"trips_per_day_normalized\" in odfv_df.columns:\n", + " assert odfv_df[\"trips_per_day_normalized\"].notna().any(), \"trips_per_day_normalized should have non-null values\"\n", + " print(\"✓ On-demand feature 'trips_per_day_normalized' is computed\")\n", + " \n", + " print(f\"✓ Retrieved {len(odfv_df)} rows with on-demand transformations\")\n", + " \n", + " # Display results\n", + " print(\"\\nFeatures with On-Demand Transformations:\")\n", + " display(odfv_df)\n", + " \n", + " # Show specific transformed features\n", + " if \"conv_rate_plus_acc_rate\" in odfv_df.columns:\n", + " print(\"\\nSample with on-demand features:\")\n", + " display(\n", + " odfv_df[[\"driver_id\", \"conv_rate\", \"acc_rate\", \"conv_rate_plus_acc_rate\"]]\n", + " )\n", + "\n", + "except Exception as e:\n", + " print(f\"⚠ On-demand features failed: {e}\")\n", + " raise\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 5. Materialize Features to Online Store\n", + "\n", + "Materialize features to the online store using Ray compute engine for efficient batch processing.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from datetime import timezone\n", + "print(\"Materializing features to online store...\")\n", + "store.materialize(\n", + "\tstart_date=datetime(2025, 1, 1, tzinfo=timezone.utc),\n", + "\tend_date=end_date,\n", + ")\n", + "\n", + "# Minimal output assertion: materialization succeeded if no exception\n", + "assert True, \"Materialization completed successfully\"\n", + "print(\"✓ Initial materialization successful\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 6. Test Online Feature Serving\n", + "\n", + "Retrieve features from the online store for low-latency serving.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Cell 7: Test Online Feature Serving\n", + "print(\"Testing online feature serving...\")\n", + "\n", + "try:\n", + " entity_rows = [\n", + " {\"driver_id\": 1001, \"customer_id\": 2001},\n", + " {\"driver_id\": 1002, \"customer_id\": 2002},\n", + " ]\n", + " \n", + " # Assertion: Verify entity rows are valid\n", + " assert len(entity_rows) == 2, \"Should have 2 entity rows\"\n", + " assert all(\"driver_id\" in row for row in entity_rows), \"All entity rows should have driver_id\"\n", + " assert all(\"customer_id\" in row for row in entity_rows), \"All entity rows should have customer_id\"\n", + " \n", + " online_features = store.get_online_features(\n", + " features=[\n", + " \"driver_hourly_stats:conv_rate\",\n", + " \"driver_hourly_stats:acc_rate\",\n", + " \"customer_daily_profile:current_balance\",\n", + " ],\n", + " entity_rows=entity_rows,\n", + " )\n", + "\n", + " online_df = online_features.to_df()\n", + " \n", + " # Assertions: Verify online features are retrieved correctly\n", + " assert online_df is not None, \"Online features DataFrame should not be None\"\n", + " assert len(online_df) == len(entity_rows), f\"Should retrieve {len(entity_rows)} rows, got {len(online_df)}\"\n", + " assert \"driver_id\" in online_df.columns, \"driver_id should be in the result\"\n", + " assert \"customer_id\" in online_df.columns, \"customer_id should be in the result\"\n", + " \n", + " # Verify expected feature columns are present\n", + " expected_features = [\"conv_rate\", \"acc_rate\", \"current_balance\"]\n", + " feature_columns = [col for col in online_df.columns if col in expected_features]\n", + " assert len(feature_columns) > 0, f\"Should have at least one feature column, got: {online_df.columns.tolist()}\"\n", + " \n", + " # Verify entity IDs match\n", + " assert all(online_df[\"driver_id\"].isin([1001, 1002])), \"driver_id values should match entity rows\"\n", + " assert all(online_df[\"customer_id\"].isin([2001, 2002])), \"customer_id values should match entity rows\"\n", + " \n", + " print(f\"✓ Retrieved {len(online_df)} online feature rows\")\n", + " print(f\"✓ Features retrieved: {feature_columns}\")\n", + " \n", + " print(\"\\nOnline Features DataFrame:\")\n", + " display(online_df)\n", + "\n", + "except Exception as e:\n", + " print(f\"⚠ Online serving failed: {e}\")\n", + " raise\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "cluster.down()" + ] + } + ], + "metadata": { + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/infra/feast-operator/test/e2e_rhoai/resources/feast_kube_auth.yaml b/infra/feast-operator/test/e2e_rhoai/resources/feast_kube_auth.yaml new file mode 100644 index 00000000000..ce66b22e926 --- /dev/null +++ b/infra/feast-operator/test/e2e_rhoai/resources/feast_kube_auth.yaml @@ -0,0 +1,74 @@ +apiVersion: v1 +kind: Secret +metadata: + name: feast-data-stores + namespace: test-ns-feast +stringData: + redis: | + connection_string: redis.test-ns-feast.svc.cluster.local:6379 + sql: | + path: postgresql+psycopg://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres.test-ns-feast.svc.cluster.local:5432/${POSTGRES_DB} + cache_ttl_seconds: 60 + sqlalchemy_config_kwargs: + echo: false + pool_pre_ping: true +--- +apiVersion: feast.dev/v1 +kind: FeatureStore +metadata: + name: credit-scoring + namespace: test-ns-feast +spec: + authz: + kubernetes: + roles: [] + feastProject: credit_scoring_local + feastProjectDir: + git: + url: https://github.com/feast-dev/feast-credit-score-local-tutorial + ref: 598a270 + services: + offlineStore: + persistence: + file: + type: duckdb + server: + envFrom: + - secretRef: + name: postgres-secret + env: + - name: MPLCONFIGDIR + value: /tmp + resources: + requests: + cpu: 150m + memory: 128Mi + onlineStore: + persistence: + store: + type: redis + secretRef: + name: feast-data-stores + server: + envFrom: + - secretRef: + name: postgres-secret + env: + - name: MPLCONFIGDIR + value: /tmp + resources: + requests: + cpu: 150m + memory: 128Mi + registry: + local: + persistence: + store: + type: sql + secretRef: + name: feast-data-stores + server: + envFrom: + - secretRef: + name: postgres-secret + restAPI: true diff --git a/infra/feast-operator/test/e2e_rhoai/resources/feast_oidc.yaml b/infra/feast-operator/test/e2e_rhoai/resources/feast_oidc.yaml new file mode 100644 index 00000000000..2aaf5f33fcd --- /dev/null +++ b/infra/feast-operator/test/e2e_rhoai/resources/feast_oidc.yaml @@ -0,0 +1,20 @@ +apiVersion: feast.dev/v1 +kind: FeatureStore +metadata: + name: test-feast-oidc + labels: + feature-store-ui: enabled +spec: + authz: + oidc: + verifySSL: false + feastProject: driver_ranking_oidc + services: + offlineStore: + server: {} + onlineStore: + server: {} + registry: + local: + server: + restAPI: true diff --git a/infra/feast-operator/test/e2e_rhoai/resources/feast_s3.yaml b/infra/feast-operator/test/e2e_rhoai/resources/feast_s3.yaml new file mode 100644 index 00000000000..9855c590ffc --- /dev/null +++ b/infra/feast-operator/test/e2e_rhoai/resources/feast_s3.yaml @@ -0,0 +1,43 @@ +kind: Secret +apiVersion: v1 +metadata: + name: s3-credentials-secret + namespace: test-ns-feast-upgrade +stringData: + AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID} + AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY} + AWS_DEFAULT_REGION: ${AWS_DEFAULT_REGION} + dynamodb: | + type: dynamodb + region: ${AWS_DEFAULT_REGION} +--- +apiVersion: feast.dev/v1 +kind: FeatureStore +metadata: + name: test-s3 + namespace: test-ns-feast-upgrade + labels: + feature-store-ui: enabled +spec: + feastProject: driver_ranking + services: + onlineStore: + persistence: + store: + type: dynamodb + secretRef: + name: s3-credentials-secret + server: + envFrom: + - secretRef: + name: s3-credentials-secret + registry: + local: + persistence: + file: + path: s3://${AWS_S3_BUCKET}/feast-test/driver_ranking/registry.pb + server: + envFrom: + - secretRef: + name: s3-credentials-secret + restAPI: true diff --git a/infra/feast-operator/test/e2e_rhoai/resources/feature_repo/__init__.py b/infra/feast-operator/test/e2e_rhoai/resources/feature_repo/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/infra/feast-operator/test/e2e_rhoai/resources/feature_repo/example_repo.py b/infra/feast-operator/test/e2e_rhoai/resources/feature_repo/example_repo.py new file mode 100755 index 00000000000..7a37d99d495 --- /dev/null +++ b/infra/feast-operator/test/e2e_rhoai/resources/feature_repo/example_repo.py @@ -0,0 +1,42 @@ +from datetime import timedelta + +from feast import ( + FeatureView, + Field, + FileSource, +) +from feast.data_format import ParquetFormat +from feast.types import Float32, Array, String, ValueType +from feast import Entity + +item = Entity( + name="item_id", + description="Item ID", + value_type=ValueType.INT64, +) + +parquet_file_path = "./data/city_wikipedia_summaries_with_embeddings.parquet" + +source = FileSource( + file_format=ParquetFormat(), + path=parquet_file_path, + timestamp_field="event_timestamp", +) + +city_embeddings_feature_view = FeatureView( + name="city_embeddings", + entities=[item], + schema=[ + Field( + name="vector", + dtype=Array(Float32), + vector_index=True, + vector_search_metric="COSINE", + ), + Field(name="state", dtype=String), + Field(name="sentence_chunks", dtype=String), + Field(name="wiki_summary", dtype=String), + ], + source=source, + ttl=timedelta(hours=2), +) diff --git a/infra/feast-operator/test/e2e_rhoai/resources/feature_repo/feature_store.yaml b/infra/feast-operator/test/e2e_rhoai/resources/feature_repo/feature_store.yaml new file mode 100755 index 00000000000..f8f9cc293dc --- /dev/null +++ b/infra/feast-operator/test/e2e_rhoai/resources/feature_repo/feature_store.yaml @@ -0,0 +1,16 @@ +project: rag +provider: local +registry: data/registry.db +online_store: + type: milvus + path: data/online_store.db + vector_enabled: true + embedding_dim: 384 + index_type: "FLAT" + metric_type: "COSINE" +offline_store: + type: file +entity_key_serialization_version: 3 +auth: + type: no_auth + diff --git a/infra/feast-operator/test/e2e_rhoai/resources/kueue_resources_setup.yaml b/infra/feast-operator/test/e2e_rhoai/resources/kueue_resources_setup.yaml new file mode 100644 index 00000000000..ebcac54f4a0 --- /dev/null +++ b/infra/feast-operator/test/e2e_rhoai/resources/kueue_resources_setup.yaml @@ -0,0 +1,31 @@ +apiVersion: kueue.x-k8s.io/v1beta1 +kind: ResourceFlavor +metadata: + name: "fs-resource-flavor" +--- +apiVersion: kueue.x-k8s.io/v1beta1 +kind: ClusterQueue +metadata: + name: "fs-cluster-queue" +spec: + namespaceSelector: {} # match all. + resourceGroups: + - coveredResources: ["cpu", "memory","nvidia.com/gpu"] + flavors: + - name: "fs-resource-flavor" + resources: + - name: "cpu" + nominalQuota: 9 + - name: "memory" + nominalQuota: 36Gi + - name: "nvidia.com/gpu" + nominalQuota: 0 +--- +apiVersion: kueue.x-k8s.io/v1beta1 +kind: LocalQueue +metadata: + name: "fs-user-queue" + annotations: + "kueue.x-k8s.io/default-queue": "true" +spec: + clusterQueue: "fs-cluster-queue" diff --git a/infra/feast-operator/test/e2e_rhoai/resources/permissions.py b/infra/feast-operator/test/e2e_rhoai/resources/permissions.py new file mode 100644 index 00000000000..5c2746cb875 --- /dev/null +++ b/infra/feast-operator/test/e2e_rhoai/resources/permissions.py @@ -0,0 +1,24 @@ +from feast.feast_object import ALL_FEATURE_VIEW_TYPES +from feast.permissions.permission import Permission +from feast.permissions.action import READ, AuthzedAction +from feast.permissions.policy import NamespaceBasedPolicy +from feast.project import Project +from feast.entity import Entity +from feast.feature_service import FeatureService +from feast.saved_dataset import SavedDataset + +perm_namespace = ["test-ns-feast"] + +WITHOUT_DATA_SOURCE = [ + Project, + Entity, + FeatureService, + SavedDataset, +] + ALL_FEATURE_VIEW_TYPES + +test_perm = Permission( + name="feast-auth", + types=WITHOUT_DATA_SOURCE, + policy=NamespaceBasedPolicy(namespaces=perm_namespace), + actions=[AuthzedAction.DESCRIBE] + READ, +) diff --git a/infra/feast-operator/test/e2e_rhoai/resources/permissions_oidc.py b/infra/feast-operator/test/e2e_rhoai/resources/permissions_oidc.py new file mode 100644 index 00000000000..1da939e6016 --- /dev/null +++ b/infra/feast-operator/test/e2e_rhoai/resources/permissions_oidc.py @@ -0,0 +1,17 @@ +from feast.feast_object import ALL_RESOURCE_TYPES +from feast.permissions.permission import Permission +from feast.permissions.action import ALL_ACTIONS +from feast.permissions.policy import GroupBasedPolicy + +# Define admin groups with full access +admin_groups = ["dedicated-admins"] + +# Admin group permission — full control over all Feast resources +admin_group_perm = Permission( + name="admin_group_permission", + types=ALL_RESOURCE_TYPES, + policy=GroupBasedPolicy(admin_groups), + actions=ALL_ACTIONS, +) + +print("Admin group permission configured successfully.") diff --git a/infra/feast-operator/test/e2e_rhoai/resources/pvc.yaml b/infra/feast-operator/test/e2e_rhoai/resources/pvc.yaml new file mode 100644 index 00000000000..a9e8c1be299 --- /dev/null +++ b/infra/feast-operator/test/e2e_rhoai/resources/pvc.yaml @@ -0,0 +1,10 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: jupyterhub-nb-kube-3aadmin-pvc +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 10Gi diff --git a/infra/feast-operator/test/e2e_rhoai/utils/notebook_util.go b/infra/feast-operator/test/e2e_rhoai/utils/notebook_util.go new file mode 100644 index 00000000000..239dbf287a2 --- /dev/null +++ b/infra/feast-operator/test/e2e_rhoai/utils/notebook_util.go @@ -0,0 +1,393 @@ +package utils + +import ( + "bytes" + "fmt" + "os" + "os/exec" + "strings" + "text/template" + "time" + + testutils "github.com/feast-dev/feast/infra/feast-operator/test/utils" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +type NotebookTemplateParams struct { + Namespace string + IngressDomain string + OpenDataHubNamespace string + NotebookImage string + NotebookConfigMapName string + NotebookPVC string + Username string + OC_TOKEN string + OC_SERVER string + NotebookFile string + Command string + PipIndexUrl string + PipTrustedHost string + FeastVersion string + OpenAIAPIKey string + FeastProject string + AdditionalEnv map[string]string +} + +// CreateNotebook renders a notebook manifest from a template and applies it using kubectl. +func CreateNotebook(params NotebookTemplateParams) error { + content, err := os.ReadFile("test/e2e_rhoai/resources/custom-nb.yaml") + if err != nil { + return fmt.Errorf("failed to read template file: %w", err) + } + + tmpl, err := template.New("notebook").Parse(string(content)) + if err != nil { + return fmt.Errorf("failed to parse template: %w", err) + } + + var rendered bytes.Buffer + if err := tmpl.Execute(&rendered, params); err != nil { + return fmt.Errorf("failed to substitute template: %w", err) + } + + tmpFile, err := os.CreateTemp("", "notebook-*.yaml") + if err != nil { + return fmt.Errorf("failed to create temp file: %w", err) + } + + // Defer cleanup of temp file + defer func() { + if err := os.Remove(tmpFile.Name()); err != nil { + fmt.Printf("warning: failed to remove temp file %s: %v", tmpFile.Name(), err) + } + }() + + if _, err := tmpFile.Write(rendered.Bytes()); err != nil { + return fmt.Errorf("failed to write to temp file: %w", err) + } + + if err := tmpFile.Close(); err != nil { + return fmt.Errorf("failed to close temp file: %w", err) + } + + // fmt.Println("Notebook manifest applied successfully") + cmd := exec.Command("kubectl", "apply", "-f", tmpFile.Name(), "-n", params.Namespace) + output, err := testutils.Run(cmd, "/test/e2e_rhoai") + Expect(err).ToNot(HaveOccurred(), fmt.Sprintf( + "Failed to create Notebook %s.\nError: %v\nOutput: %s\n", + tmpFile.Name(), err, output, + )) + fmt.Printf("Notebook %s created successfully\n", tmpFile.Name()) + return nil +} + +// MonitorNotebookPod waits for a notebook pod to reach Running state and verifies execution logs. +func MonitorNotebookPod(namespace, podPrefix string, notebookName string) error { + const successMarker = "Notebook executed successfully" + const failureMarker = "Notebook execution failed" + const pollInterval = 5 * time.Second + var pod *PodInfo + + fmt.Println("🔄 Waiting for notebook pod to reach Running & Ready state...") + + foundRunningReady := false + for i := 0; i < 36; i++ { + var err error + pod, err = getPodByPrefix(namespace, podPrefix) + if err != nil { + fmt.Printf("⏳ Pod not created yet: %v\n", err) + time.Sleep(pollInterval) + continue + } + if pod.Status == "Running" { + fmt.Printf("✅ Pod %s is Running and Ready.\n", pod.Name) + foundRunningReady = true + break + } + fmt.Printf("⏳ Pod %s not ready yet. Phase: %s\n", pod.Name, pod.Status) + time.Sleep(pollInterval) + } + + if !foundRunningReady { + return fmt.Errorf("❌ Pod %s did not reach Running & Ready state within 3 minutes", podPrefix) + } + + // Start monitoring notebook logs + fmt.Printf("⏳ Monitoring Notebook pod %s Logs for Jupyter Notebook %s execution status\n", pod.Name, notebookName) + + for i := 0; i < 60; i++ { + logs, err := getPodLogs(namespace, pod.Name) + if err != nil { + fmt.Printf("⏳ Failed to get logs for pod %s: %v\n", pod.Name, err) + time.Sleep(pollInterval) + continue + } + + if strings.Contains(logs, successMarker) { + Expect(logs).To(ContainSubstring(successMarker)) + fmt.Printf("✅ Jupyter Notebook pod %s executed successfully.\n", pod.Name) + return nil + } + + if strings.Contains(logs, failureMarker) { + fmt.Printf("❌ Notebook pod %s failed: failure marker found.\n", pod.Name) + return fmt.Errorf("Notebook failed in execution. Logs:\n%s", logs) + } + + time.Sleep(pollInterval) + } + + return fmt.Errorf("❌ Timed out waiting for notebook pod %s to complete", podPrefix) +} + +type PodInfo struct { + Name string + Status string +} + +// returns the first pod matching a name prefix in the given namespace. +func getPodByPrefix(namespace, prefix string) (*PodInfo, error) { + cmd := exec.Command( + "kubectl", "get", "pods", "-n", namespace, + "-o", "jsonpath={range .items[*]}{.metadata.name} {.status.phase}{\"\\n\"}{end}", + ) + output, err := testutils.Run(cmd, "/test/e2e_rhoai") + if err != nil { + return nil, fmt.Errorf("failed to get pods: %w", err) + } + + lines := strings.Split(strings.TrimSpace(string(output)), "\n") + for _, line := range lines { + parts := strings.Fields(line) + if len(parts) < 2 { + continue + } + name := parts[0] + status := parts[1] + + if strings.HasPrefix(name, prefix) { + return &PodInfo{ + Name: name, + Status: status, + }, nil + } + } + + return nil, fmt.Errorf("no pod found with prefix %q in namespace %q", prefix, namespace) +} + +// retrieves the logs of a specified pod in the given namespace. +func getPodLogs(namespace, podName string) (string, error) { + cmd := exec.Command("kubectl", "logs", "-n", namespace, podName) + var out bytes.Buffer + var stderr bytes.Buffer + cmd.Stdout = &out + cmd.Stderr = &stderr + + err := cmd.Run() + if err != nil { + return "", fmt.Errorf("error getting pod logs: %v - %s", err, stderr.String()) + } + + return out.String(), nil +} + +// returns the OpenShift cluster ingress domain. +func GetIngressDomain(testDir string) string { + cmd := exec.Command("oc", "get", "ingresses.config.openshift.io", "cluster", "-o", "jsonpath={.spec.domain}") + output, _ := testutils.Run(cmd, testDir) + return string(output) +} + +// returns the current OpenShift user authentication token. +func GetOCToken(testDir string) string { + cmd := exec.Command("oc", "whoami", "--show-token") + output, _ := testutils.Run(cmd, testDir) + return string(output) +} + +// returns the OpenShift API server URL for the current user. +func GetOCServer(testDir string) string { + cmd := exec.Command("oc", "whoami", "--show-server") + output, _ := testutils.Run(cmd, testDir) + return string(output) +} + +// returns the OpenShift cluster logged in Username +func GetOCUser(testDir string) string { + cmd := exec.Command("oc", "whoami") + output, _ := testutils.Run(cmd, testDir) + return strings.TrimSpace(string(output)) +} + +// SetNamespaceContext sets the kubectl namespace context to the specified namespace +func SetNamespaceContext(namespace, testDir string) error { + cmd := exec.Command("kubectl", "config", "set-context", "--current", "--namespace", namespace) + output, err := testutils.Run(cmd, testDir) + if err != nil { + return fmt.Errorf("failed to set namespace context to %s: %w\nOutput: %s", namespace, err, output) + } + return nil +} + +// CreateNotebookConfigMap creates a ConfigMap containing the notebook file and feature repo +func CreateNotebookConfigMap(namespace, configMapName, notebookFile, featureRepoPath, testDir string) error { + cmd := exec.Command("kubectl", "create", "configmap", configMapName, + "--from-file="+notebookFile, + "--from-file="+featureRepoPath) + output, err := testutils.Run(cmd, testDir) + if err != nil { + return fmt.Errorf("failed to create ConfigMap %s: %w\nOutput: %s", configMapName, err, output) + } + return nil +} + +// CreateNotebookPVC creates a PersistentVolumeClaim for the notebook +func CreateNotebookPVC(pvcFile, testDir string) error { + cmd := exec.Command("kubectl", "apply", "-f", pvcFile) + _, err := testutils.Run(cmd, testDir) + if err != nil { + return fmt.Errorf("failed to create PVC from %s: %w", pvcFile, err) + } + return nil +} + +// CreateNotebookRoleBinding creates a rolebinding for the user in the specified namespace +func CreateNotebookRoleBinding(namespace, rolebindingName, username, testDir string) error { + cmd := exec.Command("kubectl", "create", "rolebinding", rolebindingName, + "-n", namespace, + "--role=admin", + "--user="+username) + _, err := testutils.Run(cmd, testDir) + if err != nil { + return fmt.Errorf("failed to create rolebinding %s: %w", rolebindingName, err) + } + return nil +} + +// BuildNotebookCommand builds the command array for executing a notebook with papermill +func BuildNotebookCommand(notebookName, testDir string) []string { + return []string{ + "/bin/sh", + "-c", + fmt.Sprintf( + "pip install papermill && "+ + "mkdir -p /opt/app-root/src/feature_repo && "+ + "cp -rL /opt/app-root/notebooks/* /opt/app-root/src/feature_repo/ && "+ + "oc login --token=%s --server=%s --insecure-skip-tls-verify=true && "+ + "(papermill /opt/app-root/notebooks/%s /opt/app-root/src/output.ipynb --kernel python3 && "+ + "echo '✅ Notebook executed successfully' || "+ + "(echo '❌ Notebook execution failed' && "+ + "cp /opt/app-root/src/output.ipynb /opt/app-root/src/failed_output.ipynb && "+ + "echo '📄 Copied failed notebook to failed_output.ipynb')) && "+ + "jupyter nbconvert --to notebook --stdout /opt/app-root/src/output.ipynb || echo '⚠️ nbconvert failed' && "+ + "sleep 100; exit 0", + GetOCToken(testDir), + GetOCServer(testDir), + notebookName, + ), + } +} + +// GetNotebookParams builds and returns NotebookTemplateParams from environment variables and configuration +// feastProject is optional - if provided, it will be set in the notebook annotation, otherwise it will be empty +func GetNotebookParams(namespace, configMapName, notebookPVC, notebookName, testDir string, feastProject string) NotebookTemplateParams { + username := GetOCUser(testDir) + command := BuildNotebookCommand(notebookName, testDir) + + getEnv := func(key string) string { + val, _ := os.LookupEnv(key) + return val + } + + return NotebookTemplateParams{ + Namespace: namespace, + IngressDomain: GetIngressDomain(testDir), + OpenDataHubNamespace: getEnv("APPLICATIONS_NAMESPACE"), + NotebookImage: getEnv("NOTEBOOK_IMAGE"), + NotebookConfigMapName: configMapName, + NotebookPVC: notebookPVC, + Username: username, + OC_TOKEN: GetOCToken(testDir), + OC_SERVER: GetOCServer(testDir), + NotebookFile: notebookName, + Command: "[\"" + strings.Join(command, "\",\"") + "\"]", + PipIndexUrl: getEnv("PIP_INDEX_URL"), + PipTrustedHost: getEnv("PIP_TRUSTED_HOST"), + FeastVersion: getEnv("FEAST_VERSION"), + OpenAIAPIKey: getEnv("OPENAI_API_KEY"), + FeastProject: feastProject, + AdditionalEnv: map[string]string{}, + } +} + +// SetupNotebookEnvironment performs all the setup steps required for notebook testing +func SetupNotebookEnvironment(namespace, configMapName, notebookFile, featureRepoPath, pvcFile, rolebindingName, testDir string) error { + // Set namespace context + if err := SetNamespaceContext(namespace, testDir); err != nil { + return fmt.Errorf("failed to set namespace context: %w", err) + } + + // Create config map + if err := CreateNotebookConfigMap(namespace, configMapName, notebookFile, featureRepoPath, testDir); err != nil { + return fmt.Errorf("failed to create config map: %w", err) + } + + // Create PVC + if err := CreateNotebookPVC(pvcFile, testDir); err != nil { + return fmt.Errorf("failed to create PVC: %w", err) + } + + // Create rolebinding + username := GetOCUser(testDir) + if err := CreateNotebookRoleBinding(namespace, rolebindingName, username, testDir); err != nil { + return fmt.Errorf("failed to create rolebinding: %w", err) + } + + return nil +} + +// prepareNotebookTestResources sets namespace context, ConfigMap, PVC, and rolebinding for notebook E2E tests. +func prepareNotebookTestResources(namespace, configMapName, notebookFile, featureRepoPath, pvcFile, rolebindingName, notebookPVC, testDir string) { + By(fmt.Sprintf("Setting namespace context to : %s", namespace)) + Expect(SetNamespaceContext(namespace, testDir)).To(Succeed()) + fmt.Printf("Successfully set namespace context to: %s\n", namespace) + + By(fmt.Sprintf("Creating Config map: %s", configMapName)) + Expect(CreateNotebookConfigMap(namespace, configMapName, notebookFile, featureRepoPath, testDir)).To(Succeed()) + fmt.Printf("ConfigMap %s created successfully\n", configMapName) + + By(fmt.Sprintf("Creating Persistent volume claim: %s", notebookPVC)) + Expect(CreateNotebookPVC(pvcFile, testDir)).To(Succeed()) + fmt.Printf("Persistent Volume Claim %s created successfully\n", notebookPVC) + + By(fmt.Sprintf("Creating rolebinding %s for the user", rolebindingName)) + Expect(CreateNotebookRoleBinding(namespace, rolebindingName, GetOCUser(testDir), testDir)).To(Succeed()) + fmt.Printf("Created rolebinding %s successfully\n", rolebindingName) +} + +// CreateNotebookTest performs all the setup steps and creates a notebook. +// This function handles namespace context, ConfigMap, PVC, rolebinding, and notebook creation. +// feastProject is optional - if provided, it will be set in the notebook annotation, otherwise it will be empty +func CreateNotebookTest(namespace, configMapName, notebookFile, featureRepoPath, pvcFile, rolebindingName, notebookPVC, notebookName, testDir string, feastProject string) { + prepareNotebookTestResources(namespace, configMapName, notebookFile, featureRepoPath, pvcFile, rolebindingName, notebookPVC, testDir) + + nbParams := GetNotebookParams(namespace, configMapName, notebookPVC, notebookName, testDir, feastProject) + By("Creating Jupyter Notebook") + Expect(CreateNotebook(nbParams)).To(Succeed(), "Failed to create notebook") +} + +// MonitorNotebookTest monitors the notebook execution and verifies completion. +func MonitorNotebookTest(namespace, notebookName string) { + By("Monitoring notebook logs") + Expect(MonitorNotebookPod(namespace, "jupyter-nb-", notebookName)).To(Succeed(), "Notebook execution failed") +} + +// RunNotebookTest performs all the setup steps, creates a notebook, and monitors its execution. +// This function is kept for backward compatibility. For new tests, use CreateNotebookTest and MonitorNotebookTest separately. +// feastProject is optional - if provided, it will be set in the notebook annotation, otherwise it will be empty +func RunNotebookTest(namespace, configMapName, notebookFile, featureRepoPath, pvcFile, rolebindingName, notebookPVC, notebookName, testDir string, feastProject string) { + CreateNotebookTest(namespace, configMapName, notebookFile, featureRepoPath, pvcFile, rolebindingName, notebookPVC, notebookName, testDir, feastProject) + MonitorNotebookTest(namespace, notebookName) +} diff --git a/infra/feast-operator/test/e2e_rhoai/utils/oidc_util.go b/infra/feast-operator/test/e2e_rhoai/utils/oidc_util.go new file mode 100644 index 00000000000..1c60a5f036f --- /dev/null +++ b/infra/feast-operator/test/e2e_rhoai/utils/oidc_util.go @@ -0,0 +1,181 @@ +/* +Copyright 2026 Feast Community. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package utils + +import ( + "encoding/base64" + "fmt" + "os" + "os/exec" + "strings" + + testutils "github.com/feast-dev/feast/infra/feast-operator/test/utils" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +// ApplyFeastOidcPermissions writes the OIDC permissions file into the Feast registry pod +// using kubectl exec (avoids `oc cp` which requires tar in the container), +// then runs `feast apply` to register the permissions. +func ApplyFeastOidcPermissions(fileName string, registryFilePath string, namespace string, podNamePrefix string) { + By("Applying Feast OIDC permissions to the Feast registry pod") + + By(fmt.Sprintf("Finding pod with prefix %q in namespace %q", podNamePrefix, namespace)) + pod, err := getPodByPrefix(namespace, podNamePrefix) + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + ExpectWithOffset(1, pod).NotTo(BeNil()) + + podName := pod.Name + fmt.Printf("Found pod: %s\n", podName) + + // Read the permissions file content locally + content, err := os.ReadFile(fileName) + ExpectWithOffset(1, err).NotTo(HaveOccurred(), fmt.Sprintf("Failed to read permissions file %s", fileName)) + + idx := strings.LastIndex(registryFilePath, "/") + ExpectWithOffset(1, idx).To(BeNumerically(">", 0), "registryFilePath must include a directory component") + dir := registryFilePath[:idx] + + By(fmt.Sprintf("Writing permissions file to %s in pod %s", registryFilePath, podName)) + cmd := exec.Command( + "kubectl", "exec", podName, + "-n", namespace, + "-c", "registry", + "--", + "mkdir", "-p", dir, + ) + _, err = testutils.Run(cmd, "/test/e2e_rhoai") + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + + encoded := base64.StdEncoding.EncodeToString(content) + shellCmd := fmt.Sprintf("echo '%s' | base64 -d > %s", encoded, registryFilePath) + cmd = exec.Command( + "kubectl", "exec", podName, + "-n", namespace, + "-c", "registry", + "--", + "sh", "-c", shellCmd, + ) + _, err = testutils.Run(cmd, "/test/e2e_rhoai") + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + fmt.Printf("Successfully wrote OIDC permissions file to pod: %s\n", podName) + + By("Running feast apply inside the Feast registry pod") + cmd = exec.Command( + "kubectl", "exec", podName, + "-n", namespace, + "-c", "registry", + "--", + "sh", "-c", + "cd /feast-data/driver_ranking_oidc/feature_repo && feast apply", + ) + _, err = testutils.Run(cmd, "/test/e2e_rhoai") + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + fmt.Println("Feast OIDC permissions apply executed successfully") + + By("Validating that Feast OIDC permission has been applied") + cmd = exec.Command( + "kubectl", "exec", podName, + "-n", namespace, + "-c", "registry", + "--", + "feast", "permissions", "list", + ) + + output, err := testutils.Run(cmd, "/test/e2e_rhoai") + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + + ExpectWithOffset(1, output).To(ContainSubstring("admin_group_permission"), "Expected permission 'admin_group_permission' to exist") + fmt.Println("Verified: Feast OIDC permission 'admin_group_permission' exists") +} + +// ApplyFeastOidcYamlAndVerify applies the OIDC FeatureStore manifest and waits for the +// deployment to become available. Unlike ApplyFeastYamlAndVerify, this skips postgres +// table checks and git repo init container verification since the OIDC CR uses default stores. +func ApplyFeastOidcYamlAndVerify(namespace string, testDir string, feastDeploymentName string, feastCRName string, feastYAMLFilePath string) { + By("Applying OIDC Feast yaml for Feature store CR") + cmd := exec.Command("kubectl", "apply", "-n", namespace, + "-f", feastYAMLFilePath) + _, err := testutils.Run(cmd, testDir) + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + CheckDeployment(namespace, feastDeploymentName) +} + +// CreateOidcNotebookTest performs all the setup steps and creates a notebook with OIDC token injected. +func CreateOidcNotebookTest(namespace, configMapName, notebookFile, featureRepoPath, pvcFile, rolebindingName, notebookPVC, notebookName, testDir string, feastProject string) { + prepareNotebookTestResources(namespace, configMapName, notebookFile, featureRepoPath, pvcFile, rolebindingName, notebookPVC, testDir) + + nbParams := GetOidcNotebookParams(namespace, configMapName, notebookPVC, notebookName, testDir, feastProject) + By("Creating Jupyter Notebook with OIDC token") + Expect(CreateNotebook(nbParams)).To(Succeed(), "Failed to create OIDC notebook") +} + +// BuildOidcNotebookCommand builds the command for executing the OIDC notebook. +func BuildOidcNotebookCommand(notebookName string) []string { + return []string{ + "/bin/sh", + "-c", + fmt.Sprintf( + "pip install papermill && "+ + "mkdir -p /opt/app-root/src/feature_repo && "+ + "cp -rL /opt/app-root/notebooks/* /opt/app-root/src/feature_repo/ && "+ + "(papermill /opt/app-root/notebooks/%s /opt/app-root/src/output.ipynb --kernel python3 && "+ + "echo '✅ Notebook executed successfully' || "+ + "(echo '❌ Notebook execution failed' && "+ + "cp /opt/app-root/src/output.ipynb /opt/app-root/src/failed_output.ipynb && "+ + "echo '📄 Copied failed notebook to failed_output.ipynb')) && "+ + "jupyter nbconvert --to notebook --stdout /opt/app-root/src/output.ipynb || echo '⚠️ nbconvert failed' && "+ + "sleep 100; exit 0", + notebookName, + ), + } +} + +// GetOidcNotebookParams builds NotebookTemplateParams with FEAST_OIDC_TOKEN set from TOKEN env var. +func GetOidcNotebookParams(namespace, configMapName, notebookPVC, notebookName, testDir string, feastProject string) NotebookTemplateParams { + username := GetOCUser(testDir) + command := BuildOidcNotebookCommand(notebookName) + oidcToken := strings.TrimSpace(os.Getenv("TOKEN")) + ExpectWithOffset(1, oidcToken).NotTo(BeEmpty(), "TOKEN env var must be set for OIDC notebook test") + + getEnv := func(key string) string { + val, _ := os.LookupEnv(key) + return val + } + + return NotebookTemplateParams{ + Namespace: namespace, + IngressDomain: GetIngressDomain(testDir), + OpenDataHubNamespace: getEnv("APPLICATIONS_NAMESPACE"), + NotebookImage: getEnv("NOTEBOOK_IMAGE"), + NotebookConfigMapName: configMapName, + NotebookPVC: notebookPVC, + Username: username, + OC_TOKEN: GetOCToken(testDir), + OC_SERVER: GetOCServer(testDir), + NotebookFile: notebookName, + Command: "[\"" + strings.Join(command, "\",\"") + "\"]", + PipIndexUrl: getEnv("PIP_INDEX_URL"), + PipTrustedHost: getEnv("PIP_TRUSTED_HOST"), + FeastVersion: getEnv("FEAST_VERSION"), + OpenAIAPIKey: getEnv("OPENAI_API_KEY"), + FeastProject: feastProject, + AdditionalEnv: map[string]string{ + "FEAST_OIDC_TOKEN": oidcToken, + }, + } +} diff --git a/infra/feast-operator/test/e2e_rhoai/utils/util.go b/infra/feast-operator/test/e2e_rhoai/utils/util.go new file mode 100644 index 00000000000..f5ed55c3bb2 --- /dev/null +++ b/infra/feast-operator/test/e2e_rhoai/utils/util.go @@ -0,0 +1,737 @@ +package utils + +import ( + "bytes" + "fmt" + "os" + "os/exec" + "regexp" + "strings" + + testutils "github.com/feast-dev/feast/infra/feast-operator/test/utils" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +const ( + FeastPrefix = "feast-" +) + +// logCronJobCommands prints CronJob command list read from FeatureStore CR status. +func logCronJobCommands(commands string) { + fmt.Printf("CronJob commands from CR status:\n %s\n\n", strings.TrimSpace(commands)) +} + +func ListConfigMaps(namespace string) ([]string, error) { + cmd := exec.Command("kubectl", "get", "cm", "-n", namespace, "-o", "jsonpath={range .items[*]}{.metadata.name}{\"\\n\"}{end}") + var out bytes.Buffer + var stderr bytes.Buffer + cmd.Stdout = &out + cmd.Stderr = &stderr + + if err := cmd.Run(); err != nil { + return nil, fmt.Errorf("failed to list config maps in namespace %s. Error: %v. Stderr: %s", + namespace, err, stderr.String()) + } + + configMaps := strings.Split(strings.TrimSpace(out.String()), "\n") + // Filter out empty strings + var result []string + for _, cm := range configMaps { + if cm != "" { + result = append(result, cm) + } + } + return result, nil +} + +// VerifyConfigMapExistsInList checks if a ConfigMap exists in the list of ConfigMaps +func VerifyConfigMapExistsInList(namespace, configMapName string) (bool, error) { + configMaps, err := ListConfigMaps(namespace) + if err != nil { + return false, err + } + + for _, cm := range configMaps { + if cm == configMapName { + return true, nil + } + } + + return false, nil +} + +// checkIfConfigMapExists validates if a config map exists using the kubectl CLI. +func checkIfConfigMapExists(namespace, configMapName string) error { + cmd := exec.Command("kubectl", "get", "cm", configMapName, "-n", namespace) + + var out bytes.Buffer + var stderr bytes.Buffer + cmd.Stdout = &out + cmd.Stderr = &stderr + + if err := cmd.Run(); err != nil { + return fmt.Errorf("failed to find config map %s in namespace %s. Error: %v. Stderr: %s", + configMapName, namespace, err, stderr.String()) + } + + // Check the output to confirm presence + if !strings.Contains(out.String(), configMapName) { + return fmt.Errorf("config map %s not found in namespace %s", configMapName, namespace) + } + + return nil +} + +// VerifyFeastConfigMapExists verifies that a ConfigMap exists and contains the specified key/file +func VerifyFeastConfigMapExists(namespace, configMapName, expectedKey string) error { + // First verify the ConfigMap exists + if err := checkIfConfigMapExists(namespace, configMapName); err != nil { + return fmt.Errorf("config map %s does not exist: %w", configMapName, err) + } + + // Get the ConfigMap data to verify the key exists + cmd := exec.Command("kubectl", "get", "cm", configMapName, "-n", namespace, "-o", "jsonpath={.data."+expectedKey+"}") + var out bytes.Buffer + var stderr bytes.Buffer + cmd.Stdout = &out + cmd.Stderr = &stderr + + if err := cmd.Run(); err != nil { + return fmt.Errorf("failed to get config map data for %s in namespace %s. Error: %v. Stderr: %s", + configMapName, namespace, err, stderr.String()) + } + + configContent := out.String() + if configContent == "" { + return fmt.Errorf("config map %s does not contain key %s", configMapName, expectedKey) + } + + return nil +} + +// VerifyFeastConfigMapContent verifies that a ConfigMap contains the expected feast configuration content +// This assumes the ConfigMap and key already exist (use VerifyFeastConfigMapExists first) +func VerifyFeastConfigMapContent(namespace, configMapName, expectedKey string, expectedContent []string) error { + // Get the ConfigMap data + cmd := exec.Command("kubectl", "get", "cm", configMapName, "-n", namespace, "-o", "jsonpath={.data."+expectedKey+"}") + var out bytes.Buffer + var stderr bytes.Buffer + cmd.Stdout = &out + cmd.Stderr = &stderr + + if err := cmd.Run(); err != nil { + return fmt.Errorf("failed to get config map data for %s in namespace %s. Error: %v. Stderr: %s", + configMapName, namespace, err, stderr.String()) + } + + configContent := out.String() + if configContent == "" { + return fmt.Errorf("config map %s does not contain key %s", configMapName, expectedKey) + } + + // Verify all expected content strings are present + for _, expected := range expectedContent { + if !strings.Contains(configContent, expected) { + return fmt.Errorf("config map %s content does not contain expected string: %s. Content:\n%s", + configMapName, expected, configContent) + } + } + + return nil +} + +func ApplyFeastPermissions(fileName string, registryFilePath string, namespace string, podNamePrefix string) { + By("Applying Feast permissions to the Feast registry pod") + + // 1. Get the pod by prefix + By(fmt.Sprintf("Finding pod with prefix %q in namespace %q", podNamePrefix, namespace)) + pod, err := getPodByPrefix(namespace, podNamePrefix) + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + ExpectWithOffset(1, pod).NotTo(BeNil()) + + podName := pod.Name + fmt.Printf("Found pod: %s\n", podName) + + ExpectWithOffset(1, registryFilePath).To(And( + Not(BeEmpty()), + HavePrefix("/"), + Not(ContainSubstring("\x00")), + ), "registryFilePath must be a non-empty absolute container path") + + cmd := exec.Command( + "oc", "exec", "-i", podName, + "-n", namespace, + "-c", "registry", + "--", + "sh", "-c", `cat > "$1"`, "sh", registryFilePath, + ) + + file, err := os.Open(fileName) + ExpectWithOffset(1, err).NotTo(HaveOccurred(), "Failed to open permissions file") + defer func() { + if cerr := file.Close(); cerr != nil { + fmt.Printf("Warning: failed to close file: %v\n", cerr) + } + }() + cmd.Stdin = file + + output, err := cmd.CombinedOutput() + ExpectWithOffset(1, err).NotTo(HaveOccurred(), + fmt.Sprintf("Failed to copy file to pod: %s", string(output))) + + fmt.Printf("Successfully copied file to pod: %s\n", podName) + + // Run `feast apply` inside the pod to apply updated permissions + By("Running feast apply inside the Feast registry pod") + cmd = exec.Command( + "oc", "exec", podName, + "-n", namespace, + "-c", "registry", + "--", + "bash", "-c", + "cd /feast-data/credit_scoring_local/feature_repo && feast apply", + ) + _, err = testutils.Run(cmd, "/test/e2e_rhoai") + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + fmt.Println("Feast permissions apply executed successfully") + + By("Validating that Feast permission has been applied") + + cmd = exec.Command( + "oc", "exec", podName, + "-n", namespace, + "-c", "registry", + "--", + "feast", "permissions", "list", + ) + + output, err = testutils.Run(cmd, "/test/e2e_rhoai") + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + + // Change "feast-auth" if your permission name is different + ExpectWithOffset(1, output).To(ContainSubstring("feast-auth"), "Expected permission 'feast-auth' to exist") + + fmt.Println("Verified: Feast permission 'feast-auth' exists") +} + +// CreateNamespace - create the namespace for tests +func CreateNamespace(namespace string, testDir string) error { + cmd := exec.Command("kubectl", "create", "ns", namespace) + output, err := testutils.Run(cmd, testDir) + if err != nil { + return fmt.Errorf("failed to create namespace %s: %v\nOutput: %s", namespace, err, output) + } + return nil +} + +// DeleteNamespace - Delete the namespace for tests +func DeleteNamespace(namespace string, testDir string) error { + cmd := exec.Command("kubectl", "delete", "ns", namespace, "--timeout=180s") + output, err := testutils.Run(cmd, testDir) + if err != nil { + return fmt.Errorf("failed to delete namespace %s: %v\nOutput: %s", namespace, err, output) + } + return nil +} + +// applies the manifests for Redis and Postgres and checks whether the deployments become available +func ApplyFeastInfraManifestsAndVerify(namespace string, testDir string) { + By("Applying postgres.yaml and redis.yaml manifests") + cmd := exec.Command("kubectl", "apply", "-n", namespace, "-f", "test/testdata/feast_integration_test_crs/postgres.yaml", "-f", "test/testdata/feast_integration_test_crs/redis.yaml") + _, cmdOutputerr := testutils.Run(cmd, testDir) + ExpectWithOffset(1, cmdOutputerr).NotTo(HaveOccurred()) + CheckDeployment(namespace, "postgres") + CheckDeployment(namespace, "redis") +} + +// substituteFeastS3Credentials replaces ${AWS_ACCESS_KEY_ID}, ${AWS_S3_BUCKET}, etc. from process environment variables. +// Returns an error if any required variable is unset or whitespace-only so the test fails before kubectl apply. +func substituteFeastS3Credentials(content string) (string, error) { + required := []struct { + env, placeholder string + }{ + {"AWS_ACCESS_KEY_ID", "${AWS_ACCESS_KEY_ID}"}, + {"AWS_SECRET_ACCESS_KEY", "${AWS_SECRET_ACCESS_KEY}"}, + {"AWS_DEFAULT_REGION", "${AWS_DEFAULT_REGION}"}, + {"AWS_S3_BUCKET", "${AWS_S3_BUCKET}"}, + } + repl := make(map[string]string, len(required)) + var missing []string + for _, r := range required { + v := strings.TrimSpace(os.Getenv(r.env)) + if v == "" { + missing = append(missing, r.env) + continue + } + repl[r.placeholder] = v + } + if len(missing) > 0 { + return "", fmt.Errorf( + "feast S3 e2e requires non-empty environment variables: %s", + strings.Join(missing, ", "), + ) + } + out := content + for ph, v := range repl { + out = strings.ReplaceAll(out, ph, v) + } + return out, nil +} + +// ApplyFeastS3YamlAndVerify applies the S3-based FeatureStore manifest (no postgres/redis), +// waits for deployment FeastPrefix+feastCRName, validates CR Ready and feature_store.yaml for S3 driver_ranking. +// feastCRName must match FeatureStore metadata.name in the manifest (e.g. "test-s3"). +func ApplyFeastS3YamlAndVerify(namespace string, testDir string, feastS3YamlPath string, feastCRName string) { + By("Applying Feast S3 manifest (secrets + FeatureStore CR)") + data, err := os.ReadFile(feastS3YamlPath) + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + + rendered, err := substituteFeastS3Credentials(string(data)) + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + tmp, err := os.CreateTemp("", "feast-s3-rendered-*.yaml") + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + tmpPath := tmp.Name() + defer func() { + if rmErr := os.Remove(tmpPath); rmErr != nil { + fmt.Fprintf(os.Stderr, "warning: could not remove temp file %s: %v\n", tmpPath, rmErr) + } + }() + _, err = tmp.WriteString(rendered) + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + ExpectWithOffset(1, tmp.Close()).To(Succeed()) + + cmd := exec.Command("kubectl", "apply", "-n", namespace, "-f", tmpPath) + _, err = testutils.Run(cmd, testDir) + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + fmt.Printf("Applied rendered Feast S3 manifest to namespace %q\n", namespace) + + feastDeploymentName := FeastPrefix + feastCRName + CheckDeployment(namespace, feastDeploymentName) + ValidateFeatureStoreCRStatus(namespace, feastCRName) + + By("Verifying client feature_store.yaml for S3-backed registry and driver_ranking project") + validateFeatureStoreYamlS3(namespace, feastDeploymentName) +} + +// CheckDeployment verifies the specified deployment exists and is in the "Available" state. +func CheckDeployment(namespace, name string) { + By(fmt.Sprintf("Waiting for %s deployment to become available", name)) + err := testutils.CheckIfDeploymentExistsAndAvailable(namespace, name, 2*testutils.Timeout) + Expect(err).ToNot(HaveOccurred(), fmt.Sprintf( + "Deployment %s is not available but expected to be.\nError: %v", name, err, + )) + fmt.Printf("Deployment %s is available\n", name) +} + +// validate that the status of the FeatureStore CR is "Ready". +func ValidateFeatureStoreCRStatus(namespace, crName string) { + lastObservation := "no observation captured yet" + Eventually(func() string { + cmd := exec.Command("kubectl", "get", "feast", crName, "-n", namespace, "-o", "jsonpath={.status.phase}") + output, err := cmd.CombinedOutput() + if err != nil { + lastObservation = fmt.Sprintf("kubectl get failed: %v; output: %s", err, strings.TrimSpace(string(output))) + return "" + } + phase := strings.TrimSpace(string(output)) + lastObservation = fmt.Sprintf("status.phase=%q", phase) + return phase + }, "5m", "5s").Should( + Equal("Ready"), + fmt.Sprintf( + "Feature Store CR %s/%s did not reach 'Ready' state in time; last observation: %s", + namespace, crName, lastObservation, + ), + ) + + fmt.Printf("Feature Store CR is Ready: %s/%s\n", namespace, crName) +} + +// verifyApplyFeatureStoreCronJob checks CronJob commands on the FeatureStore CR and runs a one-off apply job with expected log substrings. +func verifyApplyFeatureStoreCronJob(namespace, feastCRName, feastDeploymentName, testDir string, expectedJobLogs []string) { + By("Verify CronJob commands in FeatureStore CR") + cmd := exec.Command("kubectl", "get", "-n", namespace, "feast/"+feastCRName, "-o", "jsonpath={.status.applied.cronJob.containerConfigs.commands}") + output, err := cmd.CombinedOutput() + Expect(err).NotTo(HaveOccurred(), fmt.Sprintf("Failed to fetch CronJob commands:\n%s", output)) + commands := string(output) + logCronJobCommands(commands) + Expect(commands).To(ContainSubstring(`feast apply`)) + Expect(commands).To(ContainSubstring(`feast materialize-incremental $(date -u +'%Y-%m-%dT%H:%M:%S')`)) + + CreateAndVerifyJobFromCron(namespace, feastDeploymentName, "feast-test-apply", testDir, expectedJobLogs) +} + +// validates the `feast apply` and `feast materialize-incremental commands were configured in the FeatureStore CR's CronJob config. +func VerifyApplyFeatureStoreDefinitions(namespace string, feastCRName string, feastDeploymentName string) { + verifyApplyFeatureStoreCronJob(namespace, feastCRName, feastDeploymentName, "", []string{ + "No project found in the repository", + "Applying changes for project credit_scoring_local", + "Deploying infrastructure for credit_history", + "Deploying infrastructure for zipcode_features", + "Materializing 2 feature views to", + "into the redis online store", + "credit_history from", + "zipcode_features from", + }) +} + +// VerifyApplyFeatureStoreDefinitionsS3 validates the apply/materialize CronJob for the S3 + driver_ranking FeatureStore. +func VerifyApplyFeatureStoreDefinitionsS3(namespace string, feastCRName string, feastDeploymentName string) { + verifyApplyFeatureStoreCronJob(namespace, feastCRName, feastDeploymentName, "", []string{ + "Applying changes for project driver_ranking", + "Materializing", + }) +} + +// Create a Job and verifies its logs contain expected substrings +func CreateAndVerifyJobFromCron(namespace, cronName, jobName, testDir string, expectedLogSubstrings []string) { + By(fmt.Sprintf("Creating Job %s from CronJob %s", jobName, cronName)) + cmd := exec.Command("kubectl", "create", "job", "--from=cronjob/"+cronName, jobName, "-n", namespace) + _, err := testutils.Run(cmd, testDir) + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + fmt.Printf("Created one-off job %q from CronJob %q\n", jobName, cronName) + + By("Waiting for Job completion") + cmd = exec.Command("kubectl", "wait", "--for=condition=complete", "--timeout=5m", "job/"+jobName, "-n", namespace) + _, err = testutils.Run(cmd, testDir) + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + fmt.Printf("Job %q completed successfully\n", jobName) + + By("Checking logs of completed job") + cmd = exec.Command("kubectl", "logs", "job/"+jobName, "-n", namespace, "--all-containers=true") + output, err := testutils.Run(cmd, testDir) + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + + outputStr := string(output) + ansi := regexp.MustCompile(`\x1b\[[0-9;]*m`) + outputStr = ansi.ReplaceAllString(outputStr, "") + fmt.Printf("----- begin logs: job/%s namespace/%s -----\n%s\n----- end logs: job/%s -----\n", + jobName, namespace, strings.TrimRight(outputStr, "\n"), jobName) + + for _, expected := range expectedLogSubstrings { + ExpectWithOffset(1, outputStr).To(ContainSubstring(expected), + fmt.Sprintf("job %q logs should contain substring %q", jobName, expected)) + } + fmt.Printf("Job %q: all %d expected log substring(s) found: %v\n\n", + jobName, len(expectedLogSubstrings), expectedLogSubstrings) +} + +// validate the feature store yaml +func validateFeatureStoreYaml(namespace, deployment string) { + cmd := exec.Command("kubectl", "exec", "deploy/"+deployment, "-n", namespace, "-c", "online", "--", "cat", "feature_store.yaml") + output, err := cmd.CombinedOutput() + Expect(err).NotTo(HaveOccurred(), "Failed to read feature_store.yaml") + + content := string(output) + Expect(content).To(ContainSubstring("offline_store:\n type: duckdb")) + Expect(content).To(ContainSubstring("online_store:\n type: redis")) + Expect(content).To(ContainSubstring("registry_type: sql")) +} + +func validateFeatureStoreYamlS3(namespace, deployment string) { + cmd := exec.Command("kubectl", "exec", "deploy/"+deployment, "-n", namespace, "-c", "online", "--", "cat", "feature_store.yaml") + output, err := cmd.CombinedOutput() + Expect(err).NotTo(HaveOccurred(), "Failed to read feature_store.yaml") + + content := string(output) + Expect(content).To(ContainSubstring("project: driver_ranking")) + Expect(content).To(ContainSubstring("s3://")) + fmt.Printf("feature_store.yaml in online pod: contains project driver_ranking and s3:// registry path\n") +} + +// apply and verifies the Feast deployment becomes available, the CR status is "Ready +func ApplyFeastYamlAndVerify(namespace string, testDir string, feastDeploymentName string, feastCRName string, feastYAMLFilePath string) { + By("Applying Feast yaml for secrets and Feature store CR") + cmd := exec.Command("kubectl", "apply", "-n", namespace, + "-f", feastYAMLFilePath) + _, err := testutils.Run(cmd, testDir) + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + CheckDeployment(namespace, feastDeploymentName) + + By("Verify Feature Store CR is in Ready state") + ValidateFeatureStoreCRStatus(namespace, feastCRName) + + By("Verifying that the Postgres DB contains the expected Feast tables") + cmd = exec.Command("kubectl", "exec", "deploy/postgres", "-n", namespace, "--", "psql", "-h", "localhost", "-U", "feast", "feast", "-c", `\dt`) + output, err := cmd.CombinedOutput() + Expect(err).NotTo(HaveOccurred(), fmt.Sprintf("Failed to get tables from Postgres. Output:\n%s", output)) + outputStr := string(output) + fmt.Println("Postgres Tables:\n", outputStr) + // List of expected tables + expectedTables := []string{ + "data_sources", "entities", "feast_metadata", "feature_services", "feature_views", + "managed_infra", "on_demand_feature_views", "permissions", "projects", + "saved_datasets", "stream_feature_views", "validation_references", + } + for _, table := range expectedTables { + Expect(outputStr).To(ContainSubstring(table), fmt.Sprintf("Expected table %q not found in output:\n%s", table, outputStr)) + } + + By("Verifying that the Feast repo was successfully cloned by the init container") + cmd = exec.Command("kubectl", "logs", "-f", "-n", namespace, "deploy/"+feastDeploymentName, "-c", "feast-init") + output, err = cmd.CombinedOutput() + Expect(err).NotTo(HaveOccurred(), fmt.Sprintf("Failed to get logs from init container. Output:\n%s", output)) + outputStr = string(output) + fmt.Println("Init Container Logs:\n", outputStr) + // Assert that the logs contain success indicators + Expect(outputStr).To(ContainSubstring("Feast repo creation complete"), "Expected Feast repo creation message not found") + + By("Verifying client feature_store.yaml for expected store types") + validateFeatureStoreYaml(namespace, feastDeploymentName) +} + +// checks for the presence of expected entities, features, feature views, data sources, etc. +func VerifyFeastMethods(namespace string, feastDeploymentName string, testDir string) { + type feastCheck struct { + command []string + expected []string + logPrefix string + } + checks := []feastCheck{ + { + command: []string{"feast", "projects", "list"}, + expected: []string{"credit_scoring_local"}, + logPrefix: "Projects List", + }, + { + command: []string{"feast", "feature-views", "list"}, + expected: []string{"credit_history", "zipcode_features", "total_debt_calc"}, + logPrefix: "Feature Views List", + }, + { + command: []string{"feast", "entities", "list"}, + expected: []string{"zipcode", "dob_ssn"}, + logPrefix: "Entities List", + }, + { + command: []string{"feast", "data-sources", "list"}, + expected: []string{"Zipcode source", "Credit history", "application_data"}, + logPrefix: "Data Sources List", + }, + { + command: []string{"feast", "features", "list"}, + expected: []string{ + "credit_card_due", "mortgage_due", "student_loan_due", "vehicle_loan_due", + "hard_pulls", "missed_payments_2y", "missed_payments_1y", "missed_payments_6m", + "bankruptcies", "city", "state", "location_type", "tax_returns_filed", + "population", "total_wages", "total_debt_due", + }, + logPrefix: "Features List", + }, + } + + for _, check := range checks { + cmd := exec.Command("kubectl", "exec", "deploy/"+feastDeploymentName, "-n", namespace, "-c", "online", "--") + cmd.Args = append(cmd.Args, check.command...) + output, err := testutils.Run(cmd, testDir) + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + + fmt.Printf("%s:\n%s\n", check.logPrefix, string(output)) + VerifyOutputContains(output, check.expected) + } +} + +// ReplaceNamespaceInYaml reads a YAML file, replaces all existingNamespace with the actual namespace +func ReplaceNamespaceInYamlFilesInPlace(filePaths []string, existingNamespace string, actualNamespace string) error { + for _, filePath := range filePaths { + data, err := os.ReadFile(filePath) + if err != nil { + return fmt.Errorf("failed to read YAML file %s: %w", filePath, err) + } + updated := strings.ReplaceAll(string(data), existingNamespace, actualNamespace) + + err = os.WriteFile(filePath, []byte(updated), 0644) + if err != nil { + return fmt.Errorf("failed to write updated YAML file %s: %w", filePath, err) + } + } + return nil +} + +// asserts that all expected substrings are present in the given output. +func VerifyOutputContains(output []byte, expectedSubstrings []string) { + outputStr := string(output) + for _, expected := range expectedSubstrings { + Expect(outputStr).To(ContainSubstring(expected), fmt.Sprintf("Expected output to contain: %s", expected)) + } +} + +func ValidateFeatureStoreYamlS3(namespace, deployment string) { + cmd := exec.Command("kubectl", "exec", "deploy/"+deployment, "-n", namespace, "-c", "online", "--", "cat", "feature_store.yaml") + output, err := cmd.CombinedOutput() + Expect(err).NotTo(HaveOccurred(), "Failed to read feature_store.yaml") + + content := string(output) + Expect(content).To(ContainSubstring("project: driver_ranking")) + Expect(content).To(ContainSubstring("s3://")) + fmt.Printf("feature_store.yaml in online pod: contains project driver_ranking and s3:// registry path\n") +} + +// ValidateMaterializationIntervals verifies that two properties survived the operator upgrade: +// +// 1. The K8s CronJob that drives feast materialize-incremental still has a non-empty +// schedule — a schedule wipe would silently stop all future materialization. +// +// 2. The per-feature-view materialization bookmark (materializationIntervals in the S3 registry) +// is intact. feast materialize-incremental uses the most recent interval endTime as its +// start window; if the bookmark is wiped, the next run re-materializes from epoch (full scan). +// If the bookmark was never written pre-upgrade (CronJob had not yet fired), the +// materializationIntervals list is empty and this check is skipped with a warning. +func ValidateMaterializationIntervals(namespace string, feastDeploymentName string, testDir string) { + By("Asserting CronJob schedule was not wiped by the upgrade") + cmd := exec.Command("kubectl", "get", "cronjob", feastDeploymentName, + "-n", namespace, "-o", "jsonpath={.spec.schedule}") + output, err := testutils.Run(cmd, testDir) + ExpectWithOffset(1, err).NotTo(HaveOccurred(), + fmt.Sprintf("CronJob %s not found in namespace %s — upgrade may have deleted it", feastDeploymentName, namespace)) + schedule := strings.TrimSpace(string(output)) + ExpectWithOffset(1, schedule).NotTo(BeEmpty(), + "CronJob schedule is empty — upgrade reset the materialization schedule") + fmt.Printf("CronJob %s schedule: %q (preserved)\n", feastDeploymentName, schedule) + + for _, fv := range []string{"driver_hourly_stats", "driver_hourly_stats_fresh"} { + By(fmt.Sprintf("Asserting materialization bookmark for feature view %s", fv)) + cmd = exec.Command( + "kubectl", "exec", "deploy/"+feastDeploymentName, + "-n", namespace, "-c", "online", "--", + "feast", "feature-views", "describe", fv, + ) + output, err = testutils.Run(cmd, testDir) + ExpectWithOffset(1, err).NotTo(HaveOccurred(), + fmt.Sprintf("feast feature-views describe %s failed — S3 registry may be inaccessible", fv)) + + described := string(output) + fmt.Printf("feast feature-views describe %s:\n%s\n", fv, described) + + ExpectWithOffset(1, described).To(ContainSubstring("materializationIntervals"), + fmt.Sprintf("feature view %s: materializationIntervals field missing — "+ + "registry entry may have been reset by the upgrade", fv)) + + if strings.Contains(described, "startTime") { + ExpectWithOffset(1, described).NotTo(ContainSubstring("1970-01-01"), + fmt.Sprintf("feature view %s: materializationIntervals contain epoch timestamp (1970-01-01) — "+ + "bookmark was reset by the upgrade; next incremental job will re-materialize from epoch", fv)) + fmt.Printf("Feature view %s: materialization bookmark intact (non-epoch startTime present)\n", fv) + } else { + fmt.Printf("NOTICE: feature view %s has no materializationIntervals — "+ + "CronJob may not have fired before the upgrade; bookmark check skipped\n", fv) + } + } +} + +func ValidateRegistryIntact(namespace string, feastDeploymentName string, testDir string) { + type feastCheck struct { + command []string + expected []string + logPrefix string + } + + checks := []feastCheck{ + { + command: []string{"feast", "projects", "list"}, + expected: []string{"driver_ranking"}, + logPrefix: "Projects List", + }, + { + command: []string{"feast", "feature-views", "list"}, + expected: []string{"driver_hourly_stats", "driver_hourly_stats_fresh", "transformed_conv_rate", "transformed_conv_rate_fresh"}, + logPrefix: "Feature Views List", + }, + { + command: []string{"feast", "entities", "list"}, + expected: []string{"driver"}, + logPrefix: "Entities List", + }, + { + command: []string{"feast", "data-sources", "list"}, + expected: []string{"driver_hourly_stats_source", "vals_to_add", "driver_stats_push_source"}, + logPrefix: "Data Sources List", + }, + { + command: []string{"feast", "features", "list"}, + expected: []string{ + "conv_rate", "acc_rate", "avg_daily_trips", + "driver_metadata", "driver_config", "driver_profile", + "conv_rate_plus_val1", "conv_rate_plus_val2", + }, + logPrefix: "Features List", + }, + { + command: []string{"feast", "feature-services", "list"}, + expected: []string{"driver_activity_v1", "driver_activity_v2"}, + logPrefix: "Feature Services List", + }, + } + + for _, check := range checks { + cmd := exec.Command("kubectl", "exec", "deploy/"+feastDeploymentName, "-n", namespace, "-c", "online", "--") + cmd.Args = append(cmd.Args, check.command...) + output, err := testutils.Run(cmd, testDir) + ExpectWithOffset(1, err).NotTo(HaveOccurred(), + fmt.Sprintf("feast %s list failed — registry may be inaccessible or objects wiped by upgrade", check.logPrefix)) + + fmt.Printf("%s:\n%s\n", check.logPrefix, string(output)) + for _, expected := range check.expected { + ExpectWithOffset(1, string(output)).To(ContainSubstring(expected), + fmt.Sprintf("registry intact check: %s list missing %q — was the S3 registry wiped by the upgrade?", check.logPrefix, expected)) + } + } + fmt.Printf("Registry intact: all pre-upgrade objects present in S3 registry\n") +} + +// VerifyFeastMethodsForDriverRanking confirms the driver_ranking project is listed post-apply. +// Unlike ValidateRegistryIntact it runs after feast apply and serves as a write-path smoke test. +func VerifyFeastMethodsForDriverRanking(namespace string, feastDeploymentName string, testDir string) { + cmd := exec.Command("kubectl", "exec", "deploy/"+feastDeploymentName, "-n", namespace, "-c", "online", "--", + "feast", "projects", "list") + output, err := testutils.Run(cmd, testDir) + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + + fmt.Printf("Command: feast projects list\nOutput:\n%s\n", string(output)) + VerifyOutputContains(output, []string{"driver_ranking"}) + fmt.Printf("Assertion OK: output contains expected substring driver_ranking\n") +} + +func VerifyOnlineFeatureServing(namespace string, feastDeploymentName string, testDir string) { + var output []byte + var err error + + // Retry up to 3 times — the online store may take a moment to become ready + // after the deployment rolls out during an upgrade. + for attempt := 1; attempt <= 3; attempt++ { + cmd := exec.Command( + "kubectl", "exec", "deploy/"+feastDeploymentName, + "-n", namespace, "-c", "online", "--", + "feast", "get-online-features", + "-e", "driver_id=1001", + "-e", "driver_id=1002", + "-f", "driver_hourly_stats:conv_rate", + "-f", "driver_hourly_stats:acc_rate", + "-f", "driver_hourly_stats:avg_daily_trips", + ) + output, err = testutils.Run(cmd, testDir) + if err == nil { + break + } + if attempt < 3 { + fmt.Printf("feast get-online-features attempt %d/3 failed: %v — retrying\n", attempt, err) + cmd2 := exec.Command("sleep", "5") + _, _ = testutils.Run(cmd2, testDir) + } + } + ExpectWithOffset(1, err).NotTo(HaveOccurred(), + fmt.Sprintf("feast get-online-features failed after 3 attempts — "+ + "online store may be inaccessible post-upgrade:\n%s", string(output))) + + response := string(output) + fmt.Printf("feast get-online-features response:\n%s\n", response) + + for _, feature := range []string{"conv_rate", "acc_rate", "avg_daily_trips", "driver_id"} { + ExpectWithOffset(1, response).To(ContainSubstring(feature), + fmt.Sprintf("%q missing from get-online-features output — "+ + "online store schema may have changed or materialization did not write data", feature)) + } + + fmt.Printf("VerifyOnlineFeatureServing: online feature serving verified via feast get-online-features\n") +} diff --git a/sdk/python/feast/infra/feature_servers/multicloud/Dockerfile b/sdk/python/feast/infra/feature_servers/multicloud/Dockerfile index 6eb4cc96418..c894f9542fe 100644 --- a/sdk/python/feast/infra/feature_servers/multicloud/Dockerfile +++ b/sdk/python/feast/infra/feature_servers/multicloud/Dockerfile @@ -5,9 +5,30 @@ RUN microdnf install -y git gcc libpq-devel python3.12-devel && microdnf clean a USER 1001 COPY --from=ghcr.io/astral-sh/uv:latest /uv /usr/local/bin/uv + +# OS Packages needs to be installed as root +USER 0 + +# Copy Power prebuild script +COPY prebuild-power-base.sh /prebuild-power.sh +RUN chmod +x /prebuild-power.sh + ENV UV_CACHE_DIR=/tmp/uv-cache COPY requirements.txt requirements.txt + +# Run prebuild script conditionally (Power only) +RUN if [ "$(uname -m)" = "ppc64le" ]; then \ + echo "Power architecture detected. Running prebuild script..."; \ + /prebuild-power.sh && uv pip install /wheelhouse/*.whl; \ + echo "Listing built wheels:" && ls -lh /wheelhouse; \ + else \ + echo "Non-Power architecture detected; skipping Power prebuild"; \ + fi + RUN uv pip install -r requirements.txt # modify permissions to support running with a random uid RUN chmod g+w $(python -c "import feast.ui as ui; print(ui.__path__)" | tr -d "[']")/build/projects-list.json + +# Switch back to the default non-root user +USER 1001 diff --git a/sdk/python/feast/infra/feature_servers/multicloud/prebuild-power-base.sh b/sdk/python/feast/infra/feature_servers/multicloud/prebuild-power-base.sh new file mode 100644 index 00000000000..9ba044b1480 --- /dev/null +++ b/sdk/python/feast/infra/feature_servers/multicloud/prebuild-power-base.sh @@ -0,0 +1,144 @@ +#!/bin/bash +set -Eeuo pipefail +trap 'echo "[prebuild-power] failed at line $LINENO"; exit 1' ERR +shopt -s dotglob nullglob +PYTHON_VERSION=3.11 +WORKDIR=$(pwd) + +echo "[prebuild-power] Starting prebuild script..." + +# Detect release version from requirements.txt if not already provided +if [[ -z "${RELEASE_VERSION:-}" && -f "requirements.txt" ]]; then + RELEASE_VERSION=$(grep -E '^feast\[minimal\]' requirements.txt | sed -E 's/.*==\s*([0-9]+\.[0-9]+\.[0-9]+).*/v\1/') + echo "[prebuild-power] Detected RELEASE_VERSION=$RELEASE_VERSION from requirements.txt" +fi + +# URL to requirements file (using the release version) +REQ_FILE_URL="https://raw.githubusercontent.com/opendatahub-io/feast/${RELEASE_VERSION}/sdk/python/requirements/py${PYTHON_VERSION}-ci-requirements.txt" + +# Fetch the file +echo "[prebuild-power] Fetching package versions from $REQ_FILE_URL ..." +if ! REQ_CONTENT=$(curl -fsSL "$REQ_FILE_URL"); then + echo "[prebuild-power] Failed to fetch from '$RELEASE_VERSION'. Falling back to master..." + REQ_FILE_URL="https://raw.githubusercontent.com/opendatahub-io/feast/master/sdk/python/requirements/py${PYTHON_VERSION}-ci-requirements.txt" + REQ_CONTENT=$(curl -fsSL "$REQ_FILE_URL") +fi + +# Extract package versions +DUCKDB_VER=$(echo "$REQ_CONTENT" | grep -E "^duckdb==" | head -n1 | sed -E 's/.*==([0-9a-zA-Z\.\-]+).*/\1/') +GRPCIO_VER=$(echo "$REQ_CONTENT" | grep -E "^grpcio==" | head -n1 | sed -E 's/.*==([0-9a-zA-Z\.\-]+).*/\1/') +PYARROW_VER=$(echo "$REQ_CONTENT" | grep -E "^pyarrow==" | head -n1 | sed -E 's/.*==([0-9a-zA-Z\.\-]+).*/\1/') +MILVUS_VER=$(echo "$REQ_CONTENT" | grep -E "^milvus-lite==" | head -n1 | sed -E 's/.*==([0-9a-zA-Z\.\-]+).*/\1/') + +echo "[prebuild-power] Detected versions:" +echo " duckdb=$DUCKDB_VER" +echo " grpcio=$GRPCIO_VER" +echo " pyarrow=$PYARROW_VER" +echo " milvus-lite=$MILVUS_VER" + +# Ensure all versions were detected +if [[ -z "$DUCKDB_VER" || -z "$GRPCIO_VER" || -z "$PYARROW_VER" || -z "$MILVUS_VER" ]]; then + echo "[prebuild-power] Error: One or more package versions could not be detected." + exit 1 +fi + +dnf install -y gcc-toolset-13 make cmake ninja-build libomp-devel \ + git python${PYTHON_VERSION} python${PYTHON_VERSION}-devel python${PYTHON_VERSION}-pip \ + openssl openssl-devel zlib-devel libuuid-devel + +# Enable GCC toolset +source /opt/rh/gcc-toolset-13/enable +export CXX=/opt/rh/gcc-toolset-13/root/usr/bin/g++ + +# Ensure CXXFLAGS and LINKFLAGS are initialized +: "${CMAKE_ARGS:=""}" +: "${CXXFLAGS:=""}" +: "${CFLAGS:=""}" +: "${LINKFLAGS:=""}" + +# Installing Python build dependencies +python${PYTHON_VERSION} -m pip install build wheel setuptools ninja pybind11 numpy setuptools_scm Cython==3.0.8 + +# Directory to collect built wheels +mkdir -p /wheelhouse + +####################################################### +# Build DuckDB (Python package) +####################################################### +echo "[prebuild-power] Building duckdb==$DUCKDB_VER ..." +git clone https://github.com/duckdb/duckdb.git +cd duckdb +git checkout "v${DUCKDB_VER}" +cd tools/pythonpkg +python${PYTHON_VERSION} -m build --wheel --no-isolation +ls dist/*.whl >/dev/null +cp -v dist/*.whl /wheelhouse/ +cd $WORKDIR + +####################################################### +# Build gRPC (Python package) +####################################################### +echo "[prebuild-power] Building grpcio==$GRPCIO_VER ..." +GRPC_PYTHON_BUILD_SYSTEM_OPENSSL=1 python${PYTHON_VERSION} -m pip install --no-binary=:all: "grpcio==${GRPCIO_VER}" + +####################################################### +# Build Pyarrow (Python package) +####################################################### +echo "[prebuild-power] Building pyarrow==$PYARROW_VER ..." +git clone https://github.com/apache/arrow.git +cd arrow +git checkout "apache-arrow-${PYARROW_VER}" +git submodule update --init --recursive +cd cpp +mkdir -p release && cd release +cmake -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX=/usr/local \ + -DARROW_PYTHON=ON \ + -DARROW_PARQUET=ON \ + -DARROW_ORC=ON \ + -DARROW_FILESYSTEM=ON \ + -DARROW_WITH_LZ4=ON \ + -DARROW_WITH_ZSTD=ON \ + -DARROW_WITH_SNAPPY=ON \ + -DARROW_JSON=ON \ + -DARROW_CSV=ON \ + -DARROW_DATASET=ON \ + -DARROW_S3=ON \ + -DARROW_BUILD_TESTS=OFF \ + -DARROW_SUBSTRAIT=ON \ + -DProtobuf_SOURCE=BUNDLED \ + -DARROW_DEPENDENCY_SOURCE=BUNDLED \ + .. +make -j"$(nproc)" +make install +cd ../../python +export BUILD_TYPE=release +python${PYTHON_VERSION} setup.py build_ext --build-type=$BUILD_TYPE --bundle-arrow-cpp bdist_wheel +ls dist/*.whl >/dev/null +cp -v dist/*.whl /wheelhouse/ +cd $WORKDIR + +####################################################### +# Build Milvus-Lite (Python package) +####################################################### +echo "[prebuild-power] Building milvus-lite==$MILVUS_VER ..." +# Remove gcc-toolset-13; Milvus-Lite build (via Conan) requires standard gcc +dnf remove -y gcc-toolset-13 + +dnf install -y perl ncurses-devel wget openblas-devel cargo gcc gcc-c++ libstdc++-static which libaio \ + libtool m4 autoconf automake zlib-devel libffi-devel scl-utils xz + +export CC=gcc +export CXX=g++ +export CXXFLAGS="-std=c++17" + +python${PYTHON_VERSION} -m pip install conan==1.64.1 + +git clone https://github.com/milvus-io/milvus-lite +cd milvus-lite/python +git checkout "v${MILVUS_VER}" +git submodule update --init --recursive +python${PYTHON_VERSION} -m pip install -v -e . +cd $WORKDIR + +echo "[prebuild-power] All packages built successfully." diff --git a/sdk/python/feast/metrics.py b/sdk/python/feast/metrics.py index 13a855d587b..6ad3d2f9937 100644 --- a/sdk/python/feast/metrics.py +++ b/sdk/python/feast/metrics.py @@ -54,8 +54,6 @@ from datetime import datetime, timezone from typing import TYPE_CHECKING, List, Optional -import psutil - if TYPE_CHECKING: from feast.feature_store import FeatureStore @@ -517,6 +515,8 @@ def update_feature_freshness( def monitor_resources(interval: int = 5): """Background thread target that updates CPU and memory usage gauges.""" + import psutil + logger.debug("Starting resource monitoring with interval %d seconds", interval) p = psutil.Process() logger.debug("PID is %d", p.pid) diff --git a/sdk/python/feast/permissions/auth/kubernetes_token_parser.py b/sdk/python/feast/permissions/auth/kubernetes_token_parser.py index c126a90cfcc..bba92790410 100644 --- a/sdk/python/feast/permissions/auth/kubernetes_token_parser.py +++ b/sdk/python/feast/permissions/auth/kubernetes_token_parser.py @@ -41,57 +41,53 @@ async def user_details_from_access_token(self, access_token: str) -> User: Raises: AuthenticationError if any error happens. """ - # First, try to extract user information using Token Access Review + # Check for intra-communication token before hitting the K8s TokenReview API + intra_user = self._check_intra_communication_token(access_token) + if intra_user is not None: + return intra_user + + # Extract groups/namespaces via Token Access Review groups, namespaces = self._extract_groups_and_namespaces_from_token( access_token ) # Try to determine if this is a service account or regular user try: - # Attempt to decode as JWT (for service accounts) sa_namespace, sa_name = _decode_token(access_token) current_user = f"{sa_namespace}:{sa_name}" logger.info( f"Request received from ServiceAccount: {sa_name} in namespace: {sa_namespace}" ) - intra_communication_base64 = os.getenv("INTRA_COMMUNICATION_BASE64") - if sa_name is not None and sa_name == intra_communication_base64: - return User(username=sa_name, roles=[], groups=[], namespaces=[]) - else: - current_namespace = self._read_namespace_from_file() - logger.info( - f"Looking for ServiceAccount roles of {sa_namespace}:{sa_name} in {current_namespace}" - ) + current_namespace = self._read_namespace_from_file() + logger.info( + f"Looking for ServiceAccount roles of {sa_namespace}:{sa_name} in {current_namespace}" + ) - # Get roles using existing method - roles = self.get_roles( - current_namespace=current_namespace, - service_account_namespace=sa_namespace, - service_account_name=sa_name, - ) - logger.info(f"Roles: {roles}") + roles = self.get_roles( + current_namespace=current_namespace, + service_account_namespace=sa_namespace, + service_account_name=sa_name, + ) + logger.info(f"Roles: {roles}") - return User( - username=current_user, - roles=roles, - groups=groups, - namespaces=namespaces, - ) + return User( + username=current_user, + roles=roles, + groups=groups, + namespaces=namespaces, + ) except AuthenticationError as e: # If JWT decoding fails, this is likely a user token - # Use Token Access Review to get user information logger.info(f"Token is not a JWT (likely a user token): {e}") - # Get username from Token Access Review username = self._get_username_from_token_review(access_token) if not username: raise AuthenticationError("Could not extract username from token") logger.info(f"Request received from User: {username}") - # Extract roles for the user from RoleBindings and ClusterRoleBindings logger.info(f"Extracting roles for user {username} with groups: {groups}") roles = self.get_user_roles(username, groups) @@ -99,6 +95,20 @@ async def user_details_from_access_token(self, access_token: str) -> User: username=username, roles=roles, groups=groups, namespaces=namespaces ) + def _check_intra_communication_token(self, access_token: str) -> User | None: + """Short-circuits authentication for operator intra-communication tokens.""" + intra_communication_base64 = os.getenv("INTRA_COMMUNICATION_BASE64") + if not intra_communication_base64: + return None + try: + _, sa_name = _decode_token(access_token) + if sa_name == intra_communication_base64: + logger.info("Authenticated intra-communication request") + return User(username=sa_name, roles=[], groups=[], namespaces=[]) + except AuthenticationError: + pass + return None + def _read_namespace_from_file(self): try: with open(_namespace_file_path, "r") as file: