diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml index 8f71b41d08b..96e771b83b2 100644 --- a/.github/workflows/build_wheels.yml +++ b/.github/workflows/build_wheels.yml @@ -29,7 +29,7 @@ on: jobs: build-python-wheel: - name: Build wheels + name: Build wheels and source runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 @@ -50,7 +50,7 @@ jobs: with: custom_version: ${{ github.event.inputs.custom_version }} token: ${{ github.event.inputs.token }} - - name: Build wheels + - name: Checkout version and install dependencies env: VERSION: ${{ steps.get-version.outputs.release_version }} PYPI_PASSWORD: ${{ secrets.PYPI_PASSWORD }} @@ -58,69 +58,31 @@ jobs: git fetch --tags git checkout ${VERSION} python -m pip install build - python -m build --wheel --outdir wheelhouse/ + - name: Build feast + run: python -m build - uses: actions/upload-artifact@v4 with: name: python-wheels - path: ./wheelhouse/*.whl - - build-source-distribution: - name: Build source distribution - runs-on: macos-13 - steps: - - uses: actions/checkout@v4 - - name: Setup Python - id: setup-python - uses: actions/setup-python@v5 - with: - python-version: "3.11" - architecture: x64 - - name: Setup Node - uses: actions/setup-node@v3 - with: - node-version-file: './ui/.nvmrc' - registry-url: 'https://registry.npmjs.org' - - id: get-version - uses: ./.github/actions/get-semantic-release-version - with: - custom_version: ${{ github.event.inputs.custom_version }} - token: ${{ github.event.inputs.token }} - - name: Build and install dependencies - env: - VERSION: ${{ steps.get-version.outputs.release_version }} - # There's a `git restore` in here because `make install-go-ci-dependencies` is actually messing up go.mod & go.sum. - run: | - git fetch --tags - git checkout ${VERSION} - pip install -U pip setuptools wheel twine - make build-ui - git status - git restore go.mod go.sum - git restore sdk/python/feast/ui/yarn.lock - - name: Build - run: | - python3 setup.py sdist - - uses: actions/upload-artifact@v4 - with: - name: source-distribution path: dist/* # We add this step so the docker images can be built as part of the pre-release verification steps. build-docker-images: name: Build Docker images runs-on: ubuntu-latest - needs: [ build-python-wheel, build-source-distribution ] + needs: [ build-python-wheel ] strategy: matrix: component: [ feature-server-dev, feature-server-java, feature-transformation-server, feast-operator ] env: - REGISTRY: feastdev + REGISTRY: quay.io/feastdev steps: - uses: actions/checkout@v4 - name: Set up QEMU - uses: docker/setup-qemu-action@v1 + uses: docker/setup-qemu-action@v3 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 + uses: docker/setup-buildx-action@v3 + with: + install: true - id: get-version uses: ./.github/actions/get-semantic-release-version with: @@ -137,7 +99,7 @@ jobs: verify-python-wheels: name: Verify Python wheels runs-on: ${{ matrix.os }} - needs: [ build-python-wheel, build-source-distribution ] + needs: [ build-python-wheel ] strategy: matrix: os: [ ubuntu-latest, macos-13 ] @@ -168,10 +130,6 @@ jobs: with: name: python-wheels path: dist - - uses: actions/download-artifact@v4.1.7 - with: - name: source-distribution - path: dist - name: Install OS X dependencies if: matrix.os == 'macos-13' run: brew install coreutils diff --git a/.github/workflows/operator-e2e-integration-tests.yml b/.github/workflows/operator-e2e-integration-tests.yml index 83b38e52be3..c23e8095bf7 100644 --- a/.github/workflows/operator-e2e-integration-tests.yml +++ b/.github/workflows/operator-e2e-integration-tests.yml @@ -36,6 +36,17 @@ jobs: - name: Checkout code uses: actions/checkout@v4 + - name: Free Disk Space (Ubuntu) + uses: jlumbroso/free-disk-space@v1.3.1 + with: + android: true + dotnet: true + haskell: true + large-packages: false + docker-images: false + swap-storage: false + tool-cache: false + - name: Set up Go uses: actions/setup-go@v5 with: @@ -43,7 +54,15 @@ jobs: - name: Create KIND cluster run: | - kind create cluster --name $KIND_CLUSTER --wait 10m + cat < +- ++ res.json()) ++ }} ++ /> + + ); +``` + +Signed-off-by: Harri Lehtola + # [0.46.0](https://github.com/feast-dev/feast/compare/v0.45.0...v0.46.0) (2025-02-17) diff --git a/MANIFEST.in b/MANIFEST.in index c9828633d9d..c43708cdc6f 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -7,3 +7,4 @@ prune examples graft sdk/python/feast/ui/build graft sdk/python/feast/embedded_go/lib +recursive-include sdk/python/feast/static * diff --git a/Makefile b/Makefile index c199eb3a5ee..c33685ef2cb 100644 --- a/Makefile +++ b/Makefile @@ -402,9 +402,36 @@ test-python-universal-qdrant-online: -k "test_retrieve_online_documents" \ sdk/python/tests/integration/online_store/test_universal_online.py +# To use Couchbase as an offline store, you need to create an Couchbase Capella Columnar cluster on cloud.couchbase.com. +# Modify environment variables COUCHBASE_COLUMNAR_CONNECTION_STRING, COUCHBASE_COLUMNAR_USER, and COUCHBASE_COLUMNAR_PASSWORD +# with the details from your Couchbase Columnar Cluster. +test-python-universal-couchbase-offline: + PYTHONPATH='.' \ + FULL_REPO_CONFIGS_MODULE=sdk.python.feast.infra.offline_stores.contrib.couchbase_columnar_repo_configuration \ + PYTEST_PLUGINS=feast.infra.offline_stores.contrib.couchbase_offline_store.tests \ + COUCHBASE_COLUMNAR_CONNECTION_STRING=couchbases:// \ + COUCHBASE_COLUMNAR_USER=username \ + COUCHBASE_COLUMNAR_PASSWORD=password \ + python -m pytest -n 8 --integration \ + -k "not test_historical_retrieval_with_validation and \ + not test_historical_features_persisting and \ + not test_universal_cli and \ + not test_go_feature_server and \ + not test_feature_logging and \ + not test_reorder_columns and \ + not test_logged_features_validation and \ + not test_lambda_materialization_consistency and \ + not test_offline_write and \ + not test_push_features_to_offline_store and \ + not gcs_registry and \ + not s3_registry and \ + not test_snowflake and \ + not test_universal_types" \ + sdk/python/tests + test-python-universal-couchbase-online: PYTHONPATH='.' \ - FULL_REPO_CONFIGS_MODULE=sdk.python.feast.infra.online_stores.contrib.couchbase_repo_configuration \ + FULL_REPO_CONFIGS_MODULE=sdk.python.feast.infra.online_stores.couchbase_online_store.couchbase_repo_configuration \ PYTEST_PLUGINS=sdk.python.tests.integration.feature_repos.universal.online_store.couchbase \ python -m pytest -n 8 --integration \ -k "not test_universal_cli and \ @@ -480,9 +507,10 @@ push-feature-server-docker: docker push $(REGISTRY)/feature-server:$(VERSION) build-feature-server-docker: - docker buildx build --build-arg VERSION=$(VERSION) \ + docker buildx build \ -t $(REGISTRY)/feature-server:$(VERSION) \ - -f sdk/python/feast/infra/feature_servers/multicloud/Dockerfile --load . + -f sdk/python/feast/infra/feature_servers/multicloud/Dockerfile \ + --load sdk/python/feast/infra/feature_servers/multicloud push-feature-transformation-server-docker: docker push $(REGISTRY)/feature-transformation-server:$(VERSION) diff --git a/README.md b/README.md index 1052a93c11e..e820d3152d3 100644 --- a/README.md +++ b/README.md @@ -151,6 +151,9 @@ The list below contains the functionality that contributors are planning to deve * We welcome contribution to all items in the roadmap! +* **Natural Language Processing** + * [x] Vector Search (Alpha release. See [RFC](https://docs.google.com/document/d/18IWzLEA9i2lDWnbfbwXnMCg3StlqaLVI-uRpQjr_Vos/edit#heading=h.9gaqqtox9jg6)) + * [ ] [Enhanced Feature Server and SDK for native support for NLP](https://github.com/feast-dev/feast/issues/4964) * **Data Sources** * [x] [Snowflake source](https://docs.feast.dev/reference/data-sources/snowflake) * [x] [Redshift source](https://docs.feast.dev/reference/data-sources/redshift) @@ -160,6 +163,7 @@ The list below contains the functionality that contributors are planning to deve * [x] [Hive (community plugin)](https://github.com/baineng/feast-hive) * [x] [Postgres (contrib plugin)](https://docs.feast.dev/reference/data-sources/postgres) * [x] [Spark (contrib plugin)](https://docs.feast.dev/reference/data-sources/spark) + * [x] [Couchbase (contrib plugin)](https://docs.feast.dev/reference/data-sources/couchbase) * [x] Kafka / Kinesis sources (via [push support into the online store](https://docs.feast.dev/reference/data-sources/push)) * **Offline Stores** * [x] [Snowflake](https://docs.feast.dev/reference/offline-stores/snowflake) @@ -170,6 +174,7 @@ The list below contains the functionality that contributors are planning to deve * [x] [Postgres (contrib plugin)](https://docs.feast.dev/reference/offline-stores/postgres) * [x] [Trino (contrib plugin)](https://github.com/Shopify/feast-trino) * [x] [Spark (contrib plugin)](https://docs.feast.dev/reference/offline-stores/spark) + * [x] [Couchbase (contrib plugin)](https://docs.feast.dev/reference/offline-stores/couchbase) * [x] [In-memory / Pandas](https://docs.feast.dev/reference/offline-stores/file) * [x] [Custom offline store support](https://docs.feast.dev/how-to-guides/customizing-feast/adding-a-new-offline-store) * **Online Stores** @@ -185,12 +190,13 @@ The list below contains the functionality that contributors are planning to deve * [x] [Postgres (contrib plugin)](https://docs.feast.dev/reference/online-stores/postgres) * [x] [Cassandra / AstraDB (contrib plugin)](https://docs.feast.dev/reference/online-stores/cassandra) * [x] [ScyllaDB (contrib plugin)](https://docs.feast.dev/reference/online-stores/scylladb) + * [x] [Couchbase (contrib plugin)](https://docs.feast.dev/reference/online-stores/couchbase) * [x] [Custom online store support](https://docs.feast.dev/how-to-guides/customizing-feast/adding-support-for-a-new-online-store) * **Feature Engineering** - * [x] On-demand Transformations (Beta release. See [RFC](https://docs.google.com/document/d/1lgfIw0Drc65LpaxbUu49RCeJgMew547meSJttnUqz7c/edit#)) + * [x] On-demand Transformations (On Read) (Beta release. See [RFC](https://docs.google.com/document/d/1lgfIw0Drc65LpaxbUu49RCeJgMew547meSJttnUqz7c/edit#)) * [x] Streaming Transformations (Alpha release. See [RFC](https://docs.google.com/document/d/1UzEyETHUaGpn0ap4G82DHluiCj7zEbrQLkJJkKSv4e8/edit)) * [ ] Batch transformation (In progress. See [RFC](https://docs.google.com/document/d/1964OkzuBljifDvkV-0fakp2uaijnVzdwWNGdz7Vz50A/edit)) - * [ ] Persistent On-demand Transformations (Beta release. See [GitHub Issue](https://github.com/feast-dev/feast/issues/4376)) + * [x] On-demand Transformations (On Write) (Beta release. See [GitHub Issue](https://github.com/feast-dev/feast/issues/4376)) * **Streaming** * [x] [Custom streaming ingestion job support](https://docs.feast.dev/how-to-guides/customizing-feast/creating-a-custom-provider) * [x] [Push based streaming data ingestion to online store](https://docs.feast.dev/reference/data-sources/push) @@ -213,8 +219,6 @@ The list below contains the functionality that contributors are planning to deve * [x] DataHub integration (see [DataHub Feast docs](https://datahubproject.io/docs/generated/ingestion/sources/feast/)) * [x] Feast Web UI (Beta release. See [docs](https://docs.feast.dev/reference/alpha-web-ui)) * [ ] Feast Lineage Explorer -* **Natural Language Processing** - * [x] Vector Search (Alpha release. See [RFC](https://docs.google.com/document/d/18IWzLEA9i2lDWnbfbwXnMCg3StlqaLVI-uRpQjr_Vos/edit#heading=h.9gaqqtox9jg6)) ## 🎓 Important Resources diff --git a/docs/SUMMARY.md b/docs/SUMMARY.md index 127b27463ec..8db4143697e 100644 --- a/docs/SUMMARY.md +++ b/docs/SUMMARY.md @@ -37,6 +37,7 @@ * [Batch Materialization Engine](getting-started/components/batch-materialization-engine.md) * [Provider](getting-started/components/provider.md) * [Authorization Manager](getting-started/components/authz_manager.md) + * [OpenTelemetry Integration](getting-started/components/open-telemetry.md) * [Third party integrations](getting-started/third-party-integrations.md) * [FAQ](getting-started/faq.md) @@ -87,6 +88,7 @@ * [PostgreSQL (contrib)](reference/data-sources/postgres.md) * [Trino (contrib)](reference/data-sources/trino.md) * [Azure Synapse + Azure SQL (contrib)](reference/data-sources/mssql.md) + * [Couchbase (contrib)](reference/data-sources/couchbase.md) * [Offline stores](reference/offline-stores/README.md) * [Overview](reference/offline-stores/overview.md) * [Dask](reference/offline-stores/dask.md) @@ -94,6 +96,7 @@ * [BigQuery](reference/offline-stores/bigquery.md) * [Redshift](reference/offline-stores/redshift.md) * [DuckDB](reference/offline-stores/duckdb.md) + * [Couchbase Columnar (contrib)](reference/offline-stores/couchbase.md) * [Spark (contrib)](reference/offline-stores/spark.md) * [PostgreSQL (contrib)](reference/offline-stores/postgres.md) * [Trino (contrib)](reference/offline-stores/trino.md) diff --git a/docs/getting-started/architecture/push-vs-pull-model.md b/docs/getting-started/architecture/push-vs-pull-model.md index b205e97fc51..f1bd05a3e75 100644 --- a/docs/getting-started/architecture/push-vs-pull-model.md +++ b/docs/getting-started/architecture/push-vs-pull-model.md @@ -25,4 +25,4 @@ Implicit in the Push model are decisions about _how_ and _when_ to push feature From a developer's perspective, there are three ways to push feature values to the online store with different tradeoffs. -They are discussed further in the [Write Patterns](getting-started/architecture/write-patterns.md) section. +They are discussed further in the [Write Patterns](write-patterns.md) section. diff --git a/docs/getting-started/architecture/write-patterns.md b/docs/getting-started/architecture/write-patterns.md index 4674b5504d3..f92b4e9d83b 100644 --- a/docs/getting-started/architecture/write-patterns.md +++ b/docs/getting-started/architecture/write-patterns.md @@ -1,6 +1,6 @@ # Writing Data to Feast -Feast uses a [Push Model](getting-started/architecture/push-vs-pull-model.md) to push features to the online store. +Feast uses a [Push Model](push-vs-pull-model.md) to push features to the online store. This has two important consequences: (1) communication patterns between the Data Producer (i.e., the client) and Feast (i.e,. the server) and (2) feature computation and _feature value_ write patterns to Feast's online store. diff --git a/docs/getting-started/components/README.md b/docs/getting-started/components/README.md index 4c6f3a54dfc..1b224056298 100644 --- a/docs/getting-started/components/README.md +++ b/docs/getting-started/components/README.md @@ -27,3 +27,7 @@ {% content-ref url="authz_manager.md" %} [authz_manager.md](authz_manager.md) {% endcontent-ref %} + +{% content-ref url="open-telemetry.md" %} +[open-telemetry.md](open-telemetry.md) +{% endcontent-ref %} diff --git a/docs/getting-started/components/open-telemetry.md b/docs/getting-started/components/open-telemetry.md new file mode 100644 index 00000000000..bdffad1d27b --- /dev/null +++ b/docs/getting-started/components/open-telemetry.md @@ -0,0 +1,149 @@ +# OpenTelemetry Integration + +The OpenTelemetry integration in Feast provides comprehensive monitoring and observability capabilities for your feature serving infrastructure. This component enables you to track key metrics, traces, and logs from your Feast deployment. + +## Motivation + +Monitoring and observability are critical for production machine learning systems. The OpenTelemetry integration addresses these needs by: + +1. **Performance Monitoring:** Track CPU and memory usage of feature servers +2. **Operational Insights:** Collect metrics to understand system behavior and performance +3. **Troubleshooting:** Enable effective debugging through distributed tracing +4. **Resource Optimization:** Monitor resource utilization to optimize deployments +5. **Production Readiness:** Provide enterprise-grade observability capabilities + +## Architecture + +The OpenTelemetry integration in Feast consists of several components working together: + +- **OpenTelemetry Collector:** Receives, processes, and exports telemetry data +- **Prometheus Integration:** Enables metrics collection and monitoring +- **Instrumentation:** Automatic Python instrumentation for tracking metrics +- **Exporters:** Components that send telemetry data to monitoring systems + +## Key Features + +1. **Automated Instrumentation:** Python auto-instrumentation for comprehensive metric collection +2. **Metric Collection:** Track key performance indicators including: + - Memory usage + - CPU utilization + - Request latencies + - Feature retrieval statistics +3. **Flexible Configuration:** Customizable metric collection and export settings +4. **Kubernetes Integration:** Native support for Kubernetes deployments +5. **Prometheus Compatibility:** Integration with Prometheus for metrics visualization + +## Setup and Configuration + +To add monitoring to the Feast Feature Server, follow these steps: + +### 1. Deploy Prometheus Operator +Follow the [Prometheus Operator documentation](https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/user-guides/getting-started.md) to install the operator. + +### 2. Deploy OpenTelemetry Operator +Before installing the OpenTelemetry Operator: +1. Install `cert-manager` +2. Validate that the `pods` are running +3. Apply the OpenTelemetry operator: +```bash +kubectl apply -f https://github.com/open-telemetry/opentelemetry-operator/releases/latest/download/opentelemetry-operator.yaml +``` + +For additional installation steps, refer to the [OpenTelemetry Operator documentation](https://github.com/open-telemetry/opentelemetry-operator). + +### 3. Configure OpenTelemetry Collector +Add the OpenTelemetry Collector configuration under the metrics section in your values.yaml file: + +```yaml +metrics: + enabled: true + otelCollector: + endpoint: "otel-collector.default.svc.cluster.local:4317" # sample + headers: + api-key: "your-api-key" +``` + +### 4. Add Instrumentation Configuration +Add the following annotations and environment variables to your deployment.yaml: + +```yaml +template: + metadata: + annotations: + instrumentation.opentelemetry.io/inject-python: "true" +``` + +```yaml +- name: OTEL_EXPORTER_OTLP_ENDPOINT + value: http://{{ .Values.service.name }}-collector.{{ .Release.namespace }}.svc.cluster.local:{{ .Values.metrics.endpoint.port}} +- name: OTEL_EXPORTER_OTLP_INSECURE + value: "true" +``` + +### 5. Add Metric Checks +Add metric checks to all manifests and deployment files: + +```yaml +{{ if .Values.metrics.enabled }} +apiVersion: opentelemetry.io/v1alpha1 +kind: Instrumentation +metadata: + name: feast-instrumentation +spec: + exporter: + endpoint: http://{{ .Values.service.name }}-collector.{{ .Release.Namespace }}.svc.cluster.local:4318 + env: + propagators: + - tracecontext + - baggage + python: + env: + - name: OTEL_METRICS_EXPORTER + value: console,otlp_proto_http + - name: OTEL_LOGS_EXPORTER + value: otlp_proto_http + - name: OTEL_PYTHON_LOGGING_AUTO_INSTRUMENTATION_ENABLED + value: "true" +{{end}} +``` + +### 6. Add Required Manifests +Add the following components to your chart: +- Instrumentation +- OpenTelemetryCollector +- ServiceMonitors +- Prometheus Instance +- RBAC rules + +### 7. Deploy Feast +Deploy Feast with metrics enabled: + +```bash +helm install feast-release infra/charts/feast-feature-server --set metric=true --set feature_store_yaml_base64="" +``` + +## Usage + +To enable OpenTelemetry monitoring in your Feast deployment: + +1. Set `metrics.enabled=true` in your Helm values +2. Configure the OpenTelemetry Collector endpoint +3. Deploy with proper annotations and environment variables + +Example configuration: +```yaml +metrics: + enabled: true + otelCollector: + endpoint: "otel-collector.default.svc.cluster.local:4317" +``` + +## Monitoring + +Once configured, you can monitor various metrics including: + +- `feast_feature_server_memory_usage`: Memory utilization of the feature server +- `feast_feature_server_cpu_usage`: CPU usage statistics +- Additional custom metrics based on your configuration + +These metrics can be visualized using Prometheus and other compatible monitoring tools. diff --git a/docs/reference/alpha-vector-database.md b/docs/reference/alpha-vector-database.md index a16baf2d7f6..861c3fcb114 100644 --- a/docs/reference/alpha-vector-database.md +++ b/docs/reference/alpha-vector-database.md @@ -46,7 +46,7 @@ python batch_score_documents.py The output will be stored in `data/city_wikipedia_summaries.csv.` ### **Initialize Feast feature store and materialize the data to the online store** -Use the feature_store.yaml file to initialize the feature store. This will use the data as offline store, and Pgvector as online store. +Use the feature_store.yaml file to initialize the feature store. This will use the data as offline store, and Milvus as online store. ```yaml project: local_rag diff --git a/docs/reference/alpha-web-ui.md b/docs/reference/alpha-web-ui.md index 02dd107f1b4..80c5b824c5a 100644 --- a/docs/reference/alpha-web-ui.md +++ b/docs/reference/alpha-web-ui.md @@ -100,9 +100,9 @@ yarn start The advantage of importing Feast UI as a module is in the ease of customization. The `` component exposes a `feastUIConfigs` prop thorough which you can customize the UI. Currently it supports a few parameters. -**Fetching the Project List** +##### Fetching the Project List -You can use `projectListPromise` to provide a promise that overrides where the Feast UI fetches the project list from. +By default, the Feast UI fetches the project list from the app root path. You can use `projectListPromise` to provide a promise that overrides where it's fetched from. ```jsx ``` -**Custom Tabs** +##### Custom Tabs You can add custom tabs for any of the core Feast objects through the `tabsRegistry`. -``` +```jsx const tabsRegistry = { RegularFeatureViewCustomTabs: [ { diff --git a/docs/reference/data-sources/README.md b/docs/reference/data-sources/README.md index e69fbab8e36..09df6b861e8 100644 --- a/docs/reference/data-sources/README.md +++ b/docs/reference/data-sources/README.md @@ -34,6 +34,10 @@ Please see [Data Source](../../getting-started/concepts/data-ingestion.md) for a [kinesis.md](kinesis.md) {% endcontent-ref %} +{% content-ref url="couchbase.md" %} +[couchbase.md](couchbase.md) +{% endcontent-ref %} + {% content-ref url="spark.md" %} [spark.md](spark.md) {% endcontent-ref %} diff --git a/docs/reference/data-sources/couchbase.md b/docs/reference/data-sources/couchbase.md new file mode 100644 index 00000000000..596e33cf50d --- /dev/null +++ b/docs/reference/data-sources/couchbase.md @@ -0,0 +1,37 @@ +# Couchbase Columnar source (contrib) + +## Description + +Couchbase Columnar data sources are [Couchbase Capella Columnar](https://docs.couchbase.com/columnar/intro/intro.html) collections that can be used as a source for feature data. **Note that Couchbase Columnar is available through [Couchbase Capella](https://cloud.couchbase.com/).** + +## Disclaimer + +The Couchbase Columnar data source does not achieve full test coverage. +Please do not assume complete stability. + +## Examples + +Defining a Couchbase Columnar source: + +```python +from feast.infra.offline_stores.contrib.couchbase_offline_store.couchbase_source import ( + CouchbaseColumnarSource, +) + +driver_stats_source = CouchbaseColumnarSource( + name="driver_hourly_stats_source", + query="SELECT * FROM Default.Default.`feast_driver_hourly_stats`", + database="Default", + scope="Default", + collection="feast_driver_hourly_stats", + timestamp_field="event_timestamp", + created_timestamp_column="created", +) +``` + +The full set of configuration options is available [here](https://rtd.feast.dev/en/master/#feast.infra.offline_stores.contrib.couchbase_offline_store.couchbase_source.CouchbaseColumnarSource). + +## Supported Types + +Couchbase Capella Columnar data sources support `BOOLEAN`, `STRING`, `BIGINT`, and `DOUBLE` primitive types. +For a comparison against other batch data sources, please see [here](overview.md#functionality-matrix). diff --git a/docs/reference/data-sources/overview.md b/docs/reference/data-sources/overview.md index 5c2fdce9fd1..9880d388dde 100644 --- a/docs/reference/data-sources/overview.md +++ b/docs/reference/data-sources/overview.md @@ -18,14 +18,14 @@ Details for each specific data source can be found [here](README.md). Below is a matrix indicating which data sources support which types. -| | File | BigQuery | Snowflake | Redshift | Postgres | Spark | Trino | -| :-------------------------------- | :-- | :-- |:----------| :-- | :-- | :-- | :-- | -| `bytes` | yes | yes | yes | yes | yes | yes | yes | -| `string` | yes | yes | yes | yes | yes | yes | yes | -| `int32` | yes | yes | yes | yes | yes | yes | yes | -| `int64` | yes | yes | yes | yes | yes | yes | yes | -| `float32` | yes | yes | yes | yes | yes | yes | yes | -| `float64` | yes | yes | yes | yes | yes | yes | yes | -| `bool` | yes | yes | yes | yes | yes | yes | yes | -| `timestamp` | yes | yes | yes | yes | yes | yes | yes | -| array types | yes | yes | yes | no | yes | yes | no | \ No newline at end of file +| | File | BigQuery | Snowflake | Redshift | Postgres | Spark | Trino | Couchbase | +| :-------------------------------- | :-- | :-- |:----------| :-- | :-- | :-- | :-- |:----------| +| `bytes` | yes | yes | yes | yes | yes | yes | yes | yes | +| `string` | yes | yes | yes | yes | yes | yes | yes | yes | +| `int32` | yes | yes | yes | yes | yes | yes | yes | yes | +| `int64` | yes | yes | yes | yes | yes | yes | yes | yes | +| `float32` | yes | yes | yes | yes | yes | yes | yes | yes | +| `float64` | yes | yes | yes | yes | yes | yes | yes | yes | +| `bool` | yes | yes | yes | yes | yes | yes | yes | yes | +| `timestamp` | yes | yes | yes | yes | yes | yes | yes | yes | +| array types | yes | yes | yes | no | yes | yes | no | no | diff --git a/docs/reference/feature-servers/go-feature-server.md b/docs/reference/feature-servers/go-feature-server.md deleted file mode 100644 index 8209799086a..00000000000 --- a/docs/reference/feature-servers/go-feature-server.md +++ /dev/null @@ -1,93 +0,0 @@ -# Go feature server - -## Overview - -The Go feature server is an HTTP/gRPC endpoint that serves features. -It is written in Go, and is therefore significantly faster than the Python feature server. -See this [blog post](https://feast.dev/blog/go-feature-server-benchmarks/) for more details on the comparison between Python and Go. -In general, we recommend the Go feature server for all production use cases that require extremely low-latency feature serving. -Currently only the Redis and SQLite online stores are supported. - -## CLI - -By default, the Go feature server is turned off. -To turn it on you can add `go_feature_serving: True` to your `feature_store.yaml`: - -{% code title="feature_store.yaml" %} -```yaml -project: my_feature_repo -registry: data/registry.db -provider: local -online_store: - type: redis - connection_string: "localhost:6379" -go_feature_serving: True -``` -{% endcode %} - -Then the `feast serve` CLI command will start the Go feature server. -As with Python, the Go feature server uses port 6566 by default; the port be overridden with a `--port` flag. -Moreover, the server uses HTTP by default, but can be set to use gRPC with `--type=grpc`. - -Alternatively, if you wish to experiment with the Go feature server instead of permanently turning it on, you can just run `feast serve --go`. - -## Installation - -The Go component comes pre-compiled when you install Feast with Python versions 3.8-3.10 on macOS or Linux (on x86). -In order to install the additional Python dependencies, you should install Feast with -``` -pip install feast[go] -``` -You must also install the Apache Arrow C++ libraries. -This is because the Go feature server uses the cgo memory allocator from the Apache Arrow C++ library for interoperability between Go and Python, to prevent memory from being accidentally garbage collected when executing on-demand feature views. -You can read more about the usage of the cgo memory allocator in these [docs](https://pkg.go.dev/github.com/apache/arrow/go/arrow@v0.0.0-20211112161151-bc219186db40/cdata#ExportArrowRecordBatch). - -For macOS, run `brew install apache-arrow`. -For linux users, you have to install `libarrow-dev`. -``` -sudo apt update -sudo apt install -y -V ca-certificates lsb-release wget -wget https://apache.jfrog.io/artifactory/arrow/$(lsb_release --id --short | tr 'A-Z' 'a-z')/apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb -sudo apt install -y -V ./apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb -sudo apt update -sudo apt install -y -V libarrow-dev # For C++ -``` -For developers, if you want to build from source, run `make compile-go-lib` to build and compile the go server. In order to build the go binaries, you will need to install the `apache-arrow` c++ libraries. - -## Alpha features - -### Feature logging - -The Go feature server can log all requested entities and served features to a configured destination inside an offline store. -This allows users to create new datasets from features served online. Those datasets could be used for future trainings or for -feature validations. To enable feature logging we need to edit `feature_store.yaml`: -```yaml -project: my_feature_repo -registry: data/registry.db -provider: local -online_store: - type: redis - connection_string: "localhost:6379" -go_feature_serving: True -feature_server: - feature_logging: - enabled: True -``` - -Feature logging configuration in `feature_store.yaml` also allows to tweak some low-level parameters to achieve the best performance: -```yaml -feature_server: - feature_logging: - enabled: True - flush_interval_secs: 300 - write_to_disk_interval_secs: 30 - emit_timeout_micro_secs: 10000 - queue_capacity: 10000 -``` -All these parameters are optional. - -### Python SDK retrieval - -The logic for the Go feature server can also be used to retrieve features during a Python `get_online_features` call. -To enable this behavior, you must add `go_feature_retrieval: True` to your `feature_store.yaml`. -You must also have all the dependencies installed as detailed above. diff --git a/docs/reference/offline-stores/README.md b/docs/reference/offline-stores/README.md index 2b62c4e1f11..ab25fe9a276 100644 --- a/docs/reference/offline-stores/README.md +++ b/docs/reference/offline-stores/README.md @@ -26,6 +26,10 @@ Please see [Offline Store](../../getting-started/components/offline-store.md) fo [duckdb.md](duckdb.md) {% endcontent-ref %} +{% content-ref url="couchbase.md" %} +[couchbase.md](couchbase.md) +{% endcontent-ref %} + {% content-ref url="spark.md" %} [spark.md](spark.md) {% endcontent-ref %} diff --git a/docs/reference/offline-stores/couchbase.md b/docs/reference/offline-stores/couchbase.md new file mode 100644 index 00000000000..3ae0f68d4c2 --- /dev/null +++ b/docs/reference/offline-stores/couchbase.md @@ -0,0 +1,79 @@ +# Couchbase Columnar offline store (contrib) + +## Description + +The Couchbase Columnar offline store provides support for reading [CouchbaseColumnarSources](../data-sources/couchbase.md). **Note that Couchbase Columnar is available through [Couchbase Capella](https://cloud.couchbase.com/).** +* Entity dataframes can be provided as a SQL++ query or can be provided as a Pandas dataframe. A Pandas dataframe will be uploaded to Couchbase Capella Columnar as a collection. + +## Disclaimer + +The Couchbase Columnar offline store does not achieve full test coverage. +Please do not assume complete stability. + +## Getting started + +In order to use this offline store, you'll need to run `pip install 'feast[couchbase]'`. You can get started by then running `feast init -t couchbase`. + +To get started with Couchbase Capella Columnar: +1. Sign up for a [Couchbase Capella](https://cloud.couchbase.com/) account +2. [Deploy a Columnar cluster](https://docs.couchbase.com/columnar/admin/prepare-project.html) +3. [Create an Access Control Account](https://docs.couchbase.com/columnar/admin/auth/auth-data.html) + - This account should be able to read and write. + - For testing purposes, it is recommended to assign all roles to avoid any permission issues. +4. [Configure allowed IP addresses](https://docs.couchbase.com/columnar/admin/ip-allowed-list.html) + - You must allow the IP address of the machine running Feast. + + +## Example + +{% code title="feature_store.yaml" %} +```yaml +project: my_project +registry: data/registry.db +provider: local +offline_store: + type: couchbase.offline + connection_string: COUCHBASE_COLUMNAR_CONNECTION_STRING # Copied from Settings > Connection String page in Capella Columnar console, starts with couchbases:// + user: COUCHBASE_COLUMNAR_USER # Couchbase cluster access name from Settings > Access Control page in Capella Columnar console + password: COUCHBASE_COLUMNAR_PASSWORD # Couchbase password from Settings > Access Control page in Capella Columnar console + timeout: 120 # Timeout in seconds for Columnar operations, optional +online_store: + path: data/online_store.db +``` +{% endcode %} + +Note that `timeout`is an optional parameter. +The full set of configuration options is available in [CouchbaseColumnarOfflineStoreConfig](https://rtd.feast.dev/en/master/#feast.infra.offline_stores.contrib.couchbase_offline_store.couchbase.CouchbaseColumnarOfflineStoreConfig). + + +## Functionality Matrix + +The set of functionality supported by offline stores is described in detail [here](overview.md#functionality). +Below is a matrix indicating which functionality is supported by the Couchbase Columnar offline store. + +| | Couchbase Columnar | +| :----------------------------------------------------------------- |:-------------------| +| `get_historical_features` (point-in-time correct join) | yes | +| `pull_latest_from_table_or_query` (retrieve latest feature values) | yes | +| `pull_all_from_table_or_query` (retrieve a saved dataset) | yes | +| `offline_write_batch` (persist dataframes to offline store) | no | +| `write_logged_features` (persist logged features to offline store) | no | + +Below is a matrix indicating which functionality is supported by `CouchbaseColumnarRetrievalJob`. + +| | Couchbase Columnar | +| ----------------------------------------------------- |--------------------| +| export to dataframe | yes | +| export to arrow table | yes | +| export to arrow batches | no | +| export to SQL | yes | +| export to data lake (S3, GCS, etc.) | yes | +| export to data warehouse | yes | +| export as Spark dataframe | no | +| local execution of Python-based on-demand transforms | yes | +| remote execution of Python-based on-demand transforms | no | +| persist results in the offline store | yes | +| preview the query plan before execution | yes | +| read partitioned data | yes | + +To compare this set of functionality against other offline stores, please see the full [functionality matrix](overview.md#functionality-matrix). diff --git a/docs/reference/offline-stores/overview.md b/docs/reference/offline-stores/overview.md index 182eac65864..191ccd21a64 100644 --- a/docs/reference/offline-stores/overview.md +++ b/docs/reference/offline-stores/overview.md @@ -31,28 +31,28 @@ Details for each specific offline store, such as how to configure it in a `featu Below is a matrix indicating which offline stores support which methods. -| | Dask | BigQuery | Snowflake | Redshift | Postgres | Spark | Trino | -| :-------------------------------- | :-- | :-- | :-- | :-- | :-- | :-- | :-- | -| `get_historical_features` | yes | yes | yes | yes | yes | yes | yes | -| `pull_latest_from_table_or_query` | yes | yes | yes | yes | yes | yes | yes | -| `pull_all_from_table_or_query` | yes | yes | yes | yes | yes | yes | yes | -| `offline_write_batch` | yes | yes | yes | yes | no | no | no | -| `write_logged_features` | yes | yes | yes | yes | no | no | no | +| | Dask | BigQuery | Snowflake | Redshift | Postgres | Spark | Trino | Couchbase | +| :-------------------------------- | :-- | :-- | :-- | :-- | :-- | :-- | :-- | :-- | +| `get_historical_features` | yes | yes | yes | yes | yes | yes | yes | yes | +| `pull_latest_from_table_or_query` | yes | yes | yes | yes | yes | yes | yes | yes | +| `pull_all_from_table_or_query` | yes | yes | yes | yes | yes | yes | yes | yes | +| `offline_write_batch` | yes | yes | yes | yes | no | no | no | no | +| `write_logged_features` | yes | yes | yes | yes | no | no | no | no | Below is a matrix indicating which `RetrievalJob`s support what functionality. -| | Dask | BigQuery | Snowflake | Redshift | Postgres | Spark | Trino | DuckDB | -| --------------------------------- | --- | --- | --- | --- | --- | --- | --- | --- | -| export to dataframe | yes | yes | yes | yes | yes | yes | yes | yes | -| export to arrow table | yes | yes | yes | yes | yes | yes | yes | yes | -| export to arrow batches | no | no | no | yes | no | no | no | no | -| export to SQL | no | yes | yes | yes | yes | no | yes | no | -| export to data lake (S3, GCS, etc.) | no | no | yes | no | yes | no | no | no | -| export to data warehouse | no | yes | yes | yes | yes | no | no | no | -| export as Spark dataframe | no | no | yes | no | no | yes | no | no | -| local execution of Python-based on-demand transforms | yes | yes | yes | yes | yes | no | yes | yes | -| remote execution of Python-based on-demand transforms | no | no | no | no | no | no | no | no | -| persist results in the offline store | yes | yes | yes | yes | yes | yes | no | yes | -| preview the query plan before execution | yes | yes | yes | yes | yes | yes | yes | no | -| read partitioned data | yes | yes | yes | yes | yes | yes | yes | yes | +| | Dask | BigQuery | Snowflake | Redshift | Postgres | Spark | Trino | DuckDB | Couchbase | +| --------------------------------- | --- | --- | --- | --- | --- | --- | --- | --- | --- | +| export to dataframe | yes | yes | yes | yes | yes | yes | yes | yes | yes | +| export to arrow table | yes | yes | yes | yes | yes | yes | yes | yes | yes | +| export to arrow batches | no | no | no | yes | no | no | no | no | no | +| export to SQL | no | yes | yes | yes | yes | no | yes | no | yes | +| export to data lake (S3, GCS, etc.) | no | no | yes | no | yes | no | no | no | yes | +| export to data warehouse | no | yes | yes | yes | yes | no | no | no | yes | +| export as Spark dataframe | no | no | yes | no | no | yes | no | no | no | +| local execution of Python-based on-demand transforms | yes | yes | yes | yes | yes | no | yes | yes | yes | +| remote execution of Python-based on-demand transforms | no | no | no | no | no | no | no | no | no | +| persist results in the offline store | yes | yes | yes | yes | yes | yes | no | yes | yes | +| preview the query plan before execution | yes | yes | yes | yes | yes | yes | yes | no | yes | +| read partitioned data | yes | yes | yes | yes | yes | yes | yes | yes | yes | diff --git a/docs/reference/online-stores/couchbase.md b/docs/reference/online-stores/couchbase.md index ff8822d85d9..2878deb97ee 100644 --- a/docs/reference/online-stores/couchbase.md +++ b/docs/reference/online-stores/couchbase.md @@ -38,7 +38,7 @@ project: my_feature_repo registry: data/registry.db provider: local online_store: - type: couchbase + type: couchbase.online connection_string: couchbase://127.0.0.1 # Couchbase connection string, copied from 'Connect' page in Couchbase Capella console user: Administrator # Couchbase username from access credentials password: password # Couchbase password from access credentials diff --git a/docs/reference/registries/sql.md b/docs/reference/registries/sql.md index 631a20cbe3c..ef9993c8753 100644 --- a/docs/reference/registries/sql.md +++ b/docs/reference/registries/sql.md @@ -61,20 +61,20 @@ like we do as follows, again using `cockroachdb` as an example: ```shell cat <<'EOF' >Dockerfile -ARG DOCKER_IO_FEASTDEV_FEATURE_SERVER -FROM docker.io/feastdev/feature-server:${DOCKER_IO_FEASTDEV_FEATURE_SERVER} +ARG QUAY_IO_FEASTDEV_FEATURE_SERVER +FROM quay.io/feastdev/feature-server:${QUAY_IO_FEASTDEV_FEATURE_SERVER} ARG PYPI_ORG_PROJECT_SQLALCHEMY_COCKROACHDB RUN pip install -I --no-cache-dir \ sqlalchemy-cockroachdb==${PYPI_ORG_PROJECT_SQLALCHEMY_COCKROACHDB} EOF -export DOCKER_IO_FEASTDEV_FEATURE_SERVER=0.27.1 +export QUAY_IO_FEASTDEV_FEATURE_SERVER=0.27.1 export PYPI_ORG_PROJECT_SQLALCHEMY_COCKROACHDB=1.4.4 docker build \ - --build-arg DOCKER_IO_FEASTDEV_FEATURE_SERVER \ + --build-arg QUAY_IO_FEASTDEV_FEATURE_SERVER \ --build-arg PYPI_ORG_PROJECT_SQLALCHEMY_COCKROACHDB \ - --tag ${MY_REGISTRY}/feastdev/feature-server:${DOCKER_IO_FEASTDEV_FEATURE_SERVER} . + --tag ${MY_REGISTRY}/feastdev/feature-server:${QUAY_IO_FEASTDEV_FEATURE_SERVER} . ``` If you are running Feast in Kubernetes, set the `image.repository` and diff --git a/docs/roadmap.md b/docs/roadmap.md index ff6549a3cb1..cb55873c3fa 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -4,6 +4,9 @@ The list below contains the functionality that contributors are planning to deve * We welcome contribution to all items in the roadmap! +* **Natural Language Processing** + * [x] Vector Search (Alpha release. See [RFC](https://docs.google.com/document/d/18IWzLEA9i2lDWnbfbwXnMCg3StlqaLVI-uRpQjr_Vos/edit#heading=h.9gaqqtox9jg6)) + * [ ] [Enhanced Feature Server and SDK for native support for NLP](https://github.com/feast-dev/feast/issues/4964) * **Data Sources** * [x] [Snowflake source](https://docs.feast.dev/reference/data-sources/snowflake) * [x] [Redshift source](https://docs.feast.dev/reference/data-sources/redshift) @@ -13,6 +16,7 @@ The list below contains the functionality that contributors are planning to deve * [x] [Hive (community plugin)](https://github.com/baineng/feast-hive) * [x] [Postgres (contrib plugin)](https://docs.feast.dev/reference/data-sources/postgres) * [x] [Spark (contrib plugin)](https://docs.feast.dev/reference/data-sources/spark) + * [x] [Couchbase (contrib plugin)](https://docs.feast.dev/reference/data-sources/couchbase) * [x] Kafka / Kinesis sources (via [push support into the online store](https://docs.feast.dev/reference/data-sources/push)) * **Offline Stores** * [x] [Snowflake](https://docs.feast.dev/reference/offline-stores/snowflake) @@ -23,6 +27,7 @@ The list below contains the functionality that contributors are planning to deve * [x] [Postgres (contrib plugin)](https://docs.feast.dev/reference/offline-stores/postgres) * [x] [Trino (contrib plugin)](https://github.com/Shopify/feast-trino) * [x] [Spark (contrib plugin)](https://docs.feast.dev/reference/offline-stores/spark) + * [x] [Couchbase (contrib plugin)](https://docs.feast.dev/reference/offline-stores/couchbase) * [x] [In-memory / Pandas](https://docs.feast.dev/reference/offline-stores/file) * [x] [Custom offline store support](https://docs.feast.dev/how-to-guides/customizing-feast/adding-a-new-offline-store) * **Online Stores** @@ -37,12 +42,14 @@ The list below contains the functionality that contributors are planning to deve * [x] [Azure Cache for Redis (community plugin)](https://github.com/Azure/feast-azure) * [x] [Postgres (contrib plugin)](https://docs.feast.dev/reference/online-stores/postgres) * [x] [Cassandra / AstraDB (contrib plugin)](https://docs.feast.dev/reference/online-stores/cassandra) + * [x] [ScyllaDB (contrib plugin)](https://docs.feast.dev/reference/online-stores/scylladb) + * [x] [Couchbase (contrib plugin)](https://docs.feast.dev/reference/online-stores/couchbase) * [x] [Custom online store support](https://docs.feast.dev/how-to-guides/customizing-feast/adding-support-for-a-new-online-store) * **Feature Engineering** - * [x] On-demand Transformations (Beta release. See [RFC](https://docs.google.com/document/d/1lgfIw0Drc65LpaxbUu49RCeJgMew547meSJttnUqz7c/edit#)) + * [x] On-demand Transformations (On Read) (Beta release. See [RFC](https://docs.google.com/document/d/1lgfIw0Drc65LpaxbUu49RCeJgMew547meSJttnUqz7c/edit#)) * [x] Streaming Transformations (Alpha release. See [RFC](https://docs.google.com/document/d/1UzEyETHUaGpn0ap4G82DHluiCj7zEbrQLkJJkKSv4e8/edit)) * [ ] Batch transformation (In progress. See [RFC](https://docs.google.com/document/d/1964OkzuBljifDvkV-0fakp2uaijnVzdwWNGdz7Vz50A/edit)) - * [ ] Persistent On-demand Transformations (Beta release. See [GitHub Issue](https://github.com/feast-dev/feast/issues/4376)) + * [x] On-demand Transformations (On Write) (Beta release. See [GitHub Issue](https://github.com/feast-dev/feast/issues/4376)) * **Streaming** * [x] [Custom streaming ingestion job support](https://docs.feast.dev/how-to-guides/customizing-feast/creating-a-custom-provider) * [x] [Push based streaming data ingestion to online store](https://docs.feast.dev/reference/data-sources/push) @@ -65,5 +72,3 @@ The list below contains the functionality that contributors are planning to deve * [x] DataHub integration (see [DataHub Feast docs](https://datahubproject.io/docs/generated/ingestion/sources/feast/)) * [x] Feast Web UI (Beta release. See [docs](https://docs.feast.dev/reference/alpha-web-ui)) * [ ] Feast Lineage Explorer -* **Natural Language Processing** - * [x] Vector Search (Alpha release. See [RFC](https://docs.google.com/document/d/18IWzLEA9i2lDWnbfbwXnMCg3StlqaLVI-uRpQjr_Vos/edit#heading=h.9gaqqtox9jg6)) diff --git a/examples/README.md b/examples/README.md index f968b94b5f6..6dac867be43 100644 --- a/examples/README.md +++ b/examples/README.md @@ -1,23 +1,22 @@ # Feast Examples -1. **[Quickstart Example](quickstart)**: This is a step-by-step guide for getting started with Feast. - -2. **[Java Demo](java-demo)**: Demonstrates how to use Feast with Java feature server and deployed with Kubernetes. - -3. **[Kind Quickstart](kind-quickstart)**: Demonstrates how to install and use Feast on Kind with the Helm chart. - -4. **[Operator Quickstart](operator-quickstart)**: Demonstrates how to install and use Feast on Kubernetes with the Feast Go Operator. - -5. **[Credit Risk End-to-End](credit-risk-end-to-end)**: Demonstrates how to use Feast with Java feature server and deployed with Kubernetes. - -6. **[Python Helm Demo](python-helm-demo)**: Demonstrates Feast with Kubernetes using Helm charts and Python feature server. +The following examples illustrate various **Feast** use cases to enhance understanding of its functionality. -7. **[RBAC Local](rbac-local)**: Demonstrates using notebooks how configure and test Role-Based Access Control (RBAC) for securing access in Feast using OIDC authorization type with in a local environment. - -8. **[RBAC Remote](rbac-remote)**: Demonstrates how to configure and test Role-Based Access Control (RBAC) for securing access in Feast using Kubernetes or OIDC Authentication type with in Kubernetes environment. - -9. **[Remote Offline Store](remote-offline-store)**: Demonstrates how to set up and use remote offline server. - -10. **[Podman/Podman Compose_local](podman_local)**: Demonstrates how to deploy Feast remote server components using Podman Compose locally. - -11. **[RHOAI Feast Demo](rhoai-quickstart)**: Showcases Feast's core functionality using a Jupyter notebook, including fetching online feature data from a remote server and retrieving metadata from a remote registry. +1. **[Quickstart Example](quickstart)**: This is a step-by-step guide for getting started with Feast. +1. **[Java Demo](java-demo)**: Demonstrates how to use Feast with Java feature server and deploy it on Kubernetes. +1. **[Kind Quickstart](kind-quickstart)**: Demonstrates how to install and use Feast on Kind with the Helm chart. +1. **[Credit Risk End-to-End](credit-risk-end-to-end)**: Demonstrates how to use Feast with Java feature server and deploy it on Kubernetes. +1. **[Python Helm Demo](python-helm-demo)**: Demonstrates Feast with Kubernetes using Helm charts and Python feature server. +1. **[RBAC Local](rbac-local)**: Shows how to configure and test Role-Based Access Control (RBAC) for securing access in Feast using OIDC authorization in a local environment. +1. **[RBAC Remote](rbac-remote)**: Demonstrates how to configure and test Role-Based Access Control (RBAC) for securing access in Feast using Kubernetes or OIDC Authentication in a Kubernetes environment. +1. **[Remote Offline Store](remote-offline-store)**: Demonstrates how to set up and use a remote offline store. +1. **[Podman/Podman Compose Local](podman_local)**: Demonstrates how to deploy Feast remote server components using Podman Compose locally. +1. **[RHOAI Feast Demo](rhoai-quickstart)**: Showcases Feast's core functionality using a Jupyter notebook, including fetching online feature data from a remote server and retrieving metadata from a remote registry. + +# Feast Operator Examples + +The examples below showcase how to deploy and manage **Feast on Kubernetes** using the **Feast Go Operator**. + +1. **[Operator Quickstart](operator-quickstart)**: Demonstrates how to install and use Feast on Kubernetes with the Feast Go Operator. +1. **[Operator Quickstart with Postgres in TLS](operator-postgres-tls-demo)**: Demonstrates installing and configuring Feast with PostgreSQL in TLS mode on Kubernetes using the Feast Go Operator, with an emphasis on volumes and VolumeMounts support. +1. **[Operator RBAC with Kubernetes](operator-rbac)**: Demonstrates the Feast RBAC example on Kubernetes using the Feast Operator. diff --git a/examples/kind-quickstart/01-Install.ipynb b/examples/kind-quickstart/01-Install.ipynb index e5ece97fc26..439a78d4180 100644 --- a/examples/kind-quickstart/01-Install.ipynb +++ b/examples/kind-quickstart/01-Install.ipynb @@ -623,7 +623,7 @@ }, { "cell_type": "code", - "execution_count": 16, + "execution_count": null, "metadata": {}, "outputs": [ { @@ -636,8 +636,8 @@ } ], "source": [ - "%env FEAST_IMAGE_REPO=feastdev/feature-server\n", - "%env FEAST_IMAGE_VERSION=0.40.1" + "%env FEAST_IMAGE_REPO=quay.io/feastdev/feature-server\n", + "%env FEAST_IMAGE_VERSION=latest" ] }, { diff --git a/examples/kind-quickstart/init-job.yaml b/examples/kind-quickstart/init-job.yaml index 68df35af738..8395553ab7f 100644 --- a/examples/kind-quickstart/init-job.yaml +++ b/examples/kind-quickstart/init-job.yaml @@ -7,7 +7,7 @@ spec: spec: containers: - name: feast-apply - image: feastdev/feature-server:0.40.1 + image: quay.io/feastdev/feature-server:latest command: ["/bin/sh", "-c"] args: - | diff --git a/examples/operator-postgres-tls-demo/.gitignore b/examples/operator-postgres-tls-demo/.gitignore new file mode 100644 index 00000000000..6eb45f3fbca --- /dev/null +++ b/examples/operator-postgres-tls-demo/.gitignore @@ -0,0 +1,4 @@ +postgres-tls-certs +values.yaml +.ipynb_checkpoints +*.tar.gz \ No newline at end of file diff --git a/examples/operator-postgres-tls-demo/01-Install-postgres-tls-using-helm.ipynb b/examples/operator-postgres-tls-demo/01-Install-postgres-tls-using-helm.ipynb new file mode 100644 index 00000000000..d385f3d8de1 --- /dev/null +++ b/examples/operator-postgres-tls-demo/01-Install-postgres-tls-using-helm.ipynb @@ -0,0 +1,557 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "f16967ef", + "metadata": {}, + "source": [ + "# Deploy PostgreSQL with Helm in TLS Mode" + ] + }, + { + "cell_type": "markdown", + "id": "1247e2e7-706c-44a3-a45c-fba638e50f31", + "metadata": {}, + "source": [ + "### NOTE: This PostgreSQL setup guide is intended to demonstrate the capabilities of the Feast operator in configuring Feast with PostgreSQL in TLS mode. For ongoing assistance with Postgres setup, we recommend consulting the official Helm PostgreSQL documentation." + ] + }, + { + "cell_type": "markdown", + "id": "cce2278a", + "metadata": {}, + "source": [ + "## Step 1: Install Prerequisites" + ] + }, + { + "cell_type": "markdown", + "id": "3e4102d8", + "metadata": {}, + "source": [ + "Before starting, ensure you have the following installed:\n", + "- `kubectl` (Kubernetes CLI)\n", + "- `helm` (Helm CLI)\n", + "- A Kubernetes cluster (e.g., Minikube, GKE, EKS, or AKS)" + ] + }, + { + "cell_type": "markdown", + "id": "44b611ba-097e-4777-b77b-739116e7e4d6", + "metadata": {}, + "source": [ + "**Note:** When deploying PostgreSQL and Feast on a Kubernetes cluster, it's important to ensure that your cluster has sufficient resources to support both applications." + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "e2b40efc", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Client Version: v1.31.2\n", + "Kustomize Version: v5.4.2\n", + "version.BuildInfo{Version:\"v3.17.0\", GitCommit:\"301108edc7ac2a8ba79e4ebf5701b0b6ce6a31e4\", GitTreeState:\"clean\", GoVersion:\"go1.23.4\"}\n" + ] + } + ], + "source": [ + "# Verify kubectl and helm are installed\n", + "!kubectl version --client\n", + "!helm version" + ] + }, + { + "cell_type": "markdown", + "id": "4b72fabe", + "metadata": {}, + "source": [ + "## Step 2: Add the Bitnami Helm Repository" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "f439691e", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\"bitnami\" already exists with the same configuration, skipping\n", + "Hang tight while we grab the latest from your chart repositories...\n", + "...Successfully got an update from the \"bitnami\" chart repository\n", + "Update Complete. ⎈Happy Helming!⎈\n" + ] + } + ], + "source": [ + "# Add the Bitnami Helm repository\n", + "!helm repo add bitnami https://charts.bitnami.com/bitnami\n", + "!helm repo update" + ] + }, + { + "cell_type": "markdown", + "id": "6f51e5c8-41ba-417e-a2fc-78cf5951d9dc", + "metadata": {}, + "source": [ + "## Step 3: create kubernetes feast namespace" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "d114872a-7a43-4eca-8748-6dc7346dc176", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "namespace/feast created\n", + "Context \"kind-kind\" modified.\n" + ] + } + ], + "source": [ + "!kubectl create ns feast\n", + "!kubectl config set-context --current --namespace feast" + ] + }, + { + "cell_type": "markdown", + "id": "41f4e8db", + "metadata": {}, + "source": [ + "## Step 4: Generate Self Signed TLS Certificates" + ] + }, + { + "cell_type": "markdown", + "id": "c34957e4-dd7f-49c1-986c-eefe74dd7e22", + "metadata": {}, + "source": [ + "**Note**: \n", + "- Self signed certificates are used only for demo purpose, consider using a managed certificate service (e.g., Let's Encrypt) instead of self-signed certificates.\n", + "- \"Replace the `CN` values in the certificate generation step with your actual domain names.\"," + ] + }, + { + "cell_type": "markdown", + "id": "500f9010-6329-4868-83d5-9c063d5890f5", + "metadata": {}, + "source": [ + "Delete the directory of existing certificates if you running this demo not first time." + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "bdc71e19-0fcc-4a1f-ba94-8b5e427e45d9", + "metadata": {}, + "outputs": [], + "source": [ + "# Delete certificates directory if you are running this example not first time.\n", + "!rm -rf postgres-tls-certs" + ] + }, + { + "cell_type": "markdown", + "id": "91dc26c9-cfaa-46f5-8252-7ad463264236", + "metadata": {}, + "source": [ + "Generate the certificates by executing below scripts. " + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "8e192410", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "..+.......+.........+...+.....+......+.......+...+.....+......+.+..+......+.+.....+...+.......+...+..+.+.....+.......+........+.......+......+...........+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++*.+...+...+........+....+..+...+...+....+...+......+..+..........+..+...+...+...............+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++*...+...........+......+..........+..+.+.....+....+......+.....................+...+...+..+...+.......+..+.........+.......+.....+....+........+.+..+.............+......+....................+.........+.+......+.....+.......+........+......................+......+..+...+....+...+...+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++\n", + "..+...+......+.+.........+...+......+..+.......+.....+.+..+...+.+...+......+.....+.........+......+.+...........+....+..................+...+.........+...+.....+.+.....+...............+.+......+...+............+...+......+......+........+.+.....+.............+..+.+..+.+..............+...+...+....+............+...+.....+......+.+.....+.+...+..+...+...................+...........+....+..+.................................+..........+...........+......+.+...+..+...+.......+.....+.......+...........+.......+...+......+.....+..........+...+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++\n", + "-----\n", + ".+....+......+..+....+...+.....+......+.+........+..........+.....+............+.+...+..+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++*...+.+..............+...............+.+...........+.......+...+..+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++*....+...............+............+.....+.+......+........+...+...+.+...+.....+......+.+..............+.+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++\n", + "............+....+.....+.+...+........+..........+..............+.+..............+.........+.+...+...........+......+......+.......+........+...+.........+.+.....+.+.....+.+........+.+.....................+..+.............+........+......+.+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++\n", + "-----\n", + "Certificate request self-signature ok\n", + "subject=CN = postgresql.feast.svc.cluster.local\n", + "..+....+...+.....+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++*....+.+..+.......+......+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++*.+..+.+.....+.+...+..................+.....+...+...................+......+..+...+.+......+..+..........+..+..................+.+..+...+......+.+............+..+....+...........+..........+.....+...+......+.+...+...+..+......+.+...+...+.........+......+.....+..................+.+.....+....+..............+.+..............+.+......+....................+..........+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++\n", + "++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++*.....+.........+...+..+.......+.....+.+..+.+......+....................+......+.............+......+...+..+...+.+..+...+....+.....+...+...+.........+......+.+.....+.+..+..........+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++*..+...+.+...........+....+.....+...................+..+.+..+......+............+..........+.........+...+..+...............+..........+.....+....+............+........+.+........+.+.....+.......+.....++\n", + "-----\n", + "Certificate request self-signature ok\n", + "subject=CN = admin\n" + ] + } + ], + "source": [ + "# Create a directory for certificates\n", + "!mkdir -p postgres-tls-certs\n", + "\n", + "# Generate a CA certificate\n", + "!openssl req -new -x509 -days 365 -nodes -out postgres-tls-certs/ca.crt -keyout postgres-tls-certs/ca.key -subj \"/CN=PostgreSQL CA\"\n", + "\n", + "# Generate a server certificate\n", + "!openssl req -new -nodes -out postgres-tls-certs/server.csr -keyout postgres-tls-certs/server.key -subj \"/CN=postgresql.feast.svc.cluster.local\"\n", + "!openssl x509 -req -in postgres-tls-certs/server.csr -days 365 -CA postgres-tls-certs/ca.crt -CAkey postgres-tls-certs/ca.key -CAcreateserial -out postgres-tls-certs/server.crt\n", + "\n", + "# Generate a client certificate\n", + "!openssl req -new -nodes -out postgres-tls-certs/client.csr -keyout postgres-tls-certs/client.key -subj \"/CN=admin\"\n", + "!openssl x509 -req -in postgres-tls-certs/client.csr -days 365 -CA postgres-tls-certs/ca.crt -CAkey postgres-tls-certs/ca.key -CAcreateserial -out postgres-tls-certs/client.crt" + ] + }, + { + "cell_type": "markdown", + "id": "7e39cb28", + "metadata": {}, + "source": [ + "## Step 5: Create Kubernetes Secrets for Certificates" + ] + }, + { + "cell_type": "markdown", + "id": "a4775780-3734-40ba-ae43-48f1e47b481a", + "metadata": {}, + "source": [ + "In this step, we will create **two Kubernetes secrets** that reference the certificates generated earlier step:\n", + "\n", + "- **`postgresql-server-certs`** \n", + " This secret contains the server certificates and will be used by the PostgreSQL server.\n", + "\n", + "- **`postgresql-client-certs`** \n", + " This secret contains the client certificates and will be used by the PostgreSQL client. In our case it will be feast application." + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "d728d0d5-2ba6-4d4d-b4be-62fb020530d4", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "secret/postgresql-server-certs created\n", + "secret/postgresql-client-certs created\n" + ] + } + ], + "source": [ + "# Create a secret for the server certificates\n", + "!kubectl create secret generic postgresql-server-certs --from-file=ca.crt=./postgres-tls-certs/ca.crt --from-file=tls.crt=./postgres-tls-certs/server.crt --from-file=tls.key=./postgres-tls-certs/server.key\n", + "\n", + "# Create a secret for the client certificates\n", + "!kubectl create secret generic postgresql-client-certs --from-file=ca.crt=./postgres-tls-certs/ca.crt --from-file=tls.crt=./postgres-tls-certs/client.crt --from-file=tls.key=./postgres-tls-certs/client.key" + ] + }, + { + "cell_type": "markdown", + "id": "67d62692", + "metadata": {}, + "source": [ + "## Step 6: Deploy PostgreSQL with Helm" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "e14cae77", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "NAME: postgresql\n", + "LAST DEPLOYED: Tue Feb 25 08:12:21 2025\n", + "NAMESPACE: feast\n", + "STATUS: deployed\n", + "REVISION: 1\n", + "TEST SUITE: None\n", + "NOTES:\n", + "CHART NAME: postgresql\n", + "CHART VERSION: 16.4.9\n", + "APP VERSION: 17.3.0\n", + "\n", + "Did you know there are enterprise versions of the Bitnami catalog? For enhanced secure software supply chain features, unlimited pulls from Docker, LTS support, or application customization, see Bitnami Premium or Tanzu Application Catalog. See https://www.arrow.com/globalecs/na/vendors/bitnami for more information.\n", + "\n", + "** Please be patient while the chart is being deployed **\n", + "\n", + "PostgreSQL can be accessed via port 5432 on the following DNS names from within your cluster:\n", + "\n", + " postgresql.feast.svc.cluster.local - Read/Write connection\n", + "\n", + "To get the password for \"postgres\" run:\n", + "\n", + " export POSTGRES_ADMIN_PASSWORD=$(kubectl get secret --namespace feast postgresql -o jsonpath=\"{.data.postgres-password}\" | base64 -d)\n", + "\n", + "To get the password for \"admin\" run:\n", + "\n", + " export POSTGRES_PASSWORD=$(kubectl get secret --namespace feast postgresql -o jsonpath=\"{.data.password}\" | base64 -d)\n", + "\n", + "To connect to your database run the following command:\n", + "\n", + " kubectl run postgresql-client --rm --tty -i --restart='Never' --namespace feast --image docker.io/bitnami/postgresql:17.3.0-debian-12-r1 --env=\"PGPASSWORD=$POSTGRES_PASSWORD\" \\\n", + " --command -- psql --host postgresql -U admin -d feast -p 5432\n", + "\n", + " > NOTE: If you access the container using bash, make sure that you execute \"/opt/bitnami/scripts/postgresql/entrypoint.sh /bin/bash\" in order to avoid the error \"psql: local user with ID 1001} does not exist\"\n", + "\n", + "To connect to your database from outside the cluster execute the following commands:\n", + "\n", + " kubectl port-forward --namespace feast svc/postgresql 5432:5432 &\n", + " PGPASSWORD=\"$POSTGRES_PASSWORD\" psql --host 127.0.0.1 -U admin -d feast -p 5432\n", + "\n", + "WARNING: The configured password will be ignored on new installation in case when previous PostgreSQL release was deleted through the helm command. In that case, old PVC will have an old password, and setting it through helm won't take effect. Deleting persistent volumes (PVs) will solve the issue.\n", + "\n", + "WARNING: There are \"resources\" sections in the chart not set. Using \"resourcesPreset\" is not recommended for production. For production installations, please set the following values according to your workload needs:\n", + " - primary.resources\n", + " - readReplicas.resources\n", + " - volumePermissions.resources\n", + "+info https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/\n" + ] + } + ], + "source": [ + "# Helm values for TLS configuration\n", + "helm_values = \"\"\"\n", + "tls:\n", + " enabled: true\n", + " certificatesSecret: \"postgresql-server-certs\"\n", + " certFilename: \"tls.crt\"\n", + " certKeyFilename: \"tls.key\"\n", + " certCAFilename: \"ca.crt\"\n", + "\n", + "volumePermissions:\n", + " enabled: true\n", + "\n", + "# Set fixed PostgreSQL credentials\n", + "\n", + "global:\n", + " postgresql:\n", + " auth:\n", + " username: admin\n", + " password: password\n", + " database: feast\n", + "\"\"\"\n", + "\n", + "# Write the values to a file\n", + "with open(\"values.yaml\", \"w\") as f:\n", + " f.write(helm_values)\n", + "\n", + "# Install PostgreSQL with Helm\n", + "!helm install postgresql bitnami/postgresql --version 16.4.9 -f values.yaml -n feast " + ] + }, + { + "cell_type": "markdown", + "id": "5be34ace", + "metadata": {}, + "source": [ + "## Step 7: Verify the postgres Deployment" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "id": "132df785-762e-473a-90d2-5fdb66a59a97", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "pod/postgresql-0 condition met\n", + "\n", + "NAME READY STATUS RESTARTS AGE\n", + "postgresql-0 1/1 Running 0 14s\n", + "\n", + "Defaulted container \"postgresql\" out of: postgresql, init-chmod-data (init)\n", + "ssl = 'on'\n", + "ssl_ca_file = '/opt/bitnami/postgresql/certs/ca.crt'\n", + "ssl_cert_file = '/opt/bitnami/postgresql/certs/tls.crt'\n", + "#ssl_crl_file = ''\n", + "#ssl_crl_dir = ''\n", + "ssl_key_file = '/opt/bitnami/postgresql/certs/tls.key'\n", + "#ssl_ciphers = 'HIGH:MEDIUM:+3DES:!aNULL'\t# allowed SSL ciphers\n", + "#ssl_prefer_server_ciphers = on\n", + "#ssl_ecdh_curve = 'prime256v1'\n", + "#ssl_min_protocol_version = 'TLSv1.2'\n", + "#ssl_max_protocol_version = ''\n", + "#ssl_dh_params_file = ''\n", + "#ssl_passphrase_command = ''\n", + "#ssl_passphrase_command_supports_reload = off\n", + "\n", + "Defaulted container \"postgresql\" out of: postgresql, init-chmod-data (init)\n", + " List of databases\n", + " Name | Owner | Encoding | Locale Provider | Collate | Ctype | Locale | ICU Rules | Access privileges \n", + "-----------+----------+----------+-----------------+-------------+-------------+--------+-----------+-----------------------\n", + " feast | admin | UTF8 | libc | en_US.UTF-8 | en_US.UTF-8 | | | =Tc/admin +\n", + " | | | | | | | | admin=CTc/admin\n", + " postgres | postgres | UTF8 | libc | en_US.UTF-8 | en_US.UTF-8 | | | \n", + " template0 | postgres | UTF8 | libc | en_US.UTF-8 | en_US.UTF-8 | | | =c/postgres +\n", + " | | | | | | | | postgres=CTc/postgres\n", + " template1 | postgres | UTF8 | libc | en_US.UTF-8 | en_US.UTF-8 | | | =c/postgres +\n", + " | | | | | | | | postgres=CTc/postgres\n", + "(4 rows)\n", + "\n" + ] + } + ], + "source": [ + "# Wait for the status of the PostgreSQL pod to be in Ready status.\n", + "!kubectl wait --for=condition=Ready pod -l app.kubernetes.io/name=postgresql --timeout=60s\n", + "\n", + "# Insert an empty line in the output for verbocity.\n", + "print()\n", + "\n", + "# display the pod status.\n", + "!kubectl get pods -l app.kubernetes.io/name=postgresql\n", + "\n", + "# Insert an empty line in the output for verbocity.\n", + "print()\n", + "\n", + "# check if the ssl is on and the path to certificates is configured.\n", + "!kubectl exec postgresql-0 -- cat /opt/bitnami/postgresql/conf/postgresql.conf | grep ssl\n", + "\n", + "# Insert an empty line in the output for verbocity.\n", + "print()\n", + "\n", + "# Connect to PostgreSQL using TLS (non-interactive mode)\n", + "!kubectl exec postgresql-0 -- env PGPASSWORD=password psql -U admin -d feast -c '\\l'\n" + ] + }, + { + "cell_type": "markdown", + "id": "c921423a-81df-456e-9cca-f689070c44d2", + "metadata": {}, + "source": [ + "## Step 8: Port forwarding in the terminal for the connection testing using python" + ] + }, + { + "cell_type": "markdown", + "id": "d6a26bb4-e0e7-419e-9c91-f0d63db127bc", + "metadata": {}, + "source": [ + "**Note:** If you do not intend to test the PostgreSQL connection from outside the Kubernetes cluster, you can skip the remaining steps." + ] + }, + { + "cell_type": "markdown", + "id": "6fcad5e1-66d2-4353-aba7-3549ef21bc9f", + "metadata": {}, + "source": [ + "**Note:**\n", + "To test a connection to a PostgreSQL database outside of your Kubernetes cluster, you'll need to execute the following command in your system's terminal window. This is necessary because Jupyter Notebook does not support running commands in a separate thread." + ] + }, + { + "cell_type": "markdown", + "id": "88a4a7c1-51c4-4c5a-9472-5cace1c47a1c", + "metadata": {}, + "source": [ + "kubectl port-forward svc/postgresql 5432:5432" + ] + }, + { + "cell_type": "markdown", + "id": "a8777ca3-bf59-4f23-b7d0-60ae8c92d5a5", + "metadata": {}, + "source": [ + "## Step 9: Check the connection using Python sql alchemy" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "5a523f9f-784f-493b-b69d-5a3cb1a830af", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "postgresql+psycopg://admin:password@localhost:5432/feast?sslmode=verify-ca&sslrootcert=postgres-tls-certs/ca.crt&sslcert=postgres-tls-certs/client.crt&sslkey=postgres-tls-certs/client.key\n", + "Connected successfully!\n" + ] + } + ], + "source": [ + "# Define database connection parameters\n", + "DB_USER = \"admin\"\n", + "DB_PASSWORD = \"password\"\n", + "DB_HOST = \"localhost\"\n", + "DB_PORT = \"5432\"\n", + "DB_NAME = \"feast\"\n", + "\n", + "# TLS Certificate Paths\n", + "SSL_CERT = \"postgres-tls-certs/client.crt\"\n", + "SSL_KEY = \"postgres-tls-certs/client.key\"\n", + "SSL_ROOT_CERT = \"postgres-tls-certs/ca.crt\"\n", + "\n", + "import os\n", + "os.environ[\"FEAST_CA_CERT_FILE_PATH\"] = \"postgres-tls-certs/ca.crt\"\n", + "\n", + "from sqlalchemy import create_engine\n", + "# Create SQLAlchemy connection string\n", + "DATABASE_URL = (\n", + " f\"postgresql+psycopg://{DB_USER}:{DB_PASSWORD}@{DB_HOST}:{DB_PORT}/{DB_NAME}?\"\n", + " f\"sslmode=verify-ca&sslrootcert={SSL_ROOT_CERT}&sslcert={SSL_CERT}&sslkey={SSL_KEY}\"\n", + ")\n", + "\n", + "print(DATABASE_URL)\n", + "\n", + "# Create SQLAlchemy engine\n", + "engine = create_engine(DATABASE_URL)\n", + "\n", + "# Test connection\n", + "try:\n", + " with engine.connect() as connection:\n", + " print(\"Connected successfully!\")\n", + "except Exception as e:\n", + " print(\"Connection failed: Make sure that port forwarding step is done in the terminal.\", e)\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7503e47e-12f1-44dd-8a50-786d744bbf4c", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.10" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/examples/operator-postgres-tls-demo/02-Install-feast.ipynb b/examples/operator-postgres-tls-demo/02-Install-feast.ipynb new file mode 100644 index 00000000000..16948b3610c --- /dev/null +++ b/examples/operator-postgres-tls-demo/02-Install-feast.ipynb @@ -0,0 +1,458 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Install Feast on Kubernetes with the Feast Operator\n", + "## Objective\n", + "\n", + "Provide a reference implementation of a runbook to deploy a Feast environment on a Kubernetes cluster using [Kind](https://kind.sigs.k8s.io/docs/user/quick-start) and the [Feast Operator](../../infra/feast-operator/)." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Prerequisites\n", + "* Kubernetes Cluster\n", + "* [kubectl](https://kubernetes.io/docs/tasks/tools/#kubectl) Kubernetes CLI tool." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Install the Feast Operator" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "namespace/feast-operator-system created\n", + "customresourcedefinition.apiextensions.k8s.io/featurestores.feast.dev created\n", + "serviceaccount/feast-operator-controller-manager created\n", + "role.rbac.authorization.k8s.io/feast-operator-leader-election-role created\n", + "clusterrole.rbac.authorization.k8s.io/feast-operator-featurestore-editor-role created\n", + "clusterrole.rbac.authorization.k8s.io/feast-operator-featurestore-viewer-role created\n", + "clusterrole.rbac.authorization.k8s.io/feast-operator-manager-role created\n", + "clusterrole.rbac.authorization.k8s.io/feast-operator-metrics-auth-role created\n", + "clusterrole.rbac.authorization.k8s.io/feast-operator-metrics-reader created\n", + "rolebinding.rbac.authorization.k8s.io/feast-operator-leader-election-rolebinding created\n", + "clusterrolebinding.rbac.authorization.k8s.io/feast-operator-manager-rolebinding created\n", + "clusterrolebinding.rbac.authorization.k8s.io/feast-operator-metrics-auth-rolebinding created\n", + "service/feast-operator-controller-manager-metrics-service created\n", + "deployment.apps/feast-operator-controller-manager created\n", + "deployment.apps/feast-operator-controller-manager condition met\n" + ] + } + ], + "source": [ + "## Use this install command from a release branch (e.g. 'v0.46-branch')\n", + "!kubectl apply -f ../../infra/feast-operator/dist/install.yaml\n", + "\n", + "## OR, for the latest code/builds, use one the following commands from the 'master' branch\n", + "# !make -C ../../infra/feast-operator install deploy IMG=quay.io/feastdev-ci/feast-operator:develop FS_IMG=quay.io/feastdev-ci/feature-server:develop\n", + "# !make -C ../../infra/feast-operator install deploy IMG=quay.io/feastdev-ci/feast-operator:$(git rev-parse HEAD) FS_IMG=quay.io/feastdev-ci/feature-server:$(git rev-parse HEAD)\n", + "\n", + "!kubectl wait --for=condition=available --timeout=5m deployment/feast-operator-controller-manager -n feast-operator-system" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Install the Feast services via FeatureStore CR\n", + "Next, we'll use the running Feast Operator to install the feast services. Before doing that it is important to understand basic understanding of operator support of Volumes and volumeMounts and how to mount TLS certificates." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Mounting TLS Certificates with Volumes in Feast Operator \n", + "\n", + "The Feast operator supports **volumes** and **volumeMounts**, allowing you to mount TLS certificates onto a pod. This approach provides flexibility in how you mount these files, supporting different Kubernetes resources such as **Secrets, ConfigMaps,** and **Persistent Volumes (PVs).** \n", + "\n", + "#### Example: Mounting Certificates Using Kubernetes Secrets \n", + "\n", + "In this example, we demonstrate how to mount TLS certificates using **Kubernetes Secrets** that were created in a previous notebook. \n", + "\n", + "#### PostgreSQL Connection Parameters \n", + "\n", + "When connecting to PostgreSQL with TLS, some important parameters in the connection URL are: \n", + "\n", + "- **`sslrootcert`** – Specifies the path to the **CA certificate** file used to validate trusted certificates. \n", + "- **`sslcert`** – Provides the client certificate for **mutual TLS (mTLS) encryption**. \n", + "- **`sslkey`** – Specifies the private key for the client certificate. \n", + "\n", + "If mutual TLS authentication is not required, you can **omit** the `sslcert` and `sslkey` parameters. However, the `sslrootcert` parameter is still necessary for validating server certificates. \n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + " Note: Please deploy either option 1 or 2 only. Don't deploy both of them at the same time to avoid conflicts in the lateral steps. " + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**Option 1: Directly Setting the CA Certificate Path** \n", + "\n", + "In this approach, we specify the CA certificate path directly in the Feast PostgreSQL URL using the `sslrootcert` parameter. \n", + "\n", + "You can refer to the `v1alpha1_featurestore_postgres_db_volumes_tls.yaml` file for the complete configuration details. " + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": { + "scrolled": true + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "secret/postgres-secret created\n", + "secret/feast-data-stores created\n", + "featurestore.feast.dev/sample-db-ssl created\n" + ] + } + ], + "source": [ + "!kubectl apply -f ../../infra/feast-operator/config/samples/v1alpha1_featurestore_postgres_db_volumes_tls.yaml --namespace=feast" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**Option 2: Using an Environment Variable for the CA Certificate** \n", + "\n", + "In this approach, you define the CA certificate path as an environment variable. You can refer to the `v1alpha1_featurestore_postgres_tls_volumes_ca_env.yaml` file for the complete configuration details. \n", + "\n", + "```bash\n", + "FEAST_CA_CERT_FILE_PATH=\n" + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "secret/postgres-secret created\n", + "secret/feast-data-stores created\n", + "featurestore.feast.dev/sample-db-ssl created\n" + ] + } + ], + "source": [ + "!kubectl apply -f ../../infra/feast-operator/config/samples/v1alpha1_featurestore_postgres_tls_volumes_ca_env.yaml --namespace=feast" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Validate the running FeatureStore deployment\n", + "Validate the deployment status." + ] + }, + { + "cell_type": "code", + "execution_count": 31, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "deployment.apps/feast-sample-db-ssl condition met\n", + "NAME READY STATUS RESTARTS AGE\n", + "pod/feast-sample-db-ssl-86b47d54-hclb9 1/1 Running 0 27s\n", + "pod/postgresql-0 1/1 Running 0 13h\n", + "\n", + "NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE\n", + "service/feast-sample-db-ssl-online ClusterIP 10.96.61.65 80/TCP 27s\n", + "service/postgresql ClusterIP 10.96.228.3 5432/TCP 13h\n", + "service/postgresql-hl ClusterIP None 5432/TCP 13h\n", + "\n", + "NAME READY UP-TO-DATE AVAILABLE AGE\n", + "deployment.apps/feast-sample-db-ssl 1/1 1 1 27s\n", + "\n", + "NAME DESIRED CURRENT READY AGE\n", + "replicaset.apps/feast-sample-db-ssl-86b47d54 1 1 1 27s\n", + "\n", + "NAME READY AGE\n", + "statefulset.apps/postgresql 1/1 13h\n" + ] + } + ], + "source": [ + "!kubectl wait --for=condition=available --timeout=8m deployment/feast-sample-db-ssl -n feast\n", + "!kubectl get all" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Validate that the FeatureStore CR is in a `Ready` state." + ] + }, + { + "cell_type": "code", + "execution_count": 32, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "NAME STATUS AGE\n", + "sample-db-ssl Ready 33s\n" + ] + } + ], + "source": [ + "!kubectl get feast" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Verify that the DB includes the expected tables." + ] + }, + { + "cell_type": "code", + "execution_count": 33, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Defaulted container \"postgresql\" out of: postgresql, init-chmod-data (init)\n", + " List of relations\n", + " Schema | Name | Type | Owner \n", + "--------+------------------------------------------------------+-------+-------\n", + " public | data_sources | table | admin\n", + " public | entities | table | admin\n", + " public | feast_metadata | table | admin\n", + " public | feature_services | table | admin\n", + " public | feature_views | table | admin\n", + " public | managed_infra | table | admin\n", + " public | on_demand_feature_views | table | admin\n", + " public | permissions | table | admin\n", + " public | postgres_tls_sample_env_ca_driver_hourly_stats | table | admin\n", + " public | postgres_tls_sample_env_ca_driver_hourly_stats_fresh | table | admin\n", + " public | projects | table | admin\n", + " public | saved_datasets | table | admin\n", + " public | stream_feature_views | table | admin\n", + " public | validation_references | table | admin\n", + "(14 rows)\n", + "\n" + ] + } + ], + "source": [ + "!kubectl exec postgresql-0 -- env PGPASSWORD=password psql -U admin -d feast -c '\\dt'" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Verify the client `feature_store.yaml` and create the sample feature store definitions." + ] + }, + { + "cell_type": "code", + "execution_count": 34, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "project: postgres_tls_sample_env_ca\n", + "provider: local\n", + "offline_store:\n", + " host: ${POSTGRES_HOST}\n", + " type: postgres\n", + " port: 5432\n", + " database: ${POSTGRES_DB}\n", + " db_schema: public\n", + " password: ${POSTGRES_PASSWORD}\n", + " sslcert_path: /var/lib/postgresql/certs/tls.crt\n", + " sslkey_path: /var/lib/postgresql/certs/tls.key\n", + " sslmode: verify-full\n", + " sslrootcert_path: system\n", + " user: ${POSTGRES_USER}\n", + "online_store:\n", + " type: postgres\n", + " database: ${POSTGRES_DB}\n", + " db_schema: public\n", + " host: ${POSTGRES_HOST}\n", + " password: ${POSTGRES_PASSWORD}\n", + " port: 5432\n", + " sslcert_path: /var/lib/postgresql/certs/tls.crt\n", + " sslkey_path: /var/lib/postgresql/certs/tls.key\n", + " sslmode: verify-full\n", + " sslrootcert_path: system\n", + " user: ${POSTGRES_USER}\n", + "registry:\n", + " path: postgresql+psycopg://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${POSTGRES_HOST}:5432/${POSTGRES_DB}?sslmode=verify-full&sslrootcert=system&sslcert=/var/lib/postgresql/certs/tls.crt&sslkey=/var/lib/postgresql/certs/tls.key\n", + " registry_type: sql\n", + " cache_ttl_seconds: 60\n", + " sqlalchemy_config_kwargs:\n", + " echo: false\n", + " pool_pre_ping: true\n", + "auth:\n", + " type: no_auth\n", + "entity_key_serialization_version: 3\n", + ": MADV_DONTNEED does not work (memset will be used instead)\n", + ": (This is the expected behaviour if you are running under QEMU)\n", + "/opt/app-root/src/sdk/python/feast/feature_view.py:48: DeprecationWarning: Entity value_type will be mandatory in the next release. Please specify a value_type for entity '__dummy'.\n", + " DUMMY_ENTITY = Entity(\n", + "/feast-data/postgres_tls_sample_env_ca/feature_repo/example_repo.py:27: DeprecationWarning: Entity value_type will be mandatory in the next release. Please specify a value_type for entity 'driver'.\n", + " driver = Entity(name=\"driver\", join_keys=[\"driver_id\"])\n", + "/opt/app-root/src/sdk/python/feast/entity.py:173: DeprecationWarning: Entity value_type will be mandatory in the next release. Please specify a value_type for entity 'driver'.\n", + " entity = cls(\n", + "/opt/app-root/src/sdk/python/feast/entity.py:173: DeprecationWarning: Entity value_type will be mandatory in the next release. Please specify a value_type for entity '__dummy'.\n", + " entity = cls(\n", + "Applying changes for project postgres_tls_sample_env_ca\n", + "/opt/app-root/src/sdk/python/feast/entity.py:173: DeprecationWarning: Entity value_type will be mandatory in the next release. Please specify a value_type for entity 'driver'.\n", + " entity = cls(\n", + "/opt/app-root/src/sdk/python/feast/entity.py:173: DeprecationWarning: Entity value_type will be mandatory in the next release. Please specify a value_type for entity '__dummy'.\n", + " entity = cls(\n", + "/opt/app-root/src/sdk/python/feast/feature_store.py:579: RuntimeWarning: On demand feature view is an experimental feature. This API is stable, but the functionality does not scale well for offline retrieval\n", + " warnings.warn(\n", + "Deploying infrastructure for driver_hourly_stats\n", + "Deploying infrastructure for driver_hourly_stats_fresh\n", + " Feast apply is completed. You can go to next step.\n" + ] + } + ], + "source": [ + "!kubectl exec deploy/feast-sample-db-ssl -c online -- cat feature_store.yaml\n", + "!kubectl exec deploy/feast-sample-db-ssl -c online -- feast apply\n", + "print(\" Feast apply is completed. You can go to next step.\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "List the registered feast projects & feature views." + ] + }, + { + "cell_type": "code", + "execution_count": 35, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + ": MADV_DONTNEED does not work (memset will be used instead)\n", + ": (This is the expected behaviour if you are running under QEMU)\n", + "/opt/app-root/src/sdk/python/feast/feature_view.py:48: DeprecationWarning: Entity value_type will be mandatory in the next release. Please specify a value_type for entity '__dummy'.\n", + " DUMMY_ENTITY = Entity(\n", + "/opt/app-root/src/sdk/python/feast/entity.py:173: DeprecationWarning: Entity value_type will be mandatory in the next release. Please specify a value_type for entity 'driver'.\n", + " entity = cls(\n", + "/opt/app-root/src/sdk/python/feast/entity.py:173: DeprecationWarning: Entity value_type will be mandatory in the next release. Please specify a value_type for entity '__dummy'.\n", + " entity = cls(\n", + "NAME DESCRIPTION TAGS OWNER\n", + "postgres_tls_sample {}\n", + "postgres_tls_sample_env_ca A project for driver statistics {}\n", + ": MADV_DONTNEED does not work (memset will be used instead)\n", + ": (This is the expected behaviour if you are running under QEMU)\n", + "/opt/app-root/src/sdk/python/feast/feature_view.py:48: DeprecationWarning: Entity value_type will be mandatory in the next release. Please specify a value_type for entity '__dummy'.\n", + " DUMMY_ENTITY = Entity(\n", + "/opt/app-root/src/sdk/python/feast/entity.py:173: DeprecationWarning: Entity value_type will be mandatory in the next release. Please specify a value_type for entity 'driver'.\n", + " entity = cls(\n", + "/opt/app-root/src/sdk/python/feast/entity.py:173: DeprecationWarning: Entity value_type will be mandatory in the next release. Please specify a value_type for entity '__dummy'.\n", + " entity = cls(\n", + "NAME ENTITIES TYPE\n", + "driver_hourly_stats_fresh {'driver'} FeatureView\n", + "driver_hourly_stats {'driver'} FeatureView\n", + "transformed_conv_rate {'driver'} OnDemandFeatureView\n", + "transformed_conv_rate_fresh {'driver'} OnDemandFeatureView\n" + ] + } + ], + "source": [ + "!kubectl exec deploy/feast-sample-db-ssl -c online -- feast projects list\n", + "!kubectl exec deploy/feast-sample-db-ssl -c online -- feast feature-views list" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Finally, let's verify the feast version." + ] + }, + { + "cell_type": "code", + "execution_count": 36, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + ": MADV_DONTNEED does not work (memset will be used instead)\n", + ": (This is the expected behaviour if you are running under QEMU)\n", + "/opt/app-root/src/sdk/python/feast/feature_view.py:48: DeprecationWarning: Entity value_type will be mandatory in the next release. Please specify a value_type for entity '__dummy'.\n", + " DUMMY_ENTITY = Entity(\n", + "Feast SDK Version: \"0.1.dev1+g6c92447.d20250213\"\n" + ] + } + ], + "source": [ + "!kubectl exec deployment/feast-sample-db-ssl -c online -- feast version" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.10" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/examples/operator-postgres-tls-demo/03-Uninstall.ipynb b/examples/operator-postgres-tls-demo/03-Uninstall.ipynb new file mode 100644 index 00000000000..007b8d7bc1a --- /dev/null +++ b/examples/operator-postgres-tls-demo/03-Uninstall.ipynb @@ -0,0 +1,134 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Uninstall the Operator and all Feast related objects" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "secret \"postgres-secret\" deleted\n", + "secret \"feast-data-stores\" deleted\n", + "featurestore.feast.dev \"sample-db-ssl\" deleted\n", + "Error from server (NotFound): error when deleting \"../../infra/feast-operator/config/samples/v1alpha1_featurestore_postgres_tls_volumes_ca_env.yaml\": secrets \"postgres-secret\" not found\n", + "Error from server (NotFound): error when deleting \"../../infra/feast-operator/config/samples/v1alpha1_featurestore_postgres_tls_volumes_ca_env.yaml\": secrets \"feast-data-stores\" not found\n", + "Error from server (NotFound): error when deleting \"../../infra/feast-operator/config/samples/v1alpha1_featurestore_postgres_tls_volumes_ca_env.yaml\": featurestores.feast.dev \"sample-db-ssl\" not found\n" + ] + } + ], + "source": [ + "# If you have choosen the option 1 example earlier.\n", + "!kubectl delete -f ../../infra/feast-operator/config/samples/v1alpha1_featurestore_postgres_db_volumes_tls.yaml\n", + "\n", + "# If you have choosen the option 2 example earlier.\n", + "!kubectl delete -f ../../infra/feast-operator/config/samples/v1alpha1_featurestore_postgres_tls_volumes_ca_env.yaml\n", + "\n", + "#!kubectl delete -f ../../infra/feast-operator/dist/install.yaml" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Uninstall the Postgresql using helm" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "release \"postgresql\" uninstalled\n", + "secret \"postgresql-server-certs\" deleted\n", + "secret \"postgresql-client-certs\" deleted\n", + "persistentvolumeclaim \"data-postgresql-0\" deleted\n", + "persistentvolume \"pvc-d0c961d9-7579-4e30-842a-b46812b71f74\" deleted\n" + ] + } + ], + "source": [ + "# Uninstall the Helm release\n", + "!helm uninstall postgresql\n", + "\n", + "# Delete the secrets\n", + "!kubectl delete secret postgresql-server-certs\n", + "!kubectl delete secret postgresql-client-certs\n", + "\n", + "# Remove the certificates directory\n", + "!rm -rf postgres-tls-certs\n", + "\n", + "# Remove PV and PVC for clean up. some times those are not deleted automatically and can cause issues.\n", + "# Delete all PVCs in the default namespace\n", + "!kubectl delete pvc --all\n", + "\n", + "# Delete all PVs\n", + "!kubectl delete pv --all" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Ensure everything has been removed, or is in the process of being terminated." + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "No resources found in feast namespace.\n" + ] + } + ], + "source": [ + "!kubectl get all" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.10" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/examples/operator-postgres-tls-demo/README.md b/examples/operator-postgres-tls-demo/README.md new file mode 100644 index 00000000000..70ae00da6ab --- /dev/null +++ b/examples/operator-postgres-tls-demo/README.md @@ -0,0 +1,50 @@ +# Installing Feast on Kubernetes with PostgreSQL TLS Demo using feast operator + +This example folder contains a series of Jupyter Notebooks that guide you through setting up [Feast](https://feast.dev/) on a Kubernetes cluster. + +In this demo, Feast connects to a PostgreSQL database running in TLS mode, ensuring secure communication between services. Additionally, the example demonstrates how feast application references TLS certificates using Kubernetes volumes and volume mounts. While the focus is on mounting TLS certificates, you can also mount any other resources supported by Kubernetes volumes. + +## Prerequisites + +- A running Kubernetes cluster with sufficient resources. +- [Helm](https://helm.sh/) installed and configured. +- The [Feast Operator](https://docs.feast.dev/) for managing Feast deployments. +- Jupyter Notebook or JupyterLab to run the provided notebooks. +- Basic familiarity with Kubernetes, Helm, and TLS concepts. + +## Notebook Overview + +The following Jupyter Notebooks will walk you through the entire process: + +1. **[01-Install-postgres-tls-using-helm.ipynb](./01-Install-postgres-tls-using-helm.ipynb)** + Installs PostgreSQL in TLS mode using a Helm chart. + +2. **[02-Install-feast.ipynb](02-Install-feast.ipynb)** + Deploys Feast using the Feast Operator. + +3. **[03-Uninstall.ipynb](./03-Uninstall.ipynb)** + Uninstalls Feast, the Feast Operator, and the PostgreSQL deployments set up in this demo. + +## How to Run the Demo + +1. **Clone the Repository** + + ```shell + https://github.com/feast-dev/feast.git + cd examples/operator-postgres-tls-demo + ``` +2. Start Jupyter Notebook or JupyterLab from the repository root: + +```shell +jupyter notebook +``` +3. Execute the Notebooks +Run the notebooks in the order listed above. Each notebook contains step-by-step instructions and code to deploy, test, and eventually clean up the demo components. + + +## Troubleshooting +* **Cluster Resources:** +Verify that your Kubernetes cluster has adequate resources before starting the demo. + +* **Logs & Diagnostics:** +If you encounter issues, check the logs for the PostgreSQL and Feast pods. This can help identify problems related to TLS configurations or resource constraints. \ No newline at end of file diff --git a/examples/operator-quickstart/01-Install.ipynb b/examples/operator-quickstart/01-Install.ipynb index e2396efb4a0..7b974a721b3 100644 --- a/examples/operator-quickstart/01-Install.ipynb +++ b/examples/operator-quickstart/01-Install.ipynb @@ -74,7 +74,7 @@ "output_type": "stream", "text": [ "NAME STATUS AGE\n", - "feast Active 10s\n" + "feast Active 6s\n" ] } ], @@ -133,7 +133,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 4, "metadata": {}, "outputs": [ { @@ -141,20 +141,20 @@ "output_type": "stream", "text": [ "NAME READY STATUS RESTARTS AGE\n", - "pod/postgres-ff8d4cf48-6nqhs 1/1 Running 0 70s\n", - "pod/redis-b4756b75d-nttdm 1/1 Running 0 68s\n", + "pod/postgres-ff8d4cf48-c4znd 1/1 Running 0 2m17s\n", + "pod/redis-b4756b75d-r9nfb 1/1 Running 0 2m15s\n", "\n", "NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE\n", - "service/postgres ClusterIP 10.43.203.123 5432/TCP 70s\n", - "service/redis ClusterIP 10.43.234.211 6379/TCP 67s\n", + "service/postgres ClusterIP 10.43.151.129 5432/TCP 2m17s\n", + "service/redis ClusterIP 10.43.169.233 6379/TCP 2m15s\n", "\n", "NAME READY UP-TO-DATE AVAILABLE AGE\n", - "deployment.apps/postgres 1/1 1 1 70s\n", - "deployment.apps/redis 1/1 1 1 69s\n", + "deployment.apps/postgres 1/1 1 1 2m18s\n", + "deployment.apps/redis 1/1 1 1 2m16s\n", "\n", "NAME DESIRED CURRENT READY AGE\n", - "replicaset.apps/postgres-ff8d4cf48 1 1 1 70s\n", - "replicaset.apps/redis-b4756b75d 1 1 1 68s\n" + "replicaset.apps/postgres-ff8d4cf48 1 1 1 2m18s\n", + "replicaset.apps/redis-b4756b75d 1 1 1 2m16s\n" ] } ], @@ -217,7 +217,7 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": 5, "metadata": {}, "outputs": [ { @@ -243,32 +243,32 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 6, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "NAME READY STATUS RESTARTS AGE\n", - "pod/feast-example-556689b95c-gb227 0/1 PodInitializing 0 6m41s\n", - "pod/postgres-ff8d4cf48-6nqhs 1/1 Running 0 10m\n", - "pod/redis-b4756b75d-nttdm 1/1 Running 0 10m\n", + "NAME READY STATUS RESTARTS AGE\n", + "pod/feast-example-bbdc6cb6-rzkb4 0/1 Init:0/1 0 3s\n", + "pod/postgres-ff8d4cf48-c4znd 1/1 Running 0 4m49s\n", + "pod/redis-b4756b75d-r9nfb 1/1 Running 0 4m47s\n", "\n", "NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE\n", - "service/feast-example-online ClusterIP 10.43.254.136 80/TCP 6m43s\n", - "service/postgres ClusterIP 10.43.203.123 5432/TCP 10m\n", - "service/redis ClusterIP 10.43.234.211 6379/TCP 10m\n", + "service/feast-example-online ClusterIP 10.43.143.216 80/TCP 4s\n", + "service/postgres ClusterIP 10.43.151.129 5432/TCP 4m49s\n", + "service/redis ClusterIP 10.43.169.233 6379/TCP 4m47s\n", "\n", "NAME READY UP-TO-DATE AVAILABLE AGE\n", - "deployment.apps/feast-example 0/1 1 0 6m43s\n", - "deployment.apps/postgres 1/1 1 1 10m\n", - "deployment.apps/redis 1/1 1 1 10m\n", + "deployment.apps/feast-example 0/1 1 0 5s\n", + "deployment.apps/postgres 1/1 1 1 4m51s\n", + "deployment.apps/redis 1/1 1 1 4m49s\n", "\n", - "NAME DESIRED CURRENT READY AGE\n", - "replicaset.apps/feast-example-556689b95c 1 1 0 6m43s\n", - "replicaset.apps/postgres-ff8d4cf48 1 1 1 10m\n", - "replicaset.apps/redis-b4756b75d 1 1 1 10m\n", + "NAME DESIRED CURRENT READY AGE\n", + "replicaset.apps/feast-example-bbdc6cb6 1 1 0 4s\n", + "replicaset.apps/postgres-ff8d4cf48 1 1 1 4m51s\n", + "replicaset.apps/redis-b4756b75d 1 1 1 4m49s\n", "deployment.apps/feast-example condition met\n" ] } @@ -287,7 +287,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 2, "metadata": {}, "outputs": [ { @@ -295,7 +295,7 @@ "output_type": "stream", "text": [ "NAME STATUS AGE\n", - "example Ready 7m39s\n" + "example Ready 48m\n" ] } ], @@ -312,7 +312,7 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 8, "metadata": {}, "outputs": [ { @@ -343,99 +343,6 @@ "!kubectl exec deploy/postgres -- psql -h localhost -U feast feast -c '\\dt'" ] }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Verify the client `feature_store.yaml` and create the sample feature store definitions." - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "project: credit_scoring_local\n", - "provider: local\n", - "offline_store:\n", - " type: duckdb\n", - "online_store:\n", - " type: redis\n", - " connection_string: redis.feast.svc.cluster.local:6379\n", - "registry:\n", - " path: postgresql+psycopg://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres.feast.svc.cluster.local:5432/${POSTGRES_DB}\n", - " registry_type: sql\n", - " cache_ttl_seconds: 60\n", - " sqlalchemy_config_kwargs:\n", - " echo: false\n", - " pool_pre_ping: true\n", - "auth:\n", - " type: no_auth\n", - "entity_key_serialization_version: 3\n", - "/opt/app-root/lib64/python3.11/site-packages/feast/feature_view.py:48: DeprecationWarning: Entity value_type will be mandatory in the next release. Please specify a value_type for entity '__dummy'.\n", - " DUMMY_ENTITY = Entity(\n", - "/feast-data/credit_scoring_local/feature_repo/example_repo.py:27: DeprecationWarning: Entity value_type will be mandatory in the next release. Please specify a value_type for entity 'driver'.\n", - " driver = Entity(name=\"driver\", join_keys=[\"driver_id\"])\n", - "Applying changes for project credit_scoring_local\n", - "/opt/app-root/lib64/python3.11/site-packages/feast/feature_store.py:579: RuntimeWarning: On demand feature view is an experimental feature. This API is stable, but the functionality does not scale well for offline retrieval\n", - " warnings.warn(\n", - "Deploying infrastructure for \u001b[1m\u001b[32mdriver_hourly_stats\u001b[0m\n", - "Deploying infrastructure for \u001b[1m\u001b[32mdriver_hourly_stats_fresh\u001b[0m\n" - ] - } - ], - "source": [ - "!kubectl exec deploy/feast-example -itc online -- cat feature_store.yaml\n", - "!kubectl exec deploy/feast-example -itc online -- feast apply" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "List the registered feast projects & feature views." - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "/opt/app-root/lib64/python3.11/site-packages/feast/feature_view.py:48: DeprecationWarning: Entity value_type will be mandatory in the next release. Please specify a value_type for entity '__dummy'.\n", - " DUMMY_ENTITY = Entity(\n", - "/opt/app-root/lib64/python3.11/site-packages/feast/entity.py:173: DeprecationWarning: Entity value_type will be mandatory in the next release. Please specify a value_type for entity '__dummy'.\n", - " entity = cls(\n", - "/opt/app-root/lib64/python3.11/site-packages/feast/entity.py:173: DeprecationWarning: Entity value_type will be mandatory in the next release. Please specify a value_type for entity 'driver'.\n", - " entity = cls(\n", - "NAME DESCRIPTION TAGS OWNER\n", - "credit_scoring_local A project for driver statistics {}\n", - "/opt/app-root/lib64/python3.11/site-packages/feast/feature_view.py:48: DeprecationWarning: Entity value_type will be mandatory in the next release. Please specify a value_type for entity '__dummy'.\n", - " DUMMY_ENTITY = Entity(\n", - "/opt/app-root/lib64/python3.11/site-packages/feast/entity.py:173: DeprecationWarning: Entity value_type will be mandatory in the next release. Please specify a value_type for entity '__dummy'.\n", - " entity = cls(\n", - "/opt/app-root/lib64/python3.11/site-packages/feast/entity.py:173: DeprecationWarning: Entity value_type will be mandatory in the next release. Please specify a value_type for entity 'driver'.\n", - " entity = cls(\n", - "NAME ENTITIES TYPE\n", - "driver_hourly_stats {'driver'} FeatureView\n", - "driver_hourly_stats_fresh {'driver'} FeatureView\n", - "transformed_conv_rate_fresh {'driver'} OnDemandFeatureView\n", - "transformed_conv_rate {'driver'} OnDemandFeatureView\n" - ] - } - ], - "source": [ - "!kubectl exec deploy/feast-example -itc online -- feast projects list\n", - "!kubectl exec deploy/feast-example -itc online -- feast feature-views list" - ] - }, { "cell_type": "markdown", "metadata": {}, @@ -454,7 +361,7 @@ "text": [ "/opt/app-root/lib64/python3.11/site-packages/feast/feature_view.py:48: DeprecationWarning: Entity value_type will be mandatory in the next release. Please specify a value_type for entity '__dummy'.\n", " DUMMY_ENTITY = Entity(\n", - "Feast SDK Version: \"0.45.0\"\n" + "Feast SDK Version: \"0.46.0\"\n" ] } ], diff --git a/examples/operator-quickstart/02-Demo.ipynb b/examples/operator-quickstart/02-Demo.ipynb index 5ad4395d2fa..536e36f490f 100644 --- a/examples/operator-quickstart/02-Demo.ipynb +++ b/examples/operator-quickstart/02-Demo.ipynb @@ -13,21 +13,14 @@ "source": [ "We'll use the following tutorial as a demonstration.\n", "\n", - "https://github.com/feast-dev/feast-credit-score-local-tutorial/tree/f43b44b245ae2632b582f14176392cfe31f98da9" + "https://github.com/feast-dev/feast-credit-score-local-tutorial/tree/598a270353d8a83b37535f849a0fa000a07be8b5" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "## Upload the tutorial source code to the running Feast pod." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Upload the tutorial source code to the running feast pod and extract its contents." + "## Check the init container to ensure the repo was successfully cloned with git." ] }, { @@ -39,43 +32,34 @@ "name": "stdout", "output_type": "stream", "text": [ - "feast-credit-score-local-tutorial-f43b44b245ae2632b582f14176392cfe31f98da9/.gitignore\n", - "feast-credit-score-local-tutorial-f43b44b245ae2632b582f14176392cfe31f98da9/LICENSE\n", - "feast-credit-score-local-tutorial-f43b44b245ae2632b582f14176392cfe31f98da9/README.md\n", - "feast-credit-score-local-tutorial-f43b44b245ae2632b582f14176392cfe31f98da9/app.py\n", - "feast-credit-score-local-tutorial-f43b44b245ae2632b582f14176392cfe31f98da9/credit_model.py\n", - "feast-credit-score-local-tutorial-f43b44b245ae2632b582f14176392cfe31f98da9/data/\n", - "feast-credit-score-local-tutorial-f43b44b245ae2632b582f14176392cfe31f98da9/data/credit_history.parquet\n", - "feast-credit-score-local-tutorial-f43b44b245ae2632b582f14176392cfe31f98da9/data/credit_history_sample.csv\n", - "feast-credit-score-local-tutorial-f43b44b245ae2632b582f14176392cfe31f98da9/data/loan_table.parquet\n", - "feast-credit-score-local-tutorial-f43b44b245ae2632b582f14176392cfe31f98da9/data/loan_table_sample.csv\n", - "feast-credit-score-local-tutorial-f43b44b245ae2632b582f14176392cfe31f98da9/data/training_dataset_sample.parquet\n", - "feast-credit-score-local-tutorial-f43b44b245ae2632b582f14176392cfe31f98da9/data/zipcode_table.parquet\n", - "feast-credit-score-local-tutorial-f43b44b245ae2632b582f14176392cfe31f98da9/data/zipcode_table_sample.csv\n", - "feast-credit-score-local-tutorial-f43b44b245ae2632b582f14176392cfe31f98da9/feature_repo/\n", - "feast-credit-score-local-tutorial-f43b44b245ae2632b582f14176392cfe31f98da9/feature_repo/data/\n", - "feast-credit-score-local-tutorial-f43b44b245ae2632b582f14176392cfe31f98da9/feature_repo/data/credit_history.parquet\n", - "feast-credit-score-local-tutorial-f43b44b245ae2632b582f14176392cfe31f98da9/feature_repo/data/credit_history_sample.csv\n", - "feast-credit-score-local-tutorial-f43b44b245ae2632b582f14176392cfe31f98da9/feature_repo/data/loan_table.parquet\n", - "feast-credit-score-local-tutorial-f43b44b245ae2632b582f14176392cfe31f98da9/feature_repo/data/loan_table_sample.csv\n", - "feast-credit-score-local-tutorial-f43b44b245ae2632b582f14176392cfe31f98da9/feature_repo/data/training_dataset_sample.parquet\n", - "feast-credit-score-local-tutorial-f43b44b245ae2632b582f14176392cfe31f98da9/feature_repo/data/zipcode_table.parquet\n", - "feast-credit-score-local-tutorial-f43b44b245ae2632b582f14176392cfe31f98da9/feature_repo/data/zipcode_table_sample.csv\n", - "feast-credit-score-local-tutorial-f43b44b245ae2632b582f14176392cfe31f98da9/feature_repo/feature_store.yaml\n", - "feast-credit-score-local-tutorial-f43b44b245ae2632b582f14176392cfe31f98da9/feature_repo/features.py\n", - "feast-credit-score-local-tutorial-f43b44b245ae2632b582f14176392cfe31f98da9/requirements.txt\n", - "feast-credit-score-local-tutorial-f43b44b245ae2632b582f14176392cfe31f98da9/run.py\n", - "feast-credit-score-local-tutorial-f43b44b245ae2632b582f14176392cfe31f98da9/streamlit.png\n", - "feast-credit-score-local-tutorial-f43b44b245ae2632b582f14176392cfe31f98da9/streamlit_app.py\n" + "Creating feast repository...\n", + "git clone https://github.com/feast-dev/feast-credit-score-local-tutorial /feast-data/credit_scoring_local && cd /feast-data/credit_scoring_local && git checkout 598a270\n", + "Cloning into '/feast-data/credit_scoring_local'...\n", + "Updating files: 100% (25/25), done.\n", + "Note: switching to '598a270'.\n", + "\n", + "You are in 'detached HEAD' state. You can look around, make experimental\n", + "changes and commit them, and you can discard any commits you make in this\n", + "state without impacting any branches by switching back to a branch.\n", + "\n", + "If you want to create a new branch to retain commits you create, you may\n", + "do so (now or later) by using -c with the switch command. Example:\n", + "\n", + " git switch -c \n", + "\n", + "Or undo this operation with:\n", + "\n", + " git switch -\n", + "\n", + "Turn off this advice by setting config variable advice.detachedHead to false\n", + "\n", + "HEAD is now at 598a270 set streamlit version to 1.42.0 (#8)\n", + "Feast repo creation complete\n" ] } ], "source": [ - "![ -f f43b44b.tar.gz ] || wget https://github.com/feast-dev/feast-credit-score-local-tutorial/archive/f43b44b.tar.gz\n", - "!kubectl cp f43b44b.tar.gz $(kubectl get pods -l 'feast.dev/name=example' -ojsonpath=\"{.items[*].metadata.name}\"):/feast-data -c online\n", - "!kubectl exec deploy/feast-example -itc online -- rm -rf /feast-data/feast-credit-score-local-tutorial\n", - "!kubectl exec deploy/feast-example -itc online -- mkdir /feast-data/feast-credit-score-local-tutorial\n", - "!kubectl exec deploy/feast-example -itc online -- tar vfx /feast-data/f43b44b.tar.gz -C /feast-data/feast-credit-score-local-tutorial --strip-components 1" + "!kubectl logs -f deploy/feast-example -c feast-init" ] }, { @@ -85,13 +69,6 @@ "## Verify the client `feature_store.yaml`." ] }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Copy the `feature_store.yaml` to the tutorial directory and verify its contents." - ] - }, { "cell_type": "code", "execution_count": 2, @@ -122,8 +99,7 @@ } ], "source": [ - "!kubectl exec deploy/feast-example -itc online -- cp -f /feast-data/credit_scoring_local/feature_repo/feature_store.yaml /feast-data/feast-credit-score-local-tutorial/feature_repo/feature_store.yaml\n", - "!kubectl exec deploy/feast-example -itc online -- cat /feast-data/feast-credit-score-local-tutorial/feature_repo/feature_store.yaml" + "!kubectl exec deploy/feast-example -itc online -- cat feature_store.yaml" ] }, { @@ -152,26 +128,16 @@ "/opt/app-root/lib64/python3.11/site-packages/feast/feature_view.py:48: DeprecationWarning: Entity value_type will be mandatory in the next release. Please specify a value_type for entity '__dummy'.\n", " DUMMY_ENTITY = Entity(\n", "No project found in the repository. Using project name credit_scoring_local defined in feature_store.yaml\n", - "/opt/app-root/lib64/python3.11/site-packages/feast/entity.py:173: DeprecationWarning: Entity value_type will be mandatory in the next release. Please specify a value_type for entity '__dummy'.\n", - " entity = cls(\n", - "/opt/app-root/lib64/python3.11/site-packages/feast/entity.py:173: DeprecationWarning: Entity value_type will be mandatory in the next release. Please specify a value_type for entity 'driver'.\n", - " entity = cls(\n", "Applying changes for project credit_scoring_local\n", - "/opt/app-root/lib64/python3.11/site-packages/feast/entity.py:173: DeprecationWarning: Entity value_type will be mandatory in the next release. Please specify a value_type for entity '__dummy'.\n", - " entity = cls(\n", - "/opt/app-root/lib64/python3.11/site-packages/feast/entity.py:173: DeprecationWarning: Entity value_type will be mandatory in the next release. Please specify a value_type for entity 'driver'.\n", - " entity = cls(\n", "/opt/app-root/lib64/python3.11/site-packages/feast/feature_store.py:579: RuntimeWarning: On demand feature view is an experimental feature. This API is stable, but the functionality does not scale well for offline retrieval\n", " warnings.warn(\n", "Deploying infrastructure for \u001b[1m\u001b[32mzipcode_features\u001b[0m\n", - "Deploying infrastructure for \u001b[1m\u001b[32mcredit_history\u001b[0m\n", - "Removing infrastructure for \u001b[1m\u001b[31mdriver_hourly_stats\u001b[0m\n", - "Removing infrastructure for \u001b[1m\u001b[31mdriver_hourly_stats_fresh\u001b[0m\n" + "Deploying infrastructure for \u001b[1m\u001b[32mcredit_history\u001b[0m\n" ] } ], "source": [ - "!kubectl exec deploy/feast-example -itc online -- feast -c /feast-data/feast-credit-score-local-tutorial/feature_repo apply" + "!kubectl exec deploy/feast-example -itc online -- feast apply" ] }, { @@ -198,13 +164,13 @@ " entity = cls(\n", "/opt/app-root/lib64/python3.11/site-packages/feast/entity.py:173: DeprecationWarning: Entity value_type will be mandatory in the next release. Please specify a value_type for entity 'zipcode'.\n", " entity = cls(\n", - "Materializing \u001b[1m\u001b[32m2\u001b[0m feature views to \u001b[1m\u001b[32m2025-02-11 22:39:40+00:00\u001b[0m into the \u001b[1m\u001b[32mredis\u001b[0m online store.\n", + "Materializing \u001b[1m\u001b[32m2\u001b[0m feature views to \u001b[1m\u001b[32m2025-02-20 21:23:35+00:00\u001b[0m into the \u001b[1m\u001b[32mredis\u001b[0m online store.\n", "\n", - "\u001b[1m\u001b[32mzipcode_features\u001b[0m from \u001b[1m\u001b[32m2015-02-14 22:40:15+00:00\u001b[0m to \u001b[1m\u001b[32m2025-02-11 22:39:40+00:00\u001b[0m:\n", + "\u001b[1m\u001b[32mzipcode_features\u001b[0m from \u001b[1m\u001b[32m2015-02-23 21:24:12+00:00\u001b[0m to \u001b[1m\u001b[32m2025-02-20 21:23:35+00:00\u001b[0m:\n", "/opt/app-root/lib64/python3.11/site-packages/feast/entity.py:173: DeprecationWarning: Entity value_type will be mandatory in the next release. Please specify a value_type for entity 'zipcode'.\n", " entity = cls(\n", - "100%|███████████████████████████████████████████████████████| 28844/28844 [00:24<00:00, 1168.09it/s]\n", - "\u001b[1m\u001b[32mcredit_history\u001b[0m from \u001b[1m\u001b[32m2024-11-13 22:40:43+00:00\u001b[0m to \u001b[1m\u001b[32m2025-02-11 22:39:40+00:00\u001b[0m:\n", + "100%|███████████████████████████████████████████████████████| 28844/28844 [00:28<00:00, 1023.99it/s]\n", + "\u001b[1m\u001b[32mcredit_history\u001b[0m from \u001b[1m\u001b[32m2024-11-22 21:24:43+00:00\u001b[0m to \u001b[1m\u001b[32m2025-02-20 21:23:35+00:00\u001b[0m:\n", "/opt/app-root/lib64/python3.11/site-packages/feast/entity.py:173: DeprecationWarning: Entity value_type will be mandatory in the next release. Please specify a value_type for entity 'dob_ssn'.\n", " entity = cls(\n", "0it [00:00, ?it/s]\n" @@ -212,7 +178,7 @@ } ], "source": [ - "!kubectl exec deploy/feast-example -itc online -- bash -c 'cd /feast-data/feast-credit-score-local-tutorial/feature_repo && feast materialize-incremental $(date -u +\"%Y-%m-%dT%H:%M:%S\")'" + "!kubectl exec deploy/feast-example -itc online -- bash -c 'feast materialize-incremental $(date -u +\"%Y-%m-%dT%H:%M:%S\")'" ] }, { @@ -226,7 +192,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "List the registered feast feature views & entities." + "List the registered feast projects, feature views, & entities." ] }, { @@ -238,6 +204,16 @@ "name": "stdout", "output_type": "stream", "text": [ + "/opt/app-root/lib64/python3.11/site-packages/feast/feature_view.py:48: DeprecationWarning: Entity value_type will be mandatory in the next release. Please specify a value_type for entity '__dummy'.\n", + " DUMMY_ENTITY = Entity(\n", + "/opt/app-root/lib64/python3.11/site-packages/feast/entity.py:173: DeprecationWarning: Entity value_type will be mandatory in the next release. Please specify a value_type for entity '__dummy'.\n", + " entity = cls(\n", + "/opt/app-root/lib64/python3.11/site-packages/feast/entity.py:173: DeprecationWarning: Entity value_type will be mandatory in the next release. Please specify a value_type for entity 'dob_ssn'.\n", + " entity = cls(\n", + "/opt/app-root/lib64/python3.11/site-packages/feast/entity.py:173: DeprecationWarning: Entity value_type will be mandatory in the next release. Please specify a value_type for entity 'zipcode'.\n", + " entity = cls(\n", + "NAME DESCRIPTION TAGS OWNER\n", + "credit_scoring_local {}\n", "/opt/app-root/lib64/python3.11/site-packages/feast/feature_view.py:48: DeprecationWarning: Entity value_type will be mandatory in the next release. Please specify a value_type for entity '__dummy'.\n", " DUMMY_ENTITY = Entity(\n", "/opt/app-root/lib64/python3.11/site-packages/feast/entity.py:173: DeprecationWarning: Entity value_type will be mandatory in the next release. Please specify a value_type for entity '__dummy'.\n", @@ -271,8 +247,9 @@ } ], "source": [ - "!kubectl exec deploy/feast-example -itc online -- feast -c /feast-data/feast-credit-score-local-tutorial/feature_repo feature-views list\n", - "!kubectl exec deploy/feast-example -itc online -- feast -c /feast-data/feast-credit-score-local-tutorial/feature_repo entities list" + "!kubectl exec deploy/feast-example -itc online -- feast projects list\n", + "!kubectl exec deploy/feast-example -itc online -- feast feature-views list\n", + "!kubectl exec deploy/feast-example -itc online -- feast entities list" ] }, { @@ -291,170 +268,170 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 2, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "Collecting streamlit (from -r /feast-data/feast-credit-score-local-tutorial/requirements.txt (line 1))\n", - " Obtaining dependency information for streamlit from https://files.pythonhosted.org/packages/ad/dc/69068179e09488d0833a970d06e8bf40e35669a7bddb8a3caadc13b7dff4/streamlit-1.42.0-py2.py3-none-any.whl.metadata\n", + "Collecting streamlit==1.42.0 (from -r ../requirements.txt (line 1))\n", + " Obtaining dependency information for streamlit==1.42.0 from https://files.pythonhosted.org/packages/ad/dc/69068179e09488d0833a970d06e8bf40e35669a7bddb8a3caadc13b7dff4/streamlit-1.42.0-py2.py3-none-any.whl.metadata\n", " Downloading streamlit-1.42.0-py2.py3-none-any.whl.metadata (8.9 kB)\n", - "Collecting shap (from -r /feast-data/feast-credit-score-local-tutorial/requirements.txt (line 2))\n", + "Collecting shap (from -r ../requirements.txt (line 2))\n", " Obtaining dependency information for shap from https://files.pythonhosted.org/packages/06/6a/09e3cb9864118337c0f3c2a0dc5add6b642e9f672665062e186d67ba992d/shap-0.46.0-cp311-cp311-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata\n", " Downloading shap-0.46.0-cp311-cp311-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (24 kB)\n", - "Requirement already satisfied: pandas in /opt/app-root/lib64/python3.11/site-packages (from -r /feast-data/feast-credit-score-local-tutorial/requirements.txt (line 3)) (2.2.3)\n", - "Collecting scikit-learn (from -r /feast-data/feast-credit-score-local-tutorial/requirements.txt (line 4))\n", + "Requirement already satisfied: pandas in /opt/app-root/lib64/python3.11/site-packages (from -r ../requirements.txt (line 3)) (2.2.3)\n", + "Collecting scikit-learn (from -r ../requirements.txt (line 4))\n", " Obtaining dependency information for scikit-learn from https://files.pythonhosted.org/packages/a8/f3/62fc9a5a659bb58a03cdd7e258956a5824bdc9b4bb3c5d932f55880be569/scikit_learn-1.6.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata\n", " Downloading scikit_learn-1.6.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (18 kB)\n", - "Collecting matplotlib (from -r /feast-data/feast-credit-score-local-tutorial/requirements.txt (line 5))\n", + "Collecting matplotlib (from -r ../requirements.txt (line 5))\n", " Obtaining dependency information for matplotlib from https://files.pythonhosted.org/packages/b2/7d/2d873209536b9ee17340754118a2a17988bc18981b5b56e6715ee07373ac/matplotlib-3.10.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata\n", " Downloading matplotlib-3.10.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (11 kB)\n", - "Collecting altair<6,>=4.0 (from streamlit->-r /feast-data/feast-credit-score-local-tutorial/requirements.txt (line 1))\n", + "Collecting altair<6,>=4.0 (from streamlit==1.42.0->-r ../requirements.txt (line 1))\n", " Obtaining dependency information for altair<6,>=4.0 from https://files.pythonhosted.org/packages/aa/f3/0b6ced594e51cc95d8c1fc1640d3623770d01e4969d29c0bd09945fafefa/altair-5.5.0-py3-none-any.whl.metadata\n", " Downloading altair-5.5.0-py3-none-any.whl.metadata (11 kB)\n", - "Collecting blinker<2,>=1.0.0 (from streamlit->-r /feast-data/feast-credit-score-local-tutorial/requirements.txt (line 1))\n", + "Collecting blinker<2,>=1.0.0 (from streamlit==1.42.0->-r ../requirements.txt (line 1))\n", " Obtaining dependency information for blinker<2,>=1.0.0 from https://files.pythonhosted.org/packages/10/cb/f2ad4230dc2eb1a74edf38f1a38b9b52277f75bef262d8908e60d957e13c/blinker-1.9.0-py3-none-any.whl.metadata\n", " Downloading blinker-1.9.0-py3-none-any.whl.metadata (1.6 kB)\n", - "Requirement already satisfied: cachetools<6,>=4.0 in /opt/app-root/lib64/python3.11/site-packages (from streamlit->-r /feast-data/feast-credit-score-local-tutorial/requirements.txt (line 1)) (5.5.1)\n", - "Requirement already satisfied: click<9,>=7.0 in /opt/app-root/lib64/python3.11/site-packages (from streamlit->-r /feast-data/feast-credit-score-local-tutorial/requirements.txt (line 1)) (8.1.8)\n", - "Requirement already satisfied: numpy<3,>=1.23 in /opt/app-root/lib64/python3.11/site-packages (from streamlit->-r /feast-data/feast-credit-score-local-tutorial/requirements.txt (line 1)) (1.26.4)\n", - "Requirement already satisfied: packaging<25,>=20 in /opt/app-root/lib64/python3.11/site-packages (from streamlit->-r /feast-data/feast-credit-score-local-tutorial/requirements.txt (line 1)) (24.2)\n", - "Collecting pillow<12,>=7.1.0 (from streamlit->-r /feast-data/feast-credit-score-local-tutorial/requirements.txt (line 1))\n", + "Requirement already satisfied: cachetools<6,>=4.0 in /opt/app-root/lib64/python3.11/site-packages (from streamlit==1.42.0->-r ../requirements.txt (line 1)) (5.5.1)\n", + "Requirement already satisfied: click<9,>=7.0 in /opt/app-root/lib64/python3.11/site-packages (from streamlit==1.42.0->-r ../requirements.txt (line 1)) (8.1.8)\n", + "Requirement already satisfied: numpy<3,>=1.23 in /opt/app-root/lib64/python3.11/site-packages (from streamlit==1.42.0->-r ../requirements.txt (line 1)) (1.26.4)\n", + "Requirement already satisfied: packaging<25,>=20 in /opt/app-root/lib64/python3.11/site-packages (from streamlit==1.42.0->-r ../requirements.txt (line 1)) (24.2)\n", + "Collecting pillow<12,>=7.1.0 (from streamlit==1.42.0->-r ../requirements.txt (line 1))\n", " Obtaining dependency information for pillow<12,>=7.1.0 from https://files.pythonhosted.org/packages/48/a4/fbfe9d5581d7b111b28f1d8c2762dee92e9821bb209af9fa83c940e507a0/pillow-11.1.0-cp311-cp311-manylinux_2_28_x86_64.whl.metadata\n", " Downloading pillow-11.1.0-cp311-cp311-manylinux_2_28_x86_64.whl.metadata (9.1 kB)\n", - "Requirement already satisfied: protobuf<6,>=3.20 in /opt/app-root/lib64/python3.11/site-packages (from streamlit->-r /feast-data/feast-credit-score-local-tutorial/requirements.txt (line 1)) (5.29.3)\n", - "Requirement already satisfied: pyarrow>=7.0 in /opt/app-root/lib64/python3.11/site-packages (from streamlit->-r /feast-data/feast-credit-score-local-tutorial/requirements.txt (line 1)) (17.0.0)\n", - "Requirement already satisfied: requests<3,>=2.27 in /opt/app-root/lib64/python3.11/site-packages (from streamlit->-r /feast-data/feast-credit-score-local-tutorial/requirements.txt (line 1)) (2.32.3)\n", - "Requirement already satisfied: rich<14,>=10.14.0 in /opt/app-root/lib64/python3.11/site-packages (from streamlit->-r /feast-data/feast-credit-score-local-tutorial/requirements.txt (line 1)) (13.9.4)\n", - "Requirement already satisfied: tenacity<10,>=8.1.0 in /opt/app-root/lib64/python3.11/site-packages (from streamlit->-r /feast-data/feast-credit-score-local-tutorial/requirements.txt (line 1)) (8.5.0)\n", - "Requirement already satisfied: toml<2,>=0.10.1 in /opt/app-root/lib64/python3.11/site-packages (from streamlit->-r /feast-data/feast-credit-score-local-tutorial/requirements.txt (line 1)) (0.10.2)\n", - "Requirement already satisfied: typing-extensions<5,>=4.4.0 in /opt/app-root/lib64/python3.11/site-packages (from streamlit->-r /feast-data/feast-credit-score-local-tutorial/requirements.txt (line 1)) (4.12.2)\n", - "Collecting watchdog<7,>=2.1.5 (from streamlit->-r /feast-data/feast-credit-score-local-tutorial/requirements.txt (line 1))\n", + "Requirement already satisfied: protobuf<6,>=3.20 in /opt/app-root/lib64/python3.11/site-packages (from streamlit==1.42.0->-r ../requirements.txt (line 1)) (5.29.3)\n", + "Requirement already satisfied: pyarrow>=7.0 in /opt/app-root/lib64/python3.11/site-packages (from streamlit==1.42.0->-r ../requirements.txt (line 1)) (17.0.0)\n", + "Requirement already satisfied: requests<3,>=2.27 in /opt/app-root/lib64/python3.11/site-packages (from streamlit==1.42.0->-r ../requirements.txt (line 1)) (2.32.3)\n", + "Requirement already satisfied: rich<14,>=10.14.0 in /opt/app-root/lib64/python3.11/site-packages (from streamlit==1.42.0->-r ../requirements.txt (line 1)) (13.9.4)\n", + "Requirement already satisfied: tenacity<10,>=8.1.0 in /opt/app-root/lib64/python3.11/site-packages (from streamlit==1.42.0->-r ../requirements.txt (line 1)) (8.5.0)\n", + "Requirement already satisfied: toml<2,>=0.10.1 in /opt/app-root/lib64/python3.11/site-packages (from streamlit==1.42.0->-r ../requirements.txt (line 1)) (0.10.2)\n", + "Requirement already satisfied: typing-extensions<5,>=4.4.0 in /opt/app-root/lib64/python3.11/site-packages (from streamlit==1.42.0->-r ../requirements.txt (line 1)) (4.12.2)\n", + "Collecting watchdog<7,>=2.1.5 (from streamlit==1.42.0->-r ../requirements.txt (line 1))\n", " Obtaining dependency information for watchdog<7,>=2.1.5 from https://files.pythonhosted.org/packages/b5/e8/dbf020b4d98251a9860752a094d09a65e1b436ad181faf929983f697048f/watchdog-6.0.0-py3-none-manylinux2014_x86_64.whl.metadata\n", " Downloading watchdog-6.0.0-py3-none-manylinux2014_x86_64.whl.metadata (44 kB)\n", - "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m44.3/44.3 kB\u001b[0m \u001b[31m5.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", - "\u001b[?25hCollecting gitpython!=3.1.19,<4,>=3.0.7 (from streamlit->-r /feast-data/feast-credit-score-local-tutorial/requirements.txt (line 1))\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m44.3/44.3 kB\u001b[0m \u001b[31m13.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hCollecting gitpython!=3.1.19,<4,>=3.0.7 (from streamlit==1.42.0->-r ../requirements.txt (line 1))\n", " Obtaining dependency information for gitpython!=3.1.19,<4,>=3.0.7 from https://files.pythonhosted.org/packages/1d/9a/4114a9057db2f1462d5c8f8390ab7383925fe1ac012eaa42402ad65c2963/GitPython-3.1.44-py3-none-any.whl.metadata\n", " Downloading GitPython-3.1.44-py3-none-any.whl.metadata (13 kB)\n", - "Collecting pydeck<1,>=0.8.0b4 (from streamlit->-r /feast-data/feast-credit-score-local-tutorial/requirements.txt (line 1))\n", + "Collecting pydeck<1,>=0.8.0b4 (from streamlit==1.42.0->-r ../requirements.txt (line 1))\n", " Obtaining dependency information for pydeck<1,>=0.8.0b4 from https://files.pythonhosted.org/packages/ab/4c/b888e6cf58bd9db9c93f40d1c6be8283ff49d88919231afe93a6bcf61626/pydeck-0.9.1-py2.py3-none-any.whl.metadata\n", " Downloading pydeck-0.9.1-py2.py3-none-any.whl.metadata (4.1 kB)\n", - "Collecting tornado<7,>=6.0.3 (from streamlit->-r /feast-data/feast-credit-score-local-tutorial/requirements.txt (line 1))\n", + "Collecting tornado<7,>=6.0.3 (from streamlit==1.42.0->-r ../requirements.txt (line 1))\n", " Obtaining dependency information for tornado<7,>=6.0.3 from https://files.pythonhosted.org/packages/22/55/b78a464de78051a30599ceb6983b01d8f732e6f69bf37b4ed07f642ac0fc/tornado-6.4.2-cp38-abi3-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata\n", " Downloading tornado-6.4.2-cp38-abi3-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (2.5 kB)\n", - "Collecting scipy (from shap->-r /feast-data/feast-credit-score-local-tutorial/requirements.txt (line 2))\n", - " Obtaining dependency information for scipy from https://files.pythonhosted.org/packages/fc/da/452e1119e6f720df3feb588cce3c42c5e3d628d4bfd4aec097bd30b7de0c/scipy-1.15.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata\n", - " Downloading scipy-1.15.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (61 kB)\n", - "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m62.0/62.0 kB\u001b[0m \u001b[31m8.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", - "\u001b[?25hRequirement already satisfied: tqdm>=4.27.0 in /opt/app-root/lib64/python3.11/site-packages (from shap->-r /feast-data/feast-credit-score-local-tutorial/requirements.txt (line 2)) (4.67.1)\n", - "Collecting slicer==0.0.8 (from shap->-r /feast-data/feast-credit-score-local-tutorial/requirements.txt (line 2))\n", + "Collecting scipy (from shap->-r ../requirements.txt (line 2))\n", + " Obtaining dependency information for scipy from https://files.pythonhosted.org/packages/32/ea/564bacc26b676c06a00266a3f25fdfe91a9d9a2532ccea7ce6dd394541bc/scipy-1.15.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata\n", + " Downloading scipy-1.15.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (61 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m62.0/62.0 kB\u001b[0m \u001b[31m8.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hRequirement already satisfied: tqdm>=4.27.0 in /opt/app-root/lib64/python3.11/site-packages (from shap->-r ../requirements.txt (line 2)) (4.67.1)\n", + "Collecting slicer==0.0.8 (from shap->-r ../requirements.txt (line 2))\n", " Obtaining dependency information for slicer==0.0.8 from https://files.pythonhosted.org/packages/63/81/9ef641ff4e12cbcca30e54e72fb0951a2ba195d0cda0ba4100e532d929db/slicer-0.0.8-py3-none-any.whl.metadata\n", " Downloading slicer-0.0.8-py3-none-any.whl.metadata (4.0 kB)\n", - "Collecting numba (from shap->-r /feast-data/feast-credit-score-local-tutorial/requirements.txt (line 2))\n", + "Collecting numba (from shap->-r ../requirements.txt (line 2))\n", " Obtaining dependency information for numba from https://files.pythonhosted.org/packages/14/91/18b9f64b34ff318a14d072251480547f89ebfb864b2b7168e5dc5f64f502/numba-0.61.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl.metadata\n", " Downloading numba-0.61.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl.metadata (2.8 kB)\n", - "Requirement already satisfied: cloudpickle in /opt/app-root/lib64/python3.11/site-packages (from shap->-r /feast-data/feast-credit-score-local-tutorial/requirements.txt (line 2)) (3.1.1)\n", - "Requirement already satisfied: python-dateutil>=2.8.2 in /opt/app-root/lib64/python3.11/site-packages (from pandas->-r /feast-data/feast-credit-score-local-tutorial/requirements.txt (line 3)) (2.9.0.post0)\n", - "Requirement already satisfied: pytz>=2020.1 in /opt/app-root/lib64/python3.11/site-packages (from pandas->-r /feast-data/feast-credit-score-local-tutorial/requirements.txt (line 3)) (2025.1)\n", - "Requirement already satisfied: tzdata>=2022.7 in /opt/app-root/lib64/python3.11/site-packages (from pandas->-r /feast-data/feast-credit-score-local-tutorial/requirements.txt (line 3)) (2025.1)\n", - "Collecting joblib>=1.2.0 (from scikit-learn->-r /feast-data/feast-credit-score-local-tutorial/requirements.txt (line 4))\n", + "Requirement already satisfied: cloudpickle in /opt/app-root/lib64/python3.11/site-packages (from shap->-r ../requirements.txt (line 2)) (3.1.1)\n", + "Requirement already satisfied: python-dateutil>=2.8.2 in /opt/app-root/lib64/python3.11/site-packages (from pandas->-r ../requirements.txt (line 3)) (2.9.0.post0)\n", + "Requirement already satisfied: pytz>=2020.1 in /opt/app-root/lib64/python3.11/site-packages (from pandas->-r ../requirements.txt (line 3)) (2025.1)\n", + "Requirement already satisfied: tzdata>=2022.7 in /opt/app-root/lib64/python3.11/site-packages (from pandas->-r ../requirements.txt (line 3)) (2025.1)\n", + "Collecting joblib>=1.2.0 (from scikit-learn->-r ../requirements.txt (line 4))\n", " Obtaining dependency information for joblib>=1.2.0 from https://files.pythonhosted.org/packages/91/29/df4b9b42f2be0b623cbd5e2140cafcaa2bef0759a00b7b70104dcfe2fb51/joblib-1.4.2-py3-none-any.whl.metadata\n", " Downloading joblib-1.4.2-py3-none-any.whl.metadata (5.4 kB)\n", - "Collecting threadpoolctl>=3.1.0 (from scikit-learn->-r /feast-data/feast-credit-score-local-tutorial/requirements.txt (line 4))\n", + "Collecting threadpoolctl>=3.1.0 (from scikit-learn->-r ../requirements.txt (line 4))\n", " Obtaining dependency information for threadpoolctl>=3.1.0 from https://files.pythonhosted.org/packages/4b/2c/ffbf7a134b9ab11a67b0cf0726453cedd9c5043a4fe7a35d1cefa9a1bcfb/threadpoolctl-3.5.0-py3-none-any.whl.metadata\n", " Downloading threadpoolctl-3.5.0-py3-none-any.whl.metadata (13 kB)\n", - "Collecting contourpy>=1.0.1 (from matplotlib->-r /feast-data/feast-credit-score-local-tutorial/requirements.txt (line 5))\n", + "Collecting contourpy>=1.0.1 (from matplotlib->-r ../requirements.txt (line 5))\n", " Obtaining dependency information for contourpy>=1.0.1 from https://files.pythonhosted.org/packages/85/fc/7fa5d17daf77306840a4e84668a48ddff09e6bc09ba4e37e85ffc8e4faa3/contourpy-1.3.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata\n", " Downloading contourpy-1.3.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (5.4 kB)\n", - "Collecting cycler>=0.10 (from matplotlib->-r /feast-data/feast-credit-score-local-tutorial/requirements.txt (line 5))\n", + "Collecting cycler>=0.10 (from matplotlib->-r ../requirements.txt (line 5))\n", " Obtaining dependency information for cycler>=0.10 from https://files.pythonhosted.org/packages/e7/05/c19819d5e3d95294a6f5947fb9b9629efb316b96de511b418c53d245aae6/cycler-0.12.1-py3-none-any.whl.metadata\n", " Downloading cycler-0.12.1-py3-none-any.whl.metadata (3.8 kB)\n", - "Collecting fonttools>=4.22.0 (from matplotlib->-r /feast-data/feast-credit-score-local-tutorial/requirements.txt (line 5))\n", + "Collecting fonttools>=4.22.0 (from matplotlib->-r ../requirements.txt (line 5))\n", " Obtaining dependency information for fonttools>=4.22.0 from https://files.pythonhosted.org/packages/28/e9/47c02d5a7027e8ed841ab6a10ca00c93dadd5f16742f1af1fa3f9978adf4/fonttools-4.56.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata\n", " Downloading fonttools-4.56.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (101 kB)\n", - "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m101.9/101.9 kB\u001b[0m \u001b[31m10.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", - "\u001b[?25hCollecting kiwisolver>=1.3.1 (from matplotlib->-r /feast-data/feast-credit-score-local-tutorial/requirements.txt (line 5))\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m101.9/101.9 kB\u001b[0m \u001b[31m9.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hCollecting kiwisolver>=1.3.1 (from matplotlib->-r ../requirements.txt (line 5))\n", " Obtaining dependency information for kiwisolver>=1.3.1 from https://files.pythonhosted.org/packages/3a/97/5edbed69a9d0caa2e4aa616ae7df8127e10f6586940aa683a496c2c280b9/kiwisolver-1.4.8-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata\n", " Downloading kiwisolver-1.4.8-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (6.2 kB)\n", - "Collecting pyparsing>=2.3.1 (from matplotlib->-r /feast-data/feast-credit-score-local-tutorial/requirements.txt (line 5))\n", + "Collecting pyparsing>=2.3.1 (from matplotlib->-r ../requirements.txt (line 5))\n", " Obtaining dependency information for pyparsing>=2.3.1 from https://files.pythonhosted.org/packages/1c/a7/c8a2d361bf89c0d9577c934ebb7421b25dc84bf3a8e3ac0a40aed9acc547/pyparsing-3.2.1-py3-none-any.whl.metadata\n", " Downloading pyparsing-3.2.1-py3-none-any.whl.metadata (5.0 kB)\n", - "Requirement already satisfied: jinja2 in /opt/app-root/lib64/python3.11/site-packages (from altair<6,>=4.0->streamlit->-r /feast-data/feast-credit-score-local-tutorial/requirements.txt (line 1)) (3.1.5)\n", - "Requirement already satisfied: jsonschema>=3.0 in /opt/app-root/lib64/python3.11/site-packages (from altair<6,>=4.0->streamlit->-r /feast-data/feast-credit-score-local-tutorial/requirements.txt (line 1)) (4.23.0)\n", - "Collecting narwhals>=1.14.2 (from altair<6,>=4.0->streamlit->-r /feast-data/feast-credit-score-local-tutorial/requirements.txt (line 1))\n", - " Obtaining dependency information for narwhals>=1.14.2 from https://files.pythonhosted.org/packages/15/fc/420680ad8b0cf81372eee7a213a7b7173ec5a628f0d5b2426047fe55c3b3/narwhals-1.26.0-py3-none-any.whl.metadata\n", - " Downloading narwhals-1.26.0-py3-none-any.whl.metadata (10 kB)\n", - "Collecting gitdb<5,>=4.0.1 (from gitpython!=3.1.19,<4,>=3.0.7->streamlit->-r /feast-data/feast-credit-score-local-tutorial/requirements.txt (line 1))\n", + "Requirement already satisfied: jinja2 in /opt/app-root/lib64/python3.11/site-packages (from altair<6,>=4.0->streamlit==1.42.0->-r ../requirements.txt (line 1)) (3.1.5)\n", + "Requirement already satisfied: jsonschema>=3.0 in /opt/app-root/lib64/python3.11/site-packages (from altair<6,>=4.0->streamlit==1.42.0->-r ../requirements.txt (line 1)) (4.23.0)\n", + "Collecting narwhals>=1.14.2 (from altair<6,>=4.0->streamlit==1.42.0->-r ../requirements.txt (line 1))\n", + " Obtaining dependency information for narwhals>=1.14.2 from https://files.pythonhosted.org/packages/ed/ea/dc14822a0a75e027562f081eb638417b1b7845e1e01dd85c5b6573ebf1b2/narwhals-1.27.1-py3-none-any.whl.metadata\n", + " Downloading narwhals-1.27.1-py3-none-any.whl.metadata (10 kB)\n", + "Collecting gitdb<5,>=4.0.1 (from gitpython!=3.1.19,<4,>=3.0.7->streamlit==1.42.0->-r ../requirements.txt (line 1))\n", " Obtaining dependency information for gitdb<5,>=4.0.1 from https://files.pythonhosted.org/packages/a0/61/5c78b91c3143ed5c14207f463aecfc8f9dbb5092fb2869baf37c273b2705/gitdb-4.0.12-py3-none-any.whl.metadata\n", " Downloading gitdb-4.0.12-py3-none-any.whl.metadata (1.2 kB)\n", - "Requirement already satisfied: six>=1.5 in /opt/app-root/lib64/python3.11/site-packages (from python-dateutil>=2.8.2->pandas->-r /feast-data/feast-credit-score-local-tutorial/requirements.txt (line 3)) (1.17.0)\n", - "Requirement already satisfied: charset-normalizer<4,>=2 in /opt/app-root/lib64/python3.11/site-packages (from requests<3,>=2.27->streamlit->-r /feast-data/feast-credit-score-local-tutorial/requirements.txt (line 1)) (3.4.1)\n", - "Requirement already satisfied: idna<4,>=2.5 in /opt/app-root/lib64/python3.11/site-packages (from requests<3,>=2.27->streamlit->-r /feast-data/feast-credit-score-local-tutorial/requirements.txt (line 1)) (3.10)\n", - "Requirement already satisfied: urllib3<3,>=1.21.1 in /opt/app-root/lib64/python3.11/site-packages (from requests<3,>=2.27->streamlit->-r /feast-data/feast-credit-score-local-tutorial/requirements.txt (line 1)) (2.3.0)\n", - "Requirement already satisfied: certifi>=2017.4.17 in /opt/app-root/lib64/python3.11/site-packages (from requests<3,>=2.27->streamlit->-r /feast-data/feast-credit-score-local-tutorial/requirements.txt (line 1)) (2025.1.31)\n", - "Requirement already satisfied: markdown-it-py>=2.2.0 in /opt/app-root/lib64/python3.11/site-packages (from rich<14,>=10.14.0->streamlit->-r /feast-data/feast-credit-score-local-tutorial/requirements.txt (line 1)) (3.0.0)\n", - "Requirement already satisfied: pygments<3.0.0,>=2.13.0 in /opt/app-root/lib64/python3.11/site-packages (from rich<14,>=10.14.0->streamlit->-r /feast-data/feast-credit-score-local-tutorial/requirements.txt (line 1)) (2.19.1)\n", - "Collecting llvmlite<0.45,>=0.44.0dev0 (from numba->shap->-r /feast-data/feast-credit-score-local-tutorial/requirements.txt (line 2))\n", + "Requirement already satisfied: six>=1.5 in /opt/app-root/lib64/python3.11/site-packages (from python-dateutil>=2.8.2->pandas->-r ../requirements.txt (line 3)) (1.17.0)\n", + "Requirement already satisfied: charset-normalizer<4,>=2 in /opt/app-root/lib64/python3.11/site-packages (from requests<3,>=2.27->streamlit==1.42.0->-r ../requirements.txt (line 1)) (3.4.1)\n", + "Requirement already satisfied: idna<4,>=2.5 in /opt/app-root/lib64/python3.11/site-packages (from requests<3,>=2.27->streamlit==1.42.0->-r ../requirements.txt (line 1)) (3.10)\n", + "Requirement already satisfied: urllib3<3,>=1.21.1 in /opt/app-root/lib64/python3.11/site-packages (from requests<3,>=2.27->streamlit==1.42.0->-r ../requirements.txt (line 1)) (2.3.0)\n", + "Requirement already satisfied: certifi>=2017.4.17 in /opt/app-root/lib64/python3.11/site-packages (from requests<3,>=2.27->streamlit==1.42.0->-r ../requirements.txt (line 1)) (2025.1.31)\n", + "Requirement already satisfied: markdown-it-py>=2.2.0 in /opt/app-root/lib64/python3.11/site-packages (from rich<14,>=10.14.0->streamlit==1.42.0->-r ../requirements.txt (line 1)) (3.0.0)\n", + "Requirement already satisfied: pygments<3.0.0,>=2.13.0 in /opt/app-root/lib64/python3.11/site-packages (from rich<14,>=10.14.0->streamlit==1.42.0->-r ../requirements.txt (line 1)) (2.19.1)\n", + "Collecting llvmlite<0.45,>=0.44.0dev0 (from numba->shap->-r ../requirements.txt (line 2))\n", " Obtaining dependency information for llvmlite<0.45,>=0.44.0dev0 from https://files.pythonhosted.org/packages/99/fe/d030f1849ebb1f394bb3f7adad5e729b634fb100515594aca25c354ffc62/llvmlite-0.44.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata\n", " Downloading llvmlite-0.44.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (4.8 kB)\n", - "Collecting smmap<6,>=3.0.1 (from gitdb<5,>=4.0.1->gitpython!=3.1.19,<4,>=3.0.7->streamlit->-r /feast-data/feast-credit-score-local-tutorial/requirements.txt (line 1))\n", + "Collecting smmap<6,>=3.0.1 (from gitdb<5,>=4.0.1->gitpython!=3.1.19,<4,>=3.0.7->streamlit==1.42.0->-r ../requirements.txt (line 1))\n", " Obtaining dependency information for smmap<6,>=3.0.1 from https://files.pythonhosted.org/packages/04/be/d09147ad1ec7934636ad912901c5fd7667e1c858e19d355237db0d0cd5e4/smmap-5.0.2-py3-none-any.whl.metadata\n", " Downloading smmap-5.0.2-py3-none-any.whl.metadata (4.3 kB)\n", - "Requirement already satisfied: MarkupSafe>=2.0 in /opt/app-root/lib64/python3.11/site-packages (from jinja2->altair<6,>=4.0->streamlit->-r /feast-data/feast-credit-score-local-tutorial/requirements.txt (line 1)) (3.0.2)\n", - "Requirement already satisfied: attrs>=22.2.0 in /opt/app-root/lib64/python3.11/site-packages (from jsonschema>=3.0->altair<6,>=4.0->streamlit->-r /feast-data/feast-credit-score-local-tutorial/requirements.txt (line 1)) (25.1.0)\n", - "Requirement already satisfied: jsonschema-specifications>=2023.03.6 in /opt/app-root/lib64/python3.11/site-packages (from jsonschema>=3.0->altair<6,>=4.0->streamlit->-r /feast-data/feast-credit-score-local-tutorial/requirements.txt (line 1)) (2024.10.1)\n", - "Requirement already satisfied: referencing>=0.28.4 in /opt/app-root/lib64/python3.11/site-packages (from jsonschema>=3.0->altair<6,>=4.0->streamlit->-r /feast-data/feast-credit-score-local-tutorial/requirements.txt (line 1)) (0.36.2)\n", - "Requirement already satisfied: rpds-py>=0.7.1 in /opt/app-root/lib64/python3.11/site-packages (from jsonschema>=3.0->altair<6,>=4.0->streamlit->-r /feast-data/feast-credit-score-local-tutorial/requirements.txt (line 1)) (0.22.3)\n", - "Requirement already satisfied: mdurl~=0.1 in /opt/app-root/lib64/python3.11/site-packages (from markdown-it-py>=2.2.0->rich<14,>=10.14.0->streamlit->-r /feast-data/feast-credit-score-local-tutorial/requirements.txt (line 1)) (0.1.2)\n", + "Requirement already satisfied: MarkupSafe>=2.0 in /opt/app-root/lib64/python3.11/site-packages (from jinja2->altair<6,>=4.0->streamlit==1.42.0->-r ../requirements.txt (line 1)) (3.0.2)\n", + "Requirement already satisfied: attrs>=22.2.0 in /opt/app-root/lib64/python3.11/site-packages (from jsonschema>=3.0->altair<6,>=4.0->streamlit==1.42.0->-r ../requirements.txt (line 1)) (25.1.0)\n", + "Requirement already satisfied: jsonschema-specifications>=2023.03.6 in /opt/app-root/lib64/python3.11/site-packages (from jsonschema>=3.0->altair<6,>=4.0->streamlit==1.42.0->-r ../requirements.txt (line 1)) (2024.10.1)\n", + "Requirement already satisfied: referencing>=0.28.4 in /opt/app-root/lib64/python3.11/site-packages (from jsonschema>=3.0->altair<6,>=4.0->streamlit==1.42.0->-r ../requirements.txt (line 1)) (0.36.2)\n", + "Requirement already satisfied: rpds-py>=0.7.1 in /opt/app-root/lib64/python3.11/site-packages (from jsonschema>=3.0->altair<6,>=4.0->streamlit==1.42.0->-r ../requirements.txt (line 1)) (0.22.3)\n", + "Requirement already satisfied: mdurl~=0.1 in /opt/app-root/lib64/python3.11/site-packages (from markdown-it-py>=2.2.0->rich<14,>=10.14.0->streamlit==1.42.0->-r ../requirements.txt (line 1)) (0.1.2)\n", "Downloading streamlit-1.42.0-py2.py3-none-any.whl (9.6 MB)\n", - "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m9.6/9.6 MB\u001b[0m \u001b[31m5.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m9.6/9.6 MB\u001b[0m \u001b[31m5.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0mm\n", "\u001b[?25hDownloading shap-0.46.0-cp311-cp311-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl (540 kB)\n", - "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m540.2/540.2 kB\u001b[0m \u001b[31m8.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0mta \u001b[36m0:00:01\u001b[0m\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m540.2/540.2 kB\u001b[0m \u001b[31m7.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0ma \u001b[36m0:00:01\u001b[0m\n", "\u001b[?25hDownloading slicer-0.0.8-py3-none-any.whl (15 kB)\n", "Downloading scikit_learn-1.6.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (13.5 MB)\n", - "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m13.5/13.5 MB\u001b[0m \u001b[31m6.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0ma \u001b[36m0:00:01\u001b[0m\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m13.5/13.5 MB\u001b[0m \u001b[31m6.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n", "\u001b[?25hDownloading matplotlib-3.10.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (8.6 MB)\n", - "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m8.6/8.6 MB\u001b[0m \u001b[31m2.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m8.6/8.6 MB\u001b[0m \u001b[31m3.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n", "\u001b[?25hDownloading altair-5.5.0-py3-none-any.whl (731 kB)\n", - "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m731.2/731.2 kB\u001b[0m \u001b[31m2.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0ma \u001b[36m0:00:01\u001b[0m\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m731.2/731.2 kB\u001b[0m \u001b[31m3.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0ma \u001b[36m0:00:01\u001b[0m\n", "\u001b[?25hDownloading blinker-1.9.0-py3-none-any.whl (8.5 kB)\n", "Downloading contourpy-1.3.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (326 kB)\n", - "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m326.2/326.2 kB\u001b[0m \u001b[31m1.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0ma \u001b[36m0:00:01\u001b[0m\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m326.2/326.2 kB\u001b[0m \u001b[31m2.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0ma \u001b[36m0:00:01\u001b[0m\n", "\u001b[?25hDownloading cycler-0.12.1-py3-none-any.whl (8.3 kB)\n", "Downloading fonttools-4.56.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (4.9 MB)\n", - "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m4.9/4.9 MB\u001b[0m \u001b[31m2.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0ma \u001b[36m0:00:01\u001b[0m\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m4.9/4.9 MB\u001b[0m \u001b[31m3.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0ma \u001b[36m0:00:01\u001b[0m\n", "\u001b[?25hDownloading GitPython-3.1.44-py3-none-any.whl (207 kB)\n", - "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m207.6/207.6 kB\u001b[0m \u001b[31m1.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0ma \u001b[36m0:00:01\u001b[0m\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m207.6/207.6 kB\u001b[0m \u001b[31m15.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hDownloading joblib-1.4.2-py3-none-any.whl (301 kB)\n", - "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m301.8/301.8 kB\u001b[0m \u001b[31m4.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0ma \u001b[36m0:00:01\u001b[0m\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m301.8/301.8 kB\u001b[0m \u001b[31m15.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hDownloading kiwisolver-1.4.8-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.4 MB)\n", - "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.4/1.4 MB\u001b[0m \u001b[31m2.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0ma \u001b[36m0:00:01\u001b[0m\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.4/1.4 MB\u001b[0m \u001b[31m6.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0ma \u001b[36m0:00:01\u001b[0m\n", "\u001b[?25hDownloading pillow-11.1.0-cp311-cp311-manylinux_2_28_x86_64.whl (4.5 MB)\n", - "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m4.5/4.5 MB\u001b[0m \u001b[31m2.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0ma \u001b[36m0:00:01\u001b[0m\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m4.5/4.5 MB\u001b[0m \u001b[31m4.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0ma \u001b[36m0:00:01\u001b[0mm\n", "\u001b[?25hDownloading pydeck-0.9.1-py2.py3-none-any.whl (6.9 MB)\n", - "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m6.9/6.9 MB\u001b[0m \u001b[31m3.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0ma \u001b[36m0:00:01\u001b[0m\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m6.9/6.9 MB\u001b[0m \u001b[31m4.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0mm\n", "\u001b[?25hDownloading pyparsing-3.2.1-py3-none-any.whl (107 kB)\n", - "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m107.7/107.7 kB\u001b[0m \u001b[31m7.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", - "\u001b[?25hDownloading scipy-1.15.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (40.6 MB)\n", - "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m40.6/40.6 MB\u001b[0m \u001b[31m4.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m107.7/107.7 kB\u001b[0m \u001b[31m4.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hDownloading scipy-1.15.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (37.6 MB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m37.6/37.6 MB\u001b[0m \u001b[31m4.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n", "\u001b[?25hDownloading threadpoolctl-3.5.0-py3-none-any.whl (18 kB)\n", "Downloading tornado-6.4.2-cp38-abi3-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl (437 kB)\n", - "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m437.2/437.2 kB\u001b[0m \u001b[31m4.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0ma \u001b[36m0:00:01\u001b[0m\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m437.2/437.2 kB\u001b[0m \u001b[31m14.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hDownloading watchdog-6.0.0-py3-none-manylinux2014_x86_64.whl (79 kB)\n", - "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m79.1/79.1 kB\u001b[0m \u001b[31m7.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m79.1/79.1 kB\u001b[0m \u001b[31m14.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hDownloading numba-0.61.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl (3.8 MB)\n", - "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m3.8/3.8 MB\u001b[0m \u001b[31m2.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0ma \u001b[36m0:00:01\u001b[0m\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m3.8/3.8 MB\u001b[0m \u001b[31m2.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n", "\u001b[?25hDownloading gitdb-4.0.12-py3-none-any.whl (62 kB)\n", - "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m62.8/62.8 kB\u001b[0m \u001b[31m9.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m62.8/62.8 kB\u001b[0m \u001b[31m6.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hDownloading llvmlite-0.44.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (42.4 MB)\n", - "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m42.4/42.4 MB\u001b[0m \u001b[31m3.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n", - "\u001b[?25hDownloading narwhals-1.26.0-py3-none-any.whl (306 kB)\n", - "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m306.6/306.6 kB\u001b[0m \u001b[31m12.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m42.4/42.4 MB\u001b[0m \u001b[31m2.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n", + "\u001b[?25hDownloading narwhals-1.27.1-py3-none-any.whl (308 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m308.8/308.8 kB\u001b[0m \u001b[31m3.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0ma \u001b[36m0:00:01\u001b[0m\n", "\u001b[?25hDownloading smmap-5.0.2-py3-none-any.whl (24 kB)\n", "Installing collected packages: watchdog, tornado, threadpoolctl, smmap, slicer, scipy, pyparsing, pillow, narwhals, llvmlite, kiwisolver, joblib, fonttools, cycler, contourpy, blinker, scikit-learn, pydeck, numba, matplotlib, gitdb, shap, gitpython, altair, streamlit\n", - "Successfully installed altair-5.5.0 blinker-1.9.0 contourpy-1.3.1 cycler-0.12.1 fonttools-4.56.0 gitdb-4.0.12 gitpython-3.1.44 joblib-1.4.2 kiwisolver-1.4.8 llvmlite-0.44.0 matplotlib-3.10.0 narwhals-1.26.0 numba-0.61.0 pillow-11.1.0 pydeck-0.9.1 pyparsing-3.2.1 scikit-learn-1.6.1 scipy-1.15.1 shap-0.46.0 slicer-0.0.8 smmap-5.0.2 streamlit-1.42.0 threadpoolctl-3.5.0 tornado-6.4.2 watchdog-6.0.0\n", + "Successfully installed altair-5.5.0 blinker-1.9.0 contourpy-1.3.1 cycler-0.12.1 fonttools-4.56.0 gitdb-4.0.12 gitpython-3.1.44 joblib-1.4.2 kiwisolver-1.4.8 llvmlite-0.44.0 matplotlib-3.10.0 narwhals-1.27.1 numba-0.61.0 pillow-11.1.0 pydeck-0.9.1 pyparsing-3.2.1 scikit-learn-1.6.1 scipy-1.15.2 shap-0.46.0 slicer-0.0.8 smmap-5.0.2 streamlit-1.42.0 threadpoolctl-3.5.0 tornado-6.4.2 watchdog-6.0.0\n", "\n", "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m23.2.1\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m25.0.1\u001b[0m\n", "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpip install --upgrade pip\u001b[0m\n" @@ -462,7 +439,7 @@ } ], "source": [ - "!kubectl exec deploy/feast-example -itc online -- bash -c 'pip install -r /feast-data/feast-credit-score-local-tutorial/requirements.txt'" + "!kubectl exec deploy/feast-example -itc online -- bash -c 'pip install -r ../requirements.txt'" ] }, { @@ -474,7 +451,7 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 2, "metadata": {}, "outputs": [ { @@ -500,7 +477,7 @@ } ], "source": [ - "!kubectl exec deploy/feast-example -itc online -- bash -c 'cd /feast-data/feast-credit-score-local-tutorial && python run.py'" + "!kubectl exec deploy/feast-example -itc online -- bash -c 'cd ../ && python run.py'" ] }, { @@ -544,7 +521,7 @@ "\u001b[34m\u001b[1m You can now view your Streamlit app in your browser.\u001b[0m\n", "\u001b[0m\n", "\u001b[34m Local URL: \u001b[0m\u001b[1mhttp://localhost:8501\u001b[0m\n", - "\u001b[34m Network URL: \u001b[0m\u001b[1mhttp://10.42.0.9:8501\u001b[0m\n", + "\u001b[34m Network URL: \u001b[0m\u001b[1mhttp://10.42.0.8:8501\u001b[0m\n", "\u001b[34m External URL: \u001b[0m\u001b[1mhttp://23.112.66.217:8501\u001b[0m\n", "\u001b[0m\n", "/opt/app-root/lib64/python3.11/site-packages/feast/feature_view.py:48: DeprecationWarning: Entity value_type will be mandatory in the next release. Please specify a value_type for entity '__dummy'.\n", @@ -566,7 +543,7 @@ "/opt/app-root/lib64/python3.11/site-packages/feast/entity.py:173: DeprecationWarning: Entity value_type will be mandatory in the next release. Please specify a value_type for entity 'dob_ssn'.\n", " entity = cls(\n", "(1000, 22)\n", - "2025-02-11 22:51:06.834 \n", + "2025-02-20 21:57:48.314 \n", "Calling `st.pyplot()` without providing a figure argument has been deprecated\n", "and will be removed in a later version as it requires the use of Matplotlib's\n", "global figure object, which is not thread-safe.\n", @@ -583,7 +560,7 @@ "If you have a specific use case that requires this functionality, please let us\n", "know via [issue on Github](https://github.com/streamlit/streamlit/issues).\n", "\n", - "2025-02-11 22:51:16.333 \n", + "2025-02-20 21:57:57.474 \n", "Calling `st.pyplot()` without providing a figure argument has been deprecated\n", "and will be removed in a later version as it requires the use of Matplotlib's\n", "global figure object, which is not thread-safe.\n", @@ -617,7 +594,24 @@ "/opt/app-root/lib64/python3.11/site-packages/feast/entity.py:173: DeprecationWarning: Entity value_type will be mandatory in the next release. Please specify a value_type for entity 'dob_ssn'.\n", " entity = cls(\n", "(1000, 22)\n", - "2025-02-11 22:51:57.241 \n", + "2025-02-20 21:58:34.935 \n", + "Calling `st.pyplot()` without providing a figure argument has been deprecated\n", + "and will be removed in a later version as it requires the use of Matplotlib's\n", + "global figure object, which is not thread-safe.\n", + "\n", + "To future-proof this code, you should pass in a figure as shown below:\n", + "\n", + "```python\n", + "fig, ax = plt.subplots()\n", + "ax.scatter([1, 2, 3], [1, 2, 3])\n", + "# other plotting actions...\n", + "st.pyplot(fig)\n", + "```\n", + "\n", + "If you have a specific use case that requires this functionality, please let us\n", + "know via [issue on Github](https://github.com/streamlit/streamlit/issues).\n", + "\n", + "2025-02-20 21:58:43.709 \n", "Calling `st.pyplot()` without providing a figure argument has been deprecated\n", "and will be removed in a later version as it requires the use of Matplotlib's\n", "global figure object, which is not thread-safe.\n", @@ -638,7 +632,7 @@ } ], "source": [ - "!kubectl exec deploy/feast-example -itc online -- bash -c 'cd /feast-data/feast-credit-score-local-tutorial && streamlit run --server.port 8501 streamlit_app.py'" + "!kubectl exec deploy/feast-example -itc online -- bash -c 'cd ../ && streamlit run --server.port 8501 streamlit_app.py'" ] }, { diff --git a/examples/operator-quickstart/feast.yaml b/examples/operator-quickstart/feast.yaml index 4fa166425cb..b665ec5a8bf 100644 --- a/examples/operator-quickstart/feast.yaml +++ b/examples/operator-quickstart/feast.yaml @@ -20,6 +20,10 @@ metadata: namespace: feast spec: feastProject: credit_scoring_local + feastProjectDir: + git: + url: https://github.com/feast-dev/feast-credit-score-local-tutorial + ref: 598a270 services: offlineStore: persistence: diff --git a/examples/operator-rbac/03-uninstall.ipynb b/examples/operator-rbac/03-uninstall.ipynb new file mode 100644 index 00000000000..f9c794c03f8 --- /dev/null +++ b/examples/operator-rbac/03-uninstall.ipynb @@ -0,0 +1,175 @@ +{ + "cells": [ + { + "metadata": {}, + "cell_type": "code", + "outputs": [], + "execution_count": null, + "source": "## Uninstall", + "id": "bd1a081f3f7f5752" + }, + { + "metadata": {}, + "cell_type": "markdown", + "source": "### Uninstall the Operator and all Feast related objects##", + "id": "1175f3d6c5ee9bf0" + }, + { + "metadata": { + "ExecuteTime": { + "end_time": "2025-03-05T19:09:52.349677Z", + "start_time": "2025-03-05T19:09:46.308482Z" + } + }, + "cell_type": "code", + "source": [ + "!kubectl delete -f ../../infra/feast-operator/config/samples/v1alpha1_featurestore_kubernetes_auth.yaml\n", + "!kubectl delete -f ../../infra/feast-operator/dist/install.yaml" + ], + "id": "f4b4c6fa4a1fe0a8", + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "featurestore.feast.dev \"sample-kubernetes-auth\" deleted\r\n", + "namespace \"feast-operator-system\" deleted\r\n", + "customresourcedefinition.apiextensions.k8s.io \"featurestores.feast.dev\" deleted\r\n", + "serviceaccount \"feast-operator-controller-manager\" deleted\r\n", + "role.rbac.authorization.k8s.io \"feast-operator-leader-election-role\" deleted\r\n", + "clusterrole.rbac.authorization.k8s.io \"feast-operator-featurestore-editor-role\" deleted\r\n", + "clusterrole.rbac.authorization.k8s.io \"feast-operator-featurestore-viewer-role\" deleted\r\n", + "clusterrole.rbac.authorization.k8s.io \"feast-operator-manager-role\" deleted\r\n", + "clusterrole.rbac.authorization.k8s.io \"feast-operator-metrics-auth-role\" deleted\r\n", + "clusterrole.rbac.authorization.k8s.io \"feast-operator-metrics-reader\" deleted\r\n", + "rolebinding.rbac.authorization.k8s.io \"feast-operator-leader-election-rolebinding\" deleted\r\n", + "clusterrolebinding.rbac.authorization.k8s.io \"feast-operator-manager-rolebinding\" deleted\r\n", + "clusterrolebinding.rbac.authorization.k8s.io \"feast-operator-metrics-auth-rolebinding\" deleted\r\n", + "service \"feast-operator-controller-manager-metrics-service\" deleted\r\n", + "deployment.apps \"feast-operator-controller-manager\" deleted\r\n" + ] + } + ], + "execution_count": 6 + }, + { + "metadata": {}, + "cell_type": "markdown", + "source": "## Uninstall Client Related Objects", + "id": "2a2aa884aeddfb99" + }, + { + "metadata": { + "ExecuteTime": { + "end_time": "2025-03-05T19:09:54.655575Z", + "start_time": "2025-03-05T19:09:53.553918Z" + } + }, + "cell_type": "code", + "source": [ + "!echo \"Deleting RoleBindings...\"\n", + "!kubectl delete rolebinding feast-user-rolebinding -n feast --ignore-not-found\n", + "!kubectl delete rolebinding feast-admin-rolebinding -n feast --ignore-not-found\n", + "\n", + "!echo \"Deleting ServiceAccounts...\"\n", + "!kubectl delete serviceaccount feast-user-sa -n feast --ignore-not-found\n", + "!kubectl delete serviceaccount feast-admin-sa -n feast --ignore-not-found\n", + "!kubectl delete serviceaccount feast-unauthorized-user-sa -n feast --ignore-not-found\n" + ], + "id": "6ce30879d64bbd06", + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Deleting RoleBindings...\r\n", + "rolebinding.rbac.authorization.k8s.io \"feast-user-rolebinding\" deleted\r\n", + "rolebinding.rbac.authorization.k8s.io \"feast-admin-rolebinding\" deleted\r\n", + "Deleting ServiceAccounts...\r\n", + "serviceaccount \"feast-user-sa\" deleted\r\n", + "serviceaccount \"feast-admin-sa\" deleted\r\n", + "serviceaccount \"feast-unauthorized-user-sa\" deleted\r\n" + ] + } + ], + "execution_count": 7 + }, + { + "metadata": {}, + "cell_type": "markdown", + "source": "Ensure everything has been removed, or is in the process of being terminated.", + "id": "638421caa8ff849e" + }, + { + "metadata": { + "ExecuteTime": { + "end_time": "2025-03-05T19:09:59.868383Z", + "start_time": "2025-03-05T19:09:59.611048Z" + } + }, + "cell_type": "code", + "source": "!kubectl get all -n feast\n", + "id": "587eb85352a8a353", + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "No resources found in feast namespace.\r\n" + ] + } + ], + "execution_count": 8 + }, + { + "metadata": { + "ExecuteTime": { + "end_time": "2025-03-05T19:10:07.846749Z", + "start_time": "2025-03-05T19:10:02.561070Z" + } + }, + "cell_type": "code", + "source": "!kubectl delete namespace feast", + "id": "7a0ce2d9e4a92828", + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "namespace \"feast\" deleted\r\n" + ] + } + ], + "execution_count": 9 + }, + { + "metadata": {}, + "cell_type": "code", + "outputs": [], + "execution_count": null, + "source": "", + "id": "10707783148c5f8d" + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 2 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython2", + "version": "2.7.6" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/examples/operator-rbac/1-setup-operator-rbac.ipynb b/examples/operator-rbac/1-setup-operator-rbac.ipynb new file mode 100644 index 00000000000..69cc285a01c --- /dev/null +++ b/examples/operator-rbac/1-setup-operator-rbac.ipynb @@ -0,0 +1,760 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Feast Operator with RBAC Configuration\n", + "## Objective\n", + "\n", + "This demo provides a reference implementation of a runbook on how to enable Role-Based Access Control (RBAC) for Feast using the Feast Operator with the Kubernetes authentication type. This serves as useful reference material for a cluster admin / MLOps engineer.\n", + "\n", + "The demo steps include deploying the Feast Operator, creating Feast instances with server components (registry, offline store, online store), and Feast client testing locally. The goal is to ensure secure access control for Feast instances deployed by the Feast Operator.\n", + " \n", + "Please read these reference documents for understanding the Feast RBAC framework.\n", + "- [RBAC Architecture](https://docs.feast.dev/v/master/getting-started/architecture/rbac) \n", + "- [RBAC Permission](https://docs.feast.dev/v/master/getting-started/concepts/permission).\n", + "- [RBAC Authorization Manager](https://docs.feast.dev/v/master/getting-started/components/authz_manager)\n" + ] + }, + { + "metadata": {}, + "cell_type": "markdown", + "source": [ + "## Deployment Architecture\n", + "In this notebook, we will deploy a distributed topology of Feast services, which includes:\n", + "\n", + "* `Registry Server`: Handles metadata storage for feature definitions.\n", + "* `Online Store Server`: Uses the `Registry Server` to query metadata and is responsible for low-latency serving of features.\n", + "* `Offline Store Server`: Uses the `Registry Server` to query metadata and provides access to batch data for historical feature retrieval.\n", + "\n", + "Additionally, we will cover:\n", + "* RBAC Configuration with Kubernetes Authentication for Feast resources." + ] + }, + { + "metadata": {}, + "cell_type": "markdown", + "source": [ + "## Prerequisites\n", + "* Kubernetes Cluster\n", + "* [kubectl](https://kubernetes.io/docs/tasks/tools/#kubectl) Kubernetes CLI tool." + ] + }, + { + "metadata": {}, + "cell_type": "markdown", + "source": [ + "## Install Prerequisites\n", + "The following commands install and configure all the prerequisites on a MacOS environment. You can find the\n", + "equivalent instructions on the offical documentation pages:\n", + "* Install the `kubectl` cli.\n", + "* Install Kubernetes and Container runtime (e.g. [Colima](https://github.com/abiosoft/colima)).\n", + " * Alternatively, authenticate to an existing Kubernetes or OpenShift cluster.\n", + " \n", + "```bash\n", + "brew install colima kubectl\n", + "colima start -r containerd -k -m 3 -d 100 -c 2 --cpu-type max -a x86_64\n", + "colima list\n", + "```" + ] + }, + { + "metadata": { + "ExecuteTime": { + "end_time": "2025-03-06T18:27:31.474254Z", + "start_time": "2025-03-06T18:27:31.012088Z" + } + }, + "cell_type": "code", + "source": [ + "!kubectl create ns feast\n", + "!kubectl config set-context --current --namespace feast" + ], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "namespace/feast created\r\n", + "Context \"kind-kind\" modified.\r\n" + ] + } + ], + "execution_count": 1 + }, + { + "metadata": {}, + "cell_type": "markdown", + "source": "Validate the cluster setup:" + }, + { + "metadata": { + "ExecuteTime": { + "end_time": "2025-03-06T18:32:23.198122Z", + "start_time": "2025-03-06T18:32:22.930547Z" + } + }, + "cell_type": "code", + "source": "!kubectl get ns feast", + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "NAME STATUS AGE\r\n", + "feast Active 4m52s\r\n" + ] + } + ], + "execution_count": 2 + }, + { + "metadata": {}, + "cell_type": "markdown", + "source": [ + "## Feast Admin Steps:\n", + "Feast Admins or MLOps Engineers may require Kubernetes Cluster Admin roles when working with OpenShift or Kubernetes clusters. Below is the list of steps Required to set up Feast RBAC with the Operator by an Admin or MLOps Engineer.\n", + "\n", + "1. **Install the Feast Operator**\n", + "2. **Install the Feast services via FeatureStore CR**\n", + "3. **Configure the RBAC Permissions**\n", + "4. **Perform Feast Apply**\n", + "5. **Setting Service Account and Role Binding**\n", + "\n", + "## Install the Feast Operator" + ] + }, + { + "metadata": { + "ExecuteTime": { + "end_time": "2025-03-06T18:32:40.721042Z", + "start_time": "2025-03-06T18:32:28.484245Z" + } + }, + "cell_type": "code", + "source": [ + "## Use this install command from a stable branch \n", + "!kubectl apply -f ../../infra/feast-operator/dist/install.yaml\n", + "\n", + "## OR, for the latest code/builds, use one the following commands from the 'master' branch\n", + "# !make -C ../../infra/feast-operator install deploy IMG=quay.io/feastdev-ci/feast-operator:develop FS_IMG=quay.io/feastdev-ci/feature-server:develop\n", + "# !make -C ../../infra/feast-operator install deploy IMG=quay.io/feastdev-ci/feast-operator:$(git rev-parse HEAD) FS_IMG=quay.io/feastdev-ci/feature-server:$(git rev-parse HEAD)\n", + "\n", + "!kubectl wait --for=condition=available --timeout=5m deployment/feast-operator-controller-manager -n feast-operator-system" + ], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "namespace/feast-operator-system created\r\n", + "customresourcedefinition.apiextensions.k8s.io/featurestores.feast.dev created\r\n", + "serviceaccount/feast-operator-controller-manager created\r\n", + "role.rbac.authorization.k8s.io/feast-operator-leader-election-role created\r\n", + "clusterrole.rbac.authorization.k8s.io/feast-operator-featurestore-editor-role created\r\n", + "clusterrole.rbac.authorization.k8s.io/feast-operator-featurestore-viewer-role created\r\n", + "clusterrole.rbac.authorization.k8s.io/feast-operator-manager-role created\r\n", + "clusterrole.rbac.authorization.k8s.io/feast-operator-metrics-auth-role created\r\n", + "clusterrole.rbac.authorization.k8s.io/feast-operator-metrics-reader created\r\n", + "rolebinding.rbac.authorization.k8s.io/feast-operator-leader-election-rolebinding created\r\n", + "clusterrolebinding.rbac.authorization.k8s.io/feast-operator-manager-rolebinding created\r\n", + "clusterrolebinding.rbac.authorization.k8s.io/feast-operator-metrics-auth-rolebinding created\r\n", + "service/feast-operator-controller-manager-metrics-service created\r\n", + "deployment.apps/feast-operator-controller-manager created\r\n", + "deployment.apps/feast-operator-controller-manager condition met\r\n" + ] + } + ], + "execution_count": 3 + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Install the Feast services via FeatureStore CR\n", + "Next, we'll use the running Feast Operator to install the feast services with Server components online, offline, registry with kubernetes Authorization set. Apply the included [reference deployment](../../infra/feast-operator/config/samples/v1alpha1_featurestore_kubernetes_auth.yaml) to install and configure Feast with kubernetes Authorization ." + ] + }, + { + "cell_type": "code", + "metadata": { + "ExecuteTime": { + "end_time": "2025-03-06T18:34:39.847211Z", + "start_time": "2025-03-06T18:34:39.378680Z" + } + }, + "source": [ + "!cat ../../infra/feast-operator/config/samples/v1alpha1_featurestore_kubernetes_auth.yaml\n", + "!kubectl apply -f ../../infra/feast-operator/config/samples/v1alpha1_featurestore_kubernetes_auth.yaml -n feast" + ], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "apiVersion: feast.dev/v1alpha1\r\n", + "kind: FeatureStore\r\n", + "metadata:\r\n", + " name: sample-kubernetes-auth\r\n", + "spec:\r\n", + " feastProject: feast_rbac\r\n", + " authz:\r\n", + " kubernetes:\r\n", + " roles:\r\n", + " - feast-writer\r\n", + " - feast-reader\r\n", + " services:\r\n", + " offlineStore:\r\n", + " server: {}\r\n", + " onlineStore:\r\n", + " server: {}\r\n", + " registry:\r\n", + " local:\r\n", + " server: {}\r\n", + " ui: {}\r\n", + "featurestore.feast.dev/sample-kubernetes-auth created\r\n" + ] + } + ], + "execution_count": 4 + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Validate the running FeatureStore deployment\n", + "Validate the deployment status." + ] + }, + { + "cell_type": "code", + "metadata": { + "ExecuteTime": { + "end_time": "2025-03-06T18:35:05.202176Z", + "start_time": "2025-03-06T18:35:02.498106Z" + } + }, + "source": [ + "!kubectl get all\n", + "!kubectl wait --for=condition=available --timeout=8m deployment/feast-sample-kubernetes-auth" + ], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "NAME READY STATUS RESTARTS AGE\r\n", + "pod/feast-sample-kubernetes-auth-774f6df8df-95nc6 0/4 Running 0 22s\r\n", + "\r\n", + "NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE\r\n", + "service/feast-sample-kubernetes-auth-offline ClusterIP 10.96.38.230 80/TCP 22s\r\n", + "service/feast-sample-kubernetes-auth-online ClusterIP 10.96.140.194 80/TCP 22s\r\n", + "service/feast-sample-kubernetes-auth-registry ClusterIP 10.96.140.31 80/TCP 22s\r\n", + "service/feast-sample-kubernetes-auth-ui ClusterIP 10.96.26.21 80/TCP 22s\r\n", + "\r\n", + "NAME READY UP-TO-DATE AVAILABLE AGE\r\n", + "deployment.apps/feast-sample-kubernetes-auth 0/1 1 0 22s\r\n", + "\r\n", + "NAME DESIRED CURRENT READY AGE\r\n", + "replicaset.apps/feast-sample-kubernetes-auth-774f6df8df 1 1 0 22s\r\n", + "deployment.apps/feast-sample-kubernetes-auth condition met\r\n" + ] + } + ], + "execution_count": 5 + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Validate that the FeatureStore CR is in a `Ready` state." + ] + }, + { + "cell_type": "code", + "metadata": { + "ExecuteTime": { + "end_time": "2025-03-06T18:35:55.728523Z", + "start_time": "2025-03-06T18:35:55.452894Z" + } + }, + "source": [ + "!kubectl get feast" + ], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "NAME STATUS AGE\r\n", + "sample-kubernetes-auth Ready 76s\r\n" + ] + } + ], + "execution_count": 6 + }, + { + "metadata": {}, + "cell_type": "markdown", + "source": [ + "## Configure the RBAC Permissions\n", + "As we have created Kubernetes roles in FeatureStore CR to manage access control for Feast objects, the Python script `permissions_apply.py` will apply these roles to configure permissions. See the detailed code example below with comments." + ] + }, + { + "metadata": { + "ExecuteTime": { + "end_time": "2025-03-06T18:37:17.062072Z", + "start_time": "2025-03-06T18:37:16.930026Z" + } + }, + "cell_type": "code", + "source": [ + "#view the permissions \n", + "!cat permissions_apply.py" + ], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "# Necessary modules for permissions and policies in Feast for RBAC\r\n", + "from feast.feast_object import ALL_RESOURCE_TYPES\r\n", + "from feast.permissions.action import READ, AuthzedAction, ALL_ACTIONS\r\n", + "from feast.permissions.permission import Permission\r\n", + "from feast.permissions.policy import RoleBasedPolicy\r\n", + "\r\n", + "# Define K8s roles same as created with FeatureStore CR\r\n", + "admin_roles = [\"feast-writer\"] # Full access (can create, update, delete ) Feast Resources\r\n", + "user_roles = [\"feast-reader\"] # Read-only access on Feast Resources\r\n", + "\r\n", + "# User permissions (feast_user_permission)\r\n", + "# - Grants read and describing Feast objects access\r\n", + "user_perm = Permission(\r\n", + " name=\"feast_user_permission\",\r\n", + " types=ALL_RESOURCE_TYPES,\r\n", + " policy=RoleBasedPolicy(roles=user_roles),\r\n", + " actions=[AuthzedAction.DESCRIBE] + READ # Read access (READ_ONLINE, READ_OFFLINE) + describe other Feast Resources.\r\n", + ")\r\n", + "\r\n", + "# Admin permissions (feast_admin_permission)\r\n", + "# - Grants full control over all resources\r\n", + "admin_perm = Permission(\r\n", + " name=\"feast_admin_permission\",\r\n", + " types=ALL_RESOURCE_TYPES,\r\n", + " policy=RoleBasedPolicy(roles=admin_roles),\r\n", + " actions=ALL_ACTIONS # Full permissions: CREATE, UPDATE, DELETE, READ, WRITE\r\n", + ")\r\n" + ] + } + ], + "execution_count": 7 + }, + { + "metadata": { + "ExecuteTime": { + "end_time": "2025-03-06T18:37:31.662484Z", + "start_time": "2025-03-06T18:37:31.139869Z" + } + }, + "cell_type": "code", + "source": [ + "# Copy the Permissions to the pods under feature_repo directory\n", + "!kubectl cp permissions_apply.py $(kubectl get pods -l 'feast.dev/name=sample-kubernetes-auth' -ojsonpath=\"{.items[*].metadata.name}\"):/feast-data/feast_rbac/feature_repo -c online" + ], + "outputs": [], + "execution_count": 8 + }, + { + "cell_type": "code", + "metadata": { + "ExecuteTime": { + "end_time": "2025-03-06T18:37:38.003082Z", + "start_time": "2025-03-06T18:37:37.662378Z" + } + }, + "source": [ + "#view the feature_store.yaml configuration \n", + "!kubectl exec deploy/feast-sample-kubernetes-auth -itc online -- cat feature_store.yaml" + ], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "project: feast_rbac\r\n", + "provider: local\r\n", + "offline_store:\r\n", + " type: dask\r\n", + "online_store:\r\n", + " path: /feast-data/online_store.db\r\n", + " type: sqlite\r\n", + "registry:\r\n", + " path: /feast-data/registry.db\r\n", + " registry_type: file\r\n", + "auth:\r\n", + " type: kubernetes\r\n", + "entity_key_serialization_version: 3\r\n" + ] + } + ], + "execution_count": 9 + }, + { + "metadata": {}, + "cell_type": "markdown", + "source": "## Apply the Permissions and Feast Object to Registry" + }, + { + "metadata": { + "ExecuteTime": { + "end_time": "2025-03-06T18:37:56.131390Z", + "start_time": "2025-03-06T18:37:45.483916Z" + } + }, + "cell_type": "code", + "source": "!kubectl exec deploy/feast-sample-kubernetes-auth -itc online -- feast apply", + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + ": MADV_DONTNEED does not work (memset will be used instead)\r\n", + ": (This is the expected behaviour if you are running under QEMU)\r\n", + "/opt/app-root/lib64/python3.11/site-packages/feast/feature_view.py:48: DeprecationWarning: Entity value_type will be mandatory in the next release. Please specify a value_type for entity '__dummy'.\r\n", + " DUMMY_ENTITY = Entity(\r\n", + "/opt/app-root/lib64/python3.11/site-packages/pydantic/_internal/_fields.py:192: UserWarning: Field name \"vector_enabled\" in \"SqliteOnlineStoreConfig\" shadows an attribute in parent \"VectorStoreConfig\"\r\n", + " warnings.warn(\r\n", + "/opt/app-root/lib64/python3.11/site-packages/pydantic/_internal/_fields.py:192: UserWarning: Field name \"vector_len\" in \"SqliteOnlineStoreConfig\" shadows an attribute in parent \"VectorStoreConfig\"\r\n", + " warnings.warn(\r\n", + "/feast-data/feast_rbac/feature_repo/example_repo.py:27: DeprecationWarning: Entity value_type will be mandatory in the next release. Please specify a value_type for entity 'driver'.\r\n", + " driver = Entity(name=\"driver\", join_keys=[\"driver_id\"])\r\n", + "Applying changes for project feast_rbac\r\n", + "/opt/app-root/lib64/python3.11/site-packages/feast/feature_store.py:579: RuntimeWarning: On demand feature view is an experimental feature. This API is stable, but the functionality does not scale well for offline retrieval\r\n", + " warnings.warn(\r\n", + "Created project \u001B[1m\u001B[32mfeast_rbac\u001B[0m\r\n", + "Created entity \u001B[1m\u001B[32mdriver\u001B[0m\r\n", + "Created feature view \u001B[1m\u001B[32mdriver_hourly_stats\u001B[0m\r\n", + "Created feature view \u001B[1m\u001B[32mdriver_hourly_stats_fresh\u001B[0m\r\n", + "Created on demand feature view \u001B[1m\u001B[32mtransformed_conv_rate\u001B[0m\r\n", + "Created on demand feature view \u001B[1m\u001B[32mtransformed_conv_rate_fresh\u001B[0m\r\n", + "Created feature service \u001B[1m\u001B[32mdriver_activity_v2\u001B[0m\r\n", + "Created feature service \u001B[1m\u001B[32mdriver_activity_v1\u001B[0m\r\n", + "Created feature service \u001B[1m\u001B[32mdriver_activity_v3\u001B[0m\r\n", + "Created permission \u001B[1m\u001B[32mfeast_admin_permission\u001B[0m\r\n", + "Created permission \u001B[1m\u001B[32mfeast_user_permission\u001B[0m\r\n", + "\r\n", + "Created sqlite table \u001B[1m\u001B[32mfeast_rbac_driver_hourly_stats_fresh\u001B[0m\r\n", + "Created sqlite table \u001B[1m\u001B[32mfeast_rbac_driver_hourly_stats\u001B[0m\r\n", + "\r\n" + ] + } + ], + "execution_count": 10 + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": "**List the applied permission details permissions on Feast Resources.**" + }, + { + "cell_type": "code", + "metadata": { + "ExecuteTime": { + "end_time": "2025-03-06T18:38:45.881715Z", + "start_time": "2025-03-06T18:38:04.170364Z" + } + }, + "source": [ + "!kubectl exec deploy/feast-sample-kubernetes-auth -itc online -- feast permissions list-roles\n", + "!kubectl exec deploy/feast-sample-kubernetes-auth -itc online -- feast permissions list\n", + "!kubectl exec deploy/feast-sample-kubernetes-auth -itc online -- feast permissions describe feast_admin_permission\n", + "!kubectl exec deploy/feast-sample-kubernetes-auth -itc online -- feast permissions describe feast_user_permission" + ], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + ": MADV_DONTNEED does not work (memset will be used instead)\r\n", + ": (This is the expected behaviour if you are running under QEMU)\r\n", + "/opt/app-root/lib64/python3.11/site-packages/feast/feature_view.py:48: DeprecationWarning: Entity value_type will be mandatory in the next release. Please specify a value_type for entity '__dummy'.\r\n", + " DUMMY_ENTITY = Entity(\r\n", + "/opt/app-root/lib64/python3.11/site-packages/pydantic/_internal/_fields.py:192: UserWarning: Field name \"vector_enabled\" in \"SqliteOnlineStoreConfig\" shadows an attribute in parent \"VectorStoreConfig\"\r\n", + " warnings.warn(\r\n", + "/opt/app-root/lib64/python3.11/site-packages/pydantic/_internal/_fields.py:192: UserWarning: Field name \"vector_len\" in \"SqliteOnlineStoreConfig\" shadows an attribute in parent \"VectorStoreConfig\"\r\n", + " warnings.warn(\r\n", + "+--------------+\r\n", + "| ROLE NAME |\r\n", + "+==============+\r\n", + "| feast-reader |\r\n", + "+--------------+\r\n", + "| feast-writer |\r\n", + "+--------------+\r\n", + ": MADV_DONTNEED does not work (memset will be used instead)\r\n", + ": (This is the expected behaviour if you are running under QEMU)\r\n", + "/opt/app-root/lib64/python3.11/site-packages/feast/feature_view.py:48: DeprecationWarning: Entity value_type will be mandatory in the next release. Please specify a value_type for entity '__dummy'.\r\n", + " DUMMY_ENTITY = Entity(\r\n", + "/opt/app-root/lib64/python3.11/site-packages/pydantic/_internal/_fields.py:192: UserWarning: Field name \"vector_enabled\" in \"SqliteOnlineStoreConfig\" shadows an attribute in parent \"VectorStoreConfig\"\r\n", + " warnings.warn(\r\n", + "/opt/app-root/lib64/python3.11/site-packages/pydantic/_internal/_fields.py:192: UserWarning: Field name \"vector_len\" in \"SqliteOnlineStoreConfig\" shadows an attribute in parent \"VectorStoreConfig\"\r\n", + " warnings.warn(\r\n", + "NAME TYPES NAME_PATTERNS ACTIONS ROLES REQUIRED_TAGS\r\n", + "feast_admin_permission Project - CREATE feast-writer -\r\n", + " FeatureView DESCRIBE\r\n", + " OnDemandFeatureView UPDATE\r\n", + " BatchFeatureView DELETE\r\n", + " StreamFeatureView READ_ONLINE\r\n", + " Entity READ_OFFLINE\r\n", + " FeatureService WRITE_ONLINE\r\n", + " DataSource WRITE_OFFLINE\r\n", + " ValidationReference\r\n", + " SavedDataset\r\n", + " Permission\r\n", + "feast_user_permission Project - DESCRIBE feast-reader -\r\n", + " FeatureView READ_OFFLINE\r\n", + " OnDemandFeatureView READ_ONLINE\r\n", + " BatchFeatureView\r\n", + " StreamFeatureView\r\n", + " Entity\r\n", + " FeatureService\r\n", + " DataSource\r\n", + " ValidationReference\r\n", + " SavedDataset\r\n", + " Permission\r\n", + ": MADV_DONTNEED does not work (memset will be used instead)\r\n", + ": (This is the expected behaviour if you are running under QEMU)\r\n", + "/opt/app-root/lib64/python3.11/site-packages/feast/feature_view.py:48: DeprecationWarning: Entity value_type will be mandatory in the next release. Please specify a value_type for entity '__dummy'.\r\n", + " DUMMY_ENTITY = Entity(\r\n", + "/opt/app-root/lib64/python3.11/site-packages/pydantic/_internal/_fields.py:192: UserWarning: Field name \"vector_enabled\" in \"SqliteOnlineStoreConfig\" shadows an attribute in parent \"VectorStoreConfig\"\r\n", + " warnings.warn(\r\n", + "/opt/app-root/lib64/python3.11/site-packages/pydantic/_internal/_fields.py:192: UserWarning: Field name \"vector_len\" in \"SqliteOnlineStoreConfig\" shadows an attribute in parent \"VectorStoreConfig\"\r\n", + " warnings.warn(\r\n", + "spec:\r\n", + " name: feast_admin_permission\r\n", + " types:\r\n", + " - PROJECT\r\n", + " - FEATURE_VIEW\r\n", + " - ON_DEMAND_FEATURE_VIEW\r\n", + " - BATCH_FEATURE_VIEW\r\n", + " - STREAM_FEATURE_VIEW\r\n", + " - ENTITY\r\n", + " - FEATURE_SERVICE\r\n", + " - DATA_SOURCE\r\n", + " - VALIDATION_REFERENCE\r\n", + " - SAVED_DATASET\r\n", + " - PERMISSION\r\n", + " actions:\r\n", + " - CREATE\r\n", + " - DESCRIBE\r\n", + " - UPDATE\r\n", + " - DELETE\r\n", + " - READ_ONLINE\r\n", + " - READ_OFFLINE\r\n", + " - WRITE_ONLINE\r\n", + " - WRITE_OFFLINE\r\n", + " policy:\r\n", + " roleBasedPolicy:\r\n", + " roles:\r\n", + " - feast-writer\r\n", + "meta:\r\n", + " createdTimestamp: '2025-03-06T18:37:55.742625Z'\r\n", + " lastUpdatedTimestamp: '2025-03-06T18:37:55.742625Z'\r\n", + "\r\n", + ": MADV_DONTNEED does not work (memset will be used instead)\r\n", + ": (This is the expected behaviour if you are running under QEMU)\r\n", + "/opt/app-root/lib64/python3.11/site-packages/feast/feature_view.py:48: DeprecationWarning: Entity value_type will be mandatory in the next release. Please specify a value_type for entity '__dummy'.\r\n", + " DUMMY_ENTITY = Entity(\r\n", + "/opt/app-root/lib64/python3.11/site-packages/pydantic/_internal/_fields.py:192: UserWarning: Field name \"vector_enabled\" in \"SqliteOnlineStoreConfig\" shadows an attribute in parent \"VectorStoreConfig\"\r\n", + " warnings.warn(\r\n", + "/opt/app-root/lib64/python3.11/site-packages/pydantic/_internal/_fields.py:192: UserWarning: Field name \"vector_len\" in \"SqliteOnlineStoreConfig\" shadows an attribute in parent \"VectorStoreConfig\"\r\n", + " warnings.warn(\r\n", + "spec:\r\n", + " name: feast_user_permission\r\n", + " types:\r\n", + " - PROJECT\r\n", + " - FEATURE_VIEW\r\n", + " - ON_DEMAND_FEATURE_VIEW\r\n", + " - BATCH_FEATURE_VIEW\r\n", + " - STREAM_FEATURE_VIEW\r\n", + " - ENTITY\r\n", + " - FEATURE_SERVICE\r\n", + " - DATA_SOURCE\r\n", + " - VALIDATION_REFERENCE\r\n", + " - SAVED_DATASET\r\n", + " - PERMISSION\r\n", + " actions:\r\n", + " - DESCRIBE\r\n", + " - READ_OFFLINE\r\n", + " - READ_ONLINE\r\n", + " policy:\r\n", + " roleBasedPolicy:\r\n", + " roles:\r\n", + " - feast-reader\r\n", + "meta:\r\n", + " createdTimestamp: '2025-03-06T18:37:55.743643Z'\r\n", + " lastUpdatedTimestamp: '2025-03-06T18:37:55.743643Z'\r\n", + "\r\n" + ] + } + ], + "execution_count": 11 + }, + { + "metadata": {}, + "cell_type": "markdown", + "source": [ + "## Setting Up Service Account and RoleBinding \n", + "The steps below will:\n", + "- Create **three different ServiceAccounts** for Feast.\n", + "- Assign appropriate **RoleBindings** for access control." + ] + }, + { + "metadata": {}, + "cell_type": "markdown", + "source": [ + "## Test Cases\n", + "| User Type | ServiceAccount | RoleBinding Assigned | Expected Behavior in output |\n", + "|----------------|-----------------------------|----------------------|------------------------------------------------------------|\n", + "| **Read-Only** | `feast-user-sa` | `feast-reader` | Can **read** from the feature store, but **cannot write**. |\n", + "| **Unauthorized** | `feast-unauthorized-user-sa` | _None_ | **Access should be denied** in `test.py`. |\n", + "| **Admin** | `feast-admin-sa` | `feast-writer` | Can **read and write** feature store data. |" + ] + }, + { + "metadata": {}, + "cell_type": "markdown", + "source": "" + }, + { + "metadata": {}, + "cell_type": "markdown", + "source": "### Setup Read-Only Feast User the ServiceAccount and Role Binding (serviceaccount: feast-user-sa, role: feast-reader)" + }, + { + "metadata": { + "ExecuteTime": { + "end_time": "2025-03-06T18:42:04.122440Z", + "start_time": "2025-03-06T18:42:03.397214Z" + } + }, + "cell_type": "code", + "source": [ + "# Step 1: Create the ServiceAccount\n", + "!echo \"Creating ServiceAccount: feast-user-sa\"\n", + "!kubectl create serviceaccount feast-user-sa -n feast\n", + "\n", + "# Step 2: Assign RoleBinding (Read-Only Access for Feast)\n", + "!echo \"Assigning Read-Only RoleBinding: feast-user-rolebinding\"\n", + "!kubectl create rolebinding feast-user-rolebinding --role=feast-reader --serviceaccount=feast:feast-user-sa -n feast" + ], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Creating ServiceAccount: feast-user-sa\r\n", + "serviceaccount/feast-user-sa created\r\n", + "Assigning Read-Only RoleBinding: feast-user-rolebinding\r\n", + "rolebinding.rbac.authorization.k8s.io/feast-user-rolebinding created\r\n" + ] + } + ], + "execution_count": 12 + }, + { + "metadata": {}, + "cell_type": "markdown", + "source": "### Setup Unauthorized Feast User (serviceaccount: feast-unauthorized-user-sa, role: None)" + }, + { + "metadata": { + "ExecuteTime": { + "end_time": "2025-03-06T18:42:07.992216Z", + "start_time": "2025-03-06T18:42:07.721628Z" + } + }, + "cell_type": "code", + "source": [ + "# Create the ServiceAccount (Without RoleBinding)\n", + "!echo \"Creating Unauthorized ServiceAccount: feast-unauthorized-user-sa\"\n", + "!kubectl create serviceaccount feast-unauthorized-user-sa -n feast\n" + ], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Creating Unauthorized ServiceAccount: feast-unauthorized-user-sa\r\n", + "serviceaccount/feast-unauthorized-user-sa created\r\n" + ] + } + ], + "execution_count": 13 + }, + { + "metadata": {}, + "cell_type": "markdown", + "source": "## Setup Test Admin Feast User (serviceaccount: feast-admin-sa, role: feast-writer)" + }, + { + "metadata": { + "ExecuteTime": { + "end_time": "2025-03-06T18:42:11.651408Z", + "start_time": "2025-03-06T18:42:11.097231Z" + } + }, + "cell_type": "code", + "source": [ + "# Create the ServiceAccount\n", + "!echo \"Creating ServiceAccount: feast-admin-sa\"\n", + "!kubectl create serviceaccount feast-admin-sa -n feast\n", + "\n", + "# Assign RoleBinding (Admin Access for Feast)\n", + "!echo \"Assigning Admin RoleBinding: feast-admin-rolebinding\"\n", + "!kubectl create rolebinding feast-admin-rolebinding --role=feast-writer --serviceaccount=feast:feast-admin-sa -n feast\n" + ], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Creating ServiceAccount: feast-admin-sa\r\n", + "serviceaccount/feast-admin-sa created\r\n", + "Assigning Admin RoleBinding: feast-admin-rolebinding\r\n", + "rolebinding.rbac.authorization.k8s.io/feast-admin-rolebinding created\r\n" + ] + } + ], + "execution_count": 14 + }, + { + "metadata": {}, + "cell_type": "markdown", + "source": "[Next Run Client notebook](./2-client.ipynb)" + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.11" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/examples/operator-rbac/2-client.ipynb b/examples/operator-rbac/2-client.ipynb new file mode 100644 index 00000000000..cf9d57cb5bc --- /dev/null +++ b/examples/operator-rbac/2-client.ipynb @@ -0,0 +1,828 @@ +{ + "cells": [ + { + "metadata": {}, + "cell_type": "markdown", + "source": [ + "## Feast Client with RBAC\n", + "### Kubernetes RBAC Authorization\n", + "\n", + "## Feast Role-Based Access Control (RBAC) in Kubernetes \n", + "\n", + "Feast **Role-Based Access Control (RBAC)** in Kubernetes supports authentication both **inside a Kubernetes pod** and for **external clients** using the `LOCAL_K8S_TOKEN` environment variable. \n", + "\n", + "\n", + "### Inside a Kubernetes Pod\n", + "Feast automatically retrieves the Kubernetes ServiceAccount token from:\n", + "```\n", + "/var/run/secrets/kubernetes.io/serviceaccount/token\n", + "```\n", + "This means:\n", + "- No manual configuration is needed inside a pod.\n", + "- The token is mounted automatically and used for authentication.\n", + "- Developer just need create the binding with role and service account accordingly.\n", + "- Code Reference: \n", + "[Feast Kubernetes Auth Client Manager (Pod Token Usage)](https://github.com/feast-dev/feast/blob/master/sdk/python/feast/permissions/client/kubernetes_auth_client_manager.py#L15) \n", + "- Using a service account from a pod [Example](https://github.com/feast-dev/feast/blob/master/examples/rbac-remote/client/k8s/)\n", + "\n", + "### Outside a Kubernetes Pod (External Clients & Local Testing)\n", + " \n", + "If running Feast outside of Kubernetes, authentication requires setting the token manually to the environment variable `LOCAL_K8S_TOKEN` :\n", + "```sh\n", + "export LOCAL_K8S_TOKEN=\"your-service-account-token\"\n", + "```\n", + "\n", + "For more details, refer the user guide: [Kubernetes RBAC Authorization](https://docs.feast.dev/master/getting-started/components/authz_manager#kubernetes-rbac-authorization) \n" + ], + "id": "bb0145c9c1f6ebcc" + }, + { + "metadata": {}, + "cell_type": "markdown", + "source": [ + "## Test Cases\n", + "| User Type | ServiceAccount | RoleBinding Assigned | Expected Behavior in output |\n", + "|----------------|-----------------------------|----------------------|------------------------------------------------------------|\n", + "| **Read-Only** | `feast-user-sa` | `feast-reader` | Can **read** from the feature store, but **cannot write**. |\n", + "| **Unauthorized** | `feast-unauthorized-user-sa` | _None_ | **Access should be denied** in `test.py`. |\n", + "| **Admin** | `feast-admin-sa` | `feast-writer` | Can **read and write** feature store data. |" + ], + "id": "160681ba4ab3c2c5" + }, + { + "metadata": {}, + "cell_type": "markdown", + "source": "### Feature Store settings", + "id": "6590c081efb1fe3c" + }, + { + "metadata": { + "ExecuteTime": { + "end_time": "2025-03-06T18:47:45.151296Z", + "start_time": "2025-03-06T18:47:45.024854Z" + } + }, + "cell_type": "code", + "source": "!cat client/feature_store.yaml", + "id": "fac5f67ff391b5cf", + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "project: feast_rbac\r\n", + "provider: local\r\n", + "offline_store:\r\n", + " host: localhost\r\n", + " type: remote\r\n", + " port: 8081\r\n", + "online_store:\r\n", + " path: http://localhost:8082\r\n", + " type: remote\r\n", + "registry:\r\n", + " path: localhost:8083\r\n", + " registry_type: remote\r\n", + "auth:\r\n", + " type: kubernetes\r\n", + "entity_key_serialization_version: 3\r\n" + ] + } + ], + "execution_count": 1 + }, + { + "metadata": {}, + "cell_type": "markdown", + "source": "**The Operator client feature store ConfigMap** containing the `feature_store.yaml `settings. We can retrieve it and port froward to local as we are testing locally.", + "id": "84f73e09711bff9f" + }, + { + "metadata": { + "ExecuteTime": { + "end_time": "2025-03-06T18:46:36.029308Z", + "start_time": "2025-03-06T18:46:35.712532Z" + } + }, + "cell_type": "code", + "source": "!kubectl get configmap feast-sample-kubernetes-auth-client -n feast -o jsonpath='{.data.feature_store\\.yaml}' ", + "id": "456fb4df46f32380", + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "project: feast_rbac\r\n", + "provider: local\r\n", + "offline_store:\r\n", + " host: feast-sample-kubernetes-auth-offline.feast.svc.cluster.local\r\n", + " type: remote\r\n", + " port: 80\r\n", + "online_store:\r\n", + " path: http://feast-sample-kubernetes-auth-online.feast.svc.cluster.local:80\r\n", + " type: remote\r\n", + "registry:\r\n", + " path: feast-sample-kubernetes-auth-registry.feast.svc.cluster.local:80\r\n", + " registry_type: remote\r\n", + "auth:\r\n", + " type: kubernetes\r\n", + "entity_key_serialization_version: 3\r\n" + ] + } + ], + "execution_count": 34 + }, + { + "metadata": {}, + "cell_type": "markdown", + "source": "### The function below is executed to support the preparation of client testing.", + "id": "ae61f4dca31f3466" + }, + { + "metadata": {}, + "cell_type": "markdown", + "source": "Run Port Forwarding for All Services for local testing ", + "id": "28636825ae8f676d" + }, + { + "metadata": { + "ExecuteTime": { + "end_time": "2025-03-06T18:47:55.237205Z", + "start_time": "2025-03-06T18:47:55.226143Z" + } + }, + "cell_type": "code", + "source": [ + "import subprocess\n", + "\n", + "# Define services and their local ports\n", + "services = {\n", + " \"offline_store\": (\"feast-sample-kubernetes-auth-offline\", 8081),\n", + " \"online_store\": (\"feast-sample-kubernetes-auth-online\", 8082),\n", + " \"registry\": (\"feast-sample-kubernetes-auth-registry\", 8083),\n", + "}\n", + "\n", + "# Start port-forwarding for each service\n", + "port_forward_processes = {}\n", + "for name, (service, local_port) in services.items():\n", + " cmd = f\"kubectl port-forward svc/{service} -n feast {local_port}:80\"\n", + " process = subprocess.Popen(cmd, shell=True)\n", + " port_forward_processes[name] = process\n", + " print(f\"Port forwarding {service} -> localhost:{local_port}\")" + ], + "id": "c014248190863e8a", + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Port forwarding feast-sample-kubernetes-auth-offline -> localhost:8081\n", + "Port forwarding feast-sample-kubernetes-auth-online -> localhost:8082\n", + "Port forwarding feast-sample-kubernetes-auth-registry -> localhost:8083\n" + ] + } + ], + "execution_count": 2 + }, + { + "metadata": {}, + "cell_type": "markdown", + "source": "Function to retrieve a Kubernetes service account token and set it as an environment variable", + "id": "c0eccef6379f442c" + }, + { + "metadata": { + "ExecuteTime": { + "end_time": "2025-03-06T18:48:00.150752Z", + "start_time": "2025-03-06T18:48:00.143370Z" + } + }, + "cell_type": "code", + "source": [ + "import subprocess\n", + "import os\n", + "\n", + "def get_k8s_token(service_account):\n", + " namespace = \"feast\"\n", + "\n", + " if not service_account:\n", + " raise ValueError(\"Service account name is required.\")\n", + "\n", + " result = subprocess.run(\n", + " [\"kubectl\", \"create\", \"token\", service_account, \"-n\", namespace],\n", + " capture_output=True, text=True, check=True\n", + " )\n", + "\n", + " token = result.stdout.strip()\n", + "\n", + " if not token:\n", + " return None # Silently return None if token retrieval fails\n", + "\n", + " os.environ[\"LOCAL_K8S_TOKEN\"] = token\n", + " return \"Token Retrieved: ***** (hidden for security)\"\n" + ], + "id": "70bdbcd7b3fe44", + "outputs": [], + "execution_count": 3 + }, + { + "metadata": {}, + "cell_type": "markdown", + "source": [ + "**Generating training data**. The following test functions were copied from the `test_workflow.py` template but we added `try` blocks to print only \n", + "the relevant error messages, since we expect to receive errors from the permission enforcement modules." + ], + "id": "8c9e27ec4ed8ca2c" + }, + { + "metadata": { + "ExecuteTime": { + "end_time": "2025-03-06T20:16:04.254201Z", + "start_time": "2025-03-06T20:16:04.245605Z" + } + }, + "cell_type": "code", + "source": [ + "from feast import FeatureStore\n", + "from feast.data_source import PushMode\n", + "from datetime import datetime\n", + "import pandas as pd\n", + "\n", + "# Initialize Feature Store\n", + "store = FeatureStore(repo_path=\"client\")\n", + "\n", + "def fetch_historical_features_entity_df(store: FeatureStore, for_batch_scoring: bool):\n", + " \"\"\"Fetch historical features for training or batch scoring.\"\"\"\n", + " try:\n", + " entity_df = pd.DataFrame.from_dict(\n", + " {\n", + " \"driver_id\": [1001, 1002, 1003],\n", + " \"event_timestamp\": [\n", + " datetime(2021, 4, 12, 10, 59, 42),\n", + " datetime(2021, 4, 12, 8, 12, 10),\n", + " datetime(2021, 4, 12, 16, 40, 26),\n", + " ],\n", + " \"label_driver_reported_satisfaction\": [1, 5, 3],\n", + " \"val_to_add\": [1, 2, 3],\n", + " \"val_to_add_2\": [10, 20, 30],\n", + " }\n", + " )\n", + " if for_batch_scoring:\n", + " entity_df[\"event_timestamp\"] = pd.to_datetime(\"now\", utc=True)\n", + "\n", + " training_df = store.get_historical_features(\n", + " entity_df=entity_df,\n", + " features=[\n", + " \"driver_hourly_stats:conv_rate\",\n", + " \"driver_hourly_stats:acc_rate\",\n", + " \"driver_hourly_stats:avg_daily_trips\",\n", + " \"transformed_conv_rate:conv_rate_plus_val1\",\n", + " \"transformed_conv_rate:conv_rate_plus_val2\",\n", + " ],\n", + " ).to_df()\n", + " print(f\"Successfully fetched {'batch scoring' if for_batch_scoring else 'training'} historical features:\\n\", training_df.head())\n", + "\n", + " except PermissionError:\n", + " print(\"\\n*** PERMISSION DENIED *** Cannot fetch historical features.\")\n", + " except Exception as e:\n", + " print(f\"Unexpected error while fetching historical features: {e}\")\n", + "\n", + "def fetch_online_features(store: FeatureStore, source: str = \"\"):\n", + " \"\"\"Fetch online features from the feature store.\"\"\"\n", + " try:\n", + " entity_rows = [\n", + " {\n", + " \"driver_id\": 1001,\n", + " \"val_to_add\": 1000,\n", + " \"val_to_add_2\": 2000,\n", + " },\n", + " {\n", + " \"driver_id\": 1002,\n", + " \"val_to_add\": 1001,\n", + " \"val_to_add_2\": 2002,\n", + " },\n", + " ]\n", + " if source == \"feature_service\":\n", + " features_to_fetch = store.get_feature_service(\"driver_activity_v1\")\n", + " elif source == \"push\":\n", + " features_to_fetch = store.get_feature_service(\"driver_activity_v3\")\n", + " else:\n", + " features_to_fetch = [\n", + " \"driver_hourly_stats:acc_rate\",\n", + " \"transformed_conv_rate:conv_rate_plus_val1\",\n", + " \"transformed_conv_rate:conv_rate_plus_val2\",\n", + " ]\n", + "\n", + " returned_features = store.get_online_features(\n", + " features=features_to_fetch,\n", + " entity_rows=entity_rows,\n", + " ).to_dict()\n", + "\n", + " print(f\"Successfully fetched online features {'via feature service' if source else 'directly'}:\\n\")\n", + " for key, value in sorted(returned_features.items()):\n", + " print(f\"{key} : {value}\")\n", + "\n", + " except PermissionError:\n", + " print(\"\\n*** PERMISSION DENIED *** Cannot fetch online features.\")\n", + " except Exception as e:\n", + " print(f\"Unexpected error while fetching online features: {e}\")\n", + "\n", + "def check_permissions():\n", + " \"\"\"Check user role, test various Feast operations,.\"\"\"\n", + "\n", + " feature_views = []\n", + "\n", + " # Step 1: List feature views\n", + " print(\"\\n--- List feature views ---\")\n", + " try:\n", + " feature_views = store.list_feature_views()\n", + " if not feature_views:\n", + " print(\"No feature views found. You might not have access or they haven't been created.\")\n", + " has_feature_view_access = False\n", + " else:\n", + " print(f\"Successfully listed {len(feature_views)} feature views:\")\n", + " for fv in feature_views:\n", + " print(f\" - {fv.name}\")\n", + "\n", + " except PermissionError:\n", + " print(\"\\n*** PERMISSION DENIED *** Cannot list feature views.\")\n", + " has_feature_view_access = False\n", + " except Exception as e:\n", + " print(f\"Unexpected error listing feature views: {e}\")\n", + " has_feature_view_access = False\n", + "\n", + " # Step 2: Fetch Historical Features\n", + " print(\"\\n--- Fetching Historical Features for Training ---\")\n", + " fetch_historical_features_entity_df(store, for_batch_scoring=False)\n", + "\n", + " print(\"\\n--- Fetching Historical Features for Batch Scoring ---\")\n", + " fetch_historical_features_entity_df(store, for_batch_scoring=True)\n", + "\n", + " # Step 3: Apply Feature Store\n", + " print(\"\\n--- Write to Feature Store ---\")\n", + " try:\n", + " store.apply(feature_views)\n", + " print(\"User has write access to the feature store.\")\n", + " except PermissionError:\n", + " print(\"\\n*** PERMISSION DENIED *** User lacks permission to modify the feature store.\")\n", + " except Exception as e:\n", + " print(f\"Unexpected error testing write access: {e}\")\n", + "\n", + " # Step 4: Fetch Online Features\n", + " print(\"\\n--- Fetching Online Features ---\")\n", + " fetch_online_features(store)\n", + "\n", + " print(\"\\n--- Fetching Online Features via Feature Service ---\")\n", + " fetch_online_features(store, source=\"feature_service\")\n", + "\n", + " print(\"\\n--- Fetching Online Features via Push Source ---\")\n", + " fetch_online_features(store, source=\"push\")\n", + "\n", + " print(\"\\n--- Performing Push Source ---\")\n", + " # Step 5: Simulate Event Push (Streaming Ingestion)\n", + " try:\n", + " event_df = pd.DataFrame.from_dict(\n", + " {\n", + " \"driver_id\": [1001],\n", + " \"event_timestamp\": [datetime.now()],\n", + " \"created\": [datetime.now()],\n", + " \"conv_rate\": [1.0],\n", + " \"acc_rate\": [1.0],\n", + " \"avg_daily_trips\": [1000],\n", + " }\n", + " )\n", + " store.push(\"driver_stats_push_source\", event_df, to=PushMode.ONLINE_AND_OFFLINE)\n", + " print(\"Successfully pushed a test event.\")\n", + " except PermissionError:\n", + " print(\"\\n*** PERMISSION DENIED *** Cannot push event (no write access).\")\n", + " except Exception as e:\n", + " print(f\"Unexpected error while pushing event: {e}\")\n" + ], + "id": "934963c5f6b18930", + "outputs": [], + "execution_count": 51 + }, + { + "metadata": {}, + "cell_type": "markdown", + "source": [ + "### Test Read-Only Feast User \n", + "**Step 1: Set the Token**" + ], + "id": "84e3f83699b8d83" + }, + { + "metadata": { + "ExecuteTime": { + "end_time": "2025-03-06T20:12:44.771268Z", + "start_time": "2025-03-06T20:12:44.691353Z" + } + }, + "cell_type": "code", + "source": "get_k8s_token(\"feast-user-sa\")", + "id": "f1fe8baa02d27d38", + "outputs": [ + { + "data": { + "text/plain": [ + "'Token Retrieved: ***** (hidden for security)'" + ] + }, + "execution_count": 48, + "metadata": {}, + "output_type": "execute_result" + } + ], + "execution_count": 48 + }, + { + "metadata": {}, + "cell_type": "markdown", + "source": "**Step 2: Test misc functions from offline, online, materialize_incremental, and others**", + "id": "140c909fa8bcc6ab" + }, + { + "metadata": { + "ExecuteTime": { + "end_time": "2025-03-06T20:16:16.680582Z", + "start_time": "2025-03-06T20:16:14.930480Z" + } + }, + "cell_type": "code", + "source": [ + "# Run the permission check function\n", + "check_permissions()\n" + ], + "id": "14b7ad38368db767", + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "--- List feature views ---\n", + "Successfully listed 2 feature views:\n", + " - driver_hourly_stats\n", + " - driver_hourly_stats_fresh\n", + "\n", + "--- Fetching Historical Features for Training ---\n", + "Handling connection for 8081\n", + "Successfully fetched training historical features:\n", + " driver_id event_timestamp label_driver_reported_satisfaction \\\n", + "0 1001 2021-04-12 10:59:42+00:00 1 \n", + "1 1002 2021-04-12 08:12:10+00:00 5 \n", + "2 1003 2021-04-12 16:40:26+00:00 3 \n", + "\n", + " val_to_add val_to_add_2 conv_rate acc_rate avg_daily_trips \\\n", + "0 1 10 0.677818 0.453707 193 \n", + "1 2 20 0.328160 0.900565 929 \n", + "2 3 30 0.787191 0.958963 571 \n", + "\n", + " conv_rate_plus_val1 conv_rate_plus_val2 \n", + "0 1.677818 10.677818 \n", + "1 2.328160 20.328160 \n", + "2 3.787191 30.787191 \n", + "\n", + "--- Fetching Historical Features for Batch Scoring ---\n", + "Handling connection for 8081\n", + "Successfully fetched batch scoring historical features:\n", + " driver_id event_timestamp \\\n", + "0 1001 2025-03-06 20:16:15.556223+00:00 \n", + "1 1002 2025-03-06 20:16:15.556223+00:00 \n", + "2 1003 2025-03-06 20:16:15.556223+00:00 \n", + "\n", + " label_driver_reported_satisfaction val_to_add val_to_add_2 conv_rate \\\n", + "0 1 1 10 0.782836 \n", + "1 5 2 20 0.731948 \n", + "2 3 3 30 0.613211 \n", + "\n", + " acc_rate avg_daily_trips conv_rate_plus_val1 conv_rate_plus_val2 \n", + "0 0.729726 652 1.782836 10.782836 \n", + "1 0.384902 902 2.731948 20.731948 \n", + "2 0.075386 101 3.613211 30.613211 \n", + "\n", + "--- Write to Feature Store ---\n", + "\n", + "*** PERMISSION DENIED *** User lacks permission to modify the feature store.\n", + "\n", + "--- Fetching Online Features ---\n", + "Handling connection for 8082\n", + "Successfully fetched online features directly:\n", + "\n", + "acc_rate : [None, None]\n", + "conv_rate_plus_val1 : [None, None]\n", + "conv_rate_plus_val2 : [None, None]\n", + "driver_id : [1001, 1002]\n", + "\n", + "--- Fetching Online Features via Feature Service ---\n", + "Handling connection for 8082\n", + "Successfully fetched online features via feature service:\n", + "\n", + "conv_rate : [None, None]\n", + "conv_rate_plus_val1 : [None, None]\n", + "conv_rate_plus_val2 : [None, None]\n", + "driver_id : [1001, 1002]\n", + "\n", + "--- Fetching Online Features via Push Source ---\n", + "Handling connection for 8082\n", + "Successfully fetched online features via feature service:\n", + "\n", + "acc_rate : [None, None]\n", + "avg_daily_trips : [None, None]\n", + "conv_rate : [None, None]\n", + "conv_rate_plus_val1 : [None, None]\n", + "conv_rate_plus_val2 : [None, None]\n", + "driver_id : [1001, 1002]\n", + "\n", + "--- Performing Push Source ---\n", + "Unexpected error while pushing event: \n" + ] + } + ], + "execution_count": 53 + }, + { + "metadata": {}, + "cell_type": "markdown", + "source": "### Test Unauthorized Feast User ", + "id": "e5e63a172da6d6d7" + }, + { + "metadata": { + "ExecuteTime": { + "end_time": "2025-03-06T20:16:38.487573Z", + "start_time": "2025-03-06T20:16:38.351889Z" + } + }, + "cell_type": "code", + "source": [ + "# Retrieve and store the token\n", + "get_k8s_token(\"feast-unauthorized-user-sa\")" + ], + "id": "a7b3a6578fcf5c3c", + "outputs": [ + { + "data": { + "text/plain": [ + "'Token Retrieved: ***** (hidden for security)'" + ] + }, + "execution_count": 54, + "metadata": {}, + "output_type": "execute_result" + } + ], + "execution_count": 54 + }, + { + "metadata": { + "ExecuteTime": { + "end_time": "2025-03-06T20:16:41.522132Z", + "start_time": "2025-03-06T20:16:41.254668Z" + } + }, + "cell_type": "code", + "source": "check_permissions()", + "id": "7aea5658325ab008", + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "--- List feature views ---\n", + "No feature views found. You might not have access or they haven't been created.\n", + "\n", + "--- Fetching Historical Features for Training ---\n", + "\n", + "*** PERMISSION DENIED *** Cannot fetch historical features.\n", + "\n", + "--- Fetching Historical Features for Batch Scoring ---\n", + "\n", + "*** PERMISSION DENIED *** Cannot fetch historical features.\n", + "\n", + "--- Write to Feature Store ---\n", + "\n", + "*** PERMISSION DENIED *** User lacks permission to modify the feature store.\n", + "\n", + "--- Fetching Online Features ---\n", + "\n", + "*** PERMISSION DENIED *** Cannot fetch online features.\n", + "\n", + "--- Fetching Online Features via Feature Service ---\n", + "\n", + "*** PERMISSION DENIED *** Cannot fetch online features.\n", + "\n", + "--- Fetching Online Features via Push Source ---\n", + "\n", + "*** PERMISSION DENIED *** Cannot fetch online features.\n", + "\n", + "--- Performing Push Source ---\n", + "Unexpected error while pushing event: Unable to find push source 'driver_stats_push_source'.\n" + ] + } + ], + "execution_count": 55 + }, + { + "metadata": {}, + "cell_type": "markdown", + "source": "## Test Admin Feast User", + "id": "cb78ced7c37ceb4c" + }, + { + "metadata": { + "ExecuteTime": { + "end_time": "2025-03-06T20:17:02.206503Z", + "start_time": "2025-03-06T20:17:02.137409Z" + } + }, + "cell_type": "code", + "source": [ + "# Retrieve and store the token\n", + "get_k8s_token(\"feast-admin-sa\")" + ], + "id": "4f10aae116825619", + "outputs": [ + { + "data": { + "text/plain": [ + "'Token Retrieved: ***** (hidden for security)'" + ] + }, + "execution_count": 56, + "metadata": {}, + "output_type": "execute_result" + } + ], + "execution_count": 56 + }, + { + "metadata": { + "ExecuteTime": { + "end_time": "2025-03-06T20:17:07.799782Z", + "start_time": "2025-03-06T20:17:05.946696Z" + } + }, + "cell_type": "code", + "source": "check_permissions()", + "id": "7a6133f052b9cfe1", + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "--- List feature views ---\n", + "Successfully listed 2 feature views:\n", + " - driver_hourly_stats\n", + " - driver_hourly_stats_fresh\n", + "\n", + "--- Fetching Historical Features for Training ---\n", + "Handling connection for 8081\n", + "Successfully fetched training historical features:\n", + " driver_id event_timestamp label_driver_reported_satisfaction \\\n", + "0 1001 2021-04-12 10:59:42+00:00 1 \n", + "1 1002 2021-04-12 08:12:10+00:00 5 \n", + "2 1003 2021-04-12 16:40:26+00:00 3 \n", + "\n", + " val_to_add val_to_add_2 conv_rate acc_rate avg_daily_trips \\\n", + "0 1 10 0.677818 0.453707 193 \n", + "1 2 20 0.328160 0.900565 929 \n", + "2 3 30 0.787191 0.958963 571 \n", + "\n", + " conv_rate_plus_val1 conv_rate_plus_val2 \n", + "0 1.677818 10.677818 \n", + "1 2.328160 20.328160 \n", + "2 3.787191 30.787191 \n", + "\n", + "--- Fetching Historical Features for Batch Scoring ---\n", + "Handling connection for 8081\n", + "Successfully fetched batch scoring historical features:\n", + " driver_id event_timestamp \\\n", + "0 1001 2025-03-06 20:17:06.566035+00:00 \n", + "1 1002 2025-03-06 20:17:06.566035+00:00 \n", + "2 1003 2025-03-06 20:17:06.566035+00:00 \n", + "\n", + " label_driver_reported_satisfaction val_to_add val_to_add_2 conv_rate \\\n", + "0 1 1 10 0.782836 \n", + "1 5 2 20 0.731948 \n", + "2 3 3 30 0.613211 \n", + "\n", + " acc_rate avg_daily_trips conv_rate_plus_val1 conv_rate_plus_val2 \n", + "0 0.729726 652 1.782836 10.782836 \n", + "1 0.384902 902 2.731948 20.731948 \n", + "2 0.075386 101 3.613211 30.613211 \n", + "\n", + "--- Write to Feature Store ---\n", + "User has write access to the feature store.\n", + "\n", + "--- Fetching Online Features ---\n", + "Handling connection for 8082\n", + "Successfully fetched online features directly:\n", + "\n", + "acc_rate : [None, None]\n", + "conv_rate_plus_val1 : [None, None]\n", + "conv_rate_plus_val2 : [None, None]\n", + "driver_id : [1001, 1002]\n", + "\n", + "--- Fetching Online Features via Feature Service ---\n", + "Handling connection for 8082\n", + "Successfully fetched online features via feature service:\n", + "\n", + "conv_rate : [None, None]\n", + "conv_rate_plus_val1 : [None, None]\n", + "conv_rate_plus_val2 : [None, None]\n", + "driver_id : [1001, 1002]\n", + "\n", + "--- Fetching Online Features via Push Source ---\n", + "Handling connection for 8082\n", + "Successfully fetched online features via feature service:\n", + "\n", + "acc_rate : [None, None]\n", + "avg_daily_trips : [None, None]\n", + "conv_rate : [None, None]\n", + "conv_rate_plus_val1 : [None, None]\n", + "conv_rate_plus_val2 : [None, None]\n", + "driver_id : [1001, 1002]\n", + "\n", + "--- Performing Push Source ---\n", + "Unexpected error while pushing event: \n" + ] + } + ], + "execution_count": 57 + }, + { + "metadata": {}, + "cell_type": "markdown", + "source": [ + " **Note:**\n", + "**Currently, remote materialization not available in Feast when using the Remote Client**\n", + "**Workaround: Consider using running it from pod like**\n", + " \n", + " `kubectl exec deploy/feast-sample-kubernetes-auth -itc online -- bash -c 'feast materialize-incremental $(date -u +\"%Y-%m-%dT%H:%M:%S\")`\n" + ], + "id": "e451c30649630b2f" + }, + { + "metadata": {}, + "cell_type": "markdown", + "source": "Terminate the process", + "id": "e88442b1bae2b327" + }, + { + "metadata": { + "ExecuteTime": { + "end_time": "2025-03-05T19:09:29.743583Z", + "start_time": "2025-03-05T19:09:29.734671Z" + } + }, + "cell_type": "code", + "source": [ + "for name, process in port_forward_processes.items():\n", + " process.terminate()\n", + " print(f\"Stopped port forwarding for {name}\")" + ], + "id": "2984d62766da122a", + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Stopped port forwarding for offline_store\n", + "Stopped port forwarding for online_store\n", + "Stopped port forwarding for registry\n" + ] + } + ], + "execution_count": 25 + }, + { + "metadata": {}, + "cell_type": "markdown", + "source": "[Next: Uninstall the Operator and all Feast objects](./03-uninstall.ipynb)", + "id": "38c54e92643e0bda" + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 2 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython2", + "version": "2.7.6" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/examples/operator-rbac/README.md b/examples/operator-rbac/README.md new file mode 100644 index 00000000000..9c0a0461678 --- /dev/null +++ b/examples/operator-rbac/README.md @@ -0,0 +1,6 @@ +# Running the Feast RBAC example on Kubernetes using the Feast Operator. + +1. [1-setup-operator-rbac.ipynb](1-setup-operator-rbac.ipynb) will guide you through how to setup Role-Based Access Control (RBAC) for Feast using the [Feast Operator](../../infra/feast-operator/) and Kubernetes Authentication. This Feast Admin Step requires you to setup the operator and Feast RBAC on K8s. +2. [2-client.ipynb](2-client.ipynb) Validate the RBAC with the client example using different test cases using a service account token locally. +3. [03-uninstall.ipynb](03-uninstall.ipynb) Clear the installed deployments and K8s Objects. + diff --git a/examples/operator-rbac/client/feature_store.yaml b/examples/operator-rbac/client/feature_store.yaml new file mode 100644 index 00000000000..49a4c426363 --- /dev/null +++ b/examples/operator-rbac/client/feature_store.yaml @@ -0,0 +1,15 @@ +project: feast_rbac +provider: local +offline_store: + host: localhost + type: remote + port: 8081 +online_store: + path: http://localhost:8082 + type: remote +registry: + path: localhost:8083 + registry_type: remote +auth: + type: kubernetes +entity_key_serialization_version: 3 diff --git a/examples/operator-rbac/permissions_apply.py b/examples/operator-rbac/permissions_apply.py new file mode 100644 index 00000000000..0d46ad5260a --- /dev/null +++ b/examples/operator-rbac/permissions_apply.py @@ -0,0 +1,27 @@ +# Necessary modules for permissions and policies in Feast for RBAC +from feast.feast_object import ALL_RESOURCE_TYPES +from feast.permissions.action import READ, AuthzedAction, ALL_ACTIONS +from feast.permissions.permission import Permission +from feast.permissions.policy import RoleBasedPolicy + +# Define K8s roles same as created with FeatureStore CR +admin_roles = ["feast-writer"] # Full access (can create, update, delete ) Feast Resources +user_roles = ["feast-reader"] # Read-only access on Feast Resources + +# User permissions (feast_user_permission) +# - Grants read and describing Feast objects access +user_perm = Permission( + name="feast_user_permission", + types=ALL_RESOURCE_TYPES, + policy=RoleBasedPolicy(roles=user_roles), + actions=[AuthzedAction.DESCRIBE] + READ # Read access (READ_ONLINE, READ_OFFLINE) + describe other Feast Resources. +) + +# Admin permissions (feast_admin_permission) +# - Grants full control over all resources +admin_perm = Permission( + name="feast_admin_permission", + types=ALL_RESOURCE_TYPES, + policy=RoleBasedPolicy(roles=admin_roles), + actions=ALL_ACTIONS # Full permissions: CREATE, UPDATE, DELETE, READ, WRITE +) diff --git a/examples/podman_local/README.md b/examples/podman_local/README.md index f5b6ad40d49..4ab1d0012d2 100644 --- a/examples/podman_local/README.md +++ b/examples/podman_local/README.md @@ -18,7 +18,7 @@ This guide explains how to deploy Feast remote server components using Podman Co ### 2. **Run the Podman Compose File** -- Use the [docker-compose.yml](docker-compose.yml) file to install and run the Feast feature servers (online, offline, and registry) on podman. The docker-compose file uses the `feastdev/feature-server:latest` image. Each respective service has specific port mappings and maps the volume from the `./feature_repo` configuration. +- Use the [docker-compose.yml](docker-compose.yml) file to install and run the Feast feature servers (online, offline, and registry) on podman. The docker-compose file uses the `quay.io/feastdev/feature-server:latest` image. Each respective service has specific port mappings and maps the volume from the `./feature_repo` configuration. - To start the feature servers, run the following command: ```bash @@ -39,9 +39,9 @@ This guide explains how to deploy Feast remote server components using Podman Co ``` CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES - 61442d6d6ef3 docker.io/feastdev/feature-server:latest feast -c /feature... 2 minutes ago Up 2 minutes 0.0.0.0:6566->6566/tcp online-feature-server - 1274c21716a6 docker.io/feastdev/feature-server:latest feast -c /feature... 2 minutes ago Up 2 minutes 0.0.0.0:8815->8815/tcp offline-feature-server - 4e38ca8c39db docker.io/feastdev/feature-server:latest feast -c /feature... 2 minutes ago Up 2 minutes 0.0.0.0:6570->6570/tcp registry-feature-server + 61442d6d6ef3 quay.io/feastdev/feature-server:latest feast -c /feature... 2 minutes ago Up 2 minutes 0.0.0.0:6566->6566/tcp online-feature-server + 1274c21716a6 quay.io/feastdev/feature-server:latest feast -c /feature... 2 minutes ago Up 2 minutes 0.0.0.0:8815->8815/tcp offline-feature-server + 4e38ca8c39db quay.io/feastdev/feature-server:latest feast -c /feature... 2 minutes ago Up 2 minutes 0.0.0.0:6570->6570/tcp registry-feature-server ``` - Alternatively, you can verify the running containers through **Podman Desktop**: diff --git a/examples/podman_local/docker-compose.yml b/examples/podman_local/docker-compose.yml index 5bc1ae546a6..445021b643b 100644 --- a/examples/podman_local/docker-compose.yml +++ b/examples/podman_local/docker-compose.yml @@ -1,7 +1,7 @@ version: '3.9' x-defaults: &default-settings - image: feastdev/feature-server:latest + image: quay.io/feastdev/feature-server:latest restart: unless-stopped services: diff --git a/examples/rbac-remote/client/k8s/admin_user_resources.yaml b/examples/rbac-remote/client/k8s/admin_user_resources.yaml index d5df8bcbf24..63c537a609f 100644 --- a/examples/rbac-remote/client/k8s/admin_user_resources.yaml +++ b/examples/rbac-remote/client/k8s/admin_user_resources.yaml @@ -44,7 +44,7 @@ spec: serviceAccountName: feast-admin-sa containers: - name: client-admin-container - image: feastdev/feature-server:latest + image: quay.io/feastdev/feature-server:latest imagePullPolicy: Always command: ["sleep", "infinity"] volumeMounts: diff --git a/examples/rbac-remote/client/k8s/readonly_user_resources.yaml b/examples/rbac-remote/client/k8s/readonly_user_resources.yaml index c9094e7f2fc..9a0230a9d16 100644 --- a/examples/rbac-remote/client/k8s/readonly_user_resources.yaml +++ b/examples/rbac-remote/client/k8s/readonly_user_resources.yaml @@ -44,7 +44,7 @@ spec: serviceAccountName: feast-user-sa containers: - name: client-user-container - image: feastdev/feature-server:latest + image: quay.io/feastdev/feature-server:latest imagePullPolicy: Always command: ["sleep", "infinity"] volumeMounts: diff --git a/examples/rbac-remote/client/k8s/unauthorized_user_resources.yaml b/examples/rbac-remote/client/k8s/unauthorized_user_resources.yaml index 5068c94fd93..e1d29e62a52 100644 --- a/examples/rbac-remote/client/k8s/unauthorized_user_resources.yaml +++ b/examples/rbac-remote/client/k8s/unauthorized_user_resources.yaml @@ -24,7 +24,7 @@ spec: serviceAccountName: feast-unauthorized-user-sa containers: - name: client-unauthorized-user-container - image: feastdev/feature-server:latest + image: quay.io/feastdev/feature-server:latest imagePullPolicy: Always command: ["sleep", "infinity"] volumeMounts: diff --git a/examples/rbac-remote/client/oidc/admin_user_resources.yaml b/examples/rbac-remote/client/oidc/admin_user_resources.yaml index 7843ce3c9d0..1c4cb9c2763 100644 --- a/examples/rbac-remote/client/oidc/admin_user_resources.yaml +++ b/examples/rbac-remote/client/oidc/admin_user_resources.yaml @@ -17,7 +17,7 @@ spec: spec: containers: - name: client-admin-container - image: feastdev/feature-server:latest + image: quay.io/feastdev/feature-server:latest imagePullPolicy: Always command: ["sleep", "infinity"] env: diff --git a/examples/rbac-remote/client/oidc/readonly_user_resources.yaml b/examples/rbac-remote/client/oidc/readonly_user_resources.yaml index c43137bfba6..96427d270e8 100644 --- a/examples/rbac-remote/client/oidc/readonly_user_resources.yaml +++ b/examples/rbac-remote/client/oidc/readonly_user_resources.yaml @@ -17,7 +17,7 @@ spec: spec: containers: - name: client-admin-container - image: feastdev/feature-server:latest + image: quay.io/feastdev/feature-server:latest imagePullPolicy: Always command: ["sleep", "infinity"] env: diff --git a/examples/rbac-remote/client/oidc/unauthorized_user_resources.yaml b/examples/rbac-remote/client/oidc/unauthorized_user_resources.yaml index f99bb3e9874..859997876e3 100644 --- a/examples/rbac-remote/client/oidc/unauthorized_user_resources.yaml +++ b/examples/rbac-remote/client/oidc/unauthorized_user_resources.yaml @@ -17,7 +17,7 @@ spec: spec: containers: - name: client-admin-container - image: feastdev/feature-server:latest + image: quay.io/feastdev/feature-server:latest imagePullPolicy: Always command: ["sleep", "infinity"] env: diff --git a/go/internal/test/flexible_coyote/feature_repo/data/online_store_for_pg.db b/go/internal/test/flexible_coyote/feature_repo/data/online_store_for_pg.db new file mode 100644 index 00000000000..e69de29bb2d diff --git a/infra/charts/feast-feature-server/Chart.yaml b/infra/charts/feast-feature-server/Chart.yaml index d8ed41d2782..c1c693725a8 100644 --- a/infra/charts/feast-feature-server/Chart.yaml +++ b/infra/charts/feast-feature-server/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: feast-feature-server description: Feast Feature Server in Go or Python type: application -version: 0.46.0 +version: 0.47.0 keywords: - machine learning - big data diff --git a/infra/charts/feast-feature-server/README.md b/infra/charts/feast-feature-server/README.md index dc907ab8acf..6847721b7f7 100644 --- a/infra/charts/feast-feature-server/README.md +++ b/infra/charts/feast-feature-server/README.md @@ -1,6 +1,6 @@ # Feast Python / Go Feature Server Helm Charts -Current chart version is `0.46.0` +Current chart version is `0.47.0` ## Installation @@ -39,8 +39,8 @@ See [here](https://github.com/feast-dev/feast/tree/master/examples/python-helm-d | feature_store_yaml_base64 | string | `""` | [required] a base64 encoded version of feature_store.yaml | | fullnameOverride | string | `""` | | | image.pullPolicy | string | `"IfNotPresent"` | | -| image.repository | string | `"feastdev/feature-server"` | Docker image for Feature Server repository | -| image.tag | string | `"0.46.0"` | The Docker image tag (can be overwritten if custom feature server deps are needed for on demand transforms) | +| image.repository | string | `"quay.io/feastdev/feature-server"` | Docker image for Feature Server repository | +| image.tag | string | `"0.47.0"` | The Docker image tag (can be overwritten if custom feature server deps are needed for on demand transforms) | | imagePullSecrets | list | `[]` | | | livenessProbe.initialDelaySeconds | int | `30` | | | livenessProbe.periodSeconds | int | `30` | | diff --git a/infra/charts/feast-feature-server/values.yaml b/infra/charts/feast-feature-server/values.yaml index db2fbfb28bb..f377b8a3738 100644 --- a/infra/charts/feast-feature-server/values.yaml +++ b/infra/charts/feast-feature-server/values.yaml @@ -6,10 +6,10 @@ replicaCount: 1 image: # image.repository -- Docker image for Feature Server repository - repository: feastdev/feature-server + repository: quay.io/feastdev/feature-server pullPolicy: IfNotPresent # image.tag -- The Docker image tag (can be overwritten if custom feature server deps are needed for on demand transforms) - tag: 0.46.0 + tag: 0.47.0 logLevel: "WARNING" # Set log level DEBUG, INFO, WARNING, ERROR, and CRITICAL (case-insensitive) diff --git a/infra/charts/feast/Chart.yaml b/infra/charts/feast/Chart.yaml index d0dfebbf2b5..32e6717e528 100644 --- a/infra/charts/feast/Chart.yaml +++ b/infra/charts/feast/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v1 description: Feature store for machine learning name: feast -version: 0.46.0 +version: 0.47.0 keywords: - machine learning - big data diff --git a/infra/charts/feast/README.md b/infra/charts/feast/README.md index e89c8331d30..4d5c9526d06 100644 --- a/infra/charts/feast/README.md +++ b/infra/charts/feast/README.md @@ -8,7 +8,7 @@ This repo contains Helm charts for Feast Java components that are being installe ## Chart: Feast -Feature store for machine learning Current chart version is `0.46.0` +Feature store for machine learning Current chart version is `0.47.0` ## Installation @@ -65,8 +65,8 @@ See [here](https://github.com/feast-dev/feast/tree/master/examples/java-demo) fo | Repository | Name | Version | |------------|------|---------| | https://charts.helm.sh/stable | redis | 10.5.6 | -| https://feast-helm-charts.storage.googleapis.com | feature-server(feature-server) | 0.46.0 | -| https://feast-helm-charts.storage.googleapis.com | transformation-service(transformation-service) | 0.46.0 | +| https://feast-helm-charts.storage.googleapis.com | feature-server(feature-server) | 0.47.0 | +| https://feast-helm-charts.storage.googleapis.com | transformation-service(transformation-service) | 0.47.0 | ## Values diff --git a/infra/charts/feast/charts/feature-server/Chart.yaml b/infra/charts/feast/charts/feature-server/Chart.yaml index a4c10bdd5b4..4260c60521c 100644 --- a/infra/charts/feast/charts/feature-server/Chart.yaml +++ b/infra/charts/feast/charts/feature-server/Chart.yaml @@ -1,8 +1,8 @@ apiVersion: v1 description: "Feast Feature Server: Online feature serving service for Feast" name: feature-server -version: 0.46.0 -appVersion: v0.46.0 +version: 0.47.0 +appVersion: v0.47.0 keywords: - machine learning - big data diff --git a/infra/charts/feast/charts/feature-server/README.md b/infra/charts/feast/charts/feature-server/README.md index 697596b2eb2..cdcd398c4e0 100644 --- a/infra/charts/feast/charts/feature-server/README.md +++ b/infra/charts/feast/charts/feature-server/README.md @@ -1,6 +1,6 @@ # feature-server -![Version: 0.46.0](https://img.shields.io/badge/Version-0.46.0-informational?style=flat-square) ![AppVersion: v0.46.0](https://img.shields.io/badge/AppVersion-v0.46.0-informational?style=flat-square) +![Version: 0.47.0](https://img.shields.io/badge/Version-0.47.0-informational?style=flat-square) ![AppVersion: v0.47.0](https://img.shields.io/badge/AppVersion-v0.47.0-informational?style=flat-square) Feast Feature Server: Online feature serving service for Feast @@ -16,8 +16,8 @@ Feast Feature Server: Online feature serving service for Feast | "application.yaml".enabled | bool | `true` | Flag to include the default [configuration](https://github.com/feast-dev/feast/blob/master/java/serving/src/main/resources/application.yml). Please set `application-override.yaml` to override this configuration. | | envOverrides | object | `{}` | Extra environment variables to set | | image.pullPolicy | string | `"IfNotPresent"` | Image pull policy | -| image.repository | string | `"feastdev/feature-server-java"` | Docker image for Feature Server repository | -| image.tag | string | `"0.46.0"` | Image tag | +| image.repository | string | `"quay.io/feastdev/feature-server-java"` | Docker image for Feature Server repository | +| image.tag | string | `"0.47.0"` | Image tag | | ingress.grpc.annotations | object | `{}` | Extra annotations for the ingress | | ingress.grpc.auth.enabled | bool | `false` | Flag to enable auth | | ingress.grpc.class | string | `"nginx"` | Which ingress controller to use | diff --git a/infra/charts/feast/charts/feature-server/values.yaml b/infra/charts/feast/charts/feature-server/values.yaml index 48681f83ca0..86578700694 100644 --- a/infra/charts/feast/charts/feature-server/values.yaml +++ b/infra/charts/feast/charts/feature-server/values.yaml @@ -3,9 +3,9 @@ replicaCount: 1 image: # image.repository -- Docker image for Feature Server repository - repository: feastdev/feature-server-java + repository: quay.io/feastdev/feature-server-java # image.tag -- Image tag - tag: 0.46.0 + tag: 0.47.0 # image.pullPolicy -- Image pull policy pullPolicy: IfNotPresent diff --git a/infra/charts/feast/charts/transformation-service/Chart.yaml b/infra/charts/feast/charts/transformation-service/Chart.yaml index 12123e505e3..73cd6b3f00c 100644 --- a/infra/charts/feast/charts/transformation-service/Chart.yaml +++ b/infra/charts/feast/charts/transformation-service/Chart.yaml @@ -1,8 +1,8 @@ apiVersion: v1 description: "Transformation service: to compute on-demand features" name: transformation-service -version: 0.46.0 -appVersion: v0.46.0 +version: 0.47.0 +appVersion: v0.47.0 keywords: - machine learning - big data diff --git a/infra/charts/feast/charts/transformation-service/README.md b/infra/charts/feast/charts/transformation-service/README.md index 4dfd213bf2e..f4a0a50f0f2 100644 --- a/infra/charts/feast/charts/transformation-service/README.md +++ b/infra/charts/feast/charts/transformation-service/README.md @@ -1,6 +1,6 @@ # transformation-service -![Version: 0.46.0](https://img.shields.io/badge/Version-0.46.0-informational?style=flat-square) ![AppVersion: v0.46.0](https://img.shields.io/badge/AppVersion-v0.46.0-informational?style=flat-square) +![Version: 0.47.0](https://img.shields.io/badge/Version-0.47.0-informational?style=flat-square) ![AppVersion: v0.47.0](https://img.shields.io/badge/AppVersion-v0.47.0-informational?style=flat-square) Transformation service: to compute on-demand features @@ -12,8 +12,8 @@ Transformation service: to compute on-demand features |-----|------|---------|-------------| | envOverrides | object | `{}` | Extra environment variables to set | | image.pullPolicy | string | `"IfNotPresent"` | Image pull policy | -| image.repository | string | `"feastdev/feature-transformation-server"` | Docker image for Transformation Server repository | -| image.tag | string | `"0.46.0"` | Image tag | +| image.repository | string | `"quay.io/feastdev/feature-transformation-server"` | Docker image for Transformation Server repository | +| image.tag | string | `"0.47.0"` | Image tag | | nodeSelector | object | `{}` | Node labels for pod assignment | | podLabels | object | `{}` | Labels to be added to Feast Serving pods | | replicaCount | int | `1` | Number of pods that will be created | diff --git a/infra/charts/feast/charts/transformation-service/values.yaml b/infra/charts/feast/charts/transformation-service/values.yaml index 3d056d5b25f..bf2cbbc7946 100644 --- a/infra/charts/feast/charts/transformation-service/values.yaml +++ b/infra/charts/feast/charts/transformation-service/values.yaml @@ -3,9 +3,9 @@ replicaCount: 1 image: # image.repository -- Docker image for Transformation Server repository - repository: feastdev/feature-transformation-server + repository: quay.io/feastdev/feature-transformation-server # image.tag -- Image tag - tag: 0.46.0 + tag: 0.47.0 # image.pullPolicy -- Image pull policy pullPolicy: IfNotPresent diff --git a/infra/charts/feast/requirements.yaml b/infra/charts/feast/requirements.yaml index 11d9026f2df..fb8d6fccc5f 100644 --- a/infra/charts/feast/requirements.yaml +++ b/infra/charts/feast/requirements.yaml @@ -1,12 +1,12 @@ dependencies: - name: feature-server alias: feature-server - version: 0.46.0 + version: 0.47.0 condition: feature-server.enabled repository: https://feast-helm-charts.storage.googleapis.com - name: transformation-service alias: transformation-service - version: 0.46.0 + version: 0.47.0 condition: transformation-service.enabled repository: https://feast-helm-charts.storage.googleapis.com - name: redis diff --git a/infra/feast-helm-operator/Makefile b/infra/feast-helm-operator/Makefile index 76614ae37af..226e425602f 100644 --- a/infra/feast-helm-operator/Makefile +++ b/infra/feast-helm-operator/Makefile @@ -3,7 +3,7 @@ # To re-generate a bundle for another specific version without changing the standard setup, you can: # - use the VERSION as arg of the bundle target (e.g make bundle VERSION=0.0.2) # - use environment variables to overwrite this value (e.g export VERSION=0.0.2) -VERSION ?= 0.46.0 +VERSION ?= 0.47.0 # CHANNELS define the bundle channels used in the bundle. # Add a new line here if you would like to change its default config. (E.g CHANNELS = "candidate,fast,stable") @@ -24,12 +24,12 @@ BUNDLE_DEFAULT_CHANNEL := --default-channel=$(DEFAULT_CHANNEL) endif BUNDLE_METADATA_OPTS ?= $(BUNDLE_CHANNELS) $(BUNDLE_DEFAULT_CHANNEL) -# IMAGE_TAG_BASE defines the docker.io namespace and part of the image name for remote images. +# IMAGE_TAG_BASE defines the quay.io namespace and part of the image name for remote images. # This variable is used to construct full image tags for bundle and catalog images. # # For example, running 'make bundle-build bundle-push catalog-build catalog-push' will build and push both -# feastdev/feast-helm-operator-bundle:$VERSION and feastdev/feast-helm-operator-catalog:$VERSION. -IMAGE_TAG_BASE ?= feastdev/feast-helm-operator +# quay.io/feastdev/feast-helm-operator-bundle:$VERSION and quay.io/feastdev/feast-helm-operator-catalog:$VERSION. +IMAGE_TAG_BASE ?= quay.io/feastdev/feast-helm-operator # BUNDLE_IMG defines the image:tag used for the bundle. # You can use it as an arg. (E.g make bundle-build BUNDLE_IMG=/:) diff --git a/infra/feast-helm-operator/config/manager/kustomization.yaml b/infra/feast-helm-operator/config/manager/kustomization.yaml index bc970e7b408..d0181a2d532 100644 --- a/infra/feast-helm-operator/config/manager/kustomization.yaml +++ b/infra/feast-helm-operator/config/manager/kustomization.yaml @@ -4,5 +4,5 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization images: - name: controller - newName: feastdev/feast-helm-operator - newTag: 0.46.0 + newName: quay.io/feastdev/feast-helm-operator + newTag: 0.47.0 diff --git a/infra/feast-operator/.golangci.yml b/infra/feast-operator/.golangci.yml index 020e5768657..6c104980d43 100644 --- a/infra/feast-operator/.golangci.yml +++ b/infra/feast-operator/.golangci.yml @@ -19,6 +19,12 @@ issues: - path: "test/*" linters: - lll + - path: "upgrade/*" + linters: + - lll + - path: "previous-version/*" + linters: + - lll linters: disable-all: true enable: diff --git a/infra/feast-operator/Makefile b/infra/feast-operator/Makefile index 9c90f02d415..254d577beae 100644 --- a/infra/feast-operator/Makefile +++ b/infra/feast-operator/Makefile @@ -3,7 +3,7 @@ # To re-generate a bundle for another specific version without changing the standard setup, you can: # - use the VERSION as arg of the bundle target (e.g make bundle VERSION=0.0.2) # - use environment variables to overwrite this value (e.g export VERSION=0.0.2) -VERSION ?= 0.46.0 +VERSION ?= 0.47.0 # CHANNELS define the bundle channels used in the bundle. # Add a new line here if you would like to change its default config. (E.g CHANNELS = "candidate,fast,stable") @@ -24,12 +24,12 @@ BUNDLE_DEFAULT_CHANNEL := --default-channel=$(DEFAULT_CHANNEL) endif BUNDLE_METADATA_OPTS ?= $(BUNDLE_CHANNELS) $(BUNDLE_DEFAULT_CHANNEL) -# IMAGE_TAG_BASE defines the docker.io namespace and part of the image name for remote images. +# IMAGE_TAG_BASE defines the quay.io namespace and part of the image name for remote images. # This variable is used to construct full image tags for bundle and catalog images. # # For example, running 'make bundle-build bundle-push catalog-build catalog-push' will build and push both # feast.dev/feast-operator-bundle:$VERSION and feast.dev/feast-operator-catalog:$VERSION. -IMAGE_TAG_BASE ?= feastdev/feast-operator +IMAGE_TAG_BASE ?= quay.io/feastdev/feast-operator # BUNDLE_IMG defines the image:tag used for the bundle. # You can use it as an arg. (E.g make bundle-build BUNDLE_IMG=/:) @@ -51,7 +51,7 @@ endif OPERATOR_SDK_VERSION ?= v1.38.0 # Image URL to use all building/pushing image targets IMG ?= $(IMAGE_TAG_BASE):$(VERSION) -FS_IMG ?= docker.io/feastdev/feature-server:$(VERSION) +FS_IMG ?= quay.io/feastdev/feature-server:$(VERSION) # ENVTEST_K8S_VERSION refers to the version of kubebuilder assets to be downloaded by envtest binary. ENVTEST_K8S_VERSION = 1.30.0 @@ -113,12 +113,20 @@ vet: ## Run go vet against code. .PHONY: test test: build-installer vet lint envtest ## Run tests. - KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" go test $$(go list ./... | grep -v test/e2e | grep -v test/data-source-types) -coverprofile cover.out + KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" go test $$(go list ./... | grep -v test/e2e | grep -v test/data-source-types | grep -v test/upgrade | grep -v test/previous-version) -coverprofile cover.out # Utilize Kind or modify the e2e tests to load the image locally, enabling compatibility with other vendors. .PHONY: test-e2e # Run the e2e tests against a Kind k8s instance that is spun up. test-e2e: - go test -timeout 30m ./test/e2e/ -v -ginkgo.v + go test -timeout 60m ./test/e2e/ -v -ginkgo.v + +.PHONY: test-upgrade # Run the upgrade tests against a Kind k8s instance that is spun up. +test-upgrade: + go test -timeout 60m ./test/upgrade/ -v -ginkgo.v + +.PHONY: test-previous-version # Run e2e tests against previous version in a Kind k8s instance that is spun up. +test-previous-version: + go test -timeout 60m ./test/previous-version/ -v -ginkgo.v # Requires python3 .PHONY: test-datasources @@ -156,7 +164,6 @@ docker-build: ## Build docker image with the manager. feast-ci-dev-docker-img: cd ./../.. && make build-feature-server-dev - .PHONY: docker-push docker-push: ## Push docker image with the manager. $(CONTAINER_TOOL) push ${IMG} diff --git a/infra/feast-operator/api/feastversion/version.go b/infra/feast-operator/api/feastversion/version.go index 43c518ebbff..57f2bc1bc29 100644 --- a/infra/feast-operator/api/feastversion/version.go +++ b/infra/feast-operator/api/feastversion/version.go @@ -16,5 +16,5 @@ limitations under the License. package feastversion -// Feast release version -const FeastVersion = "0.46.0" +// Feast release version. Keep on line #20, this is critical to release CI +const FeastVersion = "0.47.0" diff --git a/infra/feast-operator/api/v1alpha1/featurestore_types.go b/infra/feast-operator/api/v1alpha1/featurestore_types.go index 48e70a2804d..1d00163d61b 100644 --- a/infra/feast-operator/api/v1alpha1/featurestore_types.go +++ b/infra/feast-operator/api/v1alpha1/featurestore_types.go @@ -66,9 +66,42 @@ const ( type FeatureStoreSpec struct { // +kubebuilder:validation:Pattern="^[A-Za-z0-9][A-Za-z0-9_]*$" // FeastProject is the Feast project id. This can be any alphanumeric string with underscores, but it cannot start with an underscore. Required. - FeastProject string `json:"feastProject"` - Services *FeatureStoreServices `json:"services,omitempty"` - AuthzConfig *AuthzConfig `json:"authz,omitempty"` + FeastProject string `json:"feastProject"` + FeastProjectDir *FeastProjectDir `json:"feastProjectDir,omitempty"` + Services *FeatureStoreServices `json:"services,omitempty"` + AuthzConfig *AuthzConfig `json:"authz,omitempty"` +} + +// FeastProjectDir defines how to create the feast project directory. +// +kubebuilder:validation:XValidation:rule="[has(self.git), has(self.init)].exists_one(c, c)",message="One selection required between init or git." +type FeastProjectDir struct { + Git *GitCloneOptions `json:"git,omitempty"` + Init *FeastInitOptions `json:"init,omitempty"` +} + +// GitCloneOptions describes how a clone should be performed. +// +kubebuilder:validation:XValidation:rule="has(self.featureRepoPath) ? !self.featureRepoPath.startsWith('/') : true",message="RepoPath must be a file name only, with no slashes." +type GitCloneOptions struct { + // The repository URL to clone from. + URL string `json:"url"` + // Reference to a branch / tag / commit + Ref string `json:"ref,omitempty"` + // Configs passed to git via `-c` + // e.g. http.sslVerify: 'false' + // OR 'url."https://api:\${TOKEN}@github.com/".insteadOf': 'https://github.com/' + Configs map[string]string `json:"configs,omitempty"` + // FeatureRepoPath is the relative path to the feature repo subdirectory. Default is 'feature_repo'. + FeatureRepoPath string `json:"featureRepoPath,omitempty"` + Env *[]corev1.EnvVar `json:"env,omitempty"` + EnvFrom *[]corev1.EnvFromSource `json:"envFrom,omitempty"` +} + +// FeastInitOptions defines how to run a `feast init`. +type FeastInitOptions struct { + Minimal bool `json:"minimal,omitempty"` + // Template for the created project + // +kubebuilder:validation:Enum=local;gcp;aws;snowflake;spark;postgres;hbase;cassandra;hazelcast;ikv;couchbase + Template string `json:"template,omitempty"` } // FeatureStoreServices defines the desired feast services. An ephemeral onlineStore feature server is deployed by default. @@ -115,7 +148,7 @@ var ValidOfflineStoreFilePersistenceTypes = []string{ // OfflineStoreDBStorePersistence configures the DB store persistence for the offline store service type OfflineStoreDBStorePersistence struct { // Type of the persistence type you want to use. - // +kubebuilder:validation:Enum=snowflake.offline;bigquery;redshift;spark;postgres;trino;athena;mssql + // +kubebuilder:validation:Enum=snowflake.offline;bigquery;redshift;spark;postgres;trino;athena;mssql;couchbase.offline Type string `json:"type"` // Data store parameters should be placed as-is from the "feature_store.yaml" under the secret key. "registry_type" & "type" fields should be removed. SecretRef corev1.LocalObjectReference `json:"secretRef"` @@ -132,6 +165,7 @@ var ValidOfflineStoreDBStorePersistenceTypes = []string{ "trino", "athena", "mssql", + "couchbase.offline", } // OnlineStore configures the online store service @@ -160,7 +194,7 @@ type OnlineStoreFilePersistence struct { // OnlineStoreDBStorePersistence configures the DB store persistence for the online store service type OnlineStoreDBStorePersistence struct { // Type of the persistence type you want to use. - // +kubebuilder:validation:Enum=snowflake.online;redis;ikv;datastore;dynamodb;bigtable;postgres;cassandra;mysql;hazelcast;singlestore;hbase;elasticsearch;qdrant;couchbase;milvus + // +kubebuilder:validation:Enum=snowflake.online;redis;ikv;datastore;dynamodb;bigtable;postgres;cassandra;mysql;hazelcast;singlestore;hbase;elasticsearch;qdrant;couchbase.online;milvus Type string `json:"type"` // Data store parameters should be placed as-is from the "feature_store.yaml" under the secret key. "registry_type" & "type" fields should be removed. SecretRef corev1.LocalObjectReference `json:"secretRef"` @@ -183,7 +217,7 @@ var ValidOnlineStoreDBStorePersistenceTypes = []string{ "hbase", "elasticsearch", "qdrant", - "couchbase", + "couchbase.online", "milvus", } diff --git a/infra/feast-operator/api/v1alpha1/zz_generated.deepcopy.go b/infra/feast-operator/api/v1alpha1/zz_generated.deepcopy.go index 49418e5f87e..87e5b7164af 100644 --- a/infra/feast-operator/api/v1alpha1/zz_generated.deepcopy.go +++ b/infra/feast-operator/api/v1alpha1/zz_generated.deepcopy.go @@ -21,8 +21,8 @@ limitations under the License. package v1alpha1 import ( - "k8s.io/api/apps/v1" - corev1 "k8s.io/api/core/v1" + appsv1 "k8s.io/api/apps/v1" + "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" runtime "k8s.io/apimachinery/pkg/runtime" ) @@ -89,6 +89,46 @@ func (in *DefaultCtrConfigs) DeepCopy() *DefaultCtrConfigs { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *FeastInitOptions) DeepCopyInto(out *FeastInitOptions) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new FeastInitOptions. +func (in *FeastInitOptions) DeepCopy() *FeastInitOptions { + if in == nil { + return nil + } + out := new(FeastInitOptions) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *FeastProjectDir) DeepCopyInto(out *FeastProjectDir) { + *out = *in + if in.Git != nil { + in, out := &in.Git, &out.Git + *out = new(GitCloneOptions) + (*in).DeepCopyInto(*out) + } + if in.Init != nil { + in, out := &in.Init, &out.Init + *out = new(FeastInitOptions) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new FeastProjectDir. +func (in *FeastProjectDir) DeepCopy() *FeastProjectDir { + if in == nil { + return nil + } + out := new(FeastProjectDir) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *FeatureStore) DeepCopyInto(out *FeatureStore) { *out = *in @@ -188,12 +228,12 @@ func (in *FeatureStoreServices) DeepCopyInto(out *FeatureStoreServices) { } if in.DeploymentStrategy != nil { in, out := &in.DeploymentStrategy, &out.DeploymentStrategy - *out = new(v1.DeploymentStrategy) + *out = new(appsv1.DeploymentStrategy) (*in).DeepCopyInto(*out) } if in.Volumes != nil { in, out := &in.Volumes, &out.Volumes - *out = make([]corev1.Volume, len(*in)) + *out = make([]v1.Volume, len(*in)) for i := range *in { (*in)[i].DeepCopyInto(&(*out)[i]) } @@ -213,6 +253,11 @@ func (in *FeatureStoreServices) DeepCopy() *FeatureStoreServices { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *FeatureStoreSpec) DeepCopyInto(out *FeatureStoreSpec) { *out = *in + if in.FeastProjectDir != nil { + in, out := &in.FeastProjectDir, &out.FeastProjectDir + *out = new(FeastProjectDir) + (*in).DeepCopyInto(*out) + } if in.Services != nil { in, out := &in.Services, &out.Services *out = new(FeatureStoreServices) @@ -259,6 +304,50 @@ func (in *FeatureStoreStatus) DeepCopy() *FeatureStoreStatus { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *GitCloneOptions) DeepCopyInto(out *GitCloneOptions) { + *out = *in + if in.Configs != nil { + in, out := &in.Configs, &out.Configs + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + if in.Env != nil { + in, out := &in.Env, &out.Env + *out = new([]v1.EnvVar) + if **in != nil { + in, out := *in, *out + *out = make([]v1.EnvVar, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + } + if in.EnvFrom != nil { + in, out := &in.EnvFrom, &out.EnvFrom + *out = new([]v1.EnvFromSource) + if **in != nil { + in, out := *in, *out + *out = make([]v1.EnvFromSource, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GitCloneOptions. +func (in *GitCloneOptions) DeepCopy() *GitCloneOptions { + if in == nil { + return nil + } + out := new(GitCloneOptions) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *KubernetesAuthz) DeepCopyInto(out *KubernetesAuthz) { *out = *in @@ -497,10 +586,10 @@ func (in *OptionalCtrConfigs) DeepCopyInto(out *OptionalCtrConfigs) { *out = *in if in.Env != nil { in, out := &in.Env, &out.Env - *out = new([]corev1.EnvVar) + *out = new([]v1.EnvVar) if **in != nil { in, out := *in, *out - *out = make([]corev1.EnvVar, len(*in)) + *out = make([]v1.EnvVar, len(*in)) for i := range *in { (*in)[i].DeepCopyInto(&(*out)[i]) } @@ -508,10 +597,10 @@ func (in *OptionalCtrConfigs) DeepCopyInto(out *OptionalCtrConfigs) { } if in.EnvFrom != nil { in, out := &in.EnvFrom, &out.EnvFrom - *out = new([]corev1.EnvFromSource) + *out = new([]v1.EnvFromSource) if **in != nil { in, out := *in, *out - *out = make([]corev1.EnvFromSource, len(*in)) + *out = make([]v1.EnvFromSource, len(*in)) for i := range *in { (*in)[i].DeepCopyInto(&(*out)[i]) } @@ -519,12 +608,12 @@ func (in *OptionalCtrConfigs) DeepCopyInto(out *OptionalCtrConfigs) { } if in.ImagePullPolicy != nil { in, out := &in.ImagePullPolicy, &out.ImagePullPolicy - *out = new(corev1.PullPolicy) + *out = new(v1.PullPolicy) **out = **in } if in.Resources != nil { in, out := &in.Resources, &out.Resources - *out = new(corev1.ResourceRequirements) + *out = new(v1.ResourceRequirements) (*in).DeepCopyInto(*out) } } @@ -544,7 +633,7 @@ func (in *PvcConfig) DeepCopyInto(out *PvcConfig) { *out = *in if in.Ref != nil { in, out := &in.Ref, &out.Ref - *out = new(corev1.LocalObjectReference) + *out = new(v1.LocalObjectReference) **out = **in } if in.Create != nil { @@ -569,7 +658,7 @@ func (in *PvcCreate) DeepCopyInto(out *PvcCreate) { *out = *in if in.AccessModes != nil { in, out := &in.AccessModes, &out.AccessModes - *out = make([]corev1.PersistentVolumeAccessMode, len(*in)) + *out = make([]v1.PersistentVolumeAccessMode, len(*in)) copy(*out, *in) } if in.StorageClassName != nil { @@ -748,7 +837,7 @@ func (in *ServerConfigs) DeepCopyInto(out *ServerConfigs) { } if in.VolumeMounts != nil { in, out := &in.VolumeMounts, &out.VolumeMounts - *out = make([]corev1.VolumeMount, len(*in)) + *out = make([]v1.VolumeMount, len(*in)) for i := range *in { (*in)[i].DeepCopyInto(&(*out)[i]) } @@ -785,7 +874,7 @@ func (in *TlsConfigs) DeepCopyInto(out *TlsConfigs) { *out = *in if in.SecretRef != nil { in, out := &in.SecretRef, &out.SecretRef - *out = new(corev1.LocalObjectReference) + *out = new(v1.LocalObjectReference) **out = **in } out.SecretKeyNames = in.SecretKeyNames diff --git a/infra/feast-operator/bundle/manifests/feast-operator.clusterserviceversion.yaml b/infra/feast-operator/bundle/manifests/feast-operator.clusterserviceversion.yaml index 734508cfecb..e2c9067d457 100644 --- a/infra/feast-operator/bundle/manifests/feast-operator.clusterserviceversion.yaml +++ b/infra/feast-operator/bundle/manifests/feast-operator.clusterserviceversion.yaml @@ -37,10 +37,10 @@ metadata: } ] capabilities: Basic Install - createdAt: "2025-02-17T22:19:00Z" + createdAt: "2025-03-10T19:57:05Z" operators.operatorframework.io/builder: operator-sdk-v1.38.0 operators.operatorframework.io/project_layout: go.kubebuilder.io/v4 - name: feast-operator.v0.46.0 + name: feast-operator.v0.47.0 namespace: placeholder spec: apiservicedefinitions: {} @@ -182,8 +182,8 @@ spec: - /manager env: - name: RELATED_IMAGE_FEATURE_SERVER - value: docker.io/feastdev/feature-server:0.46.0 - image: feastdev/feast-operator:0.46.0 + value: quay.io/feastdev/feature-server:0.47.0 + image: quay.io/feastdev/feast-operator:0.47.0 livenessProbe: httpGet: path: /healthz @@ -273,6 +273,6 @@ spec: name: Feast Community url: https://lf-aidata.atlassian.net/wiki/spaces/FEAST/ relatedImages: - - image: docker.io/feastdev/feature-server:0.46.0 + - image: quay.io/feastdev/feature-server:0.47.0 name: feature-server - version: 0.46.0 + version: 0.47.0 diff --git a/infra/feast-operator/bundle/manifests/feast.dev_featurestores.yaml b/infra/feast-operator/bundle/manifests/feast.dev_featurestores.yaml index 8806a5b1f3a..ab1ca8c6c2f 100644 --- a/infra/feast-operator/bundle/manifests/feast.dev_featurestores.yaml +++ b/infra/feast-operator/bundle/manifests/feast.dev_featurestores.yaml @@ -87,6 +87,215 @@ spec: description: FeastProject is the Feast project id. pattern: ^[A-Za-z0-9][A-Za-z0-9_]*$ type: string + feastProjectDir: + description: FeastProjectDir defines how to create the feast project + directory. + properties: + git: + description: GitCloneOptions describes how a clone should be performed. + properties: + configs: + additionalProperties: + type: string + description: |- + Configs passed to git via `-c` + e.g. http.sslVerify: 'false' + OR 'url."https://api:\${TOKEN}@github.com/". + type: object + env: + items: + description: EnvVar represents an environment variable present + in a Container. + properties: + name: + description: Name of the environment variable. Must + be a C_IDENTIFIER. + type: string + value: + description: |- + Variable references $(VAR_NAME) are expanded + using the previously defined environment variables in the container and + any + type: string + valueFrom: + description: Source for the environment variable's value. + Cannot be used if value is not empty. + properties: + configMapKeyRef: + description: Selects a key of a ConfigMap. + properties: + key: + description: The key to select. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. + type: string + optional: + description: Specify whether the ConfigMap or + its key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + fieldRef: + description: 'Selects a field of the pod: supports + metadata.name, metadata.namespace, `metadata.labels['''']`, + `metadata.' + properties: + apiVersion: + description: Version of the schema the FieldPath + is written in terms of, defaults to "v1". + type: string + fieldPath: + description: Path of the field to select in + the specified API version. + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + resourceFieldRef: + description: |- + Selects a resource of the container: only resources limits and requests + (limits.cpu, limits.memory, limits. + properties: + containerName: + description: 'Container name: required for volumes, + optional for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output format of + the exposed resources, defaults to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource to select' + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + secretKeyRef: + description: Selects a key of a secret in the pod's + namespace + properties: + key: + description: The key of the secret to select + from. Must be a valid secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. + type: string + optional: + description: Specify whether the Secret or its + key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + required: + - name + type: object + type: array + envFrom: + items: + description: EnvFromSource represents the source of a set + of ConfigMaps + properties: + configMapRef: + description: The ConfigMap to select from + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. + type: string + optional: + description: Specify whether the ConfigMap must + be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + prefix: + description: An optional identifier to prepend to each + key in the ConfigMap. Must be a C_IDENTIFIER. + type: string + secretRef: + description: The Secret to select from + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. + type: string + optional: + description: Specify whether the Secret must be + defined + type: boolean + type: object + x-kubernetes-map-type: atomic + type: object + type: array + featureRepoPath: + description: FeatureRepoPath is the relative path to the feature + repo subdirectory. Default is 'feature_repo'. + type: string + ref: + description: Reference to a branch / tag / commit + type: string + url: + description: The repository URL to clone from. + type: string + required: + - url + type: object + x-kubernetes-validations: + - message: RepoPath must be a file name only, with no slashes. + rule: 'has(self.featureRepoPath) ? !self.featureRepoPath.startsWith(''/'') + : true' + init: + description: FeastInitOptions defines how to run a `feast init`. + properties: + minimal: + type: boolean + template: + description: Template for the created project + enum: + - local + - gcp + - aws + - snowflake + - spark + - postgres + - hbase + - cassandra + - hazelcast + - ikv + - couchbase + type: string + type: object + type: object + x-kubernetes-validations: + - message: One selection required between init or git. + rule: '[has(self.git), has(self.init)].exists_one(c, c)' services: description: FeatureStoreServices defines the desired feast services. An ephemeral onlineStore feature server is deployed by default. @@ -252,6 +461,7 @@ spec: - trino - athena - mssql + - couchbase.offline type: string required: - secretRef @@ -705,7 +915,7 @@ spec: - hbase - elasticsearch - qdrant - - couchbase + - couchbase.online - milvus type: string required: @@ -3311,6 +3521,218 @@ spec: description: FeastProject is the Feast project id. pattern: ^[A-Za-z0-9][A-Za-z0-9_]*$ type: string + feastProjectDir: + description: FeastProjectDir defines how to create the feast project + directory. + properties: + git: + description: GitCloneOptions describes how a clone should + be performed. + properties: + configs: + additionalProperties: + type: string + description: |- + Configs passed to git via `-c` + e.g. http.sslVerify: 'false' + OR 'url."https://api:\${TOKEN}@github.com/". + type: object + env: + items: + description: EnvVar represents an environment variable + present in a Container. + properties: + name: + description: Name of the environment variable. Must + be a C_IDENTIFIER. + type: string + value: + description: |- + Variable references $(VAR_NAME) are expanded + using the previously defined environment variables in the container and + any + type: string + valueFrom: + description: Source for the environment variable's + value. Cannot be used if value is not empty. + properties: + configMapKeyRef: + description: Selects a key of a ConfigMap. + properties: + key: + description: The key to select. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. + type: string + optional: + description: Specify whether the ConfigMap + or its key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + fieldRef: + description: 'Selects a field of the pod: supports + metadata.name, metadata.namespace, `metadata.labels['''']`, + `metadata.' + properties: + apiVersion: + description: Version of the schema the FieldPath + is written in terms of, defaults to "v1". + type: string + fieldPath: + description: Path of the field to select + in the specified API version. + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + resourceFieldRef: + description: |- + Selects a resource of the container: only resources limits and requests + (limits.cpu, limits.memory, limits. + properties: + containerName: + description: 'Container name: required for + volumes, optional for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output format + of the exposed resources, defaults to + "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource to select' + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + secretKeyRef: + description: Selects a key of a secret in the + pod's namespace + properties: + key: + description: The key of the secret to select + from. Must be a valid secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. + type: string + optional: + description: Specify whether the Secret + or its key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + required: + - name + type: object + type: array + envFrom: + items: + description: EnvFromSource represents the source of + a set of ConfigMaps + properties: + configMapRef: + description: The ConfigMap to select from + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. + type: string + optional: + description: Specify whether the ConfigMap must + be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + prefix: + description: An optional identifier to prepend to + each key in the ConfigMap. Must be a C_IDENTIFIER. + type: string + secretRef: + description: The Secret to select from + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. + type: string + optional: + description: Specify whether the Secret must + be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + type: object + type: array + featureRepoPath: + description: FeatureRepoPath is the relative path to the + feature repo subdirectory. Default is 'feature_repo'. + type: string + ref: + description: Reference to a branch / tag / commit + type: string + url: + description: The repository URL to clone from. + type: string + required: + - url + type: object + x-kubernetes-validations: + - message: RepoPath must be a file name only, with no slashes. + rule: 'has(self.featureRepoPath) ? !self.featureRepoPath.startsWith(''/'') + : true' + init: + description: FeastInitOptions defines how to run a `feast + init`. + properties: + minimal: + type: boolean + template: + description: Template for the created project + enum: + - local + - gcp + - aws + - snowflake + - spark + - postgres + - hbase + - cassandra + - hazelcast + - ikv + - couchbase + type: string + type: object + type: object + x-kubernetes-validations: + - message: One selection required between init or git. + rule: '[has(self.git), has(self.init)].exists_one(c, c)' services: description: FeatureStoreServices defines the desired feast services. An ephemeral onlineStore feature server is deployed by default. @@ -3478,6 +3900,7 @@ spec: - trino - athena - mssql + - couchbase.offline type: string required: - secretRef @@ -3939,7 +4362,7 @@ spec: - hbase - elasticsearch - qdrant - - couchbase + - couchbase.online - milvus type: string required: diff --git a/infra/feast-operator/config/component_metadata.yaml b/infra/feast-operator/config/component_metadata.yaml index cbe44e473af..f45a3ab83f9 100644 --- a/infra/feast-operator/config/component_metadata.yaml +++ b/infra/feast-operator/config/component_metadata.yaml @@ -1,5 +1,5 @@ # This file is required to configure Feast release information for ODH/RHOAI Operator releases: - name: Feast - version: 0.46.0 + version: 0.47.0 repoUrl: https://github.com/feast-dev/feast diff --git a/infra/feast-operator/config/crd/bases/feast.dev_featurestores.yaml b/infra/feast-operator/config/crd/bases/feast.dev_featurestores.yaml index 54c48d70575..4bb7227f856 100644 --- a/infra/feast-operator/config/crd/bases/feast.dev_featurestores.yaml +++ b/infra/feast-operator/config/crd/bases/feast.dev_featurestores.yaml @@ -87,6 +87,215 @@ spec: description: FeastProject is the Feast project id. pattern: ^[A-Za-z0-9][A-Za-z0-9_]*$ type: string + feastProjectDir: + description: FeastProjectDir defines how to create the feast project + directory. + properties: + git: + description: GitCloneOptions describes how a clone should be performed. + properties: + configs: + additionalProperties: + type: string + description: |- + Configs passed to git via `-c` + e.g. http.sslVerify: 'false' + OR 'url."https://api:\${TOKEN}@github.com/". + type: object + env: + items: + description: EnvVar represents an environment variable present + in a Container. + properties: + name: + description: Name of the environment variable. Must + be a C_IDENTIFIER. + type: string + value: + description: |- + Variable references $(VAR_NAME) are expanded + using the previously defined environment variables in the container and + any + type: string + valueFrom: + description: Source for the environment variable's value. + Cannot be used if value is not empty. + properties: + configMapKeyRef: + description: Selects a key of a ConfigMap. + properties: + key: + description: The key to select. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. + type: string + optional: + description: Specify whether the ConfigMap or + its key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + fieldRef: + description: 'Selects a field of the pod: supports + metadata.name, metadata.namespace, `metadata.labels['''']`, + `metadata.' + properties: + apiVersion: + description: Version of the schema the FieldPath + is written in terms of, defaults to "v1". + type: string + fieldPath: + description: Path of the field to select in + the specified API version. + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + resourceFieldRef: + description: |- + Selects a resource of the container: only resources limits and requests + (limits.cpu, limits.memory, limits. + properties: + containerName: + description: 'Container name: required for volumes, + optional for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output format of + the exposed resources, defaults to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource to select' + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + secretKeyRef: + description: Selects a key of a secret in the pod's + namespace + properties: + key: + description: The key of the secret to select + from. Must be a valid secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. + type: string + optional: + description: Specify whether the Secret or its + key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + required: + - name + type: object + type: array + envFrom: + items: + description: EnvFromSource represents the source of a set + of ConfigMaps + properties: + configMapRef: + description: The ConfigMap to select from + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. + type: string + optional: + description: Specify whether the ConfigMap must + be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + prefix: + description: An optional identifier to prepend to each + key in the ConfigMap. Must be a C_IDENTIFIER. + type: string + secretRef: + description: The Secret to select from + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. + type: string + optional: + description: Specify whether the Secret must be + defined + type: boolean + type: object + x-kubernetes-map-type: atomic + type: object + type: array + featureRepoPath: + description: FeatureRepoPath is the relative path to the feature + repo subdirectory. Default is 'feature_repo'. + type: string + ref: + description: Reference to a branch / tag / commit + type: string + url: + description: The repository URL to clone from. + type: string + required: + - url + type: object + x-kubernetes-validations: + - message: RepoPath must be a file name only, with no slashes. + rule: 'has(self.featureRepoPath) ? !self.featureRepoPath.startsWith(''/'') + : true' + init: + description: FeastInitOptions defines how to run a `feast init`. + properties: + minimal: + type: boolean + template: + description: Template for the created project + enum: + - local + - gcp + - aws + - snowflake + - spark + - postgres + - hbase + - cassandra + - hazelcast + - ikv + - couchbase + type: string + type: object + type: object + x-kubernetes-validations: + - message: One selection required between init or git. + rule: '[has(self.git), has(self.init)].exists_one(c, c)' services: description: FeatureStoreServices defines the desired feast services. An ephemeral onlineStore feature server is deployed by default. @@ -252,6 +461,7 @@ spec: - trino - athena - mssql + - couchbase.offline type: string required: - secretRef @@ -705,7 +915,7 @@ spec: - hbase - elasticsearch - qdrant - - couchbase + - couchbase.online - milvus type: string required: @@ -3311,6 +3521,218 @@ spec: description: FeastProject is the Feast project id. pattern: ^[A-Za-z0-9][A-Za-z0-9_]*$ type: string + feastProjectDir: + description: FeastProjectDir defines how to create the feast project + directory. + properties: + git: + description: GitCloneOptions describes how a clone should + be performed. + properties: + configs: + additionalProperties: + type: string + description: |- + Configs passed to git via `-c` + e.g. http.sslVerify: 'false' + OR 'url."https://api:\${TOKEN}@github.com/". + type: object + env: + items: + description: EnvVar represents an environment variable + present in a Container. + properties: + name: + description: Name of the environment variable. Must + be a C_IDENTIFIER. + type: string + value: + description: |- + Variable references $(VAR_NAME) are expanded + using the previously defined environment variables in the container and + any + type: string + valueFrom: + description: Source for the environment variable's + value. Cannot be used if value is not empty. + properties: + configMapKeyRef: + description: Selects a key of a ConfigMap. + properties: + key: + description: The key to select. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. + type: string + optional: + description: Specify whether the ConfigMap + or its key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + fieldRef: + description: 'Selects a field of the pod: supports + metadata.name, metadata.namespace, `metadata.labels['''']`, + `metadata.' + properties: + apiVersion: + description: Version of the schema the FieldPath + is written in terms of, defaults to "v1". + type: string + fieldPath: + description: Path of the field to select + in the specified API version. + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + resourceFieldRef: + description: |- + Selects a resource of the container: only resources limits and requests + (limits.cpu, limits.memory, limits. + properties: + containerName: + description: 'Container name: required for + volumes, optional for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output format + of the exposed resources, defaults to + "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource to select' + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + secretKeyRef: + description: Selects a key of a secret in the + pod's namespace + properties: + key: + description: The key of the secret to select + from. Must be a valid secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. + type: string + optional: + description: Specify whether the Secret + or its key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + required: + - name + type: object + type: array + envFrom: + items: + description: EnvFromSource represents the source of + a set of ConfigMaps + properties: + configMapRef: + description: The ConfigMap to select from + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. + type: string + optional: + description: Specify whether the ConfigMap must + be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + prefix: + description: An optional identifier to prepend to + each key in the ConfigMap. Must be a C_IDENTIFIER. + type: string + secretRef: + description: The Secret to select from + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. + type: string + optional: + description: Specify whether the Secret must + be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + type: object + type: array + featureRepoPath: + description: FeatureRepoPath is the relative path to the + feature repo subdirectory. Default is 'feature_repo'. + type: string + ref: + description: Reference to a branch / tag / commit + type: string + url: + description: The repository URL to clone from. + type: string + required: + - url + type: object + x-kubernetes-validations: + - message: RepoPath must be a file name only, with no slashes. + rule: 'has(self.featureRepoPath) ? !self.featureRepoPath.startsWith(''/'') + : true' + init: + description: FeastInitOptions defines how to run a `feast + init`. + properties: + minimal: + type: boolean + template: + description: Template for the created project + enum: + - local + - gcp + - aws + - snowflake + - spark + - postgres + - hbase + - cassandra + - hazelcast + - ikv + - couchbase + type: string + type: object + type: object + x-kubernetes-validations: + - message: One selection required between init or git. + rule: '[has(self.git), has(self.init)].exists_one(c, c)' services: description: FeatureStoreServices defines the desired feast services. An ephemeral onlineStore feature server is deployed by default. @@ -3478,6 +3900,7 @@ spec: - trino - athena - mssql + - couchbase.offline type: string required: - secretRef @@ -3939,7 +4362,7 @@ spec: - hbase - elasticsearch - qdrant - - couchbase + - couchbase.online - milvus type: string required: diff --git a/infra/feast-operator/config/default/related_image_fs_patch.yaml b/infra/feast-operator/config/default/related_image_fs_patch.yaml index 890e4a8f45c..8432798da45 100644 --- a/infra/feast-operator/config/default/related_image_fs_patch.yaml +++ b/infra/feast-operator/config/default/related_image_fs_patch.yaml @@ -2,4 +2,4 @@ path: "/spec/template/spec/containers/0/env/0" value: name: RELATED_IMAGE_FEATURE_SERVER - value: docker.io/feastdev/feature-server:0.46.0 + value: quay.io/feastdev/feature-server:0.47.0 diff --git a/infra/feast-operator/config/manager/kustomization.yaml b/infra/feast-operator/config/manager/kustomization.yaml index bdf2ea9398c..5460328a76f 100644 --- a/infra/feast-operator/config/manager/kustomization.yaml +++ b/infra/feast-operator/config/manager/kustomization.yaml @@ -4,5 +4,5 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization images: - name: controller - newName: feastdev/feast-operator - newTag: 0.46.0 + newName: quay.io/feastdev/feast-operator + newTag: 0.47.0 diff --git a/infra/feast-operator/config/overlays/odh/params.env b/infra/feast-operator/config/overlays/odh/params.env index 3e846e9ccc6..48aa6f9a9a9 100644 --- a/infra/feast-operator/config/overlays/odh/params.env +++ b/infra/feast-operator/config/overlays/odh/params.env @@ -1,2 +1,2 @@ -RELATED_IMAGE_FEAST_OPERATOR=docker.io/feastdev/feast-operator:0.46.0 -RELATED_IMAGE_FEATURE_SERVER=docker.io/feastdev/feature-server:0.46.0 \ No newline at end of file +RELATED_IMAGE_FEAST_OPERATOR=quay.io/feastdev/feast-operator:0.47.0 +RELATED_IMAGE_FEATURE_SERVER=quay.io/feastdev/feature-server:0.47.0 diff --git a/infra/feast-operator/config/samples/v1alpha1_featurestore_git.yaml b/infra/feast-operator/config/samples/v1alpha1_featurestore_git.yaml new file mode 100644 index 00000000000..7730ef88518 --- /dev/null +++ b/infra/feast-operator/config/samples/v1alpha1_featurestore_git.yaml @@ -0,0 +1,10 @@ +apiVersion: feast.dev/v1alpha1 +kind: FeatureStore +metadata: + name: sample-git +spec: + feastProject: credit_scoring_local + feastProjectDir: + git: + url: https://github.com/feast-dev/feast-credit-score-local-tutorial + ref: 598a270 diff --git a/infra/feast-operator/config/samples/v1alpha1_featurestore_git_repopath.yaml b/infra/feast-operator/config/samples/v1alpha1_featurestore_git_repopath.yaml new file mode 100644 index 00000000000..6519e1bf429 --- /dev/null +++ b/infra/feast-operator/config/samples/v1alpha1_featurestore_git_repopath.yaml @@ -0,0 +1,11 @@ +apiVersion: feast.dev/v1alpha1 +kind: FeatureStore +metadata: + name: sample-git-repopath +spec: + feastProject: feast_demo_odfv + feastProjectDir: + git: + url: https://github.com/feast-dev/feast-workshop + ref: e959053 + featureRepoPath: module_2/feature_repo diff --git a/infra/feast-operator/config/samples/v1alpha1_featurestore_git_token.yaml b/infra/feast-operator/config/samples/v1alpha1_featurestore_git_token.yaml new file mode 100644 index 00000000000..f16f503c8fb --- /dev/null +++ b/infra/feast-operator/config/samples/v1alpha1_featurestore_git_token.yaml @@ -0,0 +1,21 @@ +kind: Secret +apiVersion: v1 +metadata: + name: git-token +stringData: + TOKEN: xxxxxxxxxxx +--- +apiVersion: feast.dev/v1alpha1 +kind: FeatureStore +metadata: + name: sample-git-token +spec: + feastProject: private + feastProjectDir: + git: + configs: + 'url."https://api:${TOKEN}@github.com/".insteadOf': 'https://github.com/' + envFrom: + - secretRef: + name: git-token + url: 'https://github.com/user/private' diff --git a/infra/feast-operator/config/samples/v1alpha1_featurestore_init.yaml b/infra/feast-operator/config/samples/v1alpha1_featurestore_init.yaml new file mode 100644 index 00000000000..f2324eeab2d --- /dev/null +++ b/infra/feast-operator/config/samples/v1alpha1_featurestore_init.yaml @@ -0,0 +1,9 @@ +apiVersion: feast.dev/v1alpha1 +kind: FeatureStore +metadata: + name: sample-init +spec: + feastProject: sample_init + feastProjectDir: + init: + template: spark diff --git a/infra/feast-operator/config/samples/v1alpha1_featurestore_kubernetes_auth.yaml b/infra/feast-operator/config/samples/v1alpha1_featurestore_kubernetes_auth.yaml index 6751be60e9c..33225b2edfb 100644 --- a/infra/feast-operator/config/samples/v1alpha1_featurestore_kubernetes_auth.yaml +++ b/infra/feast-operator/config/samples/v1alpha1_featurestore_kubernetes_auth.yaml @@ -3,9 +3,18 @@ kind: FeatureStore metadata: name: sample-kubernetes-auth spec: - feastProject: my_project + feastProject: feast_rbac authz: kubernetes: roles: - - reader - - writer + - feast-writer + - feast-reader + services: + offlineStore: + server: {} + onlineStore: + server: {} + registry: + local: + server: {} + ui: {} diff --git a/infra/feast-operator/config/samples/v1alpha1_featurestore_postgres_db_volumes_ssl.yaml b/infra/feast-operator/config/samples/v1alpha1_featurestore_postgres_db_volumes_tls.yaml similarity index 84% rename from infra/feast-operator/config/samples/v1alpha1_featurestore_postgres_db_volumes_ssl.yaml rename to infra/feast-operator/config/samples/v1alpha1_featurestore_postgres_db_volumes_tls.yaml index 5988a5e942c..61add153716 100644 --- a/infra/feast-operator/config/samples/v1alpha1_featurestore_postgres_db_volumes_ssl.yaml +++ b/infra/feast-operator/config/samples/v1alpha1_featurestore_postgres_db_volumes_tls.yaml @@ -2,34 +2,33 @@ apiVersion: v1 kind: Secret metadata: name: postgres-secret - namespace: default labels: app: postgres stringData: - POSTGRES_DB: mydatabase + POSTGRES_DB: feast POSTGRES_USER: admin POSTGRES_PASSWORD: password + POSTGRES_HOST: postgresql.feast.svc.cluster.local --- apiVersion: v1 kind: Secret metadata: name: feast-data-stores - namespace: default stringData: sql: | - path: postgresql+psycopg://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgresql.default.svc.cluster.local:5432/${POSTGRES_DB}?sslmode=require&sslrootcert=/var/lib/postgresql/certs/ca.crt&sslcert=/var/lib/postgresql/certs/tls.crt&sslkey=/var/lib/postgresql/certs/tls.key + path: postgresql+psycopg://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${POSTGRES_HOST}:5432/${POSTGRES_DB}?sslmode=verify-full&sslrootcert=/var/lib/postgresql/certs/ca.crt&sslcert=/var/lib/postgresql/certs/tls.crt&sslkey=/var/lib/postgresql/certs/tls.key cache_ttl_seconds: 60 sqlalchemy_config_kwargs: echo: false pool_pre_ping: true postgres: | - host: postgresql.default.svc.cluster.local + host: ${POSTGRES_HOST} port: 5432 database: ${POSTGRES_DB} db_schema: public user: ${POSTGRES_USER} password: ${POSTGRES_PASSWORD} - sslmode: require + sslmode: verify-full sslkey_path: /var/lib/postgresql/certs/tls.key sslcert_path: /var/lib/postgresql/certs/tls.crt sslrootcert_path: /var/lib/postgresql/certs/ca.crt @@ -38,7 +37,6 @@ apiVersion: feast.dev/v1alpha1 kind: FeatureStore metadata: name: sample-db-ssl - namespace: default spec: feastProject: postgres_tls_sample services: diff --git a/infra/feast-operator/config/samples/v1alpha1_featurestore_postgres_tls_volumes_ca_env.yaml b/infra/feast-operator/config/samples/v1alpha1_featurestore_postgres_tls_volumes_ca_env.yaml new file mode 100644 index 00000000000..42e1ae4b4a6 --- /dev/null +++ b/infra/feast-operator/config/samples/v1alpha1_featurestore_postgres_tls_volumes_ca_env.yaml @@ -0,0 +1,84 @@ +apiVersion: v1 +kind: Secret +metadata: + name: postgres-secret + labels: + app: postgres +stringData: + POSTGRES_DB: feast + POSTGRES_USER: admin + POSTGRES_PASSWORD: password + POSTGRES_HOST: postgresql.feast.svc.cluster.local + FEAST_CA_CERT_FILE_PATH: /var/lib/postgresql/certs/ca.crt +--- +apiVersion: v1 +kind: Secret +metadata: + name: feast-data-stores +stringData: + sql: | + path: postgresql+psycopg://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${POSTGRES_HOST}:5432/${POSTGRES_DB}?sslmode=verify-full&sslrootcert=system&sslcert=/var/lib/postgresql/certs/tls.crt&sslkey=/var/lib/postgresql/certs/tls.key + cache_ttl_seconds: 60 + sqlalchemy_config_kwargs: + echo: false + pool_pre_ping: true + postgres: | + host: ${POSTGRES_HOST} + port: 5432 + database: ${POSTGRES_DB} + db_schema: public + user: ${POSTGRES_USER} + password: ${POSTGRES_PASSWORD} + sslmode: verify-full + sslkey_path: /var/lib/postgresql/certs/tls.key + sslcert_path: /var/lib/postgresql/certs/tls.crt + sslrootcert_path: system +--- +apiVersion: feast.dev/v1alpha1 +kind: FeatureStore +metadata: + name: sample-db-ssl +spec: + feastProject: postgres_tls_sample_env_ca + services: + volumes: + - name: postgres-certs + secret: + secretName: postgresql-client-certs + items: + - key: ca.crt + path: ca.crt + mode: 0644 # Readable by all, required by PostgreSQL + - key: tls.crt + path: tls.crt + mode: 0644 # Required for the client certificate + - key: tls.key + path: tls.key + mode: 0640 # Required for the private key + offlineStore: + persistence: + store: + type: postgres + secretRef: + name: feast-data-stores + onlineStore: + persistence: + store: + type: postgres + secretRef: + name: feast-data-stores + server: + volumeMounts: + - name: postgres-certs + mountPath: /var/lib/postgresql/certs + readOnly: true + envFrom: + - secretRef: + name: postgres-secret + registry: + local: + persistence: + store: + type: sql + secretRef: + name: feast-data-stores diff --git a/infra/feast-operator/dist/install.yaml b/infra/feast-operator/dist/install.yaml index 7ab4fc3b72b..29d8059e403 100644 --- a/infra/feast-operator/dist/install.yaml +++ b/infra/feast-operator/dist/install.yaml @@ -95,6 +95,215 @@ spec: description: FeastProject is the Feast project id. pattern: ^[A-Za-z0-9][A-Za-z0-9_]*$ type: string + feastProjectDir: + description: FeastProjectDir defines how to create the feast project + directory. + properties: + git: + description: GitCloneOptions describes how a clone should be performed. + properties: + configs: + additionalProperties: + type: string + description: |- + Configs passed to git via `-c` + e.g. http.sslVerify: 'false' + OR 'url."https://api:\${TOKEN}@github.com/". + type: object + env: + items: + description: EnvVar represents an environment variable present + in a Container. + properties: + name: + description: Name of the environment variable. Must + be a C_IDENTIFIER. + type: string + value: + description: |- + Variable references $(VAR_NAME) are expanded + using the previously defined environment variables in the container and + any + type: string + valueFrom: + description: Source for the environment variable's value. + Cannot be used if value is not empty. + properties: + configMapKeyRef: + description: Selects a key of a ConfigMap. + properties: + key: + description: The key to select. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. + type: string + optional: + description: Specify whether the ConfigMap or + its key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + fieldRef: + description: 'Selects a field of the pod: supports + metadata.name, metadata.namespace, `metadata.labels['''']`, + `metadata.' + properties: + apiVersion: + description: Version of the schema the FieldPath + is written in terms of, defaults to "v1". + type: string + fieldPath: + description: Path of the field to select in + the specified API version. + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + resourceFieldRef: + description: |- + Selects a resource of the container: only resources limits and requests + (limits.cpu, limits.memory, limits. + properties: + containerName: + description: 'Container name: required for volumes, + optional for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output format of + the exposed resources, defaults to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource to select' + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + secretKeyRef: + description: Selects a key of a secret in the pod's + namespace + properties: + key: + description: The key of the secret to select + from. Must be a valid secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. + type: string + optional: + description: Specify whether the Secret or its + key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + required: + - name + type: object + type: array + envFrom: + items: + description: EnvFromSource represents the source of a set + of ConfigMaps + properties: + configMapRef: + description: The ConfigMap to select from + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. + type: string + optional: + description: Specify whether the ConfigMap must + be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + prefix: + description: An optional identifier to prepend to each + key in the ConfigMap. Must be a C_IDENTIFIER. + type: string + secretRef: + description: The Secret to select from + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. + type: string + optional: + description: Specify whether the Secret must be + defined + type: boolean + type: object + x-kubernetes-map-type: atomic + type: object + type: array + featureRepoPath: + description: FeatureRepoPath is the relative path to the feature + repo subdirectory. Default is 'feature_repo'. + type: string + ref: + description: Reference to a branch / tag / commit + type: string + url: + description: The repository URL to clone from. + type: string + required: + - url + type: object + x-kubernetes-validations: + - message: RepoPath must be a file name only, with no slashes. + rule: 'has(self.featureRepoPath) ? !self.featureRepoPath.startsWith(''/'') + : true' + init: + description: FeastInitOptions defines how to run a `feast init`. + properties: + minimal: + type: boolean + template: + description: Template for the created project + enum: + - local + - gcp + - aws + - snowflake + - spark + - postgres + - hbase + - cassandra + - hazelcast + - ikv + - couchbase + type: string + type: object + type: object + x-kubernetes-validations: + - message: One selection required between init or git. + rule: '[has(self.git), has(self.init)].exists_one(c, c)' services: description: FeatureStoreServices defines the desired feast services. An ephemeral onlineStore feature server is deployed by default. @@ -260,6 +469,7 @@ spec: - trino - athena - mssql + - couchbase.offline type: string required: - secretRef @@ -713,7 +923,7 @@ spec: - hbase - elasticsearch - qdrant - - couchbase + - couchbase.online - milvus type: string required: @@ -3319,6 +3529,218 @@ spec: description: FeastProject is the Feast project id. pattern: ^[A-Za-z0-9][A-Za-z0-9_]*$ type: string + feastProjectDir: + description: FeastProjectDir defines how to create the feast project + directory. + properties: + git: + description: GitCloneOptions describes how a clone should + be performed. + properties: + configs: + additionalProperties: + type: string + description: |- + Configs passed to git via `-c` + e.g. http.sslVerify: 'false' + OR 'url."https://api:\${TOKEN}@github.com/". + type: object + env: + items: + description: EnvVar represents an environment variable + present in a Container. + properties: + name: + description: Name of the environment variable. Must + be a C_IDENTIFIER. + type: string + value: + description: |- + Variable references $(VAR_NAME) are expanded + using the previously defined environment variables in the container and + any + type: string + valueFrom: + description: Source for the environment variable's + value. Cannot be used if value is not empty. + properties: + configMapKeyRef: + description: Selects a key of a ConfigMap. + properties: + key: + description: The key to select. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. + type: string + optional: + description: Specify whether the ConfigMap + or its key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + fieldRef: + description: 'Selects a field of the pod: supports + metadata.name, metadata.namespace, `metadata.labels['''']`, + `metadata.' + properties: + apiVersion: + description: Version of the schema the FieldPath + is written in terms of, defaults to "v1". + type: string + fieldPath: + description: Path of the field to select + in the specified API version. + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + resourceFieldRef: + description: |- + Selects a resource of the container: only resources limits and requests + (limits.cpu, limits.memory, limits. + properties: + containerName: + description: 'Container name: required for + volumes, optional for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output format + of the exposed resources, defaults to + "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource to select' + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + secretKeyRef: + description: Selects a key of a secret in the + pod's namespace + properties: + key: + description: The key of the secret to select + from. Must be a valid secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. + type: string + optional: + description: Specify whether the Secret + or its key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + required: + - name + type: object + type: array + envFrom: + items: + description: EnvFromSource represents the source of + a set of ConfigMaps + properties: + configMapRef: + description: The ConfigMap to select from + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. + type: string + optional: + description: Specify whether the ConfigMap must + be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + prefix: + description: An optional identifier to prepend to + each key in the ConfigMap. Must be a C_IDENTIFIER. + type: string + secretRef: + description: The Secret to select from + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. + type: string + optional: + description: Specify whether the Secret must + be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + type: object + type: array + featureRepoPath: + description: FeatureRepoPath is the relative path to the + feature repo subdirectory. Default is 'feature_repo'. + type: string + ref: + description: Reference to a branch / tag / commit + type: string + url: + description: The repository URL to clone from. + type: string + required: + - url + type: object + x-kubernetes-validations: + - message: RepoPath must be a file name only, with no slashes. + rule: 'has(self.featureRepoPath) ? !self.featureRepoPath.startsWith(''/'') + : true' + init: + description: FeastInitOptions defines how to run a `feast + init`. + properties: + minimal: + type: boolean + template: + description: Template for the created project + enum: + - local + - gcp + - aws + - snowflake + - spark + - postgres + - hbase + - cassandra + - hazelcast + - ikv + - couchbase + type: string + type: object + type: object + x-kubernetes-validations: + - message: One selection required between init or git. + rule: '[has(self.git), has(self.init)].exists_one(c, c)' services: description: FeatureStoreServices defines the desired feast services. An ephemeral onlineStore feature server is deployed by default. @@ -3486,6 +3908,7 @@ spec: - trino - athena - mssql + - couchbase.offline type: string required: - secretRef @@ -3947,7 +4370,7 @@ spec: - hbase - elasticsearch - qdrant - - couchbase + - couchbase.online - milvus type: string required: @@ -6933,8 +7356,8 @@ spec: - /manager env: - name: RELATED_IMAGE_FEATURE_SERVER - value: docker.io/feastdev/feature-server:0.46.0 - image: feastdev/feast-operator:0.46.0 + value: quay.io/feastdev/feature-server:0.47.0 + image: quay.io/feastdev/feast-operator:0.47.0 livenessProbe: httpGet: path: /healthz diff --git a/infra/feast-operator/docs/api/markdown/ref.md b/infra/feast-operator/docs/api/markdown/ref.md index 7a4fe04c599..aefb5abca76 100644 --- a/infra/feast-operator/docs/api/markdown/ref.md +++ b/infra/feast-operator/docs/api/markdown/ref.md @@ -61,6 +61,36 @@ _Appears in:_ | `image` _string_ | | +#### FeastInitOptions + + + +FeastInitOptions defines how to run a `feast init`. + +_Appears in:_ +- [FeastProjectDir](#feastprojectdir) + +| Field | Description | +| --- | --- | +| `minimal` _boolean_ | | +| `template` _string_ | Template for the created project | + + +#### FeastProjectDir + + + +FeastProjectDir defines how to create the feast project directory. + +_Appears in:_ +- [FeatureStoreSpec](#featurestorespec) + +| Field | Description | +| --- | --- | +| `git` _[GitCloneOptions](#gitcloneoptions)_ | | +| `init` _[FeastInitOptions](#feastinitoptions)_ | | + + #### FeatureStore @@ -126,6 +156,7 @@ _Appears in:_ | Field | Description | | --- | --- | | `feastProject` _string_ | FeastProject is the Feast project id. This can be any alphanumeric string with underscores, but it cannot start with an underscore. Required. | +| `feastProjectDir` _[FeastProjectDir](#feastprojectdir)_ | | | `services` _[FeatureStoreServices](#featurestoreservices)_ | | | `authz` _[AuthzConfig](#authzconfig)_ | | @@ -149,6 +180,27 @@ _Appears in:_ | `serviceHostnames` _[ServiceHostnames](#servicehostnames)_ | | +#### GitCloneOptions + + + +GitCloneOptions describes how a clone should be performed. + +_Appears in:_ +- [FeastProjectDir](#feastprojectdir) + +| Field | Description | +| --- | --- | +| `url` _string_ | The repository URL to clone from. | +| `ref` _string_ | Reference to a branch / tag / commit | +| `configs` _object (keys:string, values:string)_ | Configs passed to git via `-c` +e.g. http.sslVerify: 'false' +OR 'url."https://api:\${TOKEN}@github.com/".insteadOf': 'https://github.com/' | +| `featureRepoPath` _string_ | FeatureRepoPath is the relative path to the feature repo subdirectory. Default is 'feature_repo'. | +| `env` _[EnvVar](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.30/#envvar-v1-core)_ | | +| `envFrom` _[EnvFromSource](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.30/#envfromsource-v1-core)_ | | + + #### KubernetesAuthz diff --git a/infra/feast-operator/internal/controller/featurestore_controller_test.go b/infra/feast-operator/internal/controller/featurestore_controller_test.go index 23d93903d16..c73fbe1bff3 100644 --- a/infra/feast-operator/internal/controller/featurestore_controller_test.go +++ b/infra/feast-operator/internal/controller/featurestore_controller_test.go @@ -205,6 +205,7 @@ var _ = Describe("FeatureStore Controller", func() { Expect(controllerutil.HasControllerReference(deploy)).To(BeTrue()) Expect(deploy.Spec.Template.Spec.ServiceAccountName).To(Equal(deploy.Name)) Expect(deploy.Spec.Template.Spec.InitContainers).To(HaveLen(1)) + Expect(deploy.Spec.Template.Spec.InitContainers[0].Args[0]).To(ContainSubstring("feast init")) Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(1)) svc := &corev1.Service{} @@ -216,6 +217,74 @@ var _ = Describe("FeatureStore Controller", func() { Expect(err).NotTo(HaveOccurred()) Expect(controllerutil.HasControllerReference(svc)).To(BeTrue()) Expect(svc.Spec.Ports[0].TargetPort).To(Equal(intstr.FromInt(int(services.FeastServiceConstants[services.OnlineFeastType].TargetHttpPort)))) + + // change projectDir to use a git repo + featureRepoPath := "test/dir/feature_repo2" + ref := "xxxxx" + envVars := []corev1.EnvVar{ + { + Name: "test", + Value: "value", + }, + } + resource.Spec.FeastProjectDir = &feastdevv1alpha1.FeastProjectDir{ + Git: &feastdevv1alpha1.GitCloneOptions{ + URL: "test", + Ref: ref, + FeatureRepoPath: featureRepoPath, + Configs: map[string]string{ + "http.sslVerify": "false", + }, + Env: &envVars, + }, + } + err = k8sClient.Update(ctx, resource) + Expect(err).NotTo(HaveOccurred()) + _, err = controllerReconciler.Reconcile(ctx, reconcile.Request{ + NamespacedName: typeNamespacedName, + }) + Expect(err).NotTo(HaveOccurred()) + + err = k8sClient.Get(ctx, typeNamespacedName, resource) + Expect(err).NotTo(HaveOccurred()) + + err = k8sClient.Get(ctx, types.NamespacedName{ + Name: objMeta.Name, + Namespace: objMeta.Namespace, + }, deploy) + Expect(err).NotTo(HaveOccurred()) + Expect(deploy.Spec.Template.Spec.InitContainers).To(HaveLen(1)) + Expect(deploy.Spec.Template.Spec.InitContainers[0].Args[0]).To(ContainSubstring("git -c http.sslVerify=false clone")) + Expect(deploy.Spec.Template.Spec.InitContainers[0].Args[0]).To(ContainSubstring("git checkout " + ref)) + Expect(deploy.Spec.Template.Spec.InitContainers[0].Args[0]).To(ContainSubstring(featureRepoPath)) + Expect(deploy.Spec.Template.Spec.InitContainers[0].Env).To(ContainElements(envVars)) + + online := services.GetOnlineContainer(*deploy) + Expect(online.WorkingDir).To(Equal(services.EphemeralPath + "/" + resource.Spec.FeastProject + "/" + featureRepoPath)) + + // change projectDir to use an init template + resource.Spec.FeastProjectDir = &feastdevv1alpha1.FeastProjectDir{ + Init: &feastdevv1alpha1.FeastInitOptions{ + Template: "spark", + }, + } + err = k8sClient.Update(ctx, resource) + Expect(err).NotTo(HaveOccurred()) + _, err = controllerReconciler.Reconcile(ctx, reconcile.Request{ + NamespacedName: typeNamespacedName, + }) + Expect(err).NotTo(HaveOccurred()) + + err = k8sClient.Get(ctx, typeNamespacedName, resource) + Expect(err).NotTo(HaveOccurred()) + + err = k8sClient.Get(ctx, types.NamespacedName{ + Name: objMeta.Name, + Namespace: objMeta.Namespace, + }, deploy) + Expect(err).NotTo(HaveOccurred()) + Expect(deploy.Spec.Template.Spec.InitContainers).To(HaveLen(1)) + Expect(deploy.Spec.Template.Spec.InitContainers[0].Args[0]).To(ContainSubstring("feast init -t spark")) }) It("should properly encode a feature_store.yaml config", func() { diff --git a/infra/feast-operator/internal/controller/services/services.go b/infra/feast-operator/internal/controller/services/services.go index 9380415965f..d6d943568d7 100644 --- a/infra/feast-operator/internal/controller/services/services.go +++ b/infra/feast-operator/internal/controller/services/services.go @@ -390,7 +390,7 @@ func (feast *FeastServices) setContainer(containers *[]corev1.Container, feastTy container := &corev1.Container{ Name: string(feastType), Image: *defaultCtrConfigs.Image, - WorkingDir: getOfflineMountPath(feast.Handler.FeatureStore) + "/" + feast.Handler.FeatureStore.Status.Applied.FeastProject + FeatureRepoDir, + WorkingDir: feast.getFeatureRepoDir(), Command: feast.getContainerCommand(feastType), Ports: []corev1.ContainerPort{ { @@ -507,11 +507,12 @@ func (feast *FeastServices) getDeploymentStrategy() appsv1.DeploymentStrategy { } func (feast *FeastServices) setInitContainer(podSpec *corev1.PodSpec, fsYamlB64 string) { - if !feast.Handler.FeatureStore.Status.Applied.Services.DisableInitContainers { - feastProject := feast.Handler.FeatureStore.Status.Applied.FeastProject - feastRepoDir := feastProject + FeatureRepoDir + applied := feast.Handler.FeatureStore.Status.Applied + if applied.FeastProjectDir != nil && !applied.Services.DisableInitContainers { + feastProjectDir := applied.FeastProjectDir workingDir := getOfflineMountPath(feast.Handler.FeatureStore) - podSpec.InitContainers = append(podSpec.InitContainers, corev1.Container{ + projectPath := workingDir + "/" + applied.FeastProject + container := corev1.Container{ Name: "feast-init", Image: getFeatureServerImage(), Env: []corev1.EnvVar{ @@ -520,13 +521,48 @@ func (feast *FeastServices) setInitContainer(podSpec *corev1.PodSpec, fsYamlB64 Value: fsYamlB64, }, }, - Command: []string{"/bin/sh", "-c"}, - Args: []string{"echo \"Starting feast initialization job...\";\n[ -d " + - feastRepoDir + " ] || feast init " + feastProject + ";\necho $" + - TmpFeatureStoreYamlEnvVar + " | base64 -d \u003e " + workingDir + "/" + feastRepoDir + - "/feature_store.yaml;\necho \"Feast initialization complete\";\n"}, + Command: []string{"bash", "-c"}, WorkingDir: workingDir, - }) + } + + var createCommand string + if feastProjectDir.Init != nil { + initSlice := []string{"feast", "init"} + if feastProjectDir.Init.Minimal { + initSlice = append(initSlice, "-m") + } + if len(feastProjectDir.Init.Template) > 0 { + initSlice = append(initSlice, "-t", feastProjectDir.Init.Template) + } + initSlice = append(initSlice, applied.FeastProject) + createCommand = strings.Join(initSlice, " ") + } else if feastProjectDir.Git != nil { + gitSlice := []string{"git"} + for key, value := range feastProjectDir.Git.Configs { + gitSlice = append(gitSlice, "-c", key+"="+value) + } + gitSlice = append(gitSlice, "clone", feastProjectDir.Git.URL, projectPath) + + if len(feastProjectDir.Git.Ref) > 0 { + gitSlice = append(gitSlice, "&&", "cd "+projectPath, "&&", "git checkout "+feastProjectDir.Git.Ref) + } + createCommand = strings.Join(gitSlice, " ") + + if feastProjectDir.Git.Env != nil { + container.Env = envOverride(container.Env, *feastProjectDir.Git.Env) + } + if feastProjectDir.Git.EnvFrom != nil { + container.EnvFrom = *feastProjectDir.Git.EnvFrom + } + } + + featureRepoDir := feast.getFeatureRepoDir() + container.Args = []string{ + "echo \"Creating feast repository...\"\necho '" + createCommand + "'\n" + + "if [[ ! -d " + featureRepoDir + " ]]; then " + createCommand + "; fi;\n" + + "echo $" + TmpFeatureStoreYamlEnvVar + " | base64 -d \u003e " + featureRepoDir + "/feature_store.yaml;\necho \"Feast repo creation complete\";\n", + } + podSpec.InitContainers = append(podSpec.InitContainers, container) } } @@ -895,6 +931,15 @@ func (feast *FeastServices) mountEmptyDirVolumes(podSpec *corev1.PodSpec) { } } +func (feast *FeastServices) getFeatureRepoDir() string { + applied := feast.Handler.FeatureStore.Status.Applied + feastProjectDir := getOfflineMountPath(feast.Handler.FeatureStore) + "/" + applied.FeastProject + if applied.FeastProjectDir != nil && applied.FeastProjectDir.Git != nil && len(applied.FeastProjectDir.Git.FeatureRepoPath) > 0 { + return feastProjectDir + "/" + applied.FeastProjectDir.Git.FeatureRepoPath + } + return feastProjectDir + "/" + FeatureRepoDir +} + func mountEmptyDirVolume(podSpec *corev1.PodSpec) { if podSpec != nil { volName := strings.TrimPrefix(EphemeralPath, "/") diff --git a/infra/feast-operator/internal/controller/services/services_types.go b/infra/feast-operator/internal/controller/services/services_types.go index 32ff95588ae..b81860ff464 100644 --- a/infra/feast-operator/internal/controller/services/services_types.go +++ b/infra/feast-operator/internal/controller/services/services_types.go @@ -29,7 +29,7 @@ const ( feastServerImageVar = "RELATED_IMAGE_FEATURE_SERVER" FeatureStoreYamlCmKey = "feature_store.yaml" EphemeralPath = "/feast-data" - FeatureRepoDir = "/feature_repo" + FeatureRepoDir = "feature_repo" DefaultRegistryPath = "registry.db" DefaultOnlineStorePath = "online_store.db" svcDomain = ".svc.cluster.local" @@ -83,7 +83,7 @@ const ( ) var ( - DefaultImage = "feastdev/feature-server:" + feastversion.FeastVersion + DefaultImage = "quay.io/feastdev/feature-server:" + feastversion.FeastVersion DefaultReplicas = int32(1) DefaultPVCAccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce} NameLabelKey = feastdevv1alpha1.GroupVersion.Group + "/name" diff --git a/infra/feast-operator/internal/controller/services/util.go b/infra/feast-operator/internal/controller/services/util.go index 82dc65e14b5..41f3e837157 100644 --- a/infra/feast-operator/internal/controller/services/util.go +++ b/infra/feast-operator/internal/controller/services/util.go @@ -90,6 +90,11 @@ func ApplyDefaultsToStatus(cr *feastdevv1alpha1.FeatureStore) { cr.Status.FeastVersion = feastversion.FeastVersion applied := &cr.Status.Applied + if applied.FeastProjectDir == nil { + applied.FeastProjectDir = &feastdevv1alpha1.FeastProjectDir{ + Init: &feastdevv1alpha1.FeastInitOptions{}, + } + } if applied.Services == nil { applied.Services = &feastdevv1alpha1.FeatureStoreServices{} } diff --git a/infra/feast-operator/test/e2e/e2e_test.go b/infra/feast-operator/test/e2e/e2e_test.go index b2773f579f0..d1051900ae5 100644 --- a/infra/feast-operator/test/e2e/e2e_test.go +++ b/infra/feast-operator/test/e2e/e2e_test.go @@ -17,217 +17,38 @@ limitations under the License. package e2e import ( - "fmt" - "os" - "os/exec" - "time" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - "github.com/feast-dev/feast/infra/feast-operator/test/utils" -) - -const ( - feastControllerNamespace = "feast-operator-system" - timeout = 2 * time.Minute - controllerDeploymentName = "feast-operator-controller-manager" - feastPrefix = "feast-" + . "github.com/onsi/ginkgo/v2" ) var _ = Describe("controller", Ordered, func() { - BeforeAll(func() { - _, isRunOnOpenShiftCI := os.LookupEnv("RUN_ON_OPENSHIFT_CI") - if !isRunOnOpenShiftCI { - By("creating manager namespace") - cmd := exec.Command("kubectl", "create", "ns", feastControllerNamespace) - _, _ = utils.Run(cmd) - - var err error - // projectimage stores the name of the image used in the example - var projectimage = "localhost/feast-operator:v0.0.1" - - By("building the manager(Operator) image") - cmd = exec.Command("make", "docker-build", fmt.Sprintf("IMG=%s", projectimage)) - _, err = utils.Run(cmd) - ExpectWithOffset(1, err).NotTo(HaveOccurred()) - - By("loading the the manager(Operator) image on Kind") - err = utils.LoadImageToKindClusterWithName(projectimage) - ExpectWithOffset(1, err).NotTo(HaveOccurred()) - - // this image will be built in above make target. - var feastImage = "feastdev/feature-server:dev" - var feastLocalImage = "localhost/feastdev/feature-server:dev" - - By("building the feast image") - cmd = exec.Command("make", "feast-ci-dev-docker-img") - _, err = utils.Run(cmd) - ExpectWithOffset(1, err).NotTo(HaveOccurred()) - - By("Tag the local feast image for the integration tests") - cmd = exec.Command("docker", "image", "tag", feastImage, feastLocalImage) - _, err = utils.Run(cmd) - ExpectWithOffset(1, err).NotTo(HaveOccurred()) - - By("loading the the feast image on Kind cluster") - err = utils.LoadImageToKindClusterWithName(feastLocalImage) - ExpectWithOffset(1, err).NotTo(HaveOccurred()) + featureStoreName := "simple-feast-setup" + feastResourceName := utils.FeastPrefix + featureStoreName + feastK8sResourceNames := []string{ + feastResourceName + "-online", + feastResourceName + "-offline", + feastResourceName + "-ui", + } - By("installing CRDs") - cmd = exec.Command("make", "install") - _, err = utils.Run(cmd) - ExpectWithOffset(1, err).NotTo(HaveOccurred()) + runTestDeploySimpleCRFunc := utils.GetTestDeploySimpleCRFunc("/test/e2e", + "test/testdata/feast_integration_test_crs/v1alpha1_default_featurestore.yaml", + featureStoreName, feastResourceName, feastK8sResourceNames) - By("deploying the controller-manager") - cmd = exec.Command("make", "deploy", fmt.Sprintf("IMG=%s", projectimage), fmt.Sprintf("FS_IMG=%s", feastLocalImage)) - _, err = utils.Run(cmd) - ExpectWithOffset(1, err).NotTo(HaveOccurred()) + runTestWithRemoteRegistryFunction := utils.GetTestWithRemoteRegistryFunc("/test/e2e", + "test/testdata/feast_integration_test_crs/v1alpha1_default_featurestore.yaml", + "test/testdata/feast_integration_test_crs/v1alpha1_remote_registry_featurestore.yaml", + featureStoreName, feastResourceName, feastK8sResourceNames) - By("Validating that the controller-manager deployment is in available state") - err = checkIfDeploymentExistsAndAvailable(feastControllerNamespace, controllerDeploymentName, timeout) - Expect(err).ToNot(HaveOccurred(), fmt.Sprintf( - "Deployment %s is not available but expected to be available. \nError: %v\n", - controllerDeploymentName, err, - )) - fmt.Printf("Feast Control Manager Deployment %s is available\n", controllerDeploymentName) - } + BeforeAll(func() { + utils.DeployOperatorFromCode("/test/e2e", false) }) AfterAll(func() { - // Add any post clean up code here. - _, isRunOnOpenShiftCI := os.LookupEnv("RUN_ON_OPENSHIFT_CI") - if !isRunOnOpenShiftCI { - By("Uninstalling the feast CRD") - cmd := exec.Command("kubectl", "delete", "deployment", controllerDeploymentName, "-n", feastControllerNamespace) - _, err := utils.Run(cmd) - ExpectWithOffset(1, err).NotTo(HaveOccurred()) - } + utils.DeleteOperatorDeployment("/test/e2e") }) Context("Operator E2E Tests", func() { - It("Should be able to deploy and run a default feature store CR successfully", func() { - By("deploying the Simple Feast Custom Resource to Kubernetes") - namespace := "default" - - cmd := exec.Command("kubectl", "apply", "-f", - "test/testdata/feast_integration_test_crs/v1alpha1_default_featurestore.yaml", "-n", namespace) - _, cmdOutputerr := utils.Run(cmd) - ExpectWithOffset(1, cmdOutputerr).NotTo(HaveOccurred()) - - featureStoreName := "simple-feast-setup" - validateTheFeatureStoreCustomResource(namespace, featureStoreName, timeout) - - By("deleting the feast deployment") - cmd = exec.Command("kubectl", "delete", "-f", - "test/testdata/feast_integration_test_crs/v1alpha1_default_featurestore.yaml") - _, cmdOutputerr = utils.Run(cmd) - ExpectWithOffset(1, cmdOutputerr).NotTo(HaveOccurred()) - }) - - It("Should be able to deploy and run a feature store with remote registry CR successfully", func() { - By("deploying the Simple Feast Custom Resource to Kubernetes") - namespace := "default" - cmd := exec.Command("kubectl", "apply", "-f", - "test/testdata/feast_integration_test_crs/v1alpha1_default_featurestore.yaml", "-n", namespace) - _, cmdOutputErr := utils.Run(cmd) - ExpectWithOffset(1, cmdOutputErr).NotTo(HaveOccurred()) - - featureStoreName := "simple-feast-setup" - validateTheFeatureStoreCustomResource(namespace, featureStoreName, timeout) - - var remoteRegistryNs = "remote-registry" - By(fmt.Sprintf("Creating the remote registry namespace=%s", remoteRegistryNs)) - cmd = exec.Command("kubectl", "create", "ns", remoteRegistryNs) - _, _ = utils.Run(cmd) - - By("deploying the Simple Feast remote registry Custom Resource on Kubernetes") - cmd = exec.Command("kubectl", "apply", "-f", - "test/testdata/feast_integration_test_crs/v1alpha1_remote_registry_featurestore.yaml", "-n", remoteRegistryNs) - _, cmdOutputErr = utils.Run(cmd) - ExpectWithOffset(1, cmdOutputErr).NotTo(HaveOccurred()) - - remoteFeatureStoreName := "simple-feast-remote-setup" - - validateTheFeatureStoreCustomResource(remoteRegistryNs, remoteFeatureStoreName, timeout) - - By("deleting the feast remote registry deployment") - cmd = exec.Command("kubectl", "delete", "-f", - "test/testdata/feast_integration_test_crs/v1alpha1_remote_registry_featurestore.yaml", "-n", remoteRegistryNs) - _, cmdOutputErr = utils.Run(cmd) - ExpectWithOffset(1, cmdOutputErr).NotTo(HaveOccurred()) - - By("deleting the feast deployment") - cmd = exec.Command("kubectl", "delete", "-f", - "test/testdata/feast_integration_test_crs/v1alpha1_default_featurestore.yaml", "-n", namespace) - _, cmdOutputErr = utils.Run(cmd) - ExpectWithOffset(1, cmdOutputErr).NotTo(HaveOccurred()) - }) + It("Should be able to deploy and run a default feature store CR successfully", runTestDeploySimpleCRFunc) + It("Should be able to deploy and run a feature store with remote registry CR successfully", runTestWithRemoteRegistryFunction) }) }) - -func validateTheFeatureStoreCustomResource(namespace string, featureStoreName string, timeout time.Duration) { - hasRemoteRegistry, err := isFeatureStoreHavingRemoteRegistry(namespace, featureStoreName) - Expect(err).ToNot(HaveOccurred(), fmt.Sprintf( - "Error occurred while checking FeatureStore %s is having remote registry or not. \nError: %v\n", - featureStoreName, err)) - - feastResourceName := feastPrefix + featureStoreName - k8sResourceNames := []string{feastResourceName} - feastK8sResourceNames := []string{ - feastResourceName + "-online", - feastResourceName + "-offline", - feastResourceName + "-ui", - } - - if !hasRemoteRegistry { - feastK8sResourceNames = append(feastK8sResourceNames, feastResourceName+"-registry") - } - - for _, deploymentName := range k8sResourceNames { - By(fmt.Sprintf("validate the feast deployment: %s is up and in availability state.", deploymentName)) - err = checkIfDeploymentExistsAndAvailable(namespace, deploymentName, timeout) - Expect(err).ToNot(HaveOccurred(), fmt.Sprintf( - "Deployment %s is not available but expected to be available. \nError: %v\n", - deploymentName, err, - )) - fmt.Printf("Feast Deployment %s is available\n", deploymentName) - } - - By("Check if the feast client - kubernetes config map exists.") - configMapName := feastResourceName + "-client" - err = checkIfConfigMapExists(namespace, configMapName) - Expect(err).ToNot(HaveOccurred(), fmt.Sprintf( - "config map %s is not available but expected to be available. \nError: %v\n", - configMapName, err, - )) - fmt.Printf("Feast Deployment client config map %s is available\n", configMapName) - - for _, serviceAccountName := range k8sResourceNames { - By(fmt.Sprintf("validate the feast service account: %s is available.", serviceAccountName)) - err = checkIfServiceAccountExists(namespace, serviceAccountName) - Expect(err).ToNot(HaveOccurred(), fmt.Sprintf( - "Service account %s does not exist in namespace %s. Error: %v", - serviceAccountName, namespace, err, - )) - fmt.Printf("Service account %s exists in namespace %s\n", serviceAccountName, namespace) - } - - for _, serviceName := range feastK8sResourceNames { - By(fmt.Sprintf("validate the kubernetes service name: %s is available.", serviceName)) - err = checkIfKubernetesServiceExists(namespace, serviceName) - Expect(err).ToNot(HaveOccurred(), fmt.Sprintf( - "kubernetes service %s is not available but expected to be available. \nError: %v\n", - serviceName, err, - )) - fmt.Printf("kubernetes service %s is available\n", serviceName) - } - - By(fmt.Sprintf("Checking FeatureStore customer resource: %s is in Ready Status.", featureStoreName)) - err = checkIfFeatureStoreCustomResourceConditionsInReady(featureStoreName, namespace) - Expect(err).ToNot(HaveOccurred(), fmt.Sprintf( - "FeatureStore custom resource %s all conditions are not in ready state. \nError: %v\n", - featureStoreName, err, - )) - fmt.Printf("FeatureStore custom resource %s conditions are in Ready State\n", featureStoreName) -} diff --git a/infra/feast-operator/test/e2e/test_util.go b/infra/feast-operator/test/e2e/test_util.go deleted file mode 100644 index 743f04afc54..00000000000 --- a/infra/feast-operator/test/e2e/test_util.go +++ /dev/null @@ -1,209 +0,0 @@ -package e2e - -import ( - "bytes" - "encoding/json" - "errors" - "fmt" - "os/exec" - "strings" - "time" - - appsv1 "k8s.io/api/apps/v1" - - "github.com/feast-dev/feast/infra/feast-operator/api/v1alpha1" -) - -// dynamically checks if all conditions of custom resource featurestore are in "Ready" state. -func checkIfFeatureStoreCustomResourceConditionsInReady(featureStoreName, namespace string) error { - cmd := exec.Command("kubectl", "get", "featurestore", featureStoreName, "-n", namespace, "-o", "json") - - var out bytes.Buffer - var stderr bytes.Buffer - cmd.Stdout = &out - cmd.Stderr = &stderr - - if err := cmd.Run(); err != nil { - return fmt.Errorf("failed to get resource %s in namespace %s. Error: %v. Stderr: %s", - featureStoreName, namespace, err, stderr.String()) - } - - // Parse the JSON into FeatureStore - var resource v1alpha1.FeatureStore - if err := json.Unmarshal(out.Bytes(), &resource); err != nil { - return fmt.Errorf("failed to parse the resource JSON. Error: %v", err) - } - - // Validate all conditions - for _, condition := range resource.Status.Conditions { - if condition.Status != "True" { - return fmt.Errorf(" FeatureStore=%s condition '%s' is not in 'Ready' state. Status: %s", - featureStoreName, condition.Type, condition.Status) - } - } - - return nil -} - -// validates if a deployment exists and also in the availability state as True. -func checkIfDeploymentExistsAndAvailable(namespace string, deploymentName string, timeout time.Duration) error { - var output, errOutput bytes.Buffer - - ticker := time.NewTicker(2 * time.Second) - defer ticker.Stop() - - timeoutChan := time.After(timeout) - - for { - select { - case <-timeoutChan: - return fmt.Errorf("timed out waiting for deployment %s to become available", deploymentName) - case <-ticker.C: - // Run kubectl command - cmd := exec.Command("kubectl", "get", "deployment", deploymentName, "-n", namespace, "-o", "json") - cmd.Stdout = &output - cmd.Stderr = &errOutput - - if err := cmd.Run(); err != nil { - // Log error and retry - fmt.Printf("Deployment not yet found, we may try again to find the updated status: %s\n", errOutput.String()) - continue - } - - // Parse the JSON output into Deployment - var result appsv1.Deployment - if err := json.Unmarshal(output.Bytes(), &result); err != nil { - return fmt.Errorf("failed to parse deployment JSON: %v", err) - } - - // Check for Available condition - for _, condition := range result.Status.Conditions { - if condition.Type == "Available" && condition.Status == "True" { - return nil // Deployment is available - } - } - - // Reset buffers for the next loop iteration - output.Reset() - errOutput.Reset() - } - } -} - -// validates if a service account exists using the kubectl CLI. -func checkIfServiceAccountExists(namespace, saName string) error { - cmd := exec.Command("kubectl", "get", "sa", saName, "-n", namespace) - - var out bytes.Buffer - var stderr bytes.Buffer - cmd.Stdout = &out - cmd.Stderr = &stderr - - if err := cmd.Run(); err != nil { - return fmt.Errorf("failed to find service account %s in namespace %s. Error: %v. Stderr: %s", - saName, namespace, err, stderr.String()) - } - - // Check the output to confirm presence - if !strings.Contains(out.String(), saName) { - return fmt.Errorf("service account %s not found in namespace %s", saName, namespace) - } - - return nil -} - -// validates if a config map exists using the kubectl CLI. -func checkIfConfigMapExists(namespace, configMapName string) error { - cmd := exec.Command("kubectl", "get", "cm", configMapName, "-n", namespace) - - var out bytes.Buffer - var stderr bytes.Buffer - cmd.Stdout = &out - cmd.Stderr = &stderr - - if err := cmd.Run(); err != nil { - return fmt.Errorf("failed to find config map %s in namespace %s. Error: %v. Stderr: %s", - configMapName, namespace, err, stderr.String()) - } - - // Check the output to confirm presence - if !strings.Contains(out.String(), configMapName) { - return fmt.Errorf("config map %s not found in namespace %s", configMapName, namespace) - } - - return nil -} - -// validates if a kubernetes service exists using the kubectl CLI. -func checkIfKubernetesServiceExists(namespace, serviceName string) error { - cmd := exec.Command("kubectl", "get", "service", serviceName, "-n", namespace) - - var out bytes.Buffer - var stderr bytes.Buffer - cmd.Stdout = &out - cmd.Stderr = &stderr - - if err := cmd.Run(); err != nil { - return fmt.Errorf("failed to find kubernetes service %s in namespace %s. Error: %v. Stderr: %s", - serviceName, namespace, err, stderr.String()) - } - - // Check the output to confirm presence - if !strings.Contains(out.String(), serviceName) { - return fmt.Errorf("kubernetes service %s not found in namespace %s", serviceName, namespace) - } - - return nil -} - -func isFeatureStoreHavingRemoteRegistry(namespace, featureStoreName string) (bool, error) { - timeout := time.Second * 30 - interval := time.Second * 2 // Poll every 2 seconds - startTime := time.Now() - - for time.Since(startTime) < timeout { - cmd := exec.Command("kubectl", "get", "featurestore", featureStoreName, "-n", namespace, - "-o=jsonpath='{.status.applied.services.registry}'") - - output, err := cmd.Output() - if err != nil { - // Retry only on transient errors - if _, ok := err.(*exec.ExitError); ok { - time.Sleep(interval) - continue - } - return false, err // Return immediately on non-transient errors - } - - // Convert output to string and trim any extra spaces - result := strings.TrimSpace(string(output)) - - // Remove single quotes if present - if strings.HasPrefix(result, "'") && strings.HasSuffix(result, "'") { - result = strings.Trim(result, "'") - } - - if result == "" { - time.Sleep(interval) // Retry if result is empty - continue - } - - // Parse the JSON into a map - var registryConfig v1alpha1.Registry - if err := json.Unmarshal([]byte(result), ®istryConfig); err != nil { - return false, err // Return false on JSON parsing failure - } - - if registryConfig.Remote == nil { - return false, nil - } - - hasHostname := registryConfig.Remote.Hostname != nil - hasValidFeastRef := registryConfig.Remote.FeastRef != nil && - registryConfig.Remote.FeastRef.Name != "" - - return hasHostname || hasValidFeastRef, nil - } - - return false, errors.New("timeout waiting for featurestore registry status to be ready") -} diff --git a/infra/feast-operator/test/previous-version/previous_version_suite_test.go b/infra/feast-operator/test/previous-version/previous_version_suite_test.go new file mode 100644 index 00000000000..cd14c89d2d6 --- /dev/null +++ b/infra/feast-operator/test/previous-version/previous_version_suite_test.go @@ -0,0 +1,32 @@ +/* +Copyright 2024 Feast Community. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package previous_version + +import ( + "fmt" + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +// Run upgrade tests using the Ginkgo runner. +func TestPreviousVersion(t *testing.T) { + RegisterFailHandler(Fail) + _, _ = fmt.Fprintf(GinkgoWriter, "Starting test previous version suite\n") + RunSpecs(t, "previous version operator") +} diff --git a/infra/feast-operator/test/previous-version/previous_version_test.go b/infra/feast-operator/test/previous-version/previous_version_test.go new file mode 100644 index 00000000000..9775d239bcc --- /dev/null +++ b/infra/feast-operator/test/previous-version/previous_version_test.go @@ -0,0 +1,49 @@ +/* +Copyright 2024 Feast Community. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package previous_version + +import ( + "github.com/feast-dev/feast/infra/feast-operator/test/utils" + . "github.com/onsi/ginkgo/v2" +) + +var _ = Describe("previous version operator", Ordered, func() { + BeforeAll(func() { + utils.DeployPreviousVersionOperator() + }) + + AfterAll(func() { + utils.DeleteOperatorDeployment("/test/upgrade") + }) + + Context("Previous version operator Tests", func() { + feastK8sResourceNames := []string{ + utils.FeastResourceName + "-online", + utils.FeastResourceName + "-offline", + utils.FeastResourceName + "-ui", + } + + runTestDeploySimpleCRFunc := utils.GetTestDeploySimpleCRFunc("/test/upgrade", utils.GetSimplePreviousVerCR(), + utils.FeatureStoreName, utils.FeastResourceName, feastK8sResourceNames) + runTestWithRemoteRegistryFunction := utils.GetTestWithRemoteRegistryFunc("/test/upgrade", utils.GetSimplePreviousVerCR(), + utils.GetRemoteRegistryPreviousVerCR(), utils.FeatureStoreName, utils.FeastResourceName, feastK8sResourceNames) + + // Run Test on previous version operator + It("Should be able to deploy and run a default feature store CR successfully", runTestDeploySimpleCRFunc) + It("Should be able to deploy and run a feature store with remote registry CR successfully", runTestWithRemoteRegistryFunction) + }) +}) diff --git a/infra/feast-operator/test/upgrade/upgrade_suite_test.go b/infra/feast-operator/test/upgrade/upgrade_suite_test.go new file mode 100644 index 00000000000..bd0da7ab177 --- /dev/null +++ b/infra/feast-operator/test/upgrade/upgrade_suite_test.go @@ -0,0 +1,32 @@ +/* +Copyright 2024 Feast Community. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package previous_version + +import ( + "fmt" + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +// Run upgrade tests using the Ginkgo runner. +func TestUpgrade(t *testing.T) { + RegisterFailHandler(Fail) + _, _ = fmt.Fprintf(GinkgoWriter, "Starting upgrade operator suite\n") + RunSpecs(t, "operator upgrade") +} diff --git a/infra/feast-operator/test/upgrade/upgrade_test.go b/infra/feast-operator/test/upgrade/upgrade_test.go new file mode 100644 index 00000000000..313fa41213c --- /dev/null +++ b/infra/feast-operator/test/upgrade/upgrade_test.go @@ -0,0 +1,44 @@ +/* +Copyright 2024 Feast Community. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package previous_version + +import ( + "github.com/feast-dev/feast/infra/feast-operator/test/utils" + . "github.com/onsi/ginkgo/v2" +) + +var _ = Describe("operator upgrade", Ordered, func() { + BeforeAll(func() { + utils.DeployPreviousVersionOperator() + utils.DeployOperatorFromCode("/test/e2e", true) + }) + + AfterAll(func() { + utils.DeleteOperatorDeployment("/test/e2e") + }) + + Context("Operator upgrade Tests", func() { + runTestDeploySimpleCRFunc := utils.GetTestDeploySimpleCRFunc("/test/upgrade", utils.GetSimplePreviousVerCR(), + utils.FeatureStoreName, utils.FeastResourceName, []string{}) + runTestWithRemoteRegistryFunction := utils.GetTestWithRemoteRegistryFunc("/test/upgrade", utils.GetSimplePreviousVerCR(), + utils.GetRemoteRegistryPreviousVerCR(), utils.FeatureStoreName, utils.FeastResourceName, []string{}) + + // Run Test on current version operator with previous version CR + It("Should be able to deploy and run a default feature store CR successfully", runTestDeploySimpleCRFunc) + It("Should be able to deploy and run a feature store with remote registry CR successfully", runTestWithRemoteRegistryFunction) + }) +}) diff --git a/infra/feast-operator/test/utils/test_util.go b/infra/feast-operator/test/utils/test_util.go new file mode 100644 index 00000000000..b34c4272c46 --- /dev/null +++ b/infra/feast-operator/test/utils/test_util.go @@ -0,0 +1,448 @@ +package utils + +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "os" + "os/exec" + "strings" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + appsv1 "k8s.io/api/apps/v1" + + "github.com/feast-dev/feast/infra/feast-operator/api/feastversion" + "github.com/feast-dev/feast/infra/feast-operator/api/v1alpha1" +) + +const ( + FeastControllerNamespace = "feast-operator-system" + Timeout = 3 * time.Minute + ControllerDeploymentName = "feast-operator-controller-manager" + FeastPrefix = "feast-" + FeatureStoreName = "simple-feast-setup" + FeastResourceName = FeastPrefix + FeatureStoreName +) + +// dynamically checks if all conditions of custom resource featurestore are in "Ready" state. +func checkIfFeatureStoreCustomResourceConditionsInReady(featureStoreName, namespace string) error { + // Wait 10 seconds to lets the feature store status update + time.Sleep(1 * time.Minute) + + cmd := exec.Command("kubectl", "get", "featurestore", featureStoreName, "-n", namespace, "-o", "json") + + var out bytes.Buffer + var stderr bytes.Buffer + cmd.Stdout = &out + cmd.Stderr = &stderr + + if err := cmd.Run(); err != nil { + return fmt.Errorf("failed to get resource %s in namespace %s. Error: %v. Stderr: %s", + featureStoreName, namespace, err, stderr.String()) + } + + // Parse the JSON into FeatureStore + var resource v1alpha1.FeatureStore + if err := json.Unmarshal(out.Bytes(), &resource); err != nil { + return fmt.Errorf("failed to parse the resource JSON. Error: %v", err) + } + + // Validate all conditions + for _, condition := range resource.Status.Conditions { + if condition.Status != "True" { + return fmt.Errorf(" FeatureStore=%s condition '%s' is not in 'Ready' state. Status: %s", + featureStoreName, condition.Type, condition.Status) + } + } + + return nil +} + +// CheckIfDeploymentExistsAndAvailable - validates if a deployment exists and also in the availability state as True. +func CheckIfDeploymentExistsAndAvailable(namespace string, deploymentName string, timeout time.Duration) error { + var output, errOutput bytes.Buffer + + ticker := time.NewTicker(2 * time.Second) + defer ticker.Stop() + + timeoutChan := time.After(timeout) + + for { + select { + case <-timeoutChan: + return fmt.Errorf("timed out waiting for deployment %s to become available", deploymentName) + case <-ticker.C: + // Run kubectl command + cmd := exec.Command("kubectl", "get", "deployment", deploymentName, "-n", namespace, "-o", "json") + cmd.Stdout = &output + cmd.Stderr = &errOutput + + if err := cmd.Run(); err != nil { + // Log error and retry + fmt.Printf("Deployment not yet found, we may try again to find the updated status: %s\n", errOutput.String()) + continue + } + + // Parse the JSON output into Deployment + var result appsv1.Deployment + if err := json.Unmarshal(output.Bytes(), &result); err != nil { + return fmt.Errorf("failed to parse deployment JSON: %v", err) + } + + // Check for Available condition + for _, condition := range result.Status.Conditions { + if condition.Type == "Available" && condition.Status == "True" { + return nil // Deployment is available + } + } + + // Reset buffers for the next loop iteration + output.Reset() + errOutput.Reset() + } + } +} + +// validates if a service account exists using the kubectl CLI. +func checkIfServiceAccountExists(namespace, saName string) error { + cmd := exec.Command("kubectl", "get", "sa", saName, "-n", namespace) + + var out bytes.Buffer + var stderr bytes.Buffer + cmd.Stdout = &out + cmd.Stderr = &stderr + + if err := cmd.Run(); err != nil { + return fmt.Errorf("failed to find service account %s in namespace %s. Error: %v. Stderr: %s", + saName, namespace, err, stderr.String()) + } + + // Check the output to confirm presence + if !strings.Contains(out.String(), saName) { + return fmt.Errorf("service account %s not found in namespace %s", saName, namespace) + } + + return nil +} + +// validates if a config map exists using the kubectl CLI. +func checkIfConfigMapExists(namespace, configMapName string) error { + cmd := exec.Command("kubectl", "get", "cm", configMapName, "-n", namespace) + + var out bytes.Buffer + var stderr bytes.Buffer + cmd.Stdout = &out + cmd.Stderr = &stderr + + if err := cmd.Run(); err != nil { + return fmt.Errorf("failed to find config map %s in namespace %s. Error: %v. Stderr: %s", + configMapName, namespace, err, stderr.String()) + } + + // Check the output to confirm presence + if !strings.Contains(out.String(), configMapName) { + return fmt.Errorf("config map %s not found in namespace %s", configMapName, namespace) + } + + return nil +} + +// validates if a kubernetes service exists using the kubectl CLI. +func checkIfKubernetesServiceExists(namespace, serviceName string) error { + cmd := exec.Command("kubectl", "get", "service", serviceName, "-n", namespace) + + var out bytes.Buffer + var stderr bytes.Buffer + cmd.Stdout = &out + cmd.Stderr = &stderr + + if err := cmd.Run(); err != nil { + return fmt.Errorf("failed to find kubernetes service %s in namespace %s. Error: %v. Stderr: %s", + serviceName, namespace, err, stderr.String()) + } + + // Check the output to confirm presence + if !strings.Contains(out.String(), serviceName) { + return fmt.Errorf("kubernetes service %s not found in namespace %s", serviceName, namespace) + } + + return nil +} + +func isFeatureStoreHavingRemoteRegistry(namespace, featureStoreName string) (bool, error) { + timeout := time.Second * 30 + interval := time.Second * 2 // Poll every 2 seconds + startTime := time.Now() + + for time.Since(startTime) < timeout { + cmd := exec.Command("kubectl", "get", "featurestore", featureStoreName, "-n", namespace, + "-o=jsonpath='{.status.applied.services.registry}'") + + output, err := cmd.Output() + if err != nil { + // Retry only on transient errors + if _, ok := err.(*exec.ExitError); ok { + time.Sleep(interval) + continue + } + return false, err // Return immediately on non-transient errors + } + + // Convert output to string and trim any extra spaces + result := strings.TrimSpace(string(output)) + + // Remove single quotes if present + if strings.HasPrefix(result, "'") && strings.HasSuffix(result, "'") { + result = strings.Trim(result, "'") + } + + if result == "" { + time.Sleep(interval) // Retry if result is empty + continue + } + + // Parse the JSON into a map + var registryConfig v1alpha1.Registry + if err := json.Unmarshal([]byte(result), ®istryConfig); err != nil { + return false, err // Return false on JSON parsing failure + } + + if registryConfig.Remote == nil { + return false, nil + } + + hasHostname := registryConfig.Remote.Hostname != nil + hasValidFeastRef := registryConfig.Remote.FeastRef != nil && + registryConfig.Remote.FeastRef.Name != "" + + return hasHostname || hasValidFeastRef, nil + } + + return false, errors.New("timeout waiting for featurestore registry status to be ready") +} + +func validateTheFeatureStoreCustomResource(namespace string, featureStoreName string, feastResourceName string, feastK8sResourceNames []string, timeout time.Duration) { + hasRemoteRegistry, err := isFeatureStoreHavingRemoteRegistry(namespace, featureStoreName) + Expect(err).ToNot(HaveOccurred(), fmt.Sprintf( + "Error occurred while checking FeatureStore %s is having remote registry or not. \nError: %v\n", + featureStoreName, err)) + + k8sResourceNames := []string{feastResourceName} + + if !hasRemoteRegistry { + feastK8sResourceNames = append(feastK8sResourceNames, feastResourceName+"-registry") + } + + for _, deploymentName := range k8sResourceNames { + By(fmt.Sprintf("validate the feast deployment: %s is up and in availability state.", deploymentName)) + err = CheckIfDeploymentExistsAndAvailable(namespace, deploymentName, timeout) + Expect(err).ToNot(HaveOccurred(), fmt.Sprintf( + "Deployment %s is not available but expected to be available. \nError: %v\n", + deploymentName, err, + )) + fmt.Printf("Feast Deployment %s is available\n", deploymentName) + } + + By("Check if the feast client - kubernetes config map exists.") + configMapName := feastResourceName + "-client" + err = checkIfConfigMapExists(namespace, configMapName) + Expect(err).ToNot(HaveOccurred(), fmt.Sprintf( + "config map %s is not available but expected to be available. \nError: %v\n", + configMapName, err, + )) + fmt.Printf("Feast Deployment client config map %s is available\n", configMapName) + + for _, serviceAccountName := range k8sResourceNames { + By(fmt.Sprintf("validate the feast service account: %s is available.", serviceAccountName)) + err = checkIfServiceAccountExists(namespace, serviceAccountName) + Expect(err).ToNot(HaveOccurred(), fmt.Sprintf( + "Service account %s does not exist in namespace %s. Error: %v", + serviceAccountName, namespace, err, + )) + fmt.Printf("Service account %s exists in namespace %s\n", serviceAccountName, namespace) + } + + for _, serviceName := range feastK8sResourceNames { + By(fmt.Sprintf("validate the kubernetes service name: %s is available.", serviceName)) + err = checkIfKubernetesServiceExists(namespace, serviceName) + Expect(err).ToNot(HaveOccurred(), fmt.Sprintf( + "kubernetes service %s is not available but expected to be available. \nError: %v\n", + serviceName, err, + )) + fmt.Printf("kubernetes service %s is available\n", serviceName) + } + + By(fmt.Sprintf("Checking FeatureStore customer resource: %s is in Ready Status.", featureStoreName)) + err = checkIfFeatureStoreCustomResourceConditionsInReady(featureStoreName, namespace) + Expect(err).ToNot(HaveOccurred(), fmt.Sprintf( + "FeatureStore custom resource %s all conditions are not in ready state. \nError: %v\n", + featureStoreName, err, + )) + fmt.Printf("FeatureStore custom resource %s conditions are in Ready State\n", featureStoreName) +} + +// GetTestDeploySimpleCRFunc - returns a simple CR deployment function +func GetTestDeploySimpleCRFunc(testDir string, crYaml string, featureStoreName string, feastResourceName string, feastK8sResourceNames []string) func() { + return func() { + By("deploying the Simple Feast Custom Resource to Kubernetes") + namespace := "default" + + cmd := exec.Command("kubectl", "apply", "-f", crYaml, "-n", namespace) + _, cmdOutputerr := Run(cmd, testDir) + ExpectWithOffset(1, cmdOutputerr).NotTo(HaveOccurred()) + + validateTheFeatureStoreCustomResource(namespace, featureStoreName, feastResourceName, feastK8sResourceNames, Timeout) + + By("deleting the feast deployment") + cmd = exec.Command("kubectl", "delete", "-f", crYaml) + _, cmdOutputerr = Run(cmd, testDir) + ExpectWithOffset(1, cmdOutputerr).NotTo(HaveOccurred()) + } +} + +// GetTestWithRemoteRegistryFunc - returns a CR deployment with a remote registry function +func GetTestWithRemoteRegistryFunc(testDir string, crYaml string, remoteRegistryCRYaml string, featureStoreName string, feastResourceName string, feastK8sResourceNames []string) func() { + return func() { + By("deploying the Simple Feast Custom Resource to Kubernetes") + namespace := "default" + cmd := exec.Command("kubectl", "apply", "-f", crYaml, "-n", namespace) + _, cmdOutputErr := Run(cmd, testDir) + ExpectWithOffset(1, cmdOutputErr).NotTo(HaveOccurred()) + + validateTheFeatureStoreCustomResource(namespace, featureStoreName, feastResourceName, feastK8sResourceNames, Timeout) + + var remoteRegistryNs = "remote-registry" + By(fmt.Sprintf("Creating the remote registry namespace=%s", remoteRegistryNs)) + cmd = exec.Command("kubectl", "create", "ns", remoteRegistryNs) + _, _ = Run(cmd, testDir) + + By("deploying the Simple Feast remote registry Custom Resource on Kubernetes") + cmd = exec.Command("kubectl", "apply", "-f", remoteRegistryCRYaml, "-n", remoteRegistryNs) + _, cmdOutputErr = Run(cmd, testDir) + ExpectWithOffset(1, cmdOutputErr).NotTo(HaveOccurred()) + + remoteFeatureStoreName := "simple-feast-remote-setup" + remoteFeastResourceName := FeastPrefix + remoteFeatureStoreName + fixRemoteFeastK8sResourceNames(feastK8sResourceNames, remoteFeastResourceName) + validateTheFeatureStoreCustomResource(remoteRegistryNs, remoteFeatureStoreName, remoteFeastResourceName, feastK8sResourceNames, Timeout) + + By("deleting the feast remote registry deployment") + cmd = exec.Command("kubectl", "delete", "-f", remoteRegistryCRYaml, "-n", remoteRegistryNs) + _, cmdOutputErr = Run(cmd, testDir) + ExpectWithOffset(1, cmdOutputErr).NotTo(HaveOccurred()) + + By("deleting the feast deployment") + cmd = exec.Command("kubectl", "delete", "-f", crYaml, "-n", namespace) + _, cmdOutputErr = Run(cmd, testDir) + ExpectWithOffset(1, cmdOutputErr).NotTo(HaveOccurred()) + } +} + +func fixRemoteFeastK8sResourceNames(feastK8sResourceNames []string, remoteFeastResourceName string) { + for i, feastK8sResourceName := range feastK8sResourceNames { + if index := strings.LastIndex(feastK8sResourceName, "-"); index != -1 { + feastK8sResourceNames[i] = remoteFeastResourceName + feastK8sResourceName[index:] + } + } +} + +// DeployOperatorFromCode - Creates the images for the operator and deploys it +func DeployOperatorFromCode(testDir string, skipBuilds bool) { + _, isRunOnOpenShiftCI := os.LookupEnv("RUN_ON_OPENSHIFT_CI") + if !isRunOnOpenShiftCI { + By("creating manager namespace") + cmd := exec.Command("kubectl", "create", "ns", FeastControllerNamespace) + _, _ = Run(cmd, testDir) + + var err error + // projectimage stores the name of the image used in the example + var projectimage = "localhost/feast-operator:v0.0.1" + + // this image will be built in above make target. + var feastImage = "feastdev/feature-server:dev" + var feastLocalImage = "localhost/feastdev/feature-server:dev" + + if !skipBuilds { + By("building the manager(Operator) image") + cmd = exec.Command("make", "docker-build", fmt.Sprintf("IMG=%s", projectimage)) + _, err = Run(cmd, testDir) + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + + By("loading the the manager(Operator) image on Kind") + err = LoadImageToKindClusterWithName(projectimage, testDir) + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + + By("building the feast image") + cmd = exec.Command("make", "feast-ci-dev-docker-img") + _, err = Run(cmd, testDir) + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + + By("Tag the local feast image for the integration tests") + cmd = exec.Command("docker", "image", "tag", feastImage, feastLocalImage) + _, err = Run(cmd, testDir) + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + + By("loading the the feast image on Kind cluster") + err = LoadImageToKindClusterWithName(feastLocalImage, testDir) + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + } + + By("installing CRDs") + cmd = exec.Command("make", "install") + _, err = Run(cmd, testDir) + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + + By("deploying the controller-manager") + cmd = exec.Command("make", "deploy", fmt.Sprintf("IMG=%s", projectimage), fmt.Sprintf("FS_IMG=%s", feastLocalImage)) + _, err = Run(cmd, testDir) + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + } + + By("Validating that the controller-manager deployment is in available state") + err := CheckIfDeploymentExistsAndAvailable(FeastControllerNamespace, ControllerDeploymentName, Timeout) + Expect(err).ToNot(HaveOccurred(), fmt.Sprintf( + "Deployment %s is not available but expected to be available. \nError: %v\n", + ControllerDeploymentName, err, + )) + fmt.Printf("Feast Control Manager Deployment %s is available\n", ControllerDeploymentName) +} + +// DeleteOperatorDeployment - Deletes the operator deployment +func DeleteOperatorDeployment(testDir string) { + _, isRunOnOpenShiftCI := os.LookupEnv("RUN_ON_OPENSHIFT_CI") + if !isRunOnOpenShiftCI { + By("Uninstalling the feast CRD") + cmd := exec.Command("kubectl", "delete", "deployment", ControllerDeploymentName, "-n", FeastControllerNamespace) + _, err := Run(cmd, testDir) + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + } +} + +// DeployPreviousVersionOperator - Deploys the previous version of the operator +func DeployPreviousVersionOperator() { + var err error + + cmd := exec.Command("kubectl", "apply", "-f", fmt.Sprintf("https://raw.githubusercontent.com/feast-dev/feast/refs/tags/v%s/infra/feast-operator/dist/install.yaml", feastversion.FeastVersion)) + _, err = Run(cmd, "/test/upgrade") + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + + err = CheckIfDeploymentExistsAndAvailable(FeastControllerNamespace, ControllerDeploymentName, Timeout) + Expect(err).ToNot(HaveOccurred(), fmt.Sprintf( + "Deployment %s is not available but expected to be available. \nError: %v\n", + ControllerDeploymentName, err, + )) + fmt.Printf("Feast Control Manager Deployment %s is available\n", ControllerDeploymentName) +} + +// GetSimplePreviousVerCR - Get The previous version simple CR for tests +func GetSimplePreviousVerCR() string { + return fmt.Sprintf("https://raw.githubusercontent.com/feast-dev/feast/refs/tags/v%s/infra/feast-operator/test/testdata/feast_integration_test_crs/v1alpha1_default_featurestore.yaml", feastversion.FeastVersion) +} + +// GetRemoteRegistryPreviousVerCR - Get The previous version remote registry CR for tests +func GetRemoteRegistryPreviousVerCR() string { + return fmt.Sprintf("https://raw.githubusercontent.com/feast-dev/feast/refs/tags/v%s/infra/feast-operator/test/testdata/feast_integration_test_crs/v1alpha1_remote_registry_featurestore.yaml", feastversion.FeastVersion) +} diff --git a/infra/feast-operator/test/utils/utils.go b/infra/feast-operator/test/utils/utils.go index 9b57f9af61c..7529a3a0f50 100644 --- a/infra/feast-operator/test/utils/utils.go +++ b/infra/feast-operator/test/utils/utils.go @@ -39,16 +39,16 @@ func warnError(err error) { } // InstallPrometheusOperator installs the prometheus Operator to be used to export the enabled metrics. -func InstallPrometheusOperator() error { +func InstallPrometheusOperator(testDir string) error { url := fmt.Sprintf(prometheusOperatorURL, prometheusOperatorVersion) cmd := exec.Command("kubectl", "create", "-f", url) - _, err := Run(cmd) + _, err := Run(cmd, testDir) return err } // Run executes the provided command within this context -func Run(cmd *exec.Cmd) ([]byte, error) { - dir, _ := GetProjectDir() +func Run(cmd *exec.Cmd, testDir string) ([]byte, error) { + dir, _ := GetProjectDir(testDir) cmd.Dir = dir if err := os.Chdir(cmd.Dir); err != nil { @@ -67,28 +67,28 @@ func Run(cmd *exec.Cmd) ([]byte, error) { } // UninstallPrometheusOperator uninstalls the prometheus -func UninstallPrometheusOperator() { +func UninstallPrometheusOperator(testDir string) { url := fmt.Sprintf(prometheusOperatorURL, prometheusOperatorVersion) cmd := exec.Command("kubectl", "delete", "-f", url) - if _, err := Run(cmd); err != nil { + if _, err := Run(cmd, testDir); err != nil { warnError(err) } } // UninstallCertManager uninstalls the cert manager -func UninstallCertManager() { +func UninstallCertManager(testDir string) { url := fmt.Sprintf(certmanagerURLTmpl, certmanagerVersion) cmd := exec.Command("kubectl", "delete", "-f", url) - if _, err := Run(cmd); err != nil { + if _, err := Run(cmd, testDir); err != nil { warnError(err) } } // InstallCertManager installs the cert manager bundle. -func InstallCertManager() error { +func InstallCertManager(testDir string) error { url := fmt.Sprintf(certmanagerURLTmpl, certmanagerVersion) cmd := exec.Command("kubectl", "apply", "-f", url) - if _, err := Run(cmd); err != nil { + if _, err := Run(cmd, testDir); err != nil { return err } // Wait for cert-manager-webhook to be ready, which can take time if cert-manager @@ -99,12 +99,12 @@ func InstallCertManager() error { "--timeout", "5m", ) - _, err := Run(cmd) + _, err := Run(cmd, testDir) return err } // LoadImageToKindCluster loads a local docker image to the kind cluster -func LoadImageToKindClusterWithName(name string) error { +func LoadImageToKindClusterWithName(name string, testDir string) error { cluster := "kind" if v, ok := os.LookupEnv("KIND_CLUSTER"); ok { cluster = v @@ -112,7 +112,7 @@ func LoadImageToKindClusterWithName(name string) error { fmt.Println("cluster used in the test is -", cluster) kindOptions := []string{"load", "docker-image", name, "--name", cluster} cmd := exec.Command("kind", kindOptions...) - _, err := Run(cmd) + _, err := Run(cmd, testDir) return err } @@ -131,11 +131,11 @@ func GetNonEmptyLines(output string) []string { } // GetProjectDir will return the directory where the project is -func GetProjectDir() (string, error) { +func GetProjectDir(projectDir string) (string, error) { wd, err := os.Getwd() if err != nil { return wd, err } - wd = strings.Replace(wd, "/test/e2e", "", -1) + wd = strings.Replace(wd, projectDir, "", -1) return wd, nil } diff --git a/infra/scripts/pixi/pixi.lock b/infra/scripts/pixi/pixi.lock index 1ca8742026c..5f957f508c9 100644 --- a/infra/scripts/pixi/pixi.lock +++ b/infra/scripts/pixi/pixi.lock @@ -1,4 +1,4 @@ -version: 5 +version: 6 environments: default: channels: @@ -7,16 +7,16 @@ environments: linux-64: - conda: https://conda.anaconda.org/conda-forge/linux-64/_libgcc_mutex-0.1-conda_forge.tar.bz2 - conda: https://conda.anaconda.org/conda-forge/linux-64/_openmp_mutex-4.5-2_gnu.tar.bz2 - - conda: https://conda.anaconda.org/conda-forge/linux-64/libgcc-ng-13.2.0-h807b86a_5.conda - - conda: https://conda.anaconda.org/conda-forge/linux-64/libgomp-13.2.0-h807b86a_5.conda - - conda: https://conda.anaconda.org/conda-forge/linux-64/libstdcxx-ng-13.2.0-h95c4c6d_6.conda - - conda: https://conda.anaconda.org/conda-forge/linux-64/uv-0.1.39-h0ea3d13_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libgcc-14.2.0-h767d61c_2.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libgomp-14.2.0-h767d61c_2.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libstdcxx-14.2.0-h8f9b012_2.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/uv-0.6.3-h0f3a69f_0.conda osx-64: - - conda: https://conda.anaconda.org/conda-forge/osx-64/libcxx-17.0.6-heb59cac_3.conda - - conda: https://conda.anaconda.org/conda-forge/osx-64/uv-0.1.45-h4e38c46_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-64/libcxx-19.1.7-hf95d169_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-64/uv-0.6.3-h8de1528_0.conda osx-arm64: - - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libcxx-17.0.6-h5f092b4_0.conda - - conda: https://conda.anaconda.org/conda-forge/osx-arm64/uv-0.1.45-hc069d6b_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libcxx-19.1.7-ha82da77_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/uv-0.6.3-h668ec48_0.conda py310: channels: - url: https://conda.anaconda.org/conda-forge/ @@ -28,11 +28,12 @@ environments: - conda: https://conda.anaconda.org/conda-forge/linux-64/ca-certificates-2024.2.2-hbcca054_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/ld_impl_linux-64-2.40-h41732ed_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/libffi-3.4.2-h7f98852_5.tar.bz2 - - conda: https://conda.anaconda.org/conda-forge/linux-64/libgcc-ng-13.2.0-h807b86a_5.conda - - conda: https://conda.anaconda.org/conda-forge/linux-64/libgomp-13.2.0-h807b86a_5.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libgcc-14.2.0-h767d61c_2.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libgcc-ng-14.2.0-h69a702a_2.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libgomp-14.2.0-h767d61c_2.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/libnsl-2.0.1-hd590300_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/libsqlite-3.45.3-h2797004_0.conda - - conda: https://conda.anaconda.org/conda-forge/linux-64/libstdcxx-ng-13.2.0-h95c4c6d_6.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libstdcxx-14.2.0-h8f9b012_2.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/libuuid-2.38.1-h0b41bf4_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/libxcrypt-4.4.36-hd590300_1.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/libzlib-1.2.13-hd590300_5.conda @@ -42,12 +43,12 @@ environments: - conda: https://conda.anaconda.org/conda-forge/linux-64/readline-8.2-h8228510_1.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/tk-8.6.13-noxft_h4845f30_101.conda - conda: https://conda.anaconda.org/conda-forge/noarch/tzdata-2024a-h0c530f3_0.conda - - conda: https://conda.anaconda.org/conda-forge/linux-64/uv-0.1.39-h0ea3d13_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/uv-0.6.3-h0f3a69f_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/xz-5.2.6-h166bdaf_0.tar.bz2 osx-64: - conda: https://conda.anaconda.org/conda-forge/osx-64/bzip2-1.0.8-h10d778d_5.conda - conda: https://conda.anaconda.org/conda-forge/osx-64/ca-certificates-2024.7.4-h8857fd0_0.conda - - conda: https://conda.anaconda.org/conda-forge/osx-64/libcxx-17.0.6-heb59cac_3.conda + - conda: https://conda.anaconda.org/conda-forge/osx-64/libcxx-19.1.7-hf95d169_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-64/libffi-3.4.2-h0d85af4_5.tar.bz2 - conda: https://conda.anaconda.org/conda-forge/osx-64/libsqlite-3.46.0-h1b8f9f3_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-64/libzlib-1.3.1-h87427d6_1.conda @@ -57,12 +58,12 @@ environments: - conda: https://conda.anaconda.org/conda-forge/osx-64/readline-8.2-h9e318b2_1.conda - conda: https://conda.anaconda.org/conda-forge/osx-64/tk-8.6.13-h1abcd95_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/tzdata-2024a-h0c530f3_0.conda - - conda: https://conda.anaconda.org/conda-forge/osx-64/uv-0.1.45-h4e38c46_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-64/uv-0.6.3-h8de1528_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-64/xz-5.2.6-h775f41a_0.tar.bz2 osx-arm64: - conda: https://conda.anaconda.org/conda-forge/osx-arm64/bzip2-1.0.8-h93a5062_5.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/ca-certificates-2024.2.2-hf0a4a13_0.conda - - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libcxx-17.0.6-h5f092b4_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libcxx-19.1.7-ha82da77_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libffi-3.4.2-h3422bc3_5.tar.bz2 - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libsqlite-3.45.3-h091b4b1_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libzlib-1.2.13-hfb2fe0b_6.conda @@ -72,7 +73,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/osx-arm64/readline-8.2-h92ec313_1.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/tk-8.6.13-h5083fa2_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/tzdata-2024a-h0c530f3_0.conda - - conda: https://conda.anaconda.org/conda-forge/osx-arm64/uv-0.1.45-hc069d6b_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/uv-0.6.3-h668ec48_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/xz-5.2.6-h57fd34a_0.tar.bz2 py311: channels: @@ -86,11 +87,12 @@ environments: - conda: https://conda.anaconda.org/conda-forge/linux-64/ld_impl_linux-64-2.40-h55db66e_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/libexpat-2.6.2-h59595ed_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/libffi-3.4.2-h7f98852_5.tar.bz2 - - conda: https://conda.anaconda.org/conda-forge/linux-64/libgcc-ng-13.2.0-hc881cc4_6.conda - - conda: https://conda.anaconda.org/conda-forge/linux-64/libgomp-13.2.0-hc881cc4_6.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libgcc-14.2.0-h767d61c_2.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libgcc-ng-14.2.0-h69a702a_2.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libgomp-14.2.0-h767d61c_2.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/libnsl-2.0.1-hd590300_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/libsqlite-3.45.3-h2797004_0.conda - - conda: https://conda.anaconda.org/conda-forge/linux-64/libstdcxx-ng-13.2.0-h95c4c6d_6.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libstdcxx-14.2.0-h8f9b012_2.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/libuuid-2.38.1-h0b41bf4_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/libxcrypt-4.4.36-hd590300_1.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/libzlib-1.2.13-hd590300_5.conda @@ -100,12 +102,12 @@ environments: - conda: https://conda.anaconda.org/conda-forge/linux-64/readline-8.2-h8228510_1.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/tk-8.6.13-noxft_h4845f30_101.conda - conda: https://conda.anaconda.org/conda-forge/noarch/tzdata-2024a-h0c530f3_0.conda - - conda: https://conda.anaconda.org/conda-forge/linux-64/uv-0.1.39-h0ea3d13_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/uv-0.6.3-h0f3a69f_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/xz-5.2.6-h166bdaf_0.tar.bz2 osx-64: - conda: https://conda.anaconda.org/conda-forge/osx-64/bzip2-1.0.8-h10d778d_5.conda - conda: https://conda.anaconda.org/conda-forge/osx-64/ca-certificates-2024.7.4-h8857fd0_0.conda - - conda: https://conda.anaconda.org/conda-forge/osx-64/libcxx-17.0.6-heb59cac_3.conda + - conda: https://conda.anaconda.org/conda-forge/osx-64/libcxx-19.1.7-hf95d169_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-64/libexpat-2.6.2-h73e2aa4_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-64/libffi-3.4.2-h0d85af4_5.tar.bz2 - conda: https://conda.anaconda.org/conda-forge/osx-64/libsqlite-3.46.0-h1b8f9f3_0.conda @@ -116,12 +118,12 @@ environments: - conda: https://conda.anaconda.org/conda-forge/osx-64/readline-8.2-h9e318b2_1.conda - conda: https://conda.anaconda.org/conda-forge/osx-64/tk-8.6.13-h1abcd95_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/tzdata-2024a-h0c530f3_0.conda - - conda: https://conda.anaconda.org/conda-forge/osx-64/uv-0.1.45-h4e38c46_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-64/uv-0.6.3-h8de1528_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-64/xz-5.2.6-h775f41a_0.tar.bz2 osx-arm64: - conda: https://conda.anaconda.org/conda-forge/osx-arm64/bzip2-1.0.8-h93a5062_5.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/ca-certificates-2024.2.2-hf0a4a13_0.conda - - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libcxx-17.0.6-h5f092b4_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libcxx-19.1.7-ha82da77_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libexpat-2.6.2-hebf3989_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libffi-3.4.2-h3422bc3_5.tar.bz2 - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libsqlite-3.45.3-h091b4b1_0.conda @@ -132,7 +134,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/osx-arm64/readline-8.2-h92ec313_1.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/tk-8.6.13-h5083fa2_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/tzdata-2024a-h0c530f3_0.conda - - conda: https://conda.anaconda.org/conda-forge/osx-arm64/uv-0.1.45-hc069d6b_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/uv-0.6.3-h668ec48_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/xz-5.2.6-h57fd34a_0.tar.bz2 py39: channels: @@ -145,11 +147,12 @@ environments: - conda: https://conda.anaconda.org/conda-forge/linux-64/ca-certificates-2024.2.2-hbcca054_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/ld_impl_linux-64-2.40-h41732ed_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/libffi-3.4.2-h7f98852_5.tar.bz2 - - conda: https://conda.anaconda.org/conda-forge/linux-64/libgcc-ng-13.2.0-h807b86a_5.conda - - conda: https://conda.anaconda.org/conda-forge/linux-64/libgomp-13.2.0-h807b86a_5.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libgcc-14.2.0-h767d61c_2.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libgcc-ng-14.2.0-h69a702a_2.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libgomp-14.2.0-h767d61c_2.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/libnsl-2.0.1-hd590300_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/libsqlite-3.45.3-h2797004_0.conda - - conda: https://conda.anaconda.org/conda-forge/linux-64/libstdcxx-ng-13.2.0-h95c4c6d_6.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libstdcxx-14.2.0-h8f9b012_2.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/libuuid-2.38.1-h0b41bf4_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/libxcrypt-4.4.36-hd590300_1.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/libzlib-1.2.13-hd590300_5.conda @@ -159,12 +162,12 @@ environments: - conda: https://conda.anaconda.org/conda-forge/linux-64/readline-8.2-h8228510_1.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/tk-8.6.13-noxft_h4845f30_101.conda - conda: https://conda.anaconda.org/conda-forge/noarch/tzdata-2024a-h0c530f3_0.conda - - conda: https://conda.anaconda.org/conda-forge/linux-64/uv-0.1.39-h0ea3d13_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/uv-0.6.3-h0f3a69f_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/xz-5.2.6-h166bdaf_0.tar.bz2 osx-64: - conda: https://conda.anaconda.org/conda-forge/osx-64/bzip2-1.0.8-h10d778d_5.conda - conda: https://conda.anaconda.org/conda-forge/osx-64/ca-certificates-2024.7.4-h8857fd0_0.conda - - conda: https://conda.anaconda.org/conda-forge/osx-64/libcxx-17.0.6-heb59cac_3.conda + - conda: https://conda.anaconda.org/conda-forge/osx-64/libcxx-19.1.7-hf95d169_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-64/libffi-3.4.2-h0d85af4_5.tar.bz2 - conda: https://conda.anaconda.org/conda-forge/osx-64/libsqlite-3.46.0-h1b8f9f3_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-64/libzlib-1.3.1-h87427d6_1.conda @@ -174,12 +177,12 @@ environments: - conda: https://conda.anaconda.org/conda-forge/osx-64/readline-8.2-h9e318b2_1.conda - conda: https://conda.anaconda.org/conda-forge/osx-64/tk-8.6.13-h1abcd95_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/tzdata-2024a-h0c530f3_0.conda - - conda: https://conda.anaconda.org/conda-forge/osx-64/uv-0.1.45-h4e38c46_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-64/uv-0.6.3-h8de1528_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-64/xz-5.2.6-h775f41a_0.tar.bz2 osx-arm64: - conda: https://conda.anaconda.org/conda-forge/osx-arm64/bzip2-1.0.8-h93a5062_5.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/ca-certificates-2024.2.2-hf0a4a13_0.conda - - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libcxx-17.0.6-h5f092b4_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libcxx-19.1.7-ha82da77_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libffi-3.4.2-h3422bc3_5.tar.bz2 - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libsqlite-3.45.3-h091b4b1_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libzlib-1.2.13-hfb2fe0b_6.conda @@ -189,27 +192,17 @@ environments: - conda: https://conda.anaconda.org/conda-forge/osx-arm64/readline-8.2-h92ec313_1.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/tk-8.6.13-h5083fa2_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/tzdata-2024a-h0c530f3_0.conda - - conda: https://conda.anaconda.org/conda-forge/osx-arm64/uv-0.1.45-hc069d6b_0.conda + - conda: https://conda.anaconda.org/conda-forge/osx-arm64/uv-0.6.3-h668ec48_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/xz-5.2.6-h57fd34a_0.tar.bz2 packages: -- kind: conda - name: _libgcc_mutex - version: '0.1' - build: conda_forge - subdir: linux-64 - url: https://conda.anaconda.org/conda-forge/linux-64/_libgcc_mutex-0.1-conda_forge.tar.bz2 +- conda: https://conda.anaconda.org/conda-forge/linux-64/_libgcc_mutex-0.1-conda_forge.tar.bz2 sha256: fe51de6107f9edc7aa4f786a70f4a883943bc9d39b3bb7307c04c41410990726 md5: d7c89558ba9fa0495403155b64376d81 license: None size: 2562 timestamp: 1578324546067 -- kind: conda - name: _openmp_mutex - version: '4.5' - build: 2_gnu +- conda: https://conda.anaconda.org/conda-forge/linux-64/_openmp_mutex-4.5-2_gnu.tar.bz2 build_number: 16 - subdir: linux-64 - url: https://conda.anaconda.org/conda-forge/linux-64/_openmp_mutex-4.5-2_gnu.tar.bz2 sha256: fbe2c5e56a653bebb982eda4876a9178aedfc2b545f25d0ce9c4c0b508253d22 md5: 73aaf86a425cc6e73fcf236a5a46396d depends: @@ -221,86 +214,48 @@ packages: license_family: BSD size: 23621 timestamp: 1650670423406 -- kind: conda - name: bzip2 - version: 1.0.8 - build: h10d778d_5 - build_number: 5 - subdir: osx-64 - url: https://conda.anaconda.org/conda-forge/osx-64/bzip2-1.0.8-h10d778d_5.conda +- conda: https://conda.anaconda.org/conda-forge/linux-64/bzip2-1.0.8-hd590300_5.conda + sha256: 242c0c324507ee172c0e0dd2045814e746bb303d1eb78870d182ceb0abc726a8 + md5: 69b8b6202a07720f448be700e300ccf4 + depends: + - libgcc-ng >=12 + license: bzip2-1.0.6 + license_family: BSD + size: 254228 + timestamp: 1699279927352 +- conda: https://conda.anaconda.org/conda-forge/osx-64/bzip2-1.0.8-h10d778d_5.conda sha256: 61fb2b488928a54d9472113e1280b468a309561caa54f33825a3593da390b242 md5: 6097a6ca9ada32699b5fc4312dd6ef18 license: bzip2-1.0.6 license_family: BSD size: 127885 timestamp: 1699280178474 -- kind: conda - name: bzip2 - version: 1.0.8 - build: h93a5062_5 - build_number: 5 - subdir: osx-arm64 - url: https://conda.anaconda.org/conda-forge/osx-arm64/bzip2-1.0.8-h93a5062_5.conda +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/bzip2-1.0.8-h93a5062_5.conda sha256: bfa84296a638bea78a8bb29abc493ee95f2a0218775642474a840411b950fe5f md5: 1bbc659ca658bfd49a481b5ef7a0f40f license: bzip2-1.0.6 license_family: BSD size: 122325 timestamp: 1699280294368 -- kind: conda - name: bzip2 - version: 1.0.8 - build: hd590300_5 - build_number: 5 - subdir: linux-64 - url: https://conda.anaconda.org/conda-forge/linux-64/bzip2-1.0.8-hd590300_5.conda - sha256: 242c0c324507ee172c0e0dd2045814e746bb303d1eb78870d182ceb0abc726a8 - md5: 69b8b6202a07720f448be700e300ccf4 - depends: - - libgcc-ng >=12 - license: bzip2-1.0.6 - license_family: BSD - size: 254228 - timestamp: 1699279927352 -- kind: conda - name: ca-certificates - version: 2024.2.2 - build: hbcca054_0 - subdir: linux-64 - url: https://conda.anaconda.org/conda-forge/linux-64/ca-certificates-2024.2.2-hbcca054_0.conda +- conda: https://conda.anaconda.org/conda-forge/linux-64/ca-certificates-2024.2.2-hbcca054_0.conda sha256: 91d81bfecdbb142c15066df70cc952590ae8991670198f92c66b62019b251aeb md5: 2f4327a1cbe7f022401b236e915a5fef license: ISC size: 155432 timestamp: 1706843687645 -- kind: conda - name: ca-certificates - version: 2024.2.2 - build: hf0a4a13_0 - subdir: osx-arm64 - url: https://conda.anaconda.org/conda-forge/osx-arm64/ca-certificates-2024.2.2-hf0a4a13_0.conda - sha256: 49bc3439816ac72d0c0e0f144b8cc870fdcc4adec2e861407ec818d8116b2204 - md5: fb416a1795f18dcc5a038bc2dc54edf9 - license: ISC - size: 155725 - timestamp: 1706844034242 -- kind: conda - name: ca-certificates - version: 2024.7.4 - build: h8857fd0_0 - subdir: osx-64 - url: https://conda.anaconda.org/conda-forge/osx-64/ca-certificates-2024.7.4-h8857fd0_0.conda +- conda: https://conda.anaconda.org/conda-forge/osx-64/ca-certificates-2024.7.4-h8857fd0_0.conda sha256: d16f46c489cb3192305c7d25b795333c5fc17bb0986de20598ed519f8c9cc9e4 md5: 7df874a4b05b2d2b82826190170eaa0f license: ISC size: 154473 timestamp: 1720077510541 -- kind: conda - name: ld_impl_linux-64 - version: '2.40' - build: h41732ed_0 - subdir: linux-64 - url: https://conda.anaconda.org/conda-forge/linux-64/ld_impl_linux-64-2.40-h41732ed_0.conda +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/ca-certificates-2024.2.2-hf0a4a13_0.conda + sha256: 49bc3439816ac72d0c0e0f144b8cc870fdcc4adec2e861407ec818d8116b2204 + md5: fb416a1795f18dcc5a038bc2dc54edf9 + license: ISC + size: 155725 + timestamp: 1706844034242 +- conda: https://conda.anaconda.org/conda-forge/linux-64/ld_impl_linux-64-2.40-h41732ed_0.conda sha256: f6cc89d887555912d6c61b295d398cff9ec982a3417d38025c45d5dd9b9e79cd md5: 7aca3059a1729aa76c597603f10b0dd3 constrains: @@ -309,12 +264,7 @@ packages: license_family: GPL size: 704696 timestamp: 1674833944779 -- kind: conda - name: ld_impl_linux-64 - version: '2.40' - build: h55db66e_0 - subdir: linux-64 - url: https://conda.anaconda.org/conda-forge/linux-64/ld_impl_linux-64-2.40-h55db66e_0.conda +- conda: https://conda.anaconda.org/conda-forge/linux-64/ld_impl_linux-64-2.40-h55db66e_0.conda sha256: ef969eee228cfb71e55146eaecc6af065f468cb0bc0a5239bc053b39db0b5f09 md5: 10569984e7db886e4f1abc2b47ad79a1 constrains: @@ -323,41 +273,29 @@ packages: license_family: GPL size: 713322 timestamp: 1713651222435 -- kind: conda - name: libcxx - version: 17.0.6 - build: h5f092b4_0 - subdir: osx-arm64 - url: https://conda.anaconda.org/conda-forge/osx-arm64/libcxx-17.0.6-h5f092b4_0.conda - sha256: 119d3d9306f537d4c89dc99ed99b94c396d262f0b06f7833243646f68884f2c2 - md5: a96fd5dda8ce56c86a971e0fa02751d0 +- conda: https://conda.anaconda.org/conda-forge/osx-64/libcxx-19.1.7-hf95d169_0.conda + sha256: 6b2fa3fb1e8cd2000b0ed259e0c4e49cbef7b76890157fac3e494bc659a20330 + md5: 4b8f8dc448d814169dbc58fc7286057d depends: - - __osx >=11.0 + - __osx >=10.13 + arch: x86_64 + platform: osx license: Apache-2.0 WITH LLVM-exception license_family: Apache - size: 1248885 - timestamp: 1715020154867 -- kind: conda - name: libcxx - version: 17.0.6 - build: heb59cac_3 - build_number: 3 - subdir: osx-64 - url: https://conda.anaconda.org/conda-forge/osx-64/libcxx-17.0.6-heb59cac_3.conda - sha256: 9df841c64b19a3843869467ff8ff2eb3f6c5491ebaac8fd94fb8029a5b00dcbf - md5: ef15f182e353155497e13726b915bfc4 + size: 527924 + timestamp: 1736877256721 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/libcxx-19.1.7-ha82da77_0.conda + sha256: 776092346da87a2a23502e14d91eb0c32699c4a1522b7331537bd1c3751dcff5 + md5: 5b3e1610ff8bd5443476b91d618f5b77 depends: - - __osx >=10.13 + - __osx >=11.0 + arch: arm64 + platform: osx license: Apache-2.0 WITH LLVM-exception license_family: Apache - size: 1250659 - timestamp: 1720040263499 -- kind: conda - name: libexpat - version: 2.6.2 - build: h59595ed_0 - subdir: linux-64 - url: https://conda.anaconda.org/conda-forge/linux-64/libexpat-2.6.2-h59595ed_0.conda + size: 523505 + timestamp: 1736877862502 +- conda: https://conda.anaconda.org/conda-forge/linux-64/libexpat-2.6.2-h59595ed_0.conda sha256: 331bb7c7c05025343ebd79f86ae612b9e1e74d2687b8f3179faec234f986ce19 md5: e7ba12deb7020dd080c6c70e7b6f6a3d depends: @@ -368,12 +306,7 @@ packages: license_family: MIT size: 73730 timestamp: 1710362120304 -- kind: conda - name: libexpat - version: 2.6.2 - build: h73e2aa4_0 - subdir: osx-64 - url: https://conda.anaconda.org/conda-forge/osx-64/libexpat-2.6.2-h73e2aa4_0.conda +- conda: https://conda.anaconda.org/conda-forge/osx-64/libexpat-2.6.2-h73e2aa4_0.conda sha256: a188a77b275d61159a32ab547f7d17892226e7dac4518d2c6ac3ac8fc8dfde92 md5: 3d1d51c8f716d97c864d12f7af329526 constrains: @@ -382,12 +315,7 @@ packages: license_family: MIT size: 69246 timestamp: 1710362566073 -- kind: conda - name: libexpat - version: 2.6.2 - build: hebf3989_0 - subdir: osx-arm64 - url: https://conda.anaconda.org/conda-forge/osx-arm64/libexpat-2.6.2-hebf3989_0.conda +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/libexpat-2.6.2-hebf3989_0.conda sha256: ba7173ac30064ea901a4c9fb5a51846dcc25512ceb565759be7d18cbf3e5415e md5: e3cde7cfa87f82f7cb13d482d5e0ad09 constrains: @@ -396,119 +324,67 @@ packages: license_family: MIT size: 63655 timestamp: 1710362424980 -- kind: conda - name: libffi - version: 3.4.2 - build: h0d85af4_5 - build_number: 5 - subdir: osx-64 - url: https://conda.anaconda.org/conda-forge/osx-64/libffi-3.4.2-h0d85af4_5.tar.bz2 +- conda: https://conda.anaconda.org/conda-forge/linux-64/libffi-3.4.2-h7f98852_5.tar.bz2 + sha256: ab6e9856c21709b7b517e940ae7028ae0737546122f83c2aa5d692860c3b149e + md5: d645c6d2ac96843a2bfaccd2d62b3ac3 + depends: + - libgcc-ng >=9.4.0 + license: MIT + license_family: MIT + size: 58292 + timestamp: 1636488182923 +- conda: https://conda.anaconda.org/conda-forge/osx-64/libffi-3.4.2-h0d85af4_5.tar.bz2 sha256: 7a2d27a936ceee6942ea4d397f9c7d136f12549d86f7617e8b6bad51e01a941f md5: ccb34fb14960ad8b125962d3d79b31a9 license: MIT license_family: MIT size: 51348 timestamp: 1636488394370 -- kind: conda - name: libffi - version: 3.4.2 - build: h3422bc3_5 - build_number: 5 - subdir: osx-arm64 - url: https://conda.anaconda.org/conda-forge/osx-arm64/libffi-3.4.2-h3422bc3_5.tar.bz2 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/libffi-3.4.2-h3422bc3_5.tar.bz2 sha256: 41b3d13efb775e340e4dba549ab5c029611ea6918703096b2eaa9c015c0750ca md5: 086914b672be056eb70fd4285b6783b6 license: MIT license_family: MIT size: 39020 timestamp: 1636488587153 -- kind: conda - name: libffi - version: 3.4.2 - build: h7f98852_5 - build_number: 5 - subdir: linux-64 - url: https://conda.anaconda.org/conda-forge/linux-64/libffi-3.4.2-h7f98852_5.tar.bz2 - sha256: ab6e9856c21709b7b517e940ae7028ae0737546122f83c2aa5d692860c3b149e - md5: d645c6d2ac96843a2bfaccd2d62b3ac3 +- conda: https://conda.anaconda.org/conda-forge/linux-64/libgcc-14.2.0-h767d61c_2.conda + sha256: 3a572d031cb86deb541d15c1875aaa097baefc0c580b54dc61f5edab99215792 + md5: ef504d1acbd74b7cc6849ef8af47dd03 depends: - - libgcc-ng >=9.4.0 - license: MIT - license_family: MIT - size: 58292 - timestamp: 1636488182923 -- kind: conda - name: libgcc-ng - version: 13.2.0 - build: h807b86a_5 - build_number: 5 - subdir: linux-64 - url: https://conda.anaconda.org/conda-forge/linux-64/libgcc-ng-13.2.0-h807b86a_5.conda - sha256: d32f78bfaac282cfe5205f46d558704ad737b8dbf71f9227788a5ca80facaba4 - md5: d4ff227c46917d3b4565302a2bbb276b - depends: - - _libgcc_mutex 0.1 conda_forge + - __glibc >=2.17,<3.0.a0 - _openmp_mutex >=4.5 constrains: - - libgomp 13.2.0 h807b86a_5 + - libgomp 14.2.0 h767d61c_2 + - libgcc-ng ==14.2.0=*_2 + arch: x86_64 + platform: linux license: GPL-3.0-only WITH GCC-exception-3.1 license_family: GPL - size: 770506 - timestamp: 1706819192021 -- kind: conda - name: libgcc-ng - version: 13.2.0 - build: hc881cc4_6 - build_number: 6 - subdir: linux-64 - url: https://conda.anaconda.org/conda-forge/linux-64/libgcc-ng-13.2.0-hc881cc4_6.conda - sha256: 836a0057525f1414de43642d357d0ab21ac7f85e24800b010dbc17d132e6efec - md5: df88796bd09a0d2ed292e59101478ad8 - depends: - - _libgcc_mutex 0.1 conda_forge - - _openmp_mutex >=4.5 - constrains: - - libgomp 13.2.0 hc881cc4_6 + size: 847885 + timestamp: 1740240653082 +- conda: https://conda.anaconda.org/conda-forge/linux-64/libgcc-ng-14.2.0-h69a702a_2.conda + sha256: fb7558c328b38b2f9d2e412c48da7890e7721ba018d733ebdfea57280df01904 + md5: a2222a6ada71fb478682efe483ce0f92 + depends: + - libgcc 14.2.0 h767d61c_2 + arch: x86_64 + platform: linux license: GPL-3.0-only WITH GCC-exception-3.1 license_family: GPL - size: 777315 - timestamp: 1713755001744 -- kind: conda - name: libgomp - version: 13.2.0 - build: h807b86a_5 - build_number: 5 - subdir: linux-64 - url: https://conda.anaconda.org/conda-forge/linux-64/libgomp-13.2.0-h807b86a_5.conda - sha256: 0d3d4b1b0134283ea02d58e8eb5accf3655464cf7159abf098cc694002f8d34e - md5: d211c42b9ce49aee3734fdc828731689 - depends: - - _libgcc_mutex 0.1 conda_forge - license: GPL-3.0-only WITH GCC-exception-3.1 - license_family: GPL - size: 419751 - timestamp: 1706819107383 -- kind: conda - name: libgomp - version: 13.2.0 - build: hc881cc4_6 - build_number: 6 - subdir: linux-64 - url: https://conda.anaconda.org/conda-forge/linux-64/libgomp-13.2.0-hc881cc4_6.conda - sha256: e722b19b23b31a14b1592d5eceabb38dc52452ff5e4d346e330526971c22e52a - md5: aae89d3736661c36a5591788aebd0817 - depends: - - _libgcc_mutex 0.1 conda_forge + size: 53758 + timestamp: 1740240660904 +- conda: https://conda.anaconda.org/conda-forge/linux-64/libgomp-14.2.0-h767d61c_2.conda + sha256: 1a3130e0b9267e781b89399580f3163632d59fe5b0142900d63052ab1a53490e + md5: 06d02030237f4d5b3d9a7e7d348fe3c6 + depends: + - __glibc >=2.17,<3.0.a0 + arch: x86_64 + platform: linux license: GPL-3.0-only WITH GCC-exception-3.1 license_family: GPL - size: 422363 - timestamp: 1713754915251 -- kind: conda - name: libnsl - version: 2.0.1 - build: hd590300_0 - subdir: linux-64 - url: https://conda.anaconda.org/conda-forge/linux-64/libnsl-2.0.1-hd590300_0.conda + size: 459862 + timestamp: 1740240588123 +- conda: https://conda.anaconda.org/conda-forge/linux-64/libnsl-2.0.1-hd590300_0.conda sha256: 26d77a3bb4dceeedc2a41bd688564fe71bf2d149fdcf117049970bc02ff1add6 md5: 30fd6e37fe21f86f4bd26d6ee73eeec7 depends: @@ -517,25 +393,7 @@ packages: license_family: GPL size: 33408 timestamp: 1697359010159 -- kind: conda - name: libsqlite - version: 3.45.3 - build: h091b4b1_0 - subdir: osx-arm64 - url: https://conda.anaconda.org/conda-forge/osx-arm64/libsqlite-3.45.3-h091b4b1_0.conda - sha256: 4337f466eb55bbdc74e168b52ec8c38f598e3664244ec7a2536009036e2066cc - md5: c8c1186c7f3351f6ffddb97b1f54fc58 - depends: - - libzlib >=1.2.13,<2.0.0a0 - license: Unlicense - size: 824794 - timestamp: 1713367748819 -- kind: conda - name: libsqlite - version: 3.45.3 - build: h2797004_0 - subdir: linux-64 - url: https://conda.anaconda.org/conda-forge/linux-64/libsqlite-3.45.3-h2797004_0.conda +- conda: https://conda.anaconda.org/conda-forge/linux-64/libsqlite-3.45.3-h2797004_0.conda sha256: e2273d6860eadcf714a759ffb6dc24a69cfd01f2a0ea9d6c20f86049b9334e0c md5: b3316cbe90249da4f8e84cd66e1cc55b depends: @@ -544,12 +402,7 @@ packages: license: Unlicense size: 859858 timestamp: 1713367435849 -- kind: conda - name: libsqlite - version: 3.46.0 - build: h1b8f9f3_0 - subdir: osx-64 - url: https://conda.anaconda.org/conda-forge/osx-64/libsqlite-3.46.0-h1b8f9f3_0.conda +- conda: https://conda.anaconda.org/conda-forge/osx-64/libsqlite-3.46.0-h1b8f9f3_0.conda sha256: 63af1a9e3284c7e4952364bafe7267e41e2d9d8bcc0e85a4ea4b0ec02d3693f6 md5: 5dadfbc1a567fe6e475df4ce3148be09 depends: @@ -558,25 +411,27 @@ packages: license: Unlicense size: 908643 timestamp: 1718050720117 -- kind: conda - name: libstdcxx-ng - version: 13.2.0 - build: h95c4c6d_6 - build_number: 6 - subdir: linux-64 - url: https://conda.anaconda.org/conda-forge/linux-64/libstdcxx-ng-13.2.0-h95c4c6d_6.conda - sha256: 2616dbf9d28431eea20b6e307145c6a92ea0328a047c725ff34b0316de2617da - md5: 3cfab3e709f77e9f1b3d380eb622494a +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/libsqlite-3.45.3-h091b4b1_0.conda + sha256: 4337f466eb55bbdc74e168b52ec8c38f598e3664244ec7a2536009036e2066cc + md5: c8c1186c7f3351f6ffddb97b1f54fc58 + depends: + - libzlib >=1.2.13,<2.0.0a0 + license: Unlicense + size: 824794 + timestamp: 1713367748819 +- conda: https://conda.anaconda.org/conda-forge/linux-64/libstdcxx-14.2.0-h8f9b012_2.conda + sha256: 8f5bd92e4a24e1d35ba015c5252e8f818898478cb3bc50bd8b12ab54707dc4da + md5: a78c856b6dc6bf4ea8daeb9beaaa3fb0 + depends: + - __glibc >=2.17,<3.0.a0 + - libgcc 14.2.0 h767d61c_2 + arch: x86_64 + platform: linux license: GPL-3.0-only WITH GCC-exception-3.1 license_family: GPL - size: 3842900 - timestamp: 1713755068572 -- kind: conda - name: libuuid - version: 2.38.1 - build: h0b41bf4_0 - subdir: linux-64 - url: https://conda.anaconda.org/conda-forge/linux-64/libuuid-2.38.1-h0b41bf4_0.conda + size: 3884556 + timestamp: 1740240685253 +- conda: https://conda.anaconda.org/conda-forge/linux-64/libuuid-2.38.1-h0b41bf4_0.conda sha256: 787eb542f055a2b3de553614b25f09eefb0a0931b0c87dbcce6efdfd92f04f18 md5: 40b61aab5c7ba9ff276c41cfffe6b80b depends: @@ -585,13 +440,7 @@ packages: license_family: BSD size: 33601 timestamp: 1680112270483 -- kind: conda - name: libxcrypt - version: 4.4.36 - build: hd590300_1 - build_number: 1 - subdir: linux-64 - url: https://conda.anaconda.org/conda-forge/linux-64/libxcrypt-4.4.36-hd590300_1.conda +- conda: https://conda.anaconda.org/conda-forge/linux-64/libxcrypt-4.4.36-hd590300_1.conda sha256: 6ae68e0b86423ef188196fff6207ed0c8195dd84273cb5623b85aa08033a410c md5: 5aa797f8787fe7a17d1b0821485b5adc depends: @@ -599,13 +448,7 @@ packages: license: LGPL-2.1-or-later size: 100393 timestamp: 1702724383534 -- kind: conda - name: libzlib - version: 1.2.13 - build: hd590300_5 - build_number: 5 - subdir: linux-64 - url: https://conda.anaconda.org/conda-forge/linux-64/libzlib-1.2.13-hd590300_5.conda +- conda: https://conda.anaconda.org/conda-forge/linux-64/libzlib-1.2.13-hd590300_5.conda sha256: 370c7c5893b737596fd6ca0d9190c9715d89d888b8c88537ae1ef168c25e82e4 md5: f36c115f1ee199da648e0597ec2047ad depends: @@ -616,30 +459,7 @@ packages: license_family: Other size: 61588 timestamp: 1686575217516 -- kind: conda - name: libzlib - version: 1.2.13 - build: hfb2fe0b_6 - build_number: 6 - subdir: osx-arm64 - url: https://conda.anaconda.org/conda-forge/osx-arm64/libzlib-1.2.13-hfb2fe0b_6.conda - sha256: 8b29a2386d99b8f58178951dcf19117b532cd9c4aa07623bf1667eae99755d32 - md5: 9c4e121cd926cab631bd1c4a61d18b17 - depends: - - __osx >=11.0 - constrains: - - zlib 1.2.13 *_6 - license: Zlib - license_family: Other - size: 46768 - timestamp: 1716874151980 -- kind: conda - name: libzlib - version: 1.3.1 - build: h87427d6_1 - build_number: 1 - subdir: osx-64 - url: https://conda.anaconda.org/conda-forge/osx-64/libzlib-1.3.1-h87427d6_1.conda +- conda: https://conda.anaconda.org/conda-forge/osx-64/libzlib-1.3.1-h87427d6_1.conda sha256: 80a62db652b1da0ccc100812a1d86e94f75028968991bfb17f9536f3aa72d91d md5: b7575b5aa92108dcc9aaab0f05f2dbce depends: @@ -650,12 +470,18 @@ packages: license_family: Other size: 57372 timestamp: 1716874211519 -- kind: conda - name: ncurses - version: 6.4.20240210 - build: h59595ed_0 - subdir: linux-64 - url: https://conda.anaconda.org/conda-forge/linux-64/ncurses-6.4.20240210-h59595ed_0.conda +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/libzlib-1.2.13-hfb2fe0b_6.conda + sha256: 8b29a2386d99b8f58178951dcf19117b532cd9c4aa07623bf1667eae99755d32 + md5: 9c4e121cd926cab631bd1c4a61d18b17 + depends: + - __osx >=11.0 + constrains: + - zlib 1.2.13 *_6 + license: Zlib + license_family: Other + size: 46768 + timestamp: 1716874151980 +- conda: https://conda.anaconda.org/conda-forge/linux-64/ncurses-6.4.20240210-h59595ed_0.conda sha256: aa0f005b6727aac6507317ed490f0904430584fa8ca722657e7f0fb94741de81 md5: 97da8860a0da5413c7c98a3b3838a645 depends: @@ -663,35 +489,19 @@ packages: license: X11 AND BSD-3-Clause size: 895669 timestamp: 1710866638986 -- kind: conda - name: ncurses - version: '6.5' - build: h5846eda_0 - subdir: osx-64 - url: https://conda.anaconda.org/conda-forge/osx-64/ncurses-6.5-h5846eda_0.conda +- conda: https://conda.anaconda.org/conda-forge/osx-64/ncurses-6.5-h5846eda_0.conda sha256: 6ecc73db0e49143092c0934355ac41583a5d5a48c6914c5f6ca48e562d3a4b79 md5: 02a888433d165c99bf09784a7b14d900 license: X11 AND BSD-3-Clause size: 823601 timestamp: 1715195267791 -- kind: conda - name: ncurses - version: '6.5' - build: hb89a1cb_0 - subdir: osx-arm64 - url: https://conda.anaconda.org/conda-forge/osx-arm64/ncurses-6.5-hb89a1cb_0.conda +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/ncurses-6.5-hb89a1cb_0.conda sha256: 87d7cf716d9d930dab682cb57b3b8d3a61940b47d6703f3529a155c938a6990a md5: b13ad5724ac9ae98b6b4fd87e4500ba4 license: X11 AND BSD-3-Clause size: 795131 timestamp: 1715194898402 -- kind: conda - name: openssl - version: 3.2.1 - build: hd590300_1 - build_number: 1 - subdir: linux-64 - url: https://conda.anaconda.org/conda-forge/linux-64/openssl-3.2.1-hd590300_1.conda +- conda: https://conda.anaconda.org/conda-forge/linux-64/openssl-3.2.1-hd590300_1.conda sha256: 2c689444ed19a603be457284cf2115ee728a3fafb7527326e96054dee7cdc1a7 md5: 9d731343cff6ee2e5a25c4a091bf8e2a depends: @@ -703,12 +513,7 @@ packages: license_family: Apache size: 2865379 timestamp: 1710793235846 -- kind: conda - name: openssl - version: 3.3.0 - build: hd590300_0 - subdir: linux-64 - url: https://conda.anaconda.org/conda-forge/linux-64/openssl-3.3.0-hd590300_0.conda +- conda: https://conda.anaconda.org/conda-forge/linux-64/openssl-3.3.0-hd590300_0.conda sha256: fdbf05e4db88c592366c90bb82e446edbe33c6e49e5130d51c580b2629c0b5d5 md5: c0f3abb4a16477208bbd43a39bd56f18 depends: @@ -720,50 +525,33 @@ packages: license_family: Apache size: 2895187 timestamp: 1714466138265 -- kind: conda - name: openssl - version: 3.3.0 - build: hfb2fe0b_3 - build_number: 3 - subdir: osx-arm64 - url: https://conda.anaconda.org/conda-forge/osx-arm64/openssl-3.3.0-hfb2fe0b_3.conda - sha256: 6f41c163ab57e7499dff092be4498614651f0f6432e12c2b9f06859a8bc39b75 - md5: 730f618b008b3c13c1e3f973408ddd67 +- conda: https://conda.anaconda.org/conda-forge/osx-64/openssl-3.3.1-h87427d6_1.conda + sha256: 60eed5d771207bcef05e0547c8f93a61d0ad1dcf75e19f8f8d9ded8094d78477 + md5: d838ffe9ec3c6d971f110e04487466ff depends: - - __osx >=11.0 + - __osx >=10.13 - ca-certificates constrains: - pyopenssl >=22.1 license: Apache-2.0 license_family: Apache - size: 2893954 - timestamp: 1716468329572 -- kind: conda - name: openssl - version: 3.3.1 - build: h87427d6_1 - build_number: 1 - subdir: osx-64 - url: https://conda.anaconda.org/conda-forge/osx-64/openssl-3.3.1-h87427d6_1.conda - sha256: 60eed5d771207bcef05e0547c8f93a61d0ad1dcf75e19f8f8d9ded8094d78477 - md5: d838ffe9ec3c6d971f110e04487466ff + size: 2551950 + timestamp: 1719364820943 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/openssl-3.3.0-hfb2fe0b_3.conda + sha256: 6f41c163ab57e7499dff092be4498614651f0f6432e12c2b9f06859a8bc39b75 + md5: 730f618b008b3c13c1e3f973408ddd67 depends: - - __osx >=10.13 + - __osx >=11.0 - ca-certificates constrains: - pyopenssl >=22.1 license: Apache-2.0 license_family: Apache - size: 2551950 - timestamp: 1719364820943 -- kind: conda - name: python - version: 3.9.19 - build: h0755675_0_cpython - subdir: linux-64 - url: https://conda.anaconda.org/conda-forge/linux-64/python-3.9.19-h0755675_0_cpython.conda - sha256: b9253ca9ca5427e6da4b1d43353a110e0f2edfab9c951afb4bf01cbae2825b31 - md5: d9ee3647fbd9e8595b8df759b2bbefb8 + size: 2893954 + timestamp: 1716468329572 +- conda: https://conda.anaconda.org/conda-forge/linux-64/python-3.10.14-hd12c33a_0_cpython.conda + sha256: 76a5d12e73542678b70a94570f7b0f7763f9a938f77f0e75d9ea615ef22aa84c + md5: 2b4ba962994e8bd4be9ff5b64b75aff2 depends: - bzip2 >=1.0.8,<2.0a0 - ld_impl_linux-64 >=2.36.1 @@ -781,23 +569,24 @@ packages: - tzdata - xz >=5.2.6,<6.0a0 constrains: - - python_abi 3.9.* *_cp39 + - python_abi 3.10.* *_cp310 license: Python-2.0 - size: 23800555 - timestamp: 1710940120866 -- kind: conda - name: python - version: 3.9.19 - build: h7a9c478_0_cpython - subdir: osx-64 - url: https://conda.anaconda.org/conda-forge/osx-64/python-3.9.19-h7a9c478_0_cpython.conda - sha256: 58b76be84683bc03112b3ed7e377e99af24844ebf7d7568f6466a2dae7a887fe - md5: 7d53d366acd9dbfb498c69326ccb520a + size: 25517742 + timestamp: 1710939725109 +- conda: https://conda.anaconda.org/conda-forge/linux-64/python-3.11.9-hb806964_0_cpython.conda + sha256: 177f33a1fb8d3476b38f73c37b42f01c0b014fa0e039a701fd9f83d83aae6d40 + md5: ac68acfa8b558ed406c75e98d3428d7b depends: - bzip2 >=1.0.8,<2.0a0 + - ld_impl_linux-64 >=2.36.1 + - libexpat >=2.6.2,<3.0a0 - libffi >=3.4,<4.0a0 - - libsqlite >=3.45.2,<4.0a0 - - libzlib >=1.2.13,<2.0.0a0 + - libgcc-ng >=12 + - libnsl >=2.0.1,<2.1.0a0 + - libsqlite >=3.45.3,<4.0a0 + - libuuid >=2.38.1,<3.0a0 + - libxcrypt >=4.4.36 + - libzlib >=1.2.13,<1.3.0a0 - ncurses >=6.4.20240210,<7.0a0 - openssl >=3.2.1,<4.0a0 - readline >=8.2,<9.0a0 @@ -805,23 +594,23 @@ packages: - tzdata - xz >=5.2.6,<6.0a0 constrains: - - python_abi 3.9.* *_cp39 + - python_abi 3.11.* *_cp311 license: Python-2.0 - size: 12372436 - timestamp: 1710940037648 -- kind: conda - name: python - version: 3.9.19 - build: hd7ebdb9_0_cpython - subdir: osx-arm64 - url: https://conda.anaconda.org/conda-forge/osx-arm64/python-3.9.19-hd7ebdb9_0_cpython.conda - sha256: 3b93f7a405f334043758dfa8aaca050429a954a37721a6462ebd20e94ef7c5a0 - md5: 45c4d173b12154f746be3b49b1190634 + size: 30884494 + timestamp: 1713553104915 +- conda: https://conda.anaconda.org/conda-forge/linux-64/python-3.9.19-h0755675_0_cpython.conda + sha256: b9253ca9ca5427e6da4b1d43353a110e0f2edfab9c951afb4bf01cbae2825b31 + md5: d9ee3647fbd9e8595b8df759b2bbefb8 depends: - bzip2 >=1.0.8,<2.0a0 + - ld_impl_linux-64 >=2.36.1 - libffi >=3.4,<4.0a0 + - libgcc-ng >=12 + - libnsl >=2.0.1,<2.1.0a0 - libsqlite >=3.45.2,<4.0a0 - - libzlib >=1.2.13,<2.0.0a0 + - libuuid >=2.38.1,<3.0a0 + - libxcrypt >=4.4.36 + - libzlib >=1.2.13,<1.3.0a0 - ncurses >=6.4.20240210,<7.0a0 - openssl >=3.2.1,<4.0a0 - readline >=8.2,<9.0a0 @@ -831,14 +620,9 @@ packages: constrains: - python_abi 3.9.* *_cp39 license: Python-2.0 - size: 11847835 - timestamp: 1710939779164 -- kind: conda - name: python - version: 3.10.14 - build: h00d2728_0_cpython - subdir: osx-64 - url: https://conda.anaconda.org/conda-forge/osx-64/python-3.10.14-h00d2728_0_cpython.conda + size: 23800555 + timestamp: 1710940120866 +- conda: https://conda.anaconda.org/conda-forge/osx-64/python-3.10.14-h00d2728_0_cpython.conda sha256: 00c1de2d46ede26609ef4e84a44b83be7876ba6a0215b7c83bff41a0656bf694 md5: 0a1cddc4382c5c171e791c70740546dd depends: @@ -857,18 +641,15 @@ packages: license: Python-2.0 size: 11890228 timestamp: 1710940046031 -- kind: conda - name: python - version: 3.10.14 - build: h2469fbe_0_cpython - subdir: osx-arm64 - url: https://conda.anaconda.org/conda-forge/osx-arm64/python-3.10.14-h2469fbe_0_cpython.conda - sha256: 454d609fe25daedce9e886efcbfcadad103ed0362e7cb6d2bcddec90b1ecd3ee - md5: 4ae999c8227c6d8c7623d32d51d25ea9 +- conda: https://conda.anaconda.org/conda-forge/osx-64/python-3.11.9-h657bba9_0_cpython.conda + sha256: 3b50a5abb3b812875beaa9ab792dbd1bf44f335c64e9f9fedcf92d953995651c + md5: 612763bc5ede9552e4233ec518b9c9fb depends: + - __osx >=10.9 - bzip2 >=1.0.8,<2.0a0 + - libexpat >=2.6.2,<3.0a0 - libffi >=3.4,<4.0a0 - - libsqlite >=3.45.2,<4.0a0 + - libsqlite >=3.45.3,<4.0a0 - libzlib >=1.2.13,<2.0.0a0 - ncurses >=6.4.20240210,<7.0a0 - openssl >=3.2.1,<4.0a0 @@ -877,28 +658,18 @@ packages: - tzdata - xz >=5.2.6,<6.0a0 constrains: - - python_abi 3.10.* *_cp310 + - python_abi 3.11.* *_cp311 license: Python-2.0 - size: 12336005 - timestamp: 1710939659384 -- kind: conda - name: python - version: 3.10.14 - build: hd12c33a_0_cpython - subdir: linux-64 - url: https://conda.anaconda.org/conda-forge/linux-64/python-3.10.14-hd12c33a_0_cpython.conda - sha256: 76a5d12e73542678b70a94570f7b0f7763f9a938f77f0e75d9ea615ef22aa84c - md5: 2b4ba962994e8bd4be9ff5b64b75aff2 + size: 15503226 + timestamp: 1713553747073 +- conda: https://conda.anaconda.org/conda-forge/osx-64/python-3.9.19-h7a9c478_0_cpython.conda + sha256: 58b76be84683bc03112b3ed7e377e99af24844ebf7d7568f6466a2dae7a887fe + md5: 7d53d366acd9dbfb498c69326ccb520a depends: - bzip2 >=1.0.8,<2.0a0 - - ld_impl_linux-64 >=2.36.1 - libffi >=3.4,<4.0a0 - - libgcc-ng >=12 - - libnsl >=2.0.1,<2.1.0a0 - libsqlite >=3.45.2,<4.0a0 - - libuuid >=2.38.1,<3.0a0 - - libxcrypt >=4.4.36 - - libzlib >=1.2.13,<1.3.0a0 + - libzlib >=1.2.13,<2.0.0a0 - ncurses >=6.4.20240210,<7.0a0 - openssl >=3.2.1,<4.0a0 - readline >=8.2,<9.0a0 @@ -906,24 +677,17 @@ packages: - tzdata - xz >=5.2.6,<6.0a0 constrains: - - python_abi 3.10.* *_cp310 + - python_abi 3.9.* *_cp39 license: Python-2.0 - size: 25517742 - timestamp: 1710939725109 -- kind: conda - name: python - version: 3.11.9 - build: h657bba9_0_cpython - subdir: osx-64 - url: https://conda.anaconda.org/conda-forge/osx-64/python-3.11.9-h657bba9_0_cpython.conda - sha256: 3b50a5abb3b812875beaa9ab792dbd1bf44f335c64e9f9fedcf92d953995651c - md5: 612763bc5ede9552e4233ec518b9c9fb + size: 12372436 + timestamp: 1710940037648 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/python-3.10.14-h2469fbe_0_cpython.conda + sha256: 454d609fe25daedce9e886efcbfcadad103ed0362e7cb6d2bcddec90b1ecd3ee + md5: 4ae999c8227c6d8c7623d32d51d25ea9 depends: - - __osx >=10.9 - bzip2 >=1.0.8,<2.0a0 - - libexpat >=2.6.2,<3.0a0 - libffi >=3.4,<4.0a0 - - libsqlite >=3.45.3,<4.0a0 + - libsqlite >=3.45.2,<4.0a0 - libzlib >=1.2.13,<2.0.0a0 - ncurses >=6.4.20240210,<7.0a0 - openssl >=3.2.1,<4.0a0 @@ -932,16 +696,11 @@ packages: - tzdata - xz >=5.2.6,<6.0a0 constrains: - - python_abi 3.11.* *_cp311 + - python_abi 3.10.* *_cp310 license: Python-2.0 - size: 15503226 - timestamp: 1713553747073 -- kind: conda - name: python - version: 3.11.9 - build: h932a869_0_cpython - subdir: osx-arm64 - url: https://conda.anaconda.org/conda-forge/osx-arm64/python-3.11.9-h932a869_0_cpython.conda + size: 12336005 + timestamp: 1710939659384 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/python-3.11.9-h932a869_0_cpython.conda sha256: a436ceabde1f056a0ac3e347dadc780ee2a135a421ddb6e9a469370769829e3c md5: 293e0713ae804b5527a673e7605c04fc depends: @@ -962,25 +721,14 @@ packages: license: Python-2.0 size: 14644189 timestamp: 1713552154779 -- kind: conda - name: python - version: 3.11.9 - build: hb806964_0_cpython - subdir: linux-64 - url: https://conda.anaconda.org/conda-forge/linux-64/python-3.11.9-hb806964_0_cpython.conda - sha256: 177f33a1fb8d3476b38f73c37b42f01c0b014fa0e039a701fd9f83d83aae6d40 - md5: ac68acfa8b558ed406c75e98d3428d7b +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/python-3.9.19-hd7ebdb9_0_cpython.conda + sha256: 3b93f7a405f334043758dfa8aaca050429a954a37721a6462ebd20e94ef7c5a0 + md5: 45c4d173b12154f746be3b49b1190634 depends: - bzip2 >=1.0.8,<2.0a0 - - ld_impl_linux-64 >=2.36.1 - - libexpat >=2.6.2,<3.0a0 - libffi >=3.4,<4.0a0 - - libgcc-ng >=12 - - libnsl >=2.0.1,<2.1.0a0 - - libsqlite >=3.45.3,<4.0a0 - - libuuid >=2.38.1,<3.0a0 - - libxcrypt >=4.4.36 - - libzlib >=1.2.13,<1.3.0a0 + - libsqlite >=3.45.2,<4.0a0 + - libzlib >=1.2.13,<2.0.0a0 - ncurses >=6.4.20240210,<7.0a0 - openssl >=3.2.1,<4.0a0 - readline >=8.2,<9.0a0 @@ -988,17 +736,11 @@ packages: - tzdata - xz >=5.2.6,<6.0a0 constrains: - - python_abi 3.11.* *_cp311 + - python_abi 3.9.* *_cp39 license: Python-2.0 - size: 30884494 - timestamp: 1713553104915 -- kind: conda - name: readline - version: '8.2' - build: h8228510_1 - build_number: 1 - subdir: linux-64 - url: https://conda.anaconda.org/conda-forge/linux-64/readline-8.2-h8228510_1.conda + size: 11847835 + timestamp: 1710939779164 +- conda: https://conda.anaconda.org/conda-forge/linux-64/readline-8.2-h8228510_1.conda sha256: 5435cf39d039387fbdc977b0a762357ea909a7694d9528ab40f005e9208744d7 md5: 47d31b792659ce70f470b5c82fdfb7a4 depends: @@ -1008,13 +750,16 @@ packages: license_family: GPL size: 281456 timestamp: 1679532220005 -- kind: conda - name: readline - version: '8.2' - build: h92ec313_1 - build_number: 1 - subdir: osx-arm64 - url: https://conda.anaconda.org/conda-forge/osx-arm64/readline-8.2-h92ec313_1.conda +- conda: https://conda.anaconda.org/conda-forge/osx-64/readline-8.2-h9e318b2_1.conda + sha256: 41e7d30a097d9b060037f0c6a2b1d4c4ae7e942c06c943d23f9d481548478568 + md5: f17f77f2acf4d344734bda76829ce14e + depends: + - ncurses >=6.3,<7.0a0 + license: GPL-3.0-only + license_family: GPL + size: 255870 + timestamp: 1679532707590 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/readline-8.2-h92ec313_1.conda sha256: a1dfa679ac3f6007362386576a704ad2d0d7a02e98f5d0b115f207a2da63e884 md5: 8cbb776a2f641b943d413b3e19df71f4 depends: @@ -1023,28 +768,17 @@ packages: license_family: GPL size: 250351 timestamp: 1679532511311 -- kind: conda - name: readline - version: '8.2' - build: h9e318b2_1 - build_number: 1 - subdir: osx-64 - url: https://conda.anaconda.org/conda-forge/osx-64/readline-8.2-h9e318b2_1.conda - sha256: 41e7d30a097d9b060037f0c6a2b1d4c4ae7e942c06c943d23f9d481548478568 - md5: f17f77f2acf4d344734bda76829ce14e +- conda: https://conda.anaconda.org/conda-forge/linux-64/tk-8.6.13-noxft_h4845f30_101.conda + sha256: e0569c9caa68bf476bead1bed3d79650bb080b532c64a4af7d8ca286c08dea4e + md5: d453b98d9c83e71da0741bb0ff4d76bc depends: - - ncurses >=6.3,<7.0a0 - license: GPL-3.0-only - license_family: GPL - size: 255870 - timestamp: 1679532707590 -- kind: conda - name: tk - version: 8.6.13 - build: h1abcd95_1 - build_number: 1 - subdir: osx-64 - url: https://conda.anaconda.org/conda-forge/osx-64/tk-8.6.13-h1abcd95_1.conda + - libgcc-ng >=12 + - libzlib >=1.2.13,<1.3.0a0 + license: TCL + license_family: BSD + size: 3318875 + timestamp: 1699202167581 +- conda: https://conda.anaconda.org/conda-forge/osx-64/tk-8.6.13-h1abcd95_1.conda sha256: 30412b2e9de4ff82d8c2a7e5d06a15f4f4fef1809a72138b6ccb53a33b26faf5 md5: bf830ba5afc507c6232d4ef0fb1a882d depends: @@ -1053,13 +787,7 @@ packages: license_family: BSD size: 3270220 timestamp: 1699202389792 -- kind: conda - name: tk - version: 8.6.13 - build: h5083fa2_1 - build_number: 1 - subdir: osx-arm64 - url: https://conda.anaconda.org/conda-forge/osx-arm64/tk-8.6.13-h5083fa2_1.conda +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/tk-8.6.13-h5083fa2_1.conda sha256: 72457ad031b4c048e5891f3f6cb27a53cb479db68a52d965f796910e71a403a8 md5: b50a57ba89c32b62428b71a875291c9b depends: @@ -1068,86 +796,53 @@ packages: license_family: BSD size: 3145523 timestamp: 1699202432999 -- kind: conda - name: tk - version: 8.6.13 - build: noxft_h4845f30_101 - build_number: 101 - subdir: linux-64 - url: https://conda.anaconda.org/conda-forge/linux-64/tk-8.6.13-noxft_h4845f30_101.conda - sha256: e0569c9caa68bf476bead1bed3d79650bb080b532c64a4af7d8ca286c08dea4e - md5: d453b98d9c83e71da0741bb0ff4d76bc - depends: - - libgcc-ng >=12 - - libzlib >=1.2.13,<1.3.0a0 - license: TCL - license_family: BSD - size: 3318875 - timestamp: 1699202167581 -- kind: conda - name: tzdata - version: 2024a - build: h0c530f3_0 - subdir: noarch - noarch: generic - url: https://conda.anaconda.org/conda-forge/noarch/tzdata-2024a-h0c530f3_0.conda +- conda: https://conda.anaconda.org/conda-forge/noarch/tzdata-2024a-h0c530f3_0.conda sha256: 7b2b69c54ec62a243eb6fba2391b5e443421608c3ae5dbff938ad33ca8db5122 md5: 161081fc7cec0bfda0d86d7cb595f8d8 license: LicenseRef-Public-Domain size: 119815 timestamp: 1706886945727 -- kind: conda - name: uv - version: 0.1.39 - build: h0ea3d13_0 - subdir: linux-64 - url: https://conda.anaconda.org/conda-forge/linux-64/uv-0.1.39-h0ea3d13_0.conda - sha256: 763d149b6f4f5c70c91e4106d3a48409c48283ed2e27392578998fb2441f23d8 - md5: c3206e7ca254e50b3556917886f9b12b +- conda: https://conda.anaconda.org/conda-forge/linux-64/uv-0.6.3-h0f3a69f_0.conda + sha256: fc33719d8cccf555748c2cb17bede5c0c06637269a0be3979f0eaebcca9f4eb0 + md5: bfee7af0ca5d4b0397bbd9ddf386d14b depends: - - libgcc-ng >=12 - - libstdcxx-ng >=12 + - __glibc >=2.17,<3.0.a0 + - libgcc >=13 + - libstdcxx >=13 + constrains: + - __glibc >=2.17 + arch: x86_64 + platform: linux license: Apache-2.0 OR MIT - size: 11891252 - timestamp: 1714233659570 -- kind: conda - name: uv - version: 0.1.45 - build: h4e38c46_0 - subdir: osx-64 - url: https://conda.anaconda.org/conda-forge/osx-64/uv-0.1.45-h4e38c46_0.conda - sha256: 8c11774ca1940dcd90187ce240afea26b76e2942f9b18d65f6d4b483534193fd - md5: 754ce8a22c94a30c7bbd42274c7fae31 + size: 11471064 + timestamp: 1740442105821 +- conda: https://conda.anaconda.org/conda-forge/osx-64/uv-0.6.3-h8de1528_0.conda + sha256: e61ed82bb71264dc7dcf9ca1528796907b515ceec508d5c6f4d6b79e1716e0ea + md5: 861adce9aeb74e0124187afbde2f4d4a depends: - __osx >=10.13 - - libcxx >=16 + - libcxx >=18 constrains: - - __osx >=10.12 + - __osx >=10.13 + arch: x86_64 + platform: osx license: Apache-2.0 OR MIT - size: 8937335 - timestamp: 1716265195083 -- kind: conda - name: uv - version: 0.1.45 - build: hc069d6b_0 - subdir: osx-arm64 - url: https://conda.anaconda.org/conda-forge/osx-arm64/uv-0.1.45-hc069d6b_0.conda - sha256: 80dfc19f2ef473e86e718361847d1d598e95ffd0c0f5de7d07cda35d25f6aef5 - md5: 9192238a60bc6da9c41092990c31eb41 + size: 11024081 + timestamp: 1740443179556 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/uv-0.6.3-h668ec48_0.conda + sha256: b1ada7b2d26f82effe5dfddfe31f5674a39196d6ddca40f02cfd23390d5446f0 + md5: 0a3cd436a7e106362489ae2ff09db1c4 depends: - __osx >=11.0 - - libcxx >=16 + - libcxx >=18 constrains: - __osx >=11.0 + arch: arm64 + platform: osx license: Apache-2.0 OR MIT - size: 9231858 - timestamp: 1716265232676 -- kind: conda - name: xz - version: 5.2.6 - build: h166bdaf_0 - subdir: linux-64 - url: https://conda.anaconda.org/conda-forge/linux-64/xz-5.2.6-h166bdaf_0.tar.bz2 + size: 9968257 + timestamp: 1740443196241 +- conda: https://conda.anaconda.org/conda-forge/linux-64/xz-5.2.6-h166bdaf_0.tar.bz2 sha256: 03a6d28ded42af8a347345f82f3eebdd6807a08526d47899a42d62d319609162 md5: 2161070d867d1b1204ea749c8eec4ef0 depends: @@ -1155,25 +850,15 @@ packages: license: LGPL-2.1 and GPL-2.0 size: 418368 timestamp: 1660346797927 -- kind: conda - name: xz - version: 5.2.6 - build: h57fd34a_0 - subdir: osx-arm64 - url: https://conda.anaconda.org/conda-forge/osx-arm64/xz-5.2.6-h57fd34a_0.tar.bz2 - sha256: 59d78af0c3e071021cfe82dc40134c19dab8cdf804324b62940f5c8cd71803ec - md5: 39c6b54e94014701dd157f4f576ed211 - license: LGPL-2.1 and GPL-2.0 - size: 235693 - timestamp: 1660346961024 -- kind: conda - name: xz - version: 5.2.6 - build: h775f41a_0 - subdir: osx-64 - url: https://conda.anaconda.org/conda-forge/osx-64/xz-5.2.6-h775f41a_0.tar.bz2 +- conda: https://conda.anaconda.org/conda-forge/osx-64/xz-5.2.6-h775f41a_0.tar.bz2 sha256: eb09823f34cc2dd663c0ec4ab13f246f45dcd52e5b8c47b9864361de5204a1c8 md5: a72f9d4ea13d55d745ff1ed594747f10 license: LGPL-2.1 and GPL-2.0 size: 238119 timestamp: 1660346964847 +- conda: https://conda.anaconda.org/conda-forge/osx-arm64/xz-5.2.6-h57fd34a_0.tar.bz2 + sha256: 59d78af0c3e071021cfe82dc40134c19dab8cdf804324b62940f5c8cd71803ec + md5: 39c6b54e94014701dd157f4f576ed211 + license: LGPL-2.1 and GPL-2.0 + size: 235693 + timestamp: 1660346961024 diff --git a/infra/scripts/pixi/pixi.toml b/infra/scripts/pixi/pixi.toml index 487c6f7def1..89b9f0376f8 100644 --- a/infra/scripts/pixi/pixi.toml +++ b/infra/scripts/pixi/pixi.toml @@ -6,7 +6,7 @@ platforms = ["linux-64", "osx-arm64", "osx-64"] [tasks] [dependencies] -uv = ">=0.1.39,<0.2" +uv = ">=0.6.3" [feature.py39.dependencies] python = "~=3.9.0" diff --git a/infra/scripts/release/files_to_bump.txt b/infra/scripts/release/files_to_bump.txt index 7cb9dd1e8e4..71cf1746b6d 100644 --- a/infra/scripts/release/files_to_bump.txt +++ b/infra/scripts/release/files_to_bump.txt @@ -18,4 +18,5 @@ infra/feast-operator/config/component_metadata.yaml 4 infra/feast-operator/config/overlays/odh/params.env 1 2 infra/feast-operator/api/feastversion/version.go 20 java/pom.xml 38 +sdk/python/feast/infra/feature_servers/multicloud/requirements.txt 2 ui/package.json 3 diff --git a/java/pom.xml b/java/pom.xml index d7076ef501e..a51698d7e1a 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -35,7 +35,7 @@ - 0.46.0 + 0.47.0 https://github.com/feast-dev/feast UTF-8 diff --git a/protos/feast/core/FeatureService.proto b/protos/feast/core/FeatureService.proto index b143ba73f45..380b2dc3718 100644 --- a/protos/feast/core/FeatureService.proto +++ b/protos/feast/core/FeatureService.proto @@ -61,6 +61,7 @@ message LoggingConfig { SnowflakeDestination snowflake_destination = 6; CustomDestination custom_destination = 7; AthenaDestination athena_destination = 8; + CouchbaseColumnarDestination couchbase_columnar_destination = 9; } message FileDestination { @@ -95,6 +96,15 @@ message LoggingConfig { string kind = 1; map config = 2; } + + message CouchbaseColumnarDestination { + // Destination database name + string database = 1; + // Destination scope name + string scope = 2; + // Destination collection name + string collection = 3; + } } message FeatureServiceList { diff --git a/pyproject.toml b/pyproject.toml index 2a051231e2a..9eec118099a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,3 +1,170 @@ +[project] +name = "feast" +description = "Python SDK for Feast" +readme = "README.md" +requires-python = ">=3.9.0" +license = {file = "LICENSE"} +classifiers = [ + "License :: OSI Approved :: Apache Software License", + "Programming Language :: Python", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.9" +] +dynamic = [ + "version", +] +dependencies = [ + "click>=7.0.0,<9.0.0", + "colorama>=0.3.9,<1", + "dill~=0.3.0", + "protobuf>=4.24.0", + "Jinja2>=2,<4", + "jsonschema", + "mmh3", + "numpy>=1.22,<2", + "pandas>=1.4.3,<3", + "pyarrow<18.1.0", + "pydantic>=2.0.0", + "pygments>=2.12.0,<3", + "PyYAML>=5.4.0,<7", + "requests", + "SQLAlchemy[mypy]>1", + "tabulate>=0.8.0,<1", + "tenacity>=7,<9", + "toml>=0.10.0,<1", + "tqdm>=4,<5", + "typeguard>=4.0.0", + "fastapi>=0.68.0", + "uvicorn[standard]>=0.14.0,<1", + "uvicorn-worker", + "gunicorn; platform_system != 'Windows'", + "dask[dataframe]>=2024.2.1", + "prometheus_client", + "psutil", + "bigtree>=0.19.2", + "pyjwt", +] + +[project.optional-dependencies] +aws = ["boto3>=1.17.0,<2", "fsspec<=2024.9.0", "aiobotocore>2,<3"] +azure = [ + "azure-storage-blob>=0.37.0", + "azure-identity>=1.6.1", + "SQLAlchemy>=1.4.19", + "pyodbc>=4.0.30", + "pymssql" +] +cassandra = ["cassandra-driver>=3.24.0,<4"] +couchbase = ["couchbase==4.3.2", "couchbase-columnar==1.0.0"] +delta = ["deltalake"] +docling = ["docling>=2.23.0"] +duckdb = ["ibis-framework[duckdb]>=9.0.0,<10"] +elasticsearch = ["elasticsearch>=8.13.0"] +faiss = ["faiss-cpu>=1.7.0,<2"] +gcp = [ + "google-api-core>=1.23.0,<3", + "googleapis-common-protos>=1.52.0,<2", + "google-cloud-bigquery[pandas]>=2,<4", + "google-cloud-bigquery-storage >= 2.0.0,<3", + "google-cloud-datastore>=2.16.0,<3", + "google-cloud-storage>=1.34.0,<3", + "google-cloud-bigtable>=2.11.0,<3", + "fsspec<=2024.9.0", +] +ge = ["great_expectations>=0.15.41,<1"] +go = ["cffi>=1.15.0"] +grpcio = [ + "grpcio>=1.56.2,<2", + "grpcio-reflection>=1.56.2,<2", + "grpcio-health-checking>=1.56.2,<2", +] +hazelcast = ["hazelcast-python-client>=5.1"] +hbase = ["happybase>=1.2.0,<3"] +ibis = [ + "ibis-framework>=9.0.0,<10", + "ibis-substrait>=4.0.0", +] +ikv = [ + "ikvpy>=0.0.36", +] +k8s = ["kubernetes<=20.13.0"] +milvus = ["pymilvus"] +mssql = ["ibis-framework[mssql]>=9.0.0,<10"] +mysql = ["pymysql", "types-PyMySQL"] +opentelemetry = ["prometheus_client", "psutil"] +spark = ["pyspark>=3.0.0,<4"] +trino = ["trino>=0.305.0,<0.400.0", "regex"] +postgres = ["psycopg[binary,pool]>=3.0.0,<4"] +pytorch = ["torch>=2.2.2", "torchvision>=0.17.2"] +qdrant = ["qdrant-client>=1.12.0"] +redis = [ + "redis>=4.2.2,<5", + "hiredis>=2.0.0,<3", +] +singlestore = ["singlestoredb<1.8.0"] +snowflake = [ + "snowflake-connector-python[pandas]>=3.7,<4", +] +sqlite_vec = ["sqlite-vec==v0.1.6"] + +ci = [ + "build", + "virtualenv==20.23.0", + "cryptography>=43.0,<44", + "ruff>=0.8.0", + "mypy-protobuf>=3.1", + "grpcio-tools>=1.56.2,<2", + "grpcio-testing>=1.56.2,<2", + # FastAPI does not correctly pull starlette dependency on httpx see thread(https://github.com/tiangolo/fastapi/issues/5656). + "httpx==0.27.2", + "minio==7.2.11", + "mock==2.0.0", + "moto<5", + "mypy>=1.4.1,<1.11.3", + "urllib3>=1.25.4,<3", + "psutil==5.9.0", + "py>=1.11.0", # https://github.com/pytest-dev/pytest/issues/10420 + "pytest>=6.0.0,<8", + "pytest-asyncio<=0.24.0", + "pytest-cov", + "pytest-xdist", + "pytest-benchmark>=3.4.1,<4", + "pytest-lazy-fixture==0.6.3", + "pytest-timeout==1.4.2", + "pytest-ordering~=0.6.0", + "pytest-mock==1.10.4", + "pytest-env", + "Sphinx>4.0.0,<7", + "testcontainers==4.8.2", + "python-keycloak==4.2.2", + "pre-commit<3.3.2", + "assertpy==1.1", + "pip-tools", + "pybindgen", + "types-protobuf~=3.19.22", + "types-python-dateutil", + "types-pytz", + "types-PyYAML", + "types-redis", + "types-requests<2.31.0", + "types-setuptools", + "types-tabulate", + "virtualenv<20.24.2", + "feast[aws, azure, cassandra, couchbase, delta, docling, duckdb, elasticsearch, faiss, gcp, ge, go, grpcio, hazelcast, hbase, ibis, ikv, k8s, milvus, mssql, mysql, opentelemetry, spark, trino, postgres, pytorch, qdrant, redis, singlestore, snowflake, sqlite_vec]" +] +nlp = ["feast[docling, milvus, pytorch]"] +dev = ["feast[ci]"] +docs = ["feast[ci]"] + +[project.urls] +Homepage = "https://github.com/feast-dev/feast" + +[[project.authors]] +name = "Feast" + +[project.scripts] +feast = "feast.cli:cli" + [build-system] requires = [ "pybindgen==0.22.0", @@ -7,8 +174,15 @@ requires = [ "wheel", ] +[tool.setuptools] +packages = {find = {where = ["sdk/python"], exclude = ["java", "infra", "sdk/python/tests", "ui"]}} + [tool.setuptools_scm] -# Including this section is comparable to supplying use_scm_version=True in setup.py. +# Add Support for parsing tags that have a prefix containing '/' (ie 'sdk/go') to setuptools_scm. +# Regex modified from default tag regex in: +# https://github.com/pypa/setuptools_scm/blob/2a1b46d38fb2b8aeac09853e660bcd0d7c1bc7be/src/setuptools_scm/config.py#L9 +tag_regex = "^(?:[\\/\\w-]+)?(?P[vV]?\\d+(?:\\.\\d+){0,2}[^\\+]*)(?:\\+.*)?$" + [tool.ruff] line-length = 88 diff --git a/sdk/python/docs/source/feast.infra.offline_stores.contrib.couchbase_offline_store.rst b/sdk/python/docs/source/feast.infra.offline_stores.contrib.couchbase_offline_store.rst new file mode 100644 index 00000000000..7104b02bb66 --- /dev/null +++ b/sdk/python/docs/source/feast.infra.offline_stores.contrib.couchbase_offline_store.rst @@ -0,0 +1,37 @@ +feast.infra.offline\_stores.contrib.couchbase\_offline\_store package +===================================================================== + +Subpackages +----------- + +.. toctree:: + :maxdepth: 4 + + feast.infra.offline_stores.contrib.couchbase_offline_store.tests + +Submodules +---------- + +feast.infra.offline\_stores.contrib.couchbase\_offline\_store.couchbase module +------------------------------------------------------------------------------ + +.. automodule:: feast.infra.offline_stores.contrib.couchbase_offline_store.couchbase + :members: + :undoc-members: + :show-inheritance: + +feast.infra.offline\_stores.contrib.couchbase\_offline\_store.couchbase\_source module +-------------------------------------------------------------------------------------- + +.. automodule:: feast.infra.offline_stores.contrib.couchbase_offline_store.couchbase_source + :members: + :undoc-members: + :show-inheritance: + +Module contents +--------------- + +.. automodule:: feast.infra.offline_stores.contrib.couchbase_offline_store + :members: + :undoc-members: + :show-inheritance: diff --git a/sdk/python/docs/source/feast.infra.offline_stores.contrib.couchbase_offline_store.tests.rst b/sdk/python/docs/source/feast.infra.offline_stores.contrib.couchbase_offline_store.tests.rst new file mode 100644 index 00000000000..41566b5359a --- /dev/null +++ b/sdk/python/docs/source/feast.infra.offline_stores.contrib.couchbase_offline_store.tests.rst @@ -0,0 +1,21 @@ +feast.infra.offline\_stores.contrib.couchbase\_offline\_store.tests package +=========================================================================== + +Submodules +---------- + +feast.infra.offline\_stores.contrib.couchbase\_offline\_store.tests.data\_source module +--------------------------------------------------------------------------------------- + +.. automodule:: feast.infra.offline_stores.contrib.couchbase_offline_store.tests.data_source + :members: + :undoc-members: + :show-inheritance: + +Module contents +--------------- + +.. automodule:: feast.infra.offline_stores.contrib.couchbase_offline_store.tests + :members: + :undoc-members: + :show-inheritance: diff --git a/sdk/python/docs/source/feast.infra.offline_stores.contrib.rst b/sdk/python/docs/source/feast.infra.offline_stores.contrib.rst index ec74ddab05c..61e797bd6a9 100644 --- a/sdk/python/docs/source/feast.infra.offline_stores.contrib.rst +++ b/sdk/python/docs/source/feast.infra.offline_stores.contrib.rst @@ -8,6 +8,7 @@ Subpackages :maxdepth: 4 feast.infra.offline_stores.contrib.athena_offline_store + feast.infra.offline_stores.contrib.couchbase_offline_store feast.infra.offline_stores.contrib.mssql_offline_store feast.infra.offline_stores.contrib.postgres_offline_store feast.infra.offline_stores.contrib.spark_offline_store @@ -24,6 +25,14 @@ feast.infra.offline\_stores.contrib.athena\_repo\_configuration module :undoc-members: :show-inheritance: +feast.infra.offline\_stores.contrib.couchbase\_columnar\_repo\_configuration module +----------------------------------------------------------------------------------- + +.. automodule:: feast.infra.offline_stores.contrib.couchbase_columnar_repo_configuration + :members: + :undoc-members: + :show-inheritance: + feast.infra.offline\_stores.contrib.mssql\_repo\_configuration module --------------------------------------------------------------------- diff --git a/sdk/python/docs/source/feast.infra.utils.couchbase.rst b/sdk/python/docs/source/feast.infra.utils.couchbase.rst new file mode 100644 index 00000000000..d6d2025c428 --- /dev/null +++ b/sdk/python/docs/source/feast.infra.utils.couchbase.rst @@ -0,0 +1,21 @@ +feast.infra.utils.couchbase package +=================================== + +Submodules +---------- + +feast.infra.utils.couchbase.couchbase\_utils module +--------------------------------------------------- + +.. automodule:: feast.infra.utils.couchbase.couchbase_utils + :members: + :undoc-members: + :show-inheritance: + +Module contents +--------------- + +.. automodule:: feast.infra.utils.couchbase + :members: + :undoc-members: + :show-inheritance: diff --git a/sdk/python/docs/source/feast.infra.utils.rst b/sdk/python/docs/source/feast.infra.utils.rst index 083259bfaae..cfa82dc5fd2 100644 --- a/sdk/python/docs/source/feast.infra.utils.rst +++ b/sdk/python/docs/source/feast.infra.utils.rst @@ -7,6 +7,7 @@ Subpackages .. toctree:: :maxdepth: 4 + feast.infra.utils.couchbase feast.infra.utils.postgres feast.infra.utils.snowflake diff --git a/sdk/python/docs/source/feast.rst b/sdk/python/docs/source/feast.rst index ea34c3d8dd9..fdb91b2342d 100644 --- a/sdk/python/docs/source/feast.rst +++ b/sdk/python/docs/source/feast.rst @@ -332,6 +332,14 @@ feast.saved\_dataset module :undoc-members: :show-inheritance: +feast.ssl\_ca\_trust\_store\_setup module +----------------------------------------- + +.. automodule:: feast.ssl_ca_trust_store_setup + :members: + :undoc-members: + :show-inheritance: + feast.stream\_feature\_view module ---------------------------------- diff --git a/sdk/python/feast/entity.py b/sdk/python/feast/entity.py index 9c529115c8e..7f4eadc6352 100644 --- a/sdk/python/feast/entity.py +++ b/sdk/python/feast/entity.py @@ -173,13 +173,12 @@ def from_proto(cls, entity_proto: EntityProto): entity = cls( name=entity_proto.spec.name, join_keys=[entity_proto.spec.join_key], + value_type=ValueType(entity_proto.spec.value_type), description=entity_proto.spec.description, tags=dict(entity_proto.spec.tags), owner=entity_proto.spec.owner, ) - entity.value_type = ValueType(entity_proto.spec.value_type) - if entity_proto.meta.HasField("created_timestamp"): entity.created_timestamp = entity_proto.meta.created_timestamp.ToDatetime() if entity_proto.meta.HasField("last_updated_timestamp"): diff --git a/sdk/python/feast/feature_server.py b/sdk/python/feast/feature_server.py index ed742bcb98f..434efa7e44b 100644 --- a/sdk/python/feast/feature_server.py +++ b/sdk/python/feast/feature_server.py @@ -1,17 +1,29 @@ +import asyncio +import os import sys import threading import time import traceback from contextlib import asynccontextmanager +from importlib import resources as importlib_resources from typing import Any, Dict, List, Optional import pandas as pd import psutil from dateutil import parser -from fastapi import Depends, FastAPI, Request, Response, status +from fastapi import ( + Depends, + FastAPI, + Request, + Response, + WebSocket, + WebSocketDisconnect, + status, +) from fastapi.concurrency import run_in_threadpool from fastapi.logger import logger from fastapi.responses import JSONResponse +from fastapi.staticfiles import StaticFiles from google.protobuf.json_format import MessageToDict from prometheus_client import Gauge, start_http_server from pydantic import BaseModel @@ -75,6 +87,16 @@ class GetOnlineFeaturesRequest(BaseModel): features: Optional[List[str]] = None full_feature_names: bool = False query_embedding: Optional[List[float]] = None + query_string: Optional[str] = None + + +class ChatMessage(BaseModel): + role: str + content: str + + +class ChatRequest(BaseModel): + messages: List[ChatMessage] def _get_features(request: GetOnlineFeaturesRequest, store: "feast.FeatureStore"): @@ -112,6 +134,35 @@ def get_app( store: "feast.FeatureStore", registry_ttl_sec: int = DEFAULT_FEATURE_SERVER_REGISTRY_TTL, ): + """ + Creates a FastAPI app that can be used to start a feature server. + + Args: + store: The FeatureStore to use for serving features + registry_ttl_sec: The TTL in seconds for the registry cache + + Returns: + A FastAPI app + + Example: + ```python + from feast import FeatureStore + + store = FeatureStore(repo_path="feature_repo") + app = get_app(store) + ``` + + The app provides the following endpoints: + - `/get-online-features`: Get online features + - `/retrieve-online-documents`: Retrieve online documents + - `/push`: Push features to the feature store + - `/write-to-online-store`: Write to the online store + - `/health`: Health check + - `/materialize`: Materialize features + - `/materialize-incremental`: Materialize features incrementally + - `/chat`: Chat UI + - `/ws/chat`: WebSocket endpoint for chat + """ proto_json.patch() # Asynchronously refresh registry, notifying shutdown and canceling the active timer if the app is shutting down registry_proto = None @@ -195,6 +246,7 @@ async def retrieve_online_documents( entity_rows=request.entities, full_feature_names=request.full_feature_names, query=request.query_embedding, + query_string=request.query_string, ) response = await run_in_threadpool( @@ -295,6 +347,21 @@ async def health(): else Response(status_code=status.HTTP_503_SERVICE_UNAVAILABLE) ) + @app.post("/chat") + async def chat(request: ChatRequest): + # Process the chat request + # For now, just return dummy text + return {"response": "This is a dummy response from the Feast feature server."} + + @app.get("/chat") + async def chat_ui(): + # Serve the chat UI + static_dir_ref = importlib_resources.files(__spec__.parent) / "static/chat" # type: ignore[name-defined, arg-type] + with importlib_resources.as_file(static_dir_ref) as static_dir: + with open(os.path.join(static_dir, "index.html")) as f: + content = f.read() + return Response(content=content, media_type="text/html") + @app.post("/materialize", dependencies=[Depends(inject_user_details)]) def materialize(request: MaterializeRequest) -> None: for feature_view in request.feature_views or []: @@ -335,6 +402,46 @@ async def rest_exception_handler(request: Request, exc: Exception): content=str(exc), ) + # Chat WebSocket connection manager + class ConnectionManager: + def __init__(self): + self.active_connections: List[WebSocket] = [] + + async def connect(self, websocket: WebSocket): + await websocket.accept() + self.active_connections.append(websocket) + + def disconnect(self, websocket: WebSocket): + self.active_connections.remove(websocket) + + async def send_message(self, message: str, websocket: WebSocket): + await websocket.send_text(message) + + manager = ConnectionManager() + + @app.websocket("/ws/chat") + async def websocket_endpoint(websocket: WebSocket): + await manager.connect(websocket) + try: + while True: + message = await websocket.receive_text() + # Process the received message (currently unused but kept for future implementation) + # For now, just return dummy text + response = f"You sent: '{message}'. This is a dummy response from the Feast feature server." + + # Stream the response word by word + words = response.split() + for word in words: + await manager.send_message(word + " ", websocket) + await asyncio.sleep(0.1) # Add a small delay between words + except WebSocketDisconnect: + manager.disconnect(websocket) + + # Mount static files + static_dir_ref = importlib_resources.files(__spec__.parent) / "static" # type: ignore[name-defined, arg-type] + with importlib_resources.as_file(static_dir_ref) as static_dir: + app.mount("/static", StaticFiles(directory=static_dir), name="static") + return app diff --git a/sdk/python/feast/feature_store.py b/sdk/python/feast/feature_store.py index f0bdc4c1f28..7073a20d1e0 100644 --- a/sdk/python/feast/feature_store.py +++ b/sdk/python/feast/feature_store.py @@ -89,6 +89,8 @@ from feast.saved_dataset import SavedDataset, SavedDatasetStorage, ValidationReference from feast.ssl_ca_trust_store_setup import configure_ca_trust_store_env_variables from feast.stream_feature_view import StreamFeatureView +from feast.transformation.pandas_transformation import PandasTransformation +from feast.transformation.python_transformation import PythonTransformation from feast.utils import _utc_now warnings.simplefilter("once", DeprecationWarning) @@ -1546,6 +1548,64 @@ def _get_feature_view_and_df_for_online_write( df = pd.DataFrame(df) except Exception as _: raise DataFrameSerializationError(df) + + # # Apply transformations if this is an OnDemandFeatureView with write_to_online_store=True + if ( + isinstance(feature_view, OnDemandFeatureView) + and feature_view.write_to_online_store + ): + if ( + feature_view.mode == "python" + and isinstance( + feature_view.feature_transformation, PythonTransformation + ) + and df is not None + ): + input_dict = ( + df.to_dict(orient="records")[0] + if feature_view.singleton + else df.to_dict(orient="list") + ) + transformed_data = feature_view.feature_transformation.udf(input_dict) + if feature_view.write_to_online_store: + entities = [ + self.get_entity(entity) + for entity in (feature_view.entities or []) + ] + join_keys = [entity.join_key for entity in entities if entity] + join_keys = [k for k in join_keys if k in input_dict.keys()] + transformed_df = pd.DataFrame(transformed_data) + input_df = pd.DataFrame(input_dict) + if input_df.shape[0] == transformed_df.shape[0]: + for k in input_dict: + if k not in transformed_data: + transformed_data[k] = input_dict[k] + transformed_df = pd.DataFrame(transformed_data) + else: + transformed_df = pd.merge( + transformed_df, + input_df, + how="left", + on=join_keys, + ) + else: + # overwrite any transformed features and update the dictionary + for k in input_dict: + if k not in transformed_data: + transformed_data[k] = input_dict[k] + df = pd.DataFrame(transformed_data) + elif feature_view.mode == "pandas" and isinstance( + feature_view.feature_transformation, PandasTransformation + ): + transformed_df = feature_view.feature_transformation.udf(df) + if df is not None: + for col in df.columns: + transformed_df[col] = df[col] + df = transformed_df + + else: + raise Exception("Unsupported OnDemandFeatureView mode") + return feature_view, df def write_to_online_store( @@ -1863,9 +1923,10 @@ def retrieve_online_documents( def retrieve_online_documents_v2( self, - query: Union[str, List[float]], - top_k: int, features: List[str], + top_k: int, + query: Optional[List[float]] = None, + query_string: Optional[str] = None, distance_metric: Optional[str] = "L2", ) -> OnlineResponse: """ @@ -1875,18 +1936,18 @@ def retrieve_online_documents_v2( features: The list of features that should be retrieved from the online document store. These features can be specified either as a list of string document feature references or as a feature service. String feature references must have format "feature_view:feature", e.g, "document_fv:document_embeddings". - query: The query to retrieve the closest document features for. + query: The embeded query to retrieve the closest document features for (optional) top_k: The number of closest document features to retrieve. distance_metric: The distance metric to use for retrieval. + query_string: The query string to retrieve the closest document features using keyword search (bm25). """ - if isinstance(query, str): - raise ValueError( - "Using embedding functionality is not supported for document retrieval. Please embed the query before calling retrieve_online_documents." - ) + assert query is not None or query_string is not None, ( + "Either query or query_string must be provided." + ) ( available_feature_views, - _, + available_odfv_views, ) = utils._get_feature_views_to_use( registry=self._registry, project=self.project, @@ -1897,13 +1958,20 @@ def retrieve_online_documents_v2( feature_view_set = set() for feature in features: feature_view_name = feature.split(":")[0] - feature_view = self.get_feature_view(feature_view_name) + if feature_view_name in [fv.name for fv in available_odfv_views]: + feature_view: Union[OnDemandFeatureView, FeatureView] = ( + self.get_on_demand_feature_view(feature_view_name) + ) + else: + feature_view = self.get_feature_view(feature_view_name) feature_view_set.add(feature_view.name) if len(feature_view_set) > 1: raise ValueError("Document retrieval only supports a single feature view.") requested_features = [ f.split(":")[1] for f in features if isinstance(f, str) and ":" in f ] + if len(available_feature_views) == 0: + available_feature_views.extend(available_odfv_views) # type: ignore[arg-type] requested_feature_view = available_feature_views[0] if not requested_feature_view: @@ -1919,6 +1987,7 @@ def retrieve_online_documents_v2( query, top_k, distance_metric, + query_string, ) def _retrieve_from_online_store( @@ -1985,9 +2054,10 @@ def _retrieve_from_online_store_v2( provider: Provider, table: FeatureView, requested_features: List[str], - query: List[float], + query: Optional[List[float]], top_k: int, distance_metric: Optional[str], + query_string: Optional[str], ) -> OnlineResponse: """ Search and return document features from the online document store. @@ -2003,6 +2073,7 @@ def _retrieve_from_online_store_v2( query=query, top_k=top_k, distance_metric=distance_metric, + query_string=query_string, ) entity_key_dict: Dict[str, List[ValueProto]] = {} diff --git a/sdk/python/feast/feature_view.py b/sdk/python/feast/feature_view.py index 4aeb9a9c1dc..49b74893451 100644 --- a/sdk/python/feast/feature_view.py +++ b/sdk/python/feast/feature_view.py @@ -48,6 +48,7 @@ DUMMY_ENTITY = Entity( name=DUMMY_ENTITY_NAME, join_keys=[DUMMY_ENTITY_ID], + value_type=ValueType.UNKNOWN, ) DUMMY_ENTITY_FIELD = Field( name=DUMMY_ENTITY_ID, @@ -191,6 +192,10 @@ def __init__( else: features.append(field) + assert len([f for f in features if f.vector_index]) < 2, ( + f"Only one vector feature is allowed per feature view. Please update {self.name}." + ) + # TODO(felixwang9817): Add more robust validation of features. cols = [field.name for field in schema] for col in cols: @@ -343,12 +348,11 @@ def to_proto(self) -> FeatureViewProto: if self.stream_source: stream_source_proto = self.stream_source.to_proto() stream_source_proto.data_source_class_type = f"{self.stream_source.__class__.__module__}.{self.stream_source.__class__.__name__}" - spec = FeatureViewSpecProto( name=self.name, entities=self.entities, entity_columns=[field.to_proto() for field in self.entity_columns], - features=[field.to_proto() for field in self.features], + features=[feature.to_proto() for feature in self.features], description=self.description, tags=self.tags, owner=self.owner, diff --git a/sdk/python/feast/infra/feature_servers/local_process/config.py b/sdk/python/feast/infra/feature_servers/local_process/config.py index 3d97912e4bd..942927ec2e8 100644 --- a/sdk/python/feast/infra/feature_servers/local_process/config.py +++ b/sdk/python/feast/infra/feature_servers/local_process/config.py @@ -4,5 +4,8 @@ class LocalFeatureServerConfig(BaseFeatureServerConfig): + # Feature server type selector. type: Literal["local"] = "local" - """Feature server type selector.""" + + # The endpoint definition for transformation_service + transformation_service_endpoint: str = "localhost:6569" diff --git a/sdk/python/feast/infra/feature_servers/multicloud/Dockerfile b/sdk/python/feast/infra/feature_servers/multicloud/Dockerfile index 6e1c81b654f..f4096e8494d 100644 --- a/sdk/python/feast/infra/feature_servers/multicloud/Dockerfile +++ b/sdk/python/feast/infra/feature_servers/multicloud/Dockerfile @@ -1,7 +1,7 @@ FROM registry.access.redhat.com/ubi8/python-311:1 -ARG VERSION -RUN pip install "feast[aws,gcp,snowflake,redis,go,mysql,postgres,opentelemetry,grpcio,k8s,duckdb,milvus]"==${VERSION} +COPY requirements.txt requirements.txt +RUN pip install -r requirements.txt # modify permissions to support running with a random uid RUN chmod g+w $(python -c "import feast.ui as ui; print(ui.__path__)" | tr -d "[']")/build/projects-list.json diff --git a/sdk/python/feast/infra/feature_servers/multicloud/Dockerfile.dev b/sdk/python/feast/infra/feature_servers/multicloud/Dockerfile.dev index f5cef2ad9f3..31ac4a6366a 100644 --- a/sdk/python/feast/infra/feature_servers/multicloud/Dockerfile.dev +++ b/sdk/python/feast/infra/feature_servers/multicloud/Dockerfile.dev @@ -4,7 +4,11 @@ USER 0 RUN npm install -g yarn yalc && rm -rf .npm USER default -COPY --chown=default . ${APP_ROOT}/src +COPY --chown=default .git ${APP_ROOT}/src/.git +COPY --chown=default setup.py pyproject.toml README.md Makefile ${APP_ROOT}/src/ +COPY --chown=default protos ${APP_ROOT}/src/protos +COPY --chown=default ui ${APP_ROOT}/src/ui +COPY --chown=default sdk/python ${APP_ROOT}/src/sdk/python WORKDIR ${APP_ROOT}/src/ui RUN npm install && \ diff --git a/sdk/python/feast/infra/feature_servers/multicloud/requirements.txt b/sdk/python/feast/infra/feature_servers/multicloud/requirements.txt new file mode 100644 index 00000000000..20789a976d7 --- /dev/null +++ b/sdk/python/feast/infra/feature_servers/multicloud/requirements.txt @@ -0,0 +1,2 @@ +# keep VERSION on line #2, this is critical to release CI +feast[aws,gcp,snowflake,redis,go,mysql,postgres,opentelemetry,grpcio,k8s,duckdb,milvus] == 0.46.0 diff --git a/sdk/python/feast/infra/offline_stores/contrib/couchbase_columnar_repo_configuration.py b/sdk/python/feast/infra/offline_stores/contrib/couchbase_columnar_repo_configuration.py new file mode 100644 index 00000000000..745a074a757 --- /dev/null +++ b/sdk/python/feast/infra/offline_stores/contrib/couchbase_columnar_repo_configuration.py @@ -0,0 +1,20 @@ +from feast.infra.offline_stores.contrib.couchbase_offline_store.tests.data_source import ( + CouchbaseColumnarDataSourceCreator, +) +from tests.integration.feature_repos.integration_test_repo_config import ( + IntegrationTestRepoConfig, +) +from tests.integration.feature_repos.repo_configuration import REDIS_CONFIG +from tests.integration.feature_repos.universal.online_store.redis import ( + RedisOnlineStoreCreator, +) + +FULL_REPO_CONFIGS = [ + IntegrationTestRepoConfig( + provider="aws", + offline_store_creator=CouchbaseColumnarDataSourceCreator, + ), +] + +AVAILABLE_OFFLINE_STORES = [("aws", CouchbaseColumnarDataSourceCreator)] +AVAILABLE_ONLINE_STORES = {"redis": (REDIS_CONFIG, RedisOnlineStoreCreator)} diff --git a/sdk/python/feast/infra/offline_stores/contrib/couchbase_offline_store/__init__.py b/sdk/python/feast/infra/offline_stores/contrib/couchbase_offline_store/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/sdk/python/feast/infra/offline_stores/contrib/couchbase_offline_store/couchbase.py b/sdk/python/feast/infra/offline_stores/contrib/couchbase_offline_store/couchbase.py new file mode 100644 index 00000000000..a90d6c2172b --- /dev/null +++ b/sdk/python/feast/infra/offline_stores/contrib/couchbase_offline_store/couchbase.py @@ -0,0 +1,729 @@ +import contextlib +import warnings +from dataclasses import asdict +from datetime import datetime, timedelta +from typing import ( + Any, + Callable, + ContextManager, + Dict, + Iterator, + KeysView, + List, + Literal, + Optional, + Tuple, + Union, + cast, +) + +import numpy as np +import pandas as pd +import pyarrow as pa +from couchbase_columnar.cluster import Cluster +from couchbase_columnar.common.result import BlockingQueryResult +from couchbase_columnar.credential import Credential +from couchbase_columnar.options import ClusterOptions, QueryOptions, TimeoutOptions +from jinja2 import BaseLoader, Environment +from pydantic import StrictFloat, StrictStr + +from feast.data_source import DataSource +from feast.errors import InvalidEntityType, ZeroRowsQueryResult +from feast.feature_view import DUMMY_ENTITY_ID, DUMMY_ENTITY_VAL, FeatureView +from feast.infra.offline_stores.offline_store import ( + OfflineStore, + RetrievalJob, + RetrievalMetadata, +) +from feast.infra.registry.base_registry import BaseRegistry +from feast.infra.utils.couchbase.couchbase_utils import normalize_timestamp +from feast.on_demand_feature_view import OnDemandFeatureView +from feast.repo_config import FeastConfigBaseModel, RepoConfig +from feast.saved_dataset import SavedDatasetStorage + +from ... import offline_utils +from .couchbase_source import ( + CouchbaseColumnarSource, + SavedDatasetCouchbaseColumnarStorage, +) + +# Only prints out runtime warnings once. +warnings.simplefilter("once", RuntimeWarning) + + +class CouchbaseColumnarOfflineStoreConfig(FeastConfigBaseModel): + """Offline store config for Couchbase Columnar""" + + type: Literal["couchbase.offline"] = "couchbase.offline" + + connection_string: Optional[StrictStr] = None + user: Optional[StrictStr] = None + password: Optional[StrictStr] = None + timeout: StrictFloat = 120 + + +class CouchbaseColumnarOfflineStore(OfflineStore): + @staticmethod + def pull_latest_from_table_or_query( + config: RepoConfig, + data_source: DataSource, + join_key_columns: List[str], + feature_name_columns: List[str], + timestamp_field: str, + created_timestamp_column: Optional[str], + start_date: datetime, + end_date: datetime, + ) -> RetrievalJob: + """ + Fetch the latest rows for each join key. + """ + warnings.warn( + "This offline store is an experimental feature in alpha development. " + "Some functionality may still be unstable so functionality can change in the future.", + RuntimeWarning, + ) + assert isinstance(config.offline_store, CouchbaseColumnarOfflineStoreConfig) + assert isinstance(data_source, CouchbaseColumnarSource) + from_expression = data_source.get_table_query_string() + + partition_by_join_key_string = ", ".join(_append_alias(join_key_columns, "a")) + if partition_by_join_key_string != "": + partition_by_join_key_string = ( + "PARTITION BY " + partition_by_join_key_string + ) + timestamps = [timestamp_field] + if created_timestamp_column: + timestamps.append(created_timestamp_column) + timestamp_desc_string = " DESC, ".join(_append_alias(timestamps, "a")) + " DESC" + a_field_string = ", ".join( + _append_alias(join_key_columns + feature_name_columns + timestamps, "a") + ) + b_field_string = ", ".join( + _append_alias(join_key_columns + feature_name_columns + timestamps, "b") + ) + + start_date_normalized = normalize_timestamp(start_date) + end_date_normalized = normalize_timestamp(end_date) + + query = f""" + SELECT + {b_field_string} + {f", {repr(DUMMY_ENTITY_VAL)} AS {DUMMY_ENTITY_ID}" if not join_key_columns else ""} + FROM ( + SELECT {a_field_string}, + ROW_NUMBER() OVER({partition_by_join_key_string} ORDER BY {timestamp_desc_string}) AS _feast_row + FROM {from_expression} a + WHERE a.{timestamp_field} BETWEEN '{start_date_normalized}' AND '{end_date_normalized}' + ) b + WHERE _feast_row = 1 + """ + + return CouchbaseColumnarRetrievalJob( + query=query, + config=config, + full_feature_names=False, + on_demand_feature_views=None, + timestamp_field=timestamp_field, + ) + + @staticmethod + def get_historical_features( + config: RepoConfig, + feature_views: List[FeatureView], + feature_refs: List[str], + entity_df: Union[pd.DataFrame, str], + registry: BaseRegistry, + project: str, + full_feature_names: bool = False, + ) -> RetrievalJob: + """ + Retrieve historical features using point-in-time joins. + """ + warnings.warn( + "This offline store is an experimental feature in alpha development. " + "Some functionality may still be unstable so functionality can change in the future.", + RuntimeWarning, + ) + assert isinstance(config.offline_store, CouchbaseColumnarOfflineStoreConfig) + for fv in feature_views: + assert isinstance(fv.batch_source, CouchbaseColumnarSource) + + entity_schema = _get_entity_schema(entity_df, config) + + entity_df_event_timestamp_col = ( + offline_utils.infer_event_timestamp_from_entity_df(entity_schema) + ) + + entity_df_event_timestamp_range = _get_entity_df_event_timestamp_range( + entity_df, entity_df_event_timestamp_col, config + ) + + @contextlib.contextmanager + def query_generator() -> Iterator[str]: + source = cast(CouchbaseColumnarSource, feature_views[0].batch_source) + database = source.database + scope = source.scope + + table_name = ( + f"{database}.{scope}.{offline_utils.get_temp_entity_table_name()}" + ) + + _upload_entity_df(config, entity_df, table_name) + + expected_join_keys = offline_utils.get_expected_join_keys( + project, feature_views, registry + ) + + offline_utils.assert_expected_columns_in_entity_df( + entity_schema, expected_join_keys, entity_df_event_timestamp_col + ) + + query_context = offline_utils.get_feature_view_query_context( + feature_refs, + feature_views, + registry, + project, + entity_df_event_timestamp_range, + ) + + query_context_dict = [asdict(context) for context in query_context] + + try: + query = build_point_in_time_query( + query_context_dict, + left_table_query_string=table_name, + entity_df_event_timestamp_col=entity_df_event_timestamp_col, + entity_df_columns=entity_schema.keys(), + query_template=MULTIPLE_FEATURE_VIEW_POINT_IN_TIME_JOIN, + full_feature_names=full_feature_names, + ) + yield query + finally: + if table_name: + _execute_query( + config.offline_store, + f"DROP COLLECTION {table_name} IF EXISTS", + ) + + return CouchbaseColumnarRetrievalJob( + query=query_generator, + config=config, + full_feature_names=full_feature_names, + on_demand_feature_views=OnDemandFeatureView.get_requested_odfvs( + feature_refs, project, registry + ), + metadata=RetrievalMetadata( + features=feature_refs, + keys=list(entity_schema.keys() - {entity_df_event_timestamp_col}), + min_event_timestamp=entity_df_event_timestamp_range[0], + max_event_timestamp=entity_df_event_timestamp_range[1], + ), + timestamp_field=entity_df_event_timestamp_col, + ) + + @staticmethod + def pull_all_from_table_or_query( + config: RepoConfig, + data_source: DataSource, + join_key_columns: List[str], + feature_name_columns: List[str], + timestamp_field: str, + start_date: datetime, + end_date: datetime, + ) -> RetrievalJob: + """ + Fetch all rows from the specified table or query within the time range. + """ + warnings.warn( + "This offline store is an experimental feature in alpha development. " + "Some functionality may still be unstable so functionality can change in the future.", + RuntimeWarning, + ) + assert isinstance(config.offline_store, CouchbaseColumnarOfflineStoreConfig) + assert isinstance(data_source, CouchbaseColumnarSource) + from_expression = data_source.get_table_query_string() + + field_string = ", ".join( + join_key_columns + feature_name_columns + [timestamp_field] + ) + + start_date_normalized = normalize_timestamp(start_date) + end_date_normalized = normalize_timestamp(end_date) + + query = f""" + SELECT {field_string} + FROM {from_expression} + WHERE `{timestamp_field}` BETWEEN '{start_date_normalized}' AND '{end_date_normalized}' + """ + + return CouchbaseColumnarRetrievalJob( + query=query, + config=config, + full_feature_names=False, + on_demand_feature_views=None, + timestamp_field=timestamp_field, + ) + + +class CouchbaseColumnarRetrievalJob(RetrievalJob): + def __init__( + self, + query: Union[str, Callable[[], ContextManager[str]]], + config: RepoConfig, + full_feature_names: bool, + timestamp_field: str, + on_demand_feature_views: Optional[List[OnDemandFeatureView]] = None, + metadata: Optional[RetrievalMetadata] = None, + ): + if not isinstance(query, str): + self._query_generator = query + else: + + @contextlib.contextmanager + def query_generator() -> Iterator[str]: + assert isinstance(query, str) + yield query + + self._query_generator = query_generator + self._config = config + self._full_feature_names = full_feature_names + self._on_demand_feature_views = on_demand_feature_views or [] + self._metadata = metadata + self._timestamp_field = timestamp_field + + @property + def full_feature_names(self) -> bool: + return self._full_feature_names + + @property + def on_demand_feature_views(self) -> List[OnDemandFeatureView]: + return self._on_demand_feature_views + + def _to_df_internal(self, timeout: Optional[int] = None) -> pd.DataFrame: + # Use PyArrow to convert the result to a pandas DataFrame + return self._to_arrow_internal(timeout).to_pandas() + + def to_sql(self) -> str: + with self._query_generator() as query: + return query + + def _to_arrow_internal(self, timeout: Optional[int] = None) -> pa.Table: + with self._query_generator() as query: + res = _execute_query(self._config.offline_store, query) + rows = res.get_all_rows() + + processed_rows = [] + for row in rows: + processed_row = {} + for key, value in row.items(): + if key == self._timestamp_field and value is not None: + # Parse and ensure timezone-aware datetime + processed_row[key] = pd.to_datetime(value, utc=True) + else: + processed_row[key] = np.nan if value is None else value + processed_rows.append(processed_row) + + # Convert to PyArrow table + table = pa.Table.from_pylist(processed_rows) + return table + + @property + def metadata(self) -> Optional[RetrievalMetadata]: + return self._metadata + + def persist( + self, + storage: SavedDatasetStorage, + allow_overwrite: Optional[bool] = False, + timeout: Optional[int] = None, + ): + assert isinstance(storage, SavedDatasetCouchbaseColumnarStorage) + table_name = f"{storage.couchbase_options._database}.{storage.couchbase_options._scope}.{offline_utils.get_temp_entity_table_name()}" + df_to_columnar(self.to_df(), table_name, self._config.offline_store) + + +def _get_columnar_cluster(config: CouchbaseColumnarOfflineStoreConfig) -> Cluster: + assert config.connection_string is not None + assert config.user is not None + assert config.password is not None + + cred = Credential.from_username_and_password(config.user, config.password) + timeout_opts = TimeoutOptions(dispatch_timeout=timedelta(seconds=120)) + return Cluster.create_instance( + config.connection_string, cred, ClusterOptions(timeout_options=timeout_opts) + ) + + +def _execute_query( + config: CouchbaseColumnarOfflineStoreConfig, + query: str, + named_params: Optional[Dict[str, Any]] = None, +) -> BlockingQueryResult: + cluster = _get_columnar_cluster(config) + return cluster.execute_query( + query, + QueryOptions( + named_parameters=named_params, timeout=timedelta(seconds=config.timeout) + ), + ) + + +def df_to_columnar( + df: pd.DataFrame, + table_name: str, + offline_store: CouchbaseColumnarOfflineStoreConfig, +): + df_copy = df.copy() + insert_values = df_copy.apply( + lambda row: { + col: ( + normalize_timestamp(row[col], "%Y-%m-%dT%H:%M:%S.%f+00:00") + if isinstance(row[col], pd.Timestamp) + else row[col] + ) + for col in df_copy.columns + }, + axis=1, + ).tolist() + + create_collection_query = f"CREATE COLLECTION {table_name} IF NOT EXISTS PRIMARY KEY(pk: UUID) AUTOGENERATED;" + insert_query = f"INSERT INTO {table_name} ({insert_values});" + + _execute_query(offline_store, create_collection_query) + _execute_query(offline_store, insert_query) + + +def _upload_entity_df( + config: RepoConfig, entity_df: Union[pd.DataFrame, str], table_name: str +): + if isinstance(entity_df, pd.DataFrame): + df_to_columnar(entity_df, table_name, config.offline_store) + elif isinstance(entity_df, str): + # If the entity_df is a string (SQL query), create a Columnar collection out of it + create_collection_query = f""" + CREATE COLLECTION {table_name} IF NOT EXISTS + PRIMARY KEY(pk: UUID) AUTOGENERATED + AS {entity_df} + """ + _execute_query(config.offline_store, create_collection_query) + else: + raise InvalidEntityType(type(entity_df)) + + +def _get_entity_df_event_timestamp_range( + entity_df: Union[pd.DataFrame, str], + entity_df_event_timestamp_col: str, + config: RepoConfig, +) -> Tuple[datetime, datetime]: + if isinstance(entity_df, pd.DataFrame): + entity_df_event_timestamp = entity_df.loc[ + :, entity_df_event_timestamp_col + ].infer_objects() + if pd.api.types.is_string_dtype(entity_df_event_timestamp): + entity_df_event_timestamp = pd.to_datetime( + entity_df_event_timestamp, utc=True + ) + entity_df_event_timestamp_range = ( + entity_df_event_timestamp.min().to_pydatetime(), + entity_df_event_timestamp.max().to_pydatetime(), + ) + + elif isinstance(entity_df, str): + query = f""" + SELECT + MIN({entity_df_event_timestamp_col}) AS min, + MAX({entity_df_event_timestamp_col}) AS max + FROM ({entity_df}) AS tmp_alias + """ + + res = _execute_query(config.offline_store, query) + rows = res.get_all_rows() + + if not rows: + raise ZeroRowsQueryResult(query) + + # Convert the string timestamps to datetime objects + min_ts = pd.to_datetime(rows[0]["min"], utc=True).to_pydatetime() + max_ts = pd.to_datetime(rows[0]["max"], utc=True).to_pydatetime() + entity_df_event_timestamp_range = (min_ts, max_ts) + else: + raise InvalidEntityType(type(entity_df)) + return entity_df_event_timestamp_range + + +def _escape_column(column: str) -> str: + """Wrap column names in backticks to handle reserved words.""" + return f"`{column}`" + + +def _append_alias(field_names: List[str], alias: str) -> List[str]: + """Append alias to escaped column names.""" + return [f"{alias}.{_escape_column(field_name)}" for field_name in field_names] + + +def build_point_in_time_query( + feature_view_query_contexts: List[dict], + left_table_query_string: str, + entity_df_event_timestamp_col: str, + entity_df_columns: KeysView[str], + query_template: str, + full_feature_names: bool = False, +) -> str: + """Build point-in-time query between each feature view table and the entity dataframe for Couchbase Columnar""" + template = Environment(loader=BaseLoader()).from_string(source=query_template) + final_output_feature_names = list(entity_df_columns) + final_output_feature_names.extend( + [ + ( + f"{fv['name']}__{fv['field_mapping'].get(feature, feature)}" + if full_feature_names + else fv["field_mapping"].get(feature, feature) + ) + for fv in feature_view_query_contexts + for feature in fv["features"] + ] + ) + + # Add additional fields to dict + template_context = { + "left_table_query_string": left_table_query_string, + "entity_df_event_timestamp_col": entity_df_event_timestamp_col, + "unique_entity_keys": set( + [entity for fv in feature_view_query_contexts for entity in fv["entities"]] + ), + "featureviews": feature_view_query_contexts, + "full_feature_names": full_feature_names, + "final_output_feature_names": final_output_feature_names, + } + + query = template.render(template_context) + return query + + +def get_couchbase_query_schema(config, entity_df: str) -> Dict[str, np.dtype]: + df_query = f"({entity_df}) AS sub" + res = _execute_query(config.offline_store, f"SELECT sub.* FROM {df_query} LIMIT 1") + rows = res.get_all_rows() + + if rows and len(rows) > 0: + # Get the first row + first_row = rows[0] + # Create dictionary mapping each column to dtype('O') + return {key: np.dtype("O") for key in first_row.keys()} + + return {} + + +def _get_entity_schema( + entity_df: Union[pd.DataFrame, str], + config: RepoConfig, +) -> Dict[str, np.dtype]: + if isinstance(entity_df, pd.DataFrame): + return dict(zip(entity_df.columns, entity_df.dtypes)) + + elif isinstance(entity_df, str): + return get_couchbase_query_schema(config, entity_df) + else: + raise InvalidEntityType(type(entity_df)) + + +MULTIPLE_FEATURE_VIEW_POINT_IN_TIME_JOIN = """ +WITH entity_dataframe AS ( + SELECT e.*, + e.`{{entity_df_event_timestamp_col}}` AS entity_timestamp + {% for featureview in featureviews -%} + {% if featureview.entities -%} + ,CONCAT( + {% for entity in featureview.entities -%} + TOSTRING(e.`{{entity}}`), + {% endfor -%} + TOSTRING(e.`{{entity_df_event_timestamp_col}}`) + ) AS `{{featureview.name}}__entity_row_unique_id` + {% else -%} + ,TOSTRING(e.`{{entity_df_event_timestamp_col}}`) AS `{{featureview.name}}__entity_row_unique_id` + {% endif -%} + {% endfor %} + FROM {{ left_table_query_string }} e +), + +{% for featureview in featureviews %} + +`{{ featureview.name }}__entity_dataframe` AS ( + SELECT + {% if featureview.entities %}`{{ featureview.entities | join('`, `') }}`,{% endif %} + entity_timestamp, + `{{featureview.name}}__entity_row_unique_id` + FROM entity_dataframe + GROUP BY + {% if featureview.entities %}`{{ featureview.entities | join('`, `')}}`,{% endif %} + entity_timestamp, + `{{featureview.name}}__entity_row_unique_id` +), + +/* + This query template performs the point-in-time correctness join for a single feature set table + to the provided entity table. + + 1. We first join the current feature_view to the entity dataframe that has been passed. + This JOIN has the following logic: + - For each row of the entity dataframe, only keep the rows where the `timestamp_field` + is less than the one provided in the entity dataframe + - If there a TTL for the current feature_view, also keep the rows where the `timestamp_field` + is higher the the one provided minus the TTL + - For each row, Join on the entity key and retrieve the `entity_row_unique_id` that has been + computed previously + + The output of this CTE will contain all the necessary information and already filtered out most + of the data that is not relevant. +*/ +`{{ featureview.name }}__subquery` AS ( + LET max_ts = (SELECT RAW MAX(entity_timestamp) FROM entity_dataframe)[0] + SELECT s.* FROM ( + LET min_ts = (SELECT RAW MIN(entity_timestamp) FROM entity_dataframe)[0] + SELECT + `{{ featureview.timestamp_field }}` as event_timestamp, + {{ '`' ~ featureview.created_timestamp_column ~ '` as created_timestamp,' if featureview.created_timestamp_column else '' }} + {{ featureview.entity_selections | join(', ')}}{% if featureview.entity_selections %},{% else %}{% endif %} + {% for feature in featureview.features -%} + `{{ feature }}` as {% if full_feature_names %}`{{ featureview.name }}__{{featureview.field_mapping.get(feature, feature)}}`{% else %}`{{ featureview.field_mapping.get(feature, feature) }}`{% endif %}{% if not loop.last %}, {% endif %} + {%- endfor %} + FROM {{ featureview.table_subquery }} AS sub + WHERE `{{ featureview.timestamp_field }}` <= max_ts + {% if featureview.ttl == 0 %}{% else %} + AND date_diff_str(min_ts, `{{ featureview.timestamp_field }}`, "second") <= {{ featureview.ttl }} + {% endif %} + ) s +), + +`{{ featureview.name }}__base` AS ( + SELECT + subquery.*, + entity_dataframe.entity_timestamp, + entity_dataframe.`{{featureview.name}}__entity_row_unique_id` + FROM `{{ featureview.name }}__subquery` AS subquery + INNER JOIN `{{ featureview.name }}__entity_dataframe` AS entity_dataframe + ON TRUE + AND subquery.event_timestamp <= entity_dataframe.entity_timestamp + {% if featureview.ttl == 0 %}{% else %} + AND date_diff_str(entity_dataframe.entity_timestamp, subquery.event_timestamp, "second") <= {{ featureview.ttl }} + {% endif %} + {% for entity in featureview.entities %} + AND subquery.`{{ entity }}` = entity_dataframe.`{{ entity }}` + {% endfor %} +), + +/* + 2. If the `created_timestamp_column` has been set, we need to + deduplicate the data first. This is done by calculating the + `MAX(created_at_timestamp)` for each event_timestamp. + We then join the data on the next CTE +*/ +{% if featureview.created_timestamp_column %} +`{{ featureview.name }}__dedup` AS ( + SELECT + `{{featureview.name}}__entity_row_unique_id`, + event_timestamp, + MAX(created_timestamp) AS created_timestamp + FROM `{{ featureview.name }}__base` + GROUP BY `{{featureview.name}}__entity_row_unique_id`, event_timestamp +), +{% endif %} + +/* + 3. The data has been filtered during the first CTE "*__base" + Thus we only need to compute the latest timestamp of each feature. +*/ +`{{ featureview.name }}__latest` AS ( + SELECT + event_timestamp + {% if featureview.created_timestamp_column %},created_timestamp{% endif %}, + `{{featureview.name}}__entity_row_unique_id` + FROM ( + SELECT base.*, + ROW_NUMBER() OVER( + PARTITION BY base.`{{featureview.name}}__entity_row_unique_id` + ORDER BY base.event_timestamp DESC + {% if featureview.created_timestamp_column %}, base.created_timestamp DESC{% endif %} + ) AS row_number + FROM `{{ featureview.name }}__base` base + {% if featureview.created_timestamp_column %} + INNER JOIN `{{ featureview.name }}__dedup` dedup + ON base.`{{featureview.name}}__entity_row_unique_id` = dedup.`{{featureview.name}}__entity_row_unique_id` + AND base.event_timestamp = dedup.event_timestamp + AND base.created_timestamp = dedup.created_timestamp + {% endif %} + ) AS sub + WHERE sub.row_number = 1 +), + +/* + 4. Once we know the latest value of each feature for a given timestamp, + we can join again the data back to the original "base" dataset +*/ +`{{ featureview.name }}__cleaned` AS ( + SELECT base.* + FROM `{{ featureview.name }}__base` AS base + INNER JOIN `{{ featureview.name }}__latest` AS latest + ON base.`{{featureview.name}}__entity_row_unique_id` = latest.`{{featureview.name}}__entity_row_unique_id` + AND base.event_timestamp = latest.event_timestamp + {% if featureview.created_timestamp_column %} + AND base.created_timestamp = latest.created_timestamp + {% endif %} +){% if not loop.last %},{% endif %} + +{% endfor %} + +/* + Joins the outputs of multiple time travel joins to a single table. + The entity_dataframe dataset being our source of truth here. + */ +SELECT DISTINCT + {%- set fields = [] %} + {%- for feature_name in final_output_feature_names %} + {%- if '__' not in feature_name %} + {%- set ns = namespace(found=false) %} + {%- for fv in featureviews %} + {%- for feature in fv.features %} + {%- if feature == feature_name %} + {%- set ns.found = true %} + {%- if full_feature_names %} + {%- set _ = fields.append('IFMISSINGORNULL(`' ~ fv.name ~ '_final`.`' ~ fv.name ~ '__' ~ feature ~ '`, null) AS `' ~ fv.name ~ '__' ~ feature ~ '`') %} + {%- else %} + {%- set _ = fields.append('IFMISSINGORNULL(`' ~ fv.name ~ '_final`.`' ~ feature ~ '`, null) AS `' ~ feature ~ '`') %} + {%- endif %} + {%- endif %} + {%- endfor %} + {%- endfor %} + {%- if not ns.found %} + {%- if feature_name == 'feature_name' %} + {%- set _ = fields.append('IFMISSINGORNULL(`field_mapping_final`.`' ~ feature_name ~ '`, null) AS `' ~ feature_name ~ '`') %} + {%- else %} + {%- set _ = fields.append('main_entity.`' ~ feature_name ~ '`') %} + {%- endif %} + {%- endif %} + {%- else %} + {%- set feature_parts = feature_name.split('__') %} + {%- set fv_name = feature_parts[0] %} + {%- set feature = feature_parts[1] %} + {%- if feature_name == 'field_mapping__feature_name' %} + {%- set _ = fields.append('IFMISSINGORNULL(`field_mapping_final`.`field_mapping__feature_name`, null) AS `field_mapping__feature_name`') %} + {%- else %} + {%- set _ = fields.append('IFMISSINGORNULL(`' ~ fv_name ~ '_final`.`' ~ feature_name ~ '`, null) AS `' ~ feature_name ~ '`') %} + {%- endif %} + {%- endif %} + {%- endfor %} + {{ fields | reject('none') | join(',\n ') }} +FROM entity_dataframe AS main_entity + +{%- for featureview in featureviews %} +LEFT JOIN ( + SELECT + `{{featureview.name}}__entity_row_unique_id`, + {% for feature in featureview.features -%} + IFMISSINGORNULL(`{% if full_feature_names %}{{ featureview.name }}__{{ featureview.field_mapping.get(feature, feature) }}{% else %}{{ featureview.field_mapping.get(feature, feature) }}{% endif %}`, null) AS `{% if full_feature_names %}{{ featureview.name }}__{{ featureview.field_mapping.get(feature, feature) }}{% else %}{{ featureview.field_mapping.get(feature, feature) }}{% endif %}`{% if not loop.last %},{% endif %} + {% endfor %} + FROM `{{ featureview.name }}__cleaned` +) AS `{{featureview.name}}_final` +ON main_entity.`{{featureview.name}}__entity_row_unique_id` = `{{featureview.name}}_final`.`{{featureview.name}}__entity_row_unique_id` +{% endfor %} +""" diff --git a/sdk/python/feast/infra/offline_stores/contrib/couchbase_offline_store/couchbase_source.py b/sdk/python/feast/infra/offline_stores/contrib/couchbase_offline_store/couchbase_source.py new file mode 100644 index 00000000000..89e4aa2332e --- /dev/null +++ b/sdk/python/feast/infra/offline_stores/contrib/couchbase_offline_store/couchbase_source.py @@ -0,0 +1,406 @@ +import json +from datetime import timedelta +from typing import Any, Callable, Dict, Iterable, Optional, Tuple + +from couchbase_columnar.cluster import Cluster +from couchbase_columnar.credential import Credential +from couchbase_columnar.options import ClusterOptions, QueryOptions, TimeoutOptions +from typeguard import typechecked + +from feast.data_source import DataSource +from feast.errors import DataSourceNoNameException, ZeroColumnQueryResult +from feast.feature_logging import LoggingDestination +from feast.protos.feast.core.DataSource_pb2 import DataSource as DataSourceProto +from feast.protos.feast.core.FeatureService_pb2 import ( + LoggingConfig as LoggingConfigProto, +) +from feast.protos.feast.core.SavedDataset_pb2 import ( + SavedDatasetStorage as SavedDatasetStorageProto, +) +from feast.repo_config import RepoConfig +from feast.saved_dataset import SavedDatasetStorage +from feast.type_map import ValueType, cb_columnar_type_to_feast_value_type + + +@typechecked +class CouchbaseColumnarSource(DataSource): + """A CouchbaseColumnarSource object defines a data source that a CouchbaseColumnarOfflineStore class can use.""" + + def __init__( + self, + name: Optional[str] = None, + query: Optional[str] = None, + database: Optional[str] = "Default", + scope: Optional[str] = "Default", + collection: Optional[str] = None, + timestamp_field: Optional[str] = "", + created_timestamp_column: Optional[str] = "", + field_mapping: Optional[Dict[str, str]] = None, + description: Optional[str] = "", + tags: Optional[Dict[str, str]] = None, + owner: Optional[str] = "", + ): + """Creates a CouchbaseColumnarSource object. + + Args: + name: Name of CouchbaseColumnarSource, which should be unique within a project. + query: SQL++ query that will be used to fetch the data. + database: Columnar database name. + scope: Columnar scope name. + collection: Columnar collection name. + timestamp_field (optional): Event timestamp field used for point-in-time joins of + feature values. + created_timestamp_column (optional): Timestamp column indicating when the row + was created, used for deduplicating rows. + field_mapping (optional): A dictionary mapping of field names in this data + source to feature names in a feature table or view. Only used for feature + fields, not entity or timestamp fields. + description (optional): A human-readable description. + tags (optional): A dictionary of key-value pairs to store arbitrary metadata. + owner (optional): The owner of the data source, typically the email of the primary + maintainer. + """ + self._couchbase_options = CouchbaseColumnarOptions( + name=name, + query=query, + database=database, + scope=scope, + collection=collection, + ) + + # If no name, use the collection as the default name. + if name is None and collection is None: + raise DataSourceNoNameException() + name = name or collection + assert name + + super().__init__( + name=name, + timestamp_field=timestamp_field, + created_timestamp_column=created_timestamp_column, + field_mapping=field_mapping, + description=description, + tags=tags, + owner=owner, + ) + + def __hash__(self): + return super().__hash__() + + def __eq__(self, other): + if not isinstance(other, CouchbaseColumnarSource): + raise TypeError( + "Comparisons should only involve CouchbaseColumnarSource class objects." + ) + + return ( + super().__eq__(other) + and self._couchbase_options._query == other._couchbase_options._query + and self.timestamp_field == other.timestamp_field + and self.created_timestamp_column == other.created_timestamp_column + and self.field_mapping == other.field_mapping + ) + + @staticmethod + def from_proto(data_source: DataSourceProto): + assert data_source.HasField("custom_options") + + couchbase_options = json.loads(data_source.custom_options.configuration) + + return CouchbaseColumnarSource( + name=couchbase_options["name"], + query=couchbase_options["query"], + database=couchbase_options["database"], + scope=couchbase_options["scope"], + collection=couchbase_options["collection"], + field_mapping=dict(data_source.field_mapping), + timestamp_field=data_source.timestamp_field, + created_timestamp_column=data_source.created_timestamp_column, + description=data_source.description, + tags=dict(data_source.tags), + owner=data_source.owner, + ) + + def to_proto(self) -> DataSourceProto: + data_source_proto = DataSourceProto( + name=self.name, + type=DataSourceProto.CUSTOM_SOURCE, + data_source_class_type="feast.infra.offline_stores.contrib.couchbase_offline_store.couchbase_source.CouchbaseColumnarSource", + field_mapping=self.field_mapping, + custom_options=self._couchbase_options.to_proto(), + description=self.description, + tags=self.tags, + owner=self.owner, + ) + + data_source_proto.timestamp_field = self.timestamp_field + data_source_proto.created_timestamp_column = self.created_timestamp_column + + return data_source_proto + + def validate(self, config: RepoConfig): + pass + + @staticmethod + def source_datatype_to_feast_value_type() -> Callable[[str], ValueType]: + # Define the type conversion for Couchbase fields to Feast ValueType as needed + return cb_columnar_type_to_feast_value_type + + def _infer_composite_type(self, field: Dict[str, Any]) -> str: + """ + Infers type signature for a field, rejecting complex nested structures that + aren't compatible with Feast's type system. + + Args: + field: Dictionary containing field information including type and nested structures + + Returns: + String representation of the type, or raises ValueError for incompatible types + + Raises: + ValueError: If field contains complex nested structures not supported by Feast + """ + base_type = field.get("field-type", "unknown").lower() + + if base_type == "array": + if "list" not in field or not field["list"]: + return "array" + + item_type = field["list"][0] + if item_type.get("field-type") == "object": + raise ValueError( + "Complex object types in arrays are not supported by Feast. " + "Arrays must contain homogeneous primitive values." + ) + + # Only allow arrays of primitive types + inner_type = item_type.get("field-type", "unknown") + if inner_type in ["array", "multiset", "object"]: + raise ValueError( + "Nested collection types are not supported by Feast. " + "Arrays can only be one level deep." + ) + + return f"array<{inner_type}>" + + elif base_type == "object": + raise ValueError( + "Complex object types are not supported by Feast. " + "Only primitive types and homogeneous arrays are allowed." + ) + + elif base_type == "multiset": + raise ValueError( + "Multiset types are not supported by Feast. " + "Only primitive types and homogeneous arrays are allowed." + ) + + return base_type + + def get_table_column_names_and_types( + self, config: RepoConfig + ) -> Iterable[Tuple[str, str]]: + cred = Credential.from_username_and_password( + config.offline_store.user, config.offline_store.password + ) + timeout_opts = TimeoutOptions(dispatch_timeout=timedelta(seconds=120)) + cluster = Cluster.create_instance( + config.offline_store.connection_string, + cred, + ClusterOptions(timeout_options=timeout_opts), + ) + + query_context = self.get_table_query_string() + query = f""" + SELECT get_object_fields( + CASE WHEN ARRAY_LENGTH(OBJECT_PAIRS(t)) = 1 AND OBJECT_PAIRS(t)[0].`value` IS NOT MISSING + THEN OBJECT_PAIRS(t)[0].`value` + ELSE t + END + ) AS field_types + FROM {query_context} AS t + LIMIT 1; + """ + + result = cluster.execute_query( + query, QueryOptions(timeout=timedelta(seconds=config.offline_store.timeout)) + ) + if not result: + raise ZeroColumnQueryResult(query) + + rows = result.get_all_rows() + field_type_pairs = [] + if rows and rows[0]: + # Accessing the "field_types" array from the first row + field_types_list = rows[0].get("field_types", []) + for field in field_types_list: + field_name = field.get("field-name", "unknown") + field_type = field.get("field-type", "unknown") + # drop uuid fields to ensure schema matches dataframe + if field_type == "uuid": + continue + field_type = self._infer_composite_type(field) + field_type_pairs.append((field_name, field_type)) + return field_type_pairs + + def get_table_query_string(self) -> str: + if ( + self._couchbase_options._database + and self._couchbase_options._scope + and self._couchbase_options._collection + ): + return f"`{self._couchbase_options._database}`.`{self._couchbase_options._scope}`.`{self._couchbase_options._collection}`" + else: + return f"({self._couchbase_options._query})" + + @property + def database(self) -> str: + """Returns the database name.""" + return self._couchbase_options._database + + @property + def scope(self) -> str: + """Returns the scope name.""" + return self._couchbase_options._scope + + +class CouchbaseColumnarOptions: + def __init__( + self, + name: Optional[str], + query: Optional[str], + database: Optional[str], + scope: Optional[str], + collection: Optional[str], + ): + self._name = name or "" + self._query = query or "" + self._database = database or "" + self._scope = scope or "" + self._collection = collection or "" + + @classmethod + def from_proto(cls, couchbase_options_proto: DataSourceProto.CustomSourceOptions): + config = json.loads(couchbase_options_proto.configuration.decode("utf8")) + couchbase_options = cls( + name=config["name"], + query=config["query"], + database=config["database"], + scope=config["scope"], + collection=config["collection"], + ) + + return couchbase_options + + def to_proto(self) -> DataSourceProto.CustomSourceOptions: + couchbase_options_proto = DataSourceProto.CustomSourceOptions( + configuration=json.dumps( + { + "name": self._name, + "query": self._query, + "database": self._database, + "scope": self._scope, + "collection": self._collection, + } + ).encode() + ) + return couchbase_options_proto + + +class SavedDatasetCouchbaseColumnarStorage(SavedDatasetStorage): + _proto_attr_name = "custom_storage" + + couchbase_options: CouchbaseColumnarOptions + + def __init__(self, database_ref: str, scope_ref: str, collection_ref: str): + self.couchbase_options = CouchbaseColumnarOptions( + database=database_ref, + scope=scope_ref, + collection=collection_ref, + name=None, + query=None, + ) + + @staticmethod + def from_proto(storage_proto: SavedDatasetStorageProto) -> SavedDatasetStorage: + return SavedDatasetCouchbaseColumnarStorage( + database_ref=CouchbaseColumnarOptions.from_proto( + storage_proto.custom_storage + )._database, + scope_ref=CouchbaseColumnarOptions.from_proto( + storage_proto.custom_storage + )._scope, + collection_ref=CouchbaseColumnarOptions.from_proto( + storage_proto.custom_storage + )._collection, + ) + + def to_proto(self) -> SavedDatasetStorageProto: + return SavedDatasetStorageProto( + custom_storage=self.couchbase_options.to_proto() + ) + + def to_data_source(self) -> DataSource: + return CouchbaseColumnarSource( + database=self.couchbase_options._database, + scope=self.couchbase_options._scope, + collection=self.couchbase_options._collection, + ) + + +class CouchbaseColumnarLoggingDestination(LoggingDestination): + """ + Couchbase Columnar implementation of a logging destination. + """ + + database: str + scope: str + table_name: str + + _proto_kind = "couchbase_columnar_destination" + + def __init__(self, *, database: str, scope: str, table_name: str): + """ + Args: + database: The Couchbase database name + scope: The Couchbase scope name + table_name: The Couchbase collection name to log features into + """ + self.database = database + self.scope = scope + self.table_name = table_name + + def to_data_source(self) -> DataSource: + """ + Returns a data source object representing the logging destination. + """ + return CouchbaseColumnarSource( + database=self.database, + scope=self.scope, + collection=self.table_name, + ) + + def to_proto(self) -> LoggingConfigProto: + """ + Converts the logging destination to its protobuf representation. + """ + return LoggingConfigProto( + couchbase_columnar_destination=LoggingConfigProto.CouchbaseColumnarDestination( + database=self.database, + scope=self.scope, + collection=self.table_name, + ) + ) + + @classmethod + def from_proto( + cls, config_proto: LoggingConfigProto + ) -> "CouchbaseColumnarLoggingDestination": + """ + Creates a CouchbaseColumnarLoggingDestination from its protobuf representation. + """ + return CouchbaseColumnarLoggingDestination( + database=config_proto.CouchbaseColumnarDestination.database, + scope=config_proto.CouchbaseColumnarDestination.scope, + table_name=config_proto.CouchbaseColumnarDestination.collection, + ) diff --git a/sdk/python/feast/infra/offline_stores/contrib/couchbase_offline_store/tests/__init__.py b/sdk/python/feast/infra/offline_stores/contrib/couchbase_offline_store/tests/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/sdk/python/feast/infra/offline_stores/contrib/couchbase_offline_store/tests/data_source.py b/sdk/python/feast/infra/offline_stores/contrib/couchbase_offline_store/tests/data_source.py new file mode 100644 index 00000000000..c23a8301a76 --- /dev/null +++ b/sdk/python/feast/infra/offline_stores/contrib/couchbase_offline_store/tests/data_source.py @@ -0,0 +1,213 @@ +import atexit +import json +import os +import signal +import threading +import uuid +from datetime import timedelta +from typing import Dict, List, Optional + +import pandas as pd +from couchbase_columnar.cluster import Cluster +from couchbase_columnar.credential import Credential +from couchbase_columnar.options import ClusterOptions, QueryOptions, TimeoutOptions + +from feast.data_source import DataSource +from feast.feature_logging import LoggingDestination +from feast.infra.offline_stores.contrib.couchbase_offline_store.couchbase import ( + CouchbaseColumnarOfflineStoreConfig, +) +from feast.infra.offline_stores.contrib.couchbase_offline_store.couchbase_source import ( + CouchbaseColumnarLoggingDestination, + CouchbaseColumnarSource, +) +from feast.infra.utils.couchbase.couchbase_utils import normalize_timestamp +from feast.repo_config import FeastConfigBaseModel +from tests.integration.feature_repos.universal.data_source_creator import ( + DataSourceCreator, +) + +COUCHBASE_COLUMNAR_DATABASE = "Default" +COUCHBASE_COLUMNAR_SCOPE = "Default" + + +class CouchbaseColumnarDataSourceCreator(DataSourceCreator): + _shutting_down = False + _cluster = None + _cluster_lock = threading.Lock() + + @classmethod + def get_cluster(cls): + with cls._cluster_lock: + if cls._cluster is None: + cred = Credential.from_username_and_password( + os.environ["COUCHBASE_COLUMNAR_USER"], + os.environ["COUCHBASE_COLUMNAR_PASSWORD"], + ) + timeout_opts = TimeoutOptions(dispatch_timeout=timedelta(seconds=120)) + cls._cluster = Cluster.create_instance( + os.environ["COUCHBASE_COLUMNAR_CONNECTION_STRING"], + cred, + ClusterOptions(timeout_options=timeout_opts), + ) + return cls._cluster + + def __init__(self, project_name: str, **kwargs): + super().__init__(project_name) + self.project_name = project_name + self.collections: List[str] = [] + + self.offline_store_config = CouchbaseColumnarOfflineStoreConfig( + type="couchbase.offline", + connection_string=os.environ["COUCHBASE_COLUMNAR_CONNECTION_STRING"], + user=os.environ["COUCHBASE_COLUMNAR_USER"], + password=os.environ["COUCHBASE_COLUMNAR_PASSWORD"], + timeout=120, + ) + + def create_data_source( + self, + df: pd.DataFrame, + destination_name: str, + created_timestamp_column="created_ts", + field_mapping: Optional[Dict[str, str]] = None, + timestamp_field: Optional[str] = "ts", + ) -> DataSource: + def format_row(row): + """Convert row to dictionary, handling NaN and timestamps""" + return { + col: ( + normalize_timestamp(row[col]) + if isinstance(row[col], pd.Timestamp) + else None + if pd.isna(row[col]) + else row[col] + ) + for col in row.index + } + + collection_name = self.get_prefixed_collection_name(destination_name) + + create_cluster_query = f"CREATE ANALYTICS COLLECTION {COUCHBASE_COLUMNAR_DATABASE}.{COUCHBASE_COLUMNAR_SCOPE}.{collection_name} IF NOT EXISTS PRIMARY KEY(pk: UUID) AUTOGENERATED;" + self.get_cluster().execute_query( + create_cluster_query, + QueryOptions(timeout=timedelta(seconds=self.offline_store_config.timeout)), + ) + + values_list = df.apply(format_row, axis=1).apply(json.dumps).tolist() + values_clause = ",\n ".join(values_list) + + insert_query = f""" + INSERT INTO `{COUCHBASE_COLUMNAR_DATABASE}`.`{COUCHBASE_COLUMNAR_SCOPE}`.`{collection_name}` ([ + {values_clause} + ]) + """ + self.get_cluster().execute_query( + insert_query, + QueryOptions(timeout=timedelta(seconds=self.offline_store_config.timeout)), + ) + + self.collections.append(collection_name) + + return CouchbaseColumnarSource( + name=collection_name, + query=f"SELECT VALUE v FROM {COUCHBASE_COLUMNAR_DATABASE}.{COUCHBASE_COLUMNAR_SCOPE}.`{collection_name}` v", + database=COUCHBASE_COLUMNAR_DATABASE, + scope=COUCHBASE_COLUMNAR_SCOPE, + collection=collection_name, + timestamp_field=timestamp_field, + created_timestamp_column=created_timestamp_column, + field_mapping=field_mapping or {"ts_1": "ts"}, + ) + + def create_saved_dataset_destination(self): + raise NotImplementedError + + def create_logged_features_destination(self) -> LoggingDestination: + collection = self.get_prefixed_collection_name( + f"logged_features_{str(uuid.uuid4()).replace('-', '_')}" + ) + self.collections.append(collection) + return CouchbaseColumnarLoggingDestination( + table_name=collection, + database=COUCHBASE_COLUMNAR_DATABASE, + scope=COUCHBASE_COLUMNAR_SCOPE, + ) + + def create_offline_store_config(self) -> FeastConfigBaseModel: + return self.offline_store_config + + def get_prefixed_collection_name(self, suffix: str) -> str: + return f"{self.project_name}_{suffix}" + + @classmethod + def get_dangling_collections(cls) -> List[str]: + query = """ + SELECT VALUE d.DatabaseName || '.' || d.DataverseName || '.' || d.DatasetName + FROM System.Metadata.`Dataset` d + WHERE d.DataverseName <> "Metadata" + AND (REGEXP_CONTAINS(d.DatasetName, "integration_test_.*") + OR REGEXP_CONTAINS(d.DatasetName, "feast_entity_df_.*")); + """ + try: + res = cls.get_cluster().execute_query(query) + return res.get_all_rows() + except Exception as e: + print(f"Error fetching collections: {e}") + return [] + + @classmethod + def cleanup_all(cls): + if cls._shutting_down: + return + cls._shutting_down = True + try: + collections = cls.get_dangling_collections() + if len(collections) == 0: + print("No collections to clean up.") + return + + print(f"Found {len(collections)} collections to clean up.") + if len(collections) > 5: + print("This may take a few minutes...") + for collection in collections: + try: + query = f"DROP COLLECTION {collection} IF EXISTS;" + cls.get_cluster().execute_query(query) + print(f"Dropped collection: {collection}") + except Exception as e: + print(f"Error dropping collection {collection}: {e}") + finally: + print("Cleanup complete.") + cls._shutting_down = False + + def teardown(self): + for collection in self.collections: + query = f"DROP COLLECTION {COUCHBASE_COLUMNAR_DATABASE}.{COUCHBASE_COLUMNAR_SCOPE}.`{collection}` IF EXISTS;" + try: + self.get_cluster().execute_query( + query, + QueryOptions( + timeout=timedelta(seconds=self.offline_store_config.timeout) + ), + ) + print(f"Successfully dropped collection: {collection}") + except Exception as e: + print(f"Error dropping collection {collection}: {e}") + + +def cleanup_handler(signum, frame): + print("\nCleaning up dangling resources...") + try: + CouchbaseColumnarDataSourceCreator.cleanup_all() + except Exception as e: + print(f"Error during cleanup: {e}") + finally: + # Re-raise the signal to properly exit + signal.default_int_handler(signum, frame) + + +# Register both SIGINT and SIGTERM handlers +signal.signal(signal.SIGINT, cleanup_handler) +signal.signal(signal.SIGTERM, cleanup_handler) +atexit.register(CouchbaseColumnarDataSourceCreator.cleanup_all) diff --git a/sdk/python/feast/infra/offline_stores/dask.py b/sdk/python/feast/infra/offline_stores/dask.py index 51a3debb5e2..01efc492f7c 100644 --- a/sdk/python/feast/infra/offline_stores/dask.py +++ b/sdk/python/feast/infra/offline_stores/dask.py @@ -100,11 +100,9 @@ def persist( # Check if the specified location already exists. if not allow_overwrite and os.path.exists(storage.file_options.uri): raise SavedDatasetLocationAlreadyExists(location=storage.file_options.uri) - - if not Path(storage.file_options.uri).is_absolute(): - absolute_path = Path(self.repo_path) / storage.file_options.uri - else: - absolute_path = Path(storage.file_options.uri) + absolute_path = FileSource.get_uri_for_file_path( + repo_path=self.repo_path, uri=storage.file_options.uri + ) filesystem, path = FileSource.create_filesystem_and_path( str(absolute_path), diff --git a/sdk/python/feast/infra/offline_stores/duckdb.py b/sdk/python/feast/infra/offline_stores/duckdb.py index e64da029a6a..b2e3c03cb55 100644 --- a/sdk/python/feast/infra/offline_stores/duckdb.py +++ b/sdk/python/feast/infra/offline_stores/duckdb.py @@ -51,10 +51,9 @@ def _write_data_source( file_options = data_source.file_options - if not Path(file_options.uri).is_absolute(): - absolute_path = Path(repo_path) / file_options.uri - else: - absolute_path = Path(file_options.uri) + absolute_path = FileSource.get_uri_for_file_path( + repo_path=repo_path, uri=file_options.uri + ) if ( mode == "overwrite" diff --git a/sdk/python/feast/infra/offline_stores/file_source.py b/sdk/python/feast/infra/offline_stores/file_source.py index 5912cbdf3fb..af33338265b 100644 --- a/sdk/python/feast/infra/offline_stores/file_source.py +++ b/sdk/python/feast/infra/offline_stores/file_source.py @@ -1,5 +1,6 @@ from pathlib import Path -from typing import Callable, Dict, Iterable, List, Optional, Tuple +from typing import Callable, Dict, Iterable, List, Optional, Tuple, Union +from urllib.parse import urlparse import pyarrow from packaging import version @@ -154,17 +155,21 @@ def validate(self, config: RepoConfig): def source_datatype_to_feast_value_type() -> Callable[[str], ValueType]: return type_map.pa_to_feast_value_type + @staticmethod + def get_uri_for_file_path(repo_path: Union[Path, str, None], uri: str) -> str: + parsed_uri = urlparse(uri) + if parsed_uri.scheme and parsed_uri.netloc: + return uri # Keep remote URIs as they are + if repo_path is not None and not Path(uri).is_absolute(): + return str(Path(repo_path) / uri) + return str(Path(uri)) + def get_table_column_names_and_types( self, config: RepoConfig ) -> Iterable[Tuple[str, str]]: - if ( - config.repo_path is not None - and not Path(self.file_options.uri).is_absolute() - ): - absolute_path = config.repo_path / self.file_options.uri - else: - absolute_path = Path(self.file_options.uri) - + absolute_path = self.get_uri_for_file_path( + repo_path=config.repo_path, uri=self.file_options.uri + ) filesystem, path = FileSource.create_filesystem_and_path( str(absolute_path), self.file_options.s3_endpoint_override ) diff --git a/sdk/python/feast/infra/online_stores/couchbase_online_store/README.md b/sdk/python/feast/infra/online_stores/couchbase_online_store/README.md index df1b7a1382d..8f95884fe03 100644 --- a/sdk/python/feast/infra/online_stores/couchbase_online_store/README.md +++ b/sdk/python/feast/infra/online_stores/couchbase_online_store/README.md @@ -28,14 +28,14 @@ cd feature_repo #### Edit `feature_store.yaml` -Set the `online_store` type to `couchbase`, and fill in the required fields as shown below. +Set the `online_store` type to `couchbase.online`, and fill in the required fields as shown below. ```yaml project: feature_repo registry: data/registry.db provider: local online_store: - type: couchbase + type: couchbase.online connection_string: couchbase://127.0.0.1 # Couchbase connection string, copied from 'Connect' page in Couchbase Capella console user: Administrator # Couchbase username from access credentials password: password # Couchbase password from access credentials diff --git a/sdk/python/feast/infra/online_stores/couchbase_online_store/couchbase.py b/sdk/python/feast/infra/online_stores/couchbase_online_store/couchbase.py index 91ce56a5caf..c80f9e1285c 100644 --- a/sdk/python/feast/infra/online_stores/couchbase_online_store/couchbase.py +++ b/sdk/python/feast/infra/online_stores/couchbase_online_store/couchbase.py @@ -31,7 +31,7 @@ class CouchbaseOnlineStoreConfig(FeastConfigBaseModel): Configuration for the Couchbase online store. """ - type: Literal["couchbase"] = "couchbase" + type: Literal["couchbase.online"] = "couchbase.online" connection_string: Optional[StrictStr] = None user: Optional[StrictStr] = None diff --git a/sdk/python/feast/infra/online_stores/milvus_online_store/milvus.py b/sdk/python/feast/infra/online_stores/milvus_online_store/milvus.py index 4d14d826085..91e432a74fa 100644 --- a/sdk/python/feast/infra/online_stores/milvus_online_store/milvus.py +++ b/sdk/python/feast/infra/online_stores/milvus_online_store/milvus.py @@ -197,10 +197,14 @@ def _get_or_create_collection( ) index_params = self.client.prepare_index_params() for vector_field in schema.fields: - if vector_field.dtype in [ - DataType.FLOAT_VECTOR, - DataType.BINARY_VECTOR, - ]: + if ( + vector_field.dtype + in [ + DataType.FLOAT_VECTOR, + DataType.BINARY_VECTOR, + ] + and vector_field.name in vector_field_dict + ): metric = vector_field_dict[ vector_field.name ].vector_search_metric @@ -460,9 +464,10 @@ def retrieve_online_documents_v2( config: RepoConfig, table: FeatureView, requested_features: List[str], - embedding: List[float], + embedding: Optional[List[float]], top_k: int, distance_metric: Optional[str] = None, + query_string: Optional[str] = None, ) -> List[ Tuple[ Optional[datetime], @@ -470,6 +475,7 @@ def retrieve_online_documents_v2( Optional[Dict[str, ValueProto]], ] ]: + assert embedding is not None, "Key Word Search not yet implemented for Milvus" entity_name_feast_primitive_type_map = { k.name: k.dtype for k in table.entity_columns } diff --git a/sdk/python/feast/infra/online_stores/online_store.py b/sdk/python/feast/infra/online_stores/online_store.py index f5202b66f66..5111bcd47bd 100644 --- a/sdk/python/feast/infra/online_stores/online_store.py +++ b/sdk/python/feast/infra/online_stores/online_store.py @@ -436,9 +436,10 @@ def retrieve_online_documents_v2( config: RepoConfig, table: FeatureView, requested_features: List[str], - embedding: List[float], + embedding: Optional[List[float]], top_k: int, distance_metric: Optional[str] = None, + query_string: Optional[str] = None, ) -> List[ Tuple[ Optional[datetime], @@ -454,14 +455,18 @@ def retrieve_online_documents_v2( config: The config for the current feature store. table: The feature view whose feature values should be read. requested_features: The list of features whose embeddings should be used for retrieval. - embedding: The embeddings to use for retrieval. + embedding: The embeddings to use for retrieval (optional) top_k: The number of documents to retrieve. + query_string: The query string to search for using keyword search (bm25) (optional) Returns: object: A list of top k closest documents to the specified embedding. Each item in the list is a tuple where the first item is the event timestamp for the row, and the second item is a dict of feature name to embeddings. """ + assert embedding is not None or query_string is not None, ( + "Either embedding or query_string must be specified" + ) raise NotImplementedError( f"Online store {self.__class__.__name__} does not support online retrieval" ) diff --git a/sdk/python/feast/infra/online_stores/singlestore_online_store/singlestore.py b/sdk/python/feast/infra/online_stores/singlestore_online_store/singlestore.py index d78289c8671..a1535589542 100644 --- a/sdk/python/feast/infra/online_stores/singlestore_online_store/singlestore.py +++ b/sdk/python/feast/infra/online_stores/singlestore_online_store/singlestore.py @@ -50,6 +50,7 @@ def _init_conn(self, config: RepoConfig) -> Connection: password=online_store_config.password or "test", database=online_store_config.database or "feast", port=online_store_config.port or 3306, + conn_attrs={"_connector_name": "SingleStore Feast Online Store"}, autocommit=True, ) diff --git a/sdk/python/feast/infra/online_stores/sqlite.py b/sdk/python/feast/infra/online_stores/sqlite.py index 945ec965fc9..15ef81188b0 100644 --- a/sdk/python/feast/infra/online_stores/sqlite.py +++ b/sdk/python/feast/infra/online_stores/sqlite.py @@ -18,12 +18,13 @@ import sys from datetime import date, datetime from pathlib import Path -from typing import Any, Callable, Dict, List, Literal, Optional, Sequence, Tuple +from typing import Any, Callable, Dict, List, Literal, Optional, Sequence, Tuple, Union from pydantic import StrictStr from feast import Entity from feast.feature_view import FeatureView +from feast.field import Field from feast.infra.infra_object import SQLITE_INFRA_OBJECT_CLASS_TYPE, InfraObject from feast.infra.key_encoding_utils import ( deserialize_entity_key, @@ -38,7 +39,13 @@ from feast.protos.feast.types.EntityKey_pb2 import EntityKey as EntityKeyProto from feast.protos.feast.types.Value_pb2 import Value as ValueProto from feast.repo_config import FeastConfigBaseModel, RepoConfig -from feast.utils import _build_retrieve_online_document_record, to_naive_utc +from feast.type_map import feast_value_type_to_python_type +from feast.types import FEAST_VECTOR_TYPES, PrimitiveFeastType +from feast.utils import ( + _build_retrieve_online_document_record, + _serialize_vector_to_float_list, + to_naive_utc, +) def adapt_date_iso(val: date): @@ -94,6 +101,7 @@ class SqliteOnlineStoreConfig(FeastConfigBaseModel, VectorStoreConfig): vector_enabled: bool = False vector_len: Optional[int] = None + text_search_enabled: bool = False class SqliteOnlineStore(OnlineStore): @@ -144,9 +152,8 @@ def online_write_batch( progress: Optional[Callable[[int], Any]], ) -> None: conn = self._get_conn(config) - project = config.project - + feature_type_dict = {f.name: f.dtype for f in table.features} with conn: for entity_key, values, timestamp, created_ts in data: entity_key_bin = serialize_entity_key( @@ -160,71 +167,53 @@ def online_write_batch( table_name = _table_id(project, table) for feature_name, val in values.items(): if config.online_store.vector_enabled: - vector_bin = serialize_f32( - val.float_list_val.val, config.online_store.vector_len - ) # type: ignore + if ( + feature_type_dict.get(feature_name, None) + in FEAST_VECTOR_TYPES + ): + val_bin = serialize_f32( + val.float_list_val.val, config.online_store.vector_len + ) # type: ignore + else: + val_bin = feast_value_type_to_python_type(val) conn.execute( f""" - UPDATE {table_name} - SET value = ?, vector_value = ?, event_ts = ?, created_ts = ? - WHERE (entity_key = ? AND feature_name = ?) - """, - ( - # SET - val.SerializeToString(), - vector_bin, - timestamp, - created_ts, - # WHERE - entity_key_bin, - feature_name, - ), - ) - - conn.execute( - f"""INSERT OR IGNORE INTO {table_name} - (entity_key, feature_name, value, vector_value, event_ts, created_ts) - VALUES (?, ?, ?, ?, ?, ?)""", + INSERT INTO {table_name} (entity_key, feature_name, value, vector_value, event_ts, created_ts) + VALUES (?, ?, ?, ?, ?, ?) + ON CONFLICT(entity_key, feature_name) DO UPDATE SET + value = excluded.value, + vector_value = excluded.vector_value, + event_ts = excluded.event_ts, + created_ts = excluded.created_ts; + """, ( - entity_key_bin, - feature_name, - val.SerializeToString(), - vector_bin, - timestamp, - created_ts, + entity_key_bin, # entity_key + feature_name, # feature_name + val.SerializeToString(), # value + val_bin, # vector_value + timestamp, # event_ts + created_ts, # created_ts ), ) - else: conn.execute( f""" - UPDATE {table_name} - SET value = ?, event_ts = ?, created_ts = ? - WHERE (entity_key = ? AND feature_name = ?) + INSERT INTO {table_name} (entity_key, feature_name, value, event_ts, created_ts) + VALUES (?, ?, ?, ?, ?) + ON CONFLICT(entity_key, feature_name) DO UPDATE SET + value = excluded.value, + event_ts = excluded.event_ts, + created_ts = excluded.created_ts; """, ( - # SET - val.SerializeToString(), - timestamp, - created_ts, - # WHERE - entity_key_bin, - feature_name, + entity_key_bin, # entity_key + feature_name, # feature_name + val.SerializeToString(), # value + timestamp, # event_ts + created_ts, # created_ts ), ) - conn.execute( - f"""INSERT OR IGNORE INTO {table_name} - (entity_key, feature_name, value, event_ts, created_ts) - VALUES (?, ?, ?, ?, ?)""", - ( - entity_key_bin, - feature_name, - val.SerializeToString(), - timestamp, - created_ts, - ), - ) if progress: progress(1) @@ -240,22 +229,22 @@ def online_read( result: List[Tuple[Optional[datetime], Optional[Dict[str, ValueProto]]]] = [] + serialized_entity_keys = [ + serialize_entity_key( + entity_key, + entity_key_serialization_version=config.entity_key_serialization_version, + ) + for entity_key in entity_keys + ] # Fetch all entities in one go cur.execute( f"SELECT entity_key, feature_name, value, event_ts " f"FROM {_table_id(config.project, table)} " f"WHERE entity_key IN ({','.join('?' * len(entity_keys))}) " f"ORDER BY entity_key", - [ - serialize_entity_key( - entity_key, - entity_key_serialization_version=config.entity_key_serialization_version, - ) - for entity_key in entity_keys - ], + serialized_entity_keys, ) rows = cur.fetchall() - rows = { k: list(group) for k, group in itertools.groupby(rows, key=lambda r: r[0]) } @@ -369,6 +358,7 @@ def retrieve_online_documents( # Convert the embedding to a binary format instead of using SerializeToString() query_embedding_bin = serialize_f32(embedding, config.online_store.vector_len) table_name = _table_id(project, table) + vector_field = _get_vector_field(table) cur.execute( f""" @@ -383,14 +373,15 @@ def retrieve_online_documents( f""" INSERT INTO vec_table(rowid, vector_value) select rowid, vector_value from {table_name} + where feature_name = "{vector_field}" """ ) cur.execute( + f""" + CREATE VIRTUAL TABLE IF NOT EXISTS vec_table using vec0( + vector_value float[{config.online_store.vector_len}] + ); """ - INSERT INTO vec_table(rowid, vector_value) - VALUES (?, ?) - """, - (0, query_embedding_bin), ) # Have to join this with the {table_name} to get the feature name and entity_key @@ -451,9 +442,10 @@ def retrieve_online_documents_v2( config: RepoConfig, table: FeatureView, requested_features: List[str], - query: List[float], + query: Optional[List[float]], top_k: int, distance_metric: Optional[str] = None, + query_string: Optional[str] = None, ) -> List[ Tuple[ Optional[datetime], @@ -467,76 +459,141 @@ def retrieve_online_documents_v2( config: Feast configuration object table: FeatureView object as the table to search requested_features: List of requested features to retrieve - query: Query embedding to search for + query: Query embedding to search for (optional) top_k: Number of items to return distance_metric: Distance metric to use (optional) + query_string: The query string to search for using keyword search (bm25) (optional) Returns: List of tuples containing the event timestamp, entity key, and feature values """ online_store = config.online_store if not isinstance(online_store, SqliteOnlineStoreConfig): raise ValueError("online_store must be SqliteOnlineStoreConfig") - if not online_store.vector_enabled: - raise ValueError("Vector search is not enabled in the online store config") + if not online_store.vector_enabled and not online_store.text_search_enabled: + raise ValueError( + "You must enable either vector search or text search in the online store config" + ) conn = self._get_conn(config) cur = conn.cursor() - online_store = config.online_store - if not isinstance(online_store, SqliteOnlineStoreConfig): - raise ValueError("online_store must be SqliteOnlineStoreConfig") - if not online_store.vector_len: + if online_store.vector_enabled and not online_store.vector_len: raise ValueError("vector_len is not configured in the online store config") - query_embedding_bin = serialize_f32(query, online_store.vector_len) # type: ignore - table_name = _table_id(config.project, table) - cur.execute( - f""" - CREATE VIRTUAL TABLE IF NOT EXISTS vec_table using vec0( - vector_value float[{online_store.vector_len}] - ); - """ - ) + table_name = _table_id(config.project, table) + vector_field = _get_vector_field(table) + + if online_store.vector_enabled: + query_embedding_bin = serialize_f32(query, online_store.vector_len) # type: ignore + cur.execute( + f""" + CREATE VIRTUAL TABLE IF NOT EXISTS vec_table using vec0( + vector_value float[{online_store.vector_len}] + ); + """ + ) + cur.execute( + f""" + INSERT INTO vec_table (rowid, vector_value) + select rowid, vector_value from {table_name} + where feature_name = "{vector_field}" + """ + ) + elif online_store.text_search_enabled: + string_field_list = [ + f.name for f in table.features if f.dtype == PrimitiveFeastType.STRING + ] + string_fields = ", ".join(string_field_list) + # TODO: swap this for a value configurable in each Field() + BM25_DEFAULT_WEIGHTS = ", ".join( + [ + str(1.0) + for f in table.features + if f.dtype == PrimitiveFeastType.STRING + ] + ) + cur.execute( + f""" + CREATE VIRTUAL TABLE IF NOT EXISTS search_table using fts5( + entity_key, fv_rowid, {string_fields}, tokenize="porter unicode61" + ); + """ + ) + insert_query = _generate_bm25_search_insert_query( + table_name, string_field_list + ) + cur.execute(insert_query) - cur.execute( - f""" - INSERT INTO vec_table(rowid, vector_value) - select rowid, vector_value from {table_name} - """ - ) + else: + raise ValueError( + "Neither vector search nor text search are enabled in the online store config" + ) - cur.execute( - f""" + if online_store.vector_enabled: + cur.execute( + f""" + select + fv2.entity_key, + fv2.feature_name, + fv2.value, + fv.vector_value, + f.distance, + fv.event_ts, + fv.created_ts + from ( + select + rowid, + vector_value, + distance + from vec_table + where vector_value match ? + order by distance + limit ? + ) f + left join {table_name} fv + on f.rowid = fv.rowid + left join {table_name} fv2 + on fv.entity_key = fv2.entity_key + where fv2.feature_name != "{vector_field}" + """, + ( + query_embedding_bin, + top_k, + ), + ) + elif online_store.text_search_enabled: + cur.execute( + f""" select fv.entity_key, fv.feature_name, fv.value, + fv.vector_value, f.distance, fv.event_ts, fv.created_ts - from ( - select - rowid, - vector_value, - distance - from vec_table - where vector_value match ? - order by distance - limit ? - ) f - left join {table_name} fv - on f.rowid = fv.rowid - where fv.feature_name in ({",".join(["?" for _ in requested_features])}) - """, - ( - query_embedding_bin, - top_k, - *[f.split(":")[-1] for f in requested_features], - ), - ) + from {table_name} fv + inner join ( + select + fv_rowid, + entity_key, + {string_fields}, + bm25(search_table, {BM25_DEFAULT_WEIGHTS}) as distance + from search_table + where search_table match ? order by distance limit ? + ) f + on f.entity_key = fv.entity_key + """, + (query_string, top_k), + ) + + else: + raise ValueError( + "Neither vector search nor text search are enabled in the online store config" + ) rows = cur.fetchall() - result: List[ + results: List[ Tuple[ Optional[datetime], Optional[EntityKeyProto], @@ -544,20 +601,62 @@ def retrieve_online_documents_v2( ] ] = [] - for entity_key, feature_name, value_bin, distance, event_ts, created_ts in rows: - val = ValueProto() - val.ParseFromString(value_bin) - entity_key_proto = None - if entity_key: - entity_key_proto = deserialize_entity_key( - entity_key, - entity_key_serialization_version=config.entity_key_serialization_version, + entity_dict: Dict[ + str, Dict[str, Union[str, ValueProto, EntityKeyProto, datetime]] + ] = {} + for ( + entity_key, + feature_name, + value_bin, + vector_value, + distance, + event_ts, + created_ts, + ) in rows: + entity_key_proto = deserialize_entity_key( + entity_key, + entity_key_serialization_version=config.entity_key_serialization_version, + ) + if entity_key not in entity_dict: + entity_dict[entity_key] = {} + + feature_val = ValueProto() + feature_val.ParseFromString(value_bin) + entity_dict[entity_key]["entity_key_proto"] = entity_key_proto + entity_dict[entity_key][feature_name] = feature_val + if online_store.vector_enabled: + entity_dict[entity_key][vector_field] = _serialize_vector_to_float_list( + vector_value ) - res = {feature_name: val} - res["distance"] = ValueProto(float_val=distance) - result.append((event_ts, entity_key_proto, res)) - - return result + entity_dict[entity_key]["distance"] = ValueProto(float_val=distance) + entity_dict[entity_key]["event_ts"] = event_ts + entity_dict[entity_key]["created_ts"] = created_ts + + for entity_key_value in entity_dict: + res_event_ts: Optional[datetime] = None + res_entity_key_proto: Optional[EntityKeyProto] = None + if isinstance(entity_dict[entity_key_value]["event_ts"], datetime): + res_event_ts = entity_dict[entity_key_value]["event_ts"] # type: ignore[assignment] + + if isinstance( + entity_dict[entity_key_value]["entity_key_proto"], EntityKeyProto + ): + res_entity_key_proto = entity_dict[entity_key_value]["entity_key_proto"] # type: ignore[assignment] + + res_dict: Dict[str, ValueProto] = { + k: v + for k, v in entity_dict[entity_key_value].items() + if isinstance(v, ValueProto) and isinstance(k, str) + } + + results.append( + ( + res_event_ts, + res_entity_key_proto, + res_dict, + ) + ) + return results def _initialize_conn( @@ -640,7 +739,17 @@ def update(self): except ModuleNotFoundError: logging.warning("Cannot use sqlite_vec for vector search") self.conn.execute( - f"CREATE TABLE IF NOT EXISTS {self.name} (entity_key BLOB, feature_name TEXT, value BLOB, vector_value BLOB, event_ts timestamp, created_ts timestamp, PRIMARY KEY(entity_key, feature_name))" + f""" + CREATE TABLE IF NOT EXISTS {self.name} ( + entity_key BLOB, + feature_name TEXT, + value BLOB, + vector_value BLOB, + event_ts timestamp, + created_ts timestamp, + PRIMARY KEY(entity_key, feature_name) + ) + """ ) self.conn.execute( f"CREATE INDEX IF NOT EXISTS {self.name}_ek ON {self.name} (entity_key);" @@ -648,3 +757,48 @@ def update(self): def teardown(self): self.conn.execute(f"DROP TABLE IF EXISTS {self.name}") + + +def _get_vector_field(table: FeatureView) -> str: + """ + Get the vector field from the feature view. There can be only one. + """ + vector_fields: List[Field] = [ + f for f in table.features if getattr(f, "vector_index", None) + ] + assert len(vector_fields) > 0, ( + f"No vector field found, please update feature view = {table.name} to declare a vector field" + ) + assert len(vector_fields) < 2, ( + "Only one vector field is supported, please update feature view = {table.name} to declare one vector field" + ) + vector_field: str = vector_fields[0].name + return vector_field + + +def _generate_bm25_search_insert_query( + table_name: str, string_field_list: List[str] +) -> str: + """ + Generates an SQL insertion query for the given table and string fields. + + Args: + table_name (str): The name of the table to select data from. + string_field_list (List[str]): The list of string fields to be used in the insertion. + + Returns: + str: The generated SQL insertion query. + """ + _string_fields = ", ".join(string_field_list) + query = f"INSERT INTO search_table (entity_key, fv_rowid, {_string_fields})\nSELECT\n\tDISTINCT fv0.entity_key,\n\tfv0.rowid as fv_rowid" + from_query = f"\nFROM (select rowid, * from {table_name} where feature_name = '{string_field_list[0]}') fv0" + + for i, string_field in enumerate(string_field_list): + query += f"\n\t,fv{i}.value as {string_field}" + if i > 0: + from_query += ( + f"\nLEFT JOIN (select rowid, * from {table_name} where feature_name = '{string_field}') fv{i}" + + f"\n\tON fv0.entity_key = fv{i}.entity_key" + ) + + return query + from_query diff --git a/sdk/python/feast/infra/passthrough_provider.py b/sdk/python/feast/infra/passthrough_provider.py index 74b05113282..4e504997d2a 100644 --- a/sdk/python/feast/infra/passthrough_provider.py +++ b/sdk/python/feast/infra/passthrough_provider.py @@ -318,9 +318,10 @@ def retrieve_online_documents_v2( config: RepoConfig, table: FeatureView, requested_features: Optional[List[str]], - query: List[float], + query: Optional[List[float]], top_k: int, distance_metric: Optional[str] = None, + query_string: Optional[str] = None, ) -> List: result = [] if self.online_store: @@ -331,6 +332,7 @@ def retrieve_online_documents_v2( query, top_k, distance_metric, + query_string, ) return result @@ -447,7 +449,7 @@ def materialize_single_feature_view( def get_historical_features( self, config: RepoConfig, - feature_views: List[FeatureView], + feature_views: List[Union[FeatureView, OnDemandFeatureView]], feature_refs: List[str], entity_df: Union[pd.DataFrame, str], registry: BaseRegistry, diff --git a/sdk/python/feast/infra/provider.py b/sdk/python/feast/infra/provider.py index f765e754436..18fbd051771 100644 --- a/sdk/python/feast/infra/provider.py +++ b/sdk/python/feast/infra/provider.py @@ -242,7 +242,7 @@ def materialize_single_feature_view( def get_historical_features( self, config: RepoConfig, - feature_views: List[FeatureView], + feature_views: List[Union[FeatureView, OnDemandFeatureView]], feature_refs: List[str], entity_df: Union[pd.DataFrame, str], registry: BaseRegistry, @@ -456,9 +456,10 @@ def retrieve_online_documents_v2( config: RepoConfig, table: FeatureView, requested_features: List[str], - query: List[float], + query: Optional[List[float]], top_k: int, distance_metric: Optional[str] = None, + query_string: Optional[str] = None, ) -> List[ Tuple[ Optional[datetime], @@ -474,8 +475,9 @@ def retrieve_online_documents_v2( config: The config for the current feature store. table: The feature view whose embeddings should be searched. requested_features: the requested document feature names. - query: The query embedding to search for. + query: The query embedding to search for (optional). top_k: The number of documents to return. + query_string: The query string to search for using keyword search (bm25) (optional) Returns: A list of dictionaries, where each dictionary contains the datetime, entitykey, and a dictionary diff --git a/sdk/python/feast/infra/registry/caching_registry.py b/sdk/python/feast/infra/registry/caching_registry.py index 042eee06ab7..23ab80ee1d8 100644 --- a/sdk/python/feast/infra/registry/caching_registry.py +++ b/sdk/python/feast/infra/registry/caching_registry.py @@ -425,12 +425,24 @@ def list_projects( return self._list_projects(tags) def refresh(self, project: Optional[str] = None): - self.cached_registry_proto = self.proto() - self.cached_registry_proto_created = _utc_now() + if self._refresh_lock.locked(): + logger.info("Skipping refresh if already in progress") + return + try: + self.cached_registry_proto = self.proto() + self.cached_registry_proto_created = _utc_now() + except Exception as e: + logger.error(f"Error while refreshing registry: {e}", exc_info=True) def _refresh_cached_registry_if_necessary(self): if self.cache_mode == "sync": - with self._refresh_lock: + # Try acquiring the lock without blocking + if not self._refresh_lock.acquire(blocking=False): + logger.info( + "Skipping refresh if lock is already held by another thread" + ) + return + try: if self.cached_registry_proto == RegistryProto(): # Avoids the need to refresh the registry when cache is not populated yet # Specially during the __init__ phase @@ -454,6 +466,13 @@ def _refresh_cached_registry_if_necessary(self): if expired: logger.info("Registry cache expired, so refreshing") self.refresh() + except Exception as e: + logger.error( + f"Error in _refresh_cached_registry_if_necessary: {e}", + exc_info=True, + ) + finally: + self._refresh_lock.release() # Always release the lock safely def _start_thread_async_refresh(self, cache_ttl_seconds): self.refresh() diff --git a/sdk/python/feast/infra/utils/couchbase/__init__.py b/sdk/python/feast/infra/utils/couchbase/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/sdk/python/feast/infra/utils/couchbase/couchbase_utils.py b/sdk/python/feast/infra/utils/couchbase/couchbase_utils.py new file mode 100644 index 00000000000..005729274e6 --- /dev/null +++ b/sdk/python/feast/infra/utils/couchbase/couchbase_utils.py @@ -0,0 +1,13 @@ +from datetime import datetime, timezone + + +def normalize_timestamp( + dt: datetime, target_format: str = "%Y-%m-%dT%H:%M:%S%z" +) -> str: + if dt.tzinfo is None: + dt = dt.replace(tzinfo=timezone.utc) # Assume UTC for naive datetimes + # Convert to UTC + utc_dt = dt.astimezone(timezone.utc) + # Format with strftime + formatted = utc_dt.strftime(target_format) + return formatted diff --git a/sdk/python/feast/infra/utils/snowflake/snowflake_utils.py b/sdk/python/feast/infra/utils/snowflake/snowflake_utils.py index b9035b40dbf..b9254e72699 100644 --- a/sdk/python/feast/infra/utils/snowflake/snowflake_utils.py +++ b/sdk/python/feast/infra/utils/snowflake/snowflake_utils.py @@ -513,7 +513,7 @@ def chunk_helper(lst: pd.DataFrame, n: int) -> Iterator[Tuple[int, pd.DataFrame] def parse_private_key_path( - private_key_passphrase: str, + private_key_passphrase: Optional[str] = None, key_path: Optional[str] = None, private_key_content: Optional[bytes] = None, ) -> bytes: @@ -521,14 +521,18 @@ def parse_private_key_path( if private_key_content: p_key = serialization.load_pem_private_key( private_key_content, - password=private_key_passphrase.encode(), + password=private_key_passphrase.encode() + if private_key_passphrase is not None + else None, backend=default_backend(), ) elif key_path: with open(key_path, "rb") as key: p_key = serialization.load_pem_private_key( key.read(), - password=private_key_passphrase.encode(), + password=private_key_passphrase.encode() + if private_key_passphrase is not None + else None, backend=default_backend(), ) else: diff --git a/sdk/python/feast/nlp_test_data.py b/sdk/python/feast/nlp_test_data.py new file mode 100644 index 00000000000..5c0a6af4d61 --- /dev/null +++ b/sdk/python/feast/nlp_test_data.py @@ -0,0 +1,67 @@ +from datetime import datetime +from typing import Dict + +import numpy as np +import pandas as pd + + +def create_document_chunks_df( + documents: Dict[str, str], + start_date: datetime, + end_date: datetime, + embedding_size: int = 60, +) -> pd.DataFrame: + """ + Example df generated by this function: + + | event_timestamp | document_id | chunk_id | chunk_text | embedding | created | + |------------------+-------------+----------+------------------+-----------+------------------| + | 2021-03-17 19:31 | doc_1 | chunk-1 | Hello world | [0.1, ...]| 2021-03-24 19:34 | + | 2021-03-17 19:31 | doc_1 | chunk-2 | How are you? | [0.2, ...]| 2021-03-24 19:34 | + | 2021-03-17 19:31 | doc_2 | chunk-1 | This is a test | [0.3, ...]| 2021-03-24 19:34 | + | 2021-03-17 19:31 | doc_2 | chunk-2 | Document chunk | [0.4, ...]| 2021-03-24 19:34 | + """ + df_hourly = pd.DataFrame( + { + "event_timestamp": [ + pd.Timestamp(dt, unit="ms").round("ms") + for dt in pd.date_range( + start=start_date, + end=end_date, + freq="1h", + inclusive="left", + tz="UTC", + ) + ] + + [ + pd.Timestamp( + year=2021, month=4, day=12, hour=7, minute=0, second=0, tz="UTC" + ) + ] + } + ) + df_all_chunks = pd.DataFrame() + + for doc_id, doc_text in documents.items(): + chunks = doc_text.split(". ") # Simple chunking by sentence + for chunk_id, chunk_text in enumerate(chunks, start=1): + df_hourly_copy = df_hourly.copy() + df_hourly_copy["document_id"] = doc_id + df_hourly_copy["chunk_id"] = f"chunk-{chunk_id}" + df_hourly_copy["chunk_text"] = chunk_text + df_all_chunks = pd.concat([df_hourly_copy, df_all_chunks]) + + df_all_chunks.reset_index(drop=True, inplace=True) + rows = df_all_chunks["event_timestamp"].count() + + # Generate random embeddings for each chunk + df_all_chunks["embedding"] = [ + np.random.rand(embedding_size).tolist() for _ in range(rows) + ] + df_all_chunks["created"] = pd.to_datetime(pd.Timestamp.now(tz=None).round("ms")) + + # Create duplicate rows that should be filtered by created timestamp + late_row = df_all_chunks[rows // 2 : rows // 2 + 1] + df_all_chunks = pd.concat([df_all_chunks, late_row, late_row], ignore_index=True) + + return df_all_chunks diff --git a/sdk/python/feast/on_demand_feature_view.py b/sdk/python/feast/on_demand_feature_view.py index 0ae87b5e35a..f4ec0149184 100644 --- a/sdk/python/feast/on_demand_feature_view.py +++ b/sdk/python/feast/on_demand_feature_view.py @@ -339,7 +339,6 @@ def to_proto(self) -> OnDemandFeatureViewProto: write_to_online_store=self.write_to_online_store, singleton=self.singleton if self.singleton else False, ) - return OnDemandFeatureViewProto(spec=spec, meta=meta) @classmethod @@ -454,6 +453,8 @@ def from_proto( Field( name=feature.name, dtype=from_value_type(ValueType(feature.value_type)), + vector_index=feature.vector_index, + vector_search_metric=feature.vector_search_metric, ) for feature in on_demand_feature_view_proto.spec.features ], @@ -640,13 +641,25 @@ def transform_dict( def infer_features(self) -> None: random_input = self._construct_random_input(singleton=self.singleton) - inferred_features = self.feature_transformation.infer_features(random_input) + inferred_features = self.feature_transformation.infer_features( + random_input=random_input, singleton=self.singleton + ) if self.features: missing_features = [] for specified_feature in self.features: - if specified_feature not in inferred_features: + if ( + specified_feature not in inferred_features + and "Array" not in specified_feature.dtype.__str__() + ): missing_features.append(specified_feature) + elif "Array" in specified_feature.dtype.__str__(): + if specified_feature.name not in [ + f.name for f in inferred_features + ]: + missing_features.append(specified_feature) + else: + pass if missing_features: raise SpecifiedFeaturesNotPresentError( missing_features, inferred_features, self.name @@ -722,6 +735,7 @@ def get_requested_odfvs( def on_demand_feature_view( *, + name: Optional[str] = None, entities: Optional[List[Entity]] = None, schema: list[Field], sources: list[ @@ -737,11 +751,13 @@ def on_demand_feature_view( owner: str = "", write_to_online_store: bool = False, singleton: bool = False, + explode: bool = False, ): """ Creates an OnDemandFeatureView object with the given user function as udf. Args: + name (optional): The name of the on demand feature view. If not provided, the name will be the name of the user function. entities (Optional): The list of names of entities that this feature view is associated with. schema: The list of features in the output of the on demand feature view, after the transformation has been applied. @@ -757,6 +773,7 @@ def on_demand_feature_view( the online store for faster retrieval. singleton (optional): A boolean that indicates whether the transformation is executed on a singleton (only applicable when mode="python"). + explode (optional): A boolean that indicates whether the transformation explodes the input data into multiple rows. """ def mainify(obj) -> None: @@ -776,10 +793,6 @@ def decorator(user_function): ) transformation = PandasTransformation(user_function, udf_string) elif mode == "python": - if return_annotation not in (inspect._empty, dict[str, Any]): - raise TypeError( - f"return signature for {user_function} is {return_annotation} but should be dict[str, Any]" - ) transformation = PythonTransformation(user_function, udf_string) elif mode == "substrait": from ibis.expr.types.relations import Table @@ -791,7 +804,7 @@ def decorator(user_function): transformation = SubstraitTransformation.from_ibis(user_function, sources) on_demand_feature_view_obj = OnDemandFeatureView( - name=user_function.__name__, + name=name if name is not None else user_function.__name__, sources=sources, schema=schema, feature_transformation=transformation, diff --git a/sdk/python/feast/protos/feast/core/FeatureService_pb2.py b/sdk/python/feast/protos/feast/core/FeatureService_pb2.py index 642d5b010f9..7ef36079691 100644 --- a/sdk/python/feast/protos/feast/core/FeatureService_pb2.py +++ b/sdk/python/feast/protos/feast/core/FeatureService_pb2.py @@ -16,7 +16,7 @@ from feast.protos.feast.core import FeatureViewProjection_pb2 as feast_dot_core_dot_FeatureViewProjection__pb2 -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x1f\x66\x65\x61st/core/FeatureService.proto\x12\nfeast.core\x1a\x1fgoogle/protobuf/timestamp.proto\x1a&feast/core/FeatureViewProjection.proto\"l\n\x0e\x46\x65\x61tureService\x12,\n\x04spec\x18\x01 \x01(\x0b\x32\x1e.feast.core.FeatureServiceSpec\x12,\n\x04meta\x18\x02 \x01(\x0b\x32\x1e.feast.core.FeatureServiceMeta\"\xa4\x02\n\x12\x46\x65\x61tureServiceSpec\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07project\x18\x02 \x01(\t\x12\x33\n\x08\x66\x65\x61tures\x18\x03 \x03(\x0b\x32!.feast.core.FeatureViewProjection\x12\x36\n\x04tags\x18\x04 \x03(\x0b\x32(.feast.core.FeatureServiceSpec.TagsEntry\x12\x13\n\x0b\x64\x65scription\x18\x05 \x01(\t\x12\r\n\x05owner\x18\x06 \x01(\t\x12\x31\n\x0elogging_config\x18\x07 \x01(\x0b\x32\x19.feast.core.LoggingConfig\x1a+\n\tTagsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"\x87\x01\n\x12\x46\x65\x61tureServiceMeta\x12\x35\n\x11\x63reated_timestamp\x18\x01 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12:\n\x16last_updated_timestamp\x18\x02 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\"\x9a\x07\n\rLoggingConfig\x12\x13\n\x0bsample_rate\x18\x01 \x01(\x02\x12\x45\n\x10\x66ile_destination\x18\x03 \x01(\x0b\x32).feast.core.LoggingConfig.FileDestinationH\x00\x12M\n\x14\x62igquery_destination\x18\x04 \x01(\x0b\x32-.feast.core.LoggingConfig.BigQueryDestinationH\x00\x12M\n\x14redshift_destination\x18\x05 \x01(\x0b\x32-.feast.core.LoggingConfig.RedshiftDestinationH\x00\x12O\n\x15snowflake_destination\x18\x06 \x01(\x0b\x32..feast.core.LoggingConfig.SnowflakeDestinationH\x00\x12I\n\x12\x63ustom_destination\x18\x07 \x01(\x0b\x32+.feast.core.LoggingConfig.CustomDestinationH\x00\x12I\n\x12\x61thena_destination\x18\x08 \x01(\x0b\x32+.feast.core.LoggingConfig.AthenaDestinationH\x00\x1aS\n\x0f\x46ileDestination\x12\x0c\n\x04path\x18\x01 \x01(\t\x12\x1c\n\x14s3_endpoint_override\x18\x02 \x01(\t\x12\x14\n\x0cpartition_by\x18\x03 \x03(\t\x1a(\n\x13\x42igQueryDestination\x12\x11\n\ttable_ref\x18\x01 \x01(\t\x1a)\n\x13RedshiftDestination\x12\x12\n\ntable_name\x18\x01 \x01(\t\x1a\'\n\x11\x41thenaDestination\x12\x12\n\ntable_name\x18\x01 \x01(\t\x1a*\n\x14SnowflakeDestination\x12\x12\n\ntable_name\x18\x01 \x01(\t\x1a\x99\x01\n\x11\x43ustomDestination\x12\x0c\n\x04kind\x18\x01 \x01(\t\x12G\n\x06\x63onfig\x18\x02 \x03(\x0b\x32\x37.feast.core.LoggingConfig.CustomDestination.ConfigEntry\x1a-\n\x0b\x43onfigEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\x42\r\n\x0b\x64\x65stination\"I\n\x12\x46\x65\x61tureServiceList\x12\x33\n\x0f\x66\x65\x61tureservices\x18\x01 \x03(\x0b\x32\x1a.feast.core.FeatureServiceBX\n\x10\x66\x65\x61st.proto.coreB\x13\x46\x65\x61tureServiceProtoZ/github.com/feast-dev/feast/go/protos/feast/coreb\x06proto3') +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x1f\x66\x65\x61st/core/FeatureService.proto\x12\nfeast.core\x1a\x1fgoogle/protobuf/timestamp.proto\x1a&feast/core/FeatureViewProjection.proto\"l\n\x0e\x46\x65\x61tureService\x12,\n\x04spec\x18\x01 \x01(\x0b\x32\x1e.feast.core.FeatureServiceSpec\x12,\n\x04meta\x18\x02 \x01(\x0b\x32\x1e.feast.core.FeatureServiceMeta\"\xa4\x02\n\x12\x46\x65\x61tureServiceSpec\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07project\x18\x02 \x01(\t\x12\x33\n\x08\x66\x65\x61tures\x18\x03 \x03(\x0b\x32!.feast.core.FeatureViewProjection\x12\x36\n\x04tags\x18\x04 \x03(\x0b\x32(.feast.core.FeatureServiceSpec.TagsEntry\x12\x13\n\x0b\x64\x65scription\x18\x05 \x01(\t\x12\r\n\x05owner\x18\x06 \x01(\t\x12\x31\n\x0elogging_config\x18\x07 \x01(\x0b\x32\x19.feast.core.LoggingConfig\x1a+\n\tTagsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"\x87\x01\n\x12\x46\x65\x61tureServiceMeta\x12\x35\n\x11\x63reated_timestamp\x18\x01 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12:\n\x16last_updated_timestamp\x18\x02 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\"\xd1\x08\n\rLoggingConfig\x12\x13\n\x0bsample_rate\x18\x01 \x01(\x02\x12\x45\n\x10\x66ile_destination\x18\x03 \x01(\x0b\x32).feast.core.LoggingConfig.FileDestinationH\x00\x12M\n\x14\x62igquery_destination\x18\x04 \x01(\x0b\x32-.feast.core.LoggingConfig.BigQueryDestinationH\x00\x12M\n\x14redshift_destination\x18\x05 \x01(\x0b\x32-.feast.core.LoggingConfig.RedshiftDestinationH\x00\x12O\n\x15snowflake_destination\x18\x06 \x01(\x0b\x32..feast.core.LoggingConfig.SnowflakeDestinationH\x00\x12I\n\x12\x63ustom_destination\x18\x07 \x01(\x0b\x32+.feast.core.LoggingConfig.CustomDestinationH\x00\x12I\n\x12\x61thena_destination\x18\x08 \x01(\x0b\x32+.feast.core.LoggingConfig.AthenaDestinationH\x00\x12`\n\x1e\x63ouchbase_columnar_destination\x18\t \x01(\x0b\x32\x36.feast.core.LoggingConfig.CouchbaseColumnarDestinationH\x00\x1aS\n\x0f\x46ileDestination\x12\x0c\n\x04path\x18\x01 \x01(\t\x12\x1c\n\x14s3_endpoint_override\x18\x02 \x01(\t\x12\x14\n\x0cpartition_by\x18\x03 \x03(\t\x1a(\n\x13\x42igQueryDestination\x12\x11\n\ttable_ref\x18\x01 \x01(\t\x1a)\n\x13RedshiftDestination\x12\x12\n\ntable_name\x18\x01 \x01(\t\x1a\'\n\x11\x41thenaDestination\x12\x12\n\ntable_name\x18\x01 \x01(\t\x1a*\n\x14SnowflakeDestination\x12\x12\n\ntable_name\x18\x01 \x01(\t\x1a\x99\x01\n\x11\x43ustomDestination\x12\x0c\n\x04kind\x18\x01 \x01(\t\x12G\n\x06\x63onfig\x18\x02 \x03(\x0b\x32\x37.feast.core.LoggingConfig.CustomDestination.ConfigEntry\x1a-\n\x0b\x43onfigEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\x1aS\n\x1c\x43ouchbaseColumnarDestination\x12\x10\n\x08\x64\x61tabase\x18\x01 \x01(\t\x12\r\n\x05scope\x18\x02 \x01(\t\x12\x12\n\ncollection\x18\x03 \x01(\tB\r\n\x0b\x64\x65stination\"I\n\x12\x46\x65\x61tureServiceList\x12\x33\n\x0f\x66\x65\x61tureservices\x18\x01 \x03(\x0b\x32\x1a.feast.core.FeatureServiceBX\n\x10\x66\x65\x61st.proto.coreB\x13\x46\x65\x61tureServiceProtoZ/github.com/feast-dev/feast/go/protos/feast/coreb\x06proto3') _globals = globals() _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) @@ -37,21 +37,23 @@ _globals['_FEATURESERVICEMETA']._serialized_start=526 _globals['_FEATURESERVICEMETA']._serialized_end=661 _globals['_LOGGINGCONFIG']._serialized_start=664 - _globals['_LOGGINGCONFIG']._serialized_end=1586 - _globals['_LOGGINGCONFIG_FILEDESTINATION']._serialized_start=1162 - _globals['_LOGGINGCONFIG_FILEDESTINATION']._serialized_end=1245 - _globals['_LOGGINGCONFIG_BIGQUERYDESTINATION']._serialized_start=1247 - _globals['_LOGGINGCONFIG_BIGQUERYDESTINATION']._serialized_end=1287 - _globals['_LOGGINGCONFIG_REDSHIFTDESTINATION']._serialized_start=1289 - _globals['_LOGGINGCONFIG_REDSHIFTDESTINATION']._serialized_end=1330 - _globals['_LOGGINGCONFIG_ATHENADESTINATION']._serialized_start=1332 - _globals['_LOGGINGCONFIG_ATHENADESTINATION']._serialized_end=1371 - _globals['_LOGGINGCONFIG_SNOWFLAKEDESTINATION']._serialized_start=1373 - _globals['_LOGGINGCONFIG_SNOWFLAKEDESTINATION']._serialized_end=1415 - _globals['_LOGGINGCONFIG_CUSTOMDESTINATION']._serialized_start=1418 - _globals['_LOGGINGCONFIG_CUSTOMDESTINATION']._serialized_end=1571 - _globals['_LOGGINGCONFIG_CUSTOMDESTINATION_CONFIGENTRY']._serialized_start=1526 - _globals['_LOGGINGCONFIG_CUSTOMDESTINATION_CONFIGENTRY']._serialized_end=1571 - _globals['_FEATURESERVICELIST']._serialized_start=1588 - _globals['_FEATURESERVICELIST']._serialized_end=1661 + _globals['_LOGGINGCONFIG']._serialized_end=1769 + _globals['_LOGGINGCONFIG_FILEDESTINATION']._serialized_start=1260 + _globals['_LOGGINGCONFIG_FILEDESTINATION']._serialized_end=1343 + _globals['_LOGGINGCONFIG_BIGQUERYDESTINATION']._serialized_start=1345 + _globals['_LOGGINGCONFIG_BIGQUERYDESTINATION']._serialized_end=1385 + _globals['_LOGGINGCONFIG_REDSHIFTDESTINATION']._serialized_start=1387 + _globals['_LOGGINGCONFIG_REDSHIFTDESTINATION']._serialized_end=1428 + _globals['_LOGGINGCONFIG_ATHENADESTINATION']._serialized_start=1430 + _globals['_LOGGINGCONFIG_ATHENADESTINATION']._serialized_end=1469 + _globals['_LOGGINGCONFIG_SNOWFLAKEDESTINATION']._serialized_start=1471 + _globals['_LOGGINGCONFIG_SNOWFLAKEDESTINATION']._serialized_end=1513 + _globals['_LOGGINGCONFIG_CUSTOMDESTINATION']._serialized_start=1516 + _globals['_LOGGINGCONFIG_CUSTOMDESTINATION']._serialized_end=1669 + _globals['_LOGGINGCONFIG_CUSTOMDESTINATION_CONFIGENTRY']._serialized_start=1624 + _globals['_LOGGINGCONFIG_CUSTOMDESTINATION_CONFIGENTRY']._serialized_end=1669 + _globals['_LOGGINGCONFIG_COUCHBASECOLUMNARDESTINATION']._serialized_start=1671 + _globals['_LOGGINGCONFIG_COUCHBASECOLUMNARDESTINATION']._serialized_end=1754 + _globals['_FEATURESERVICELIST']._serialized_start=1771 + _globals['_FEATURESERVICELIST']._serialized_end=1844 # @@protoc_insertion_point(module_scope) diff --git a/sdk/python/feast/protos/feast/core/FeatureService_pb2.pyi b/sdk/python/feast/protos/feast/core/FeatureService_pb2.pyi index 0b1c0baa871..6d5879e52cb 100644 --- a/sdk/python/feast/protos/feast/core/FeatureService_pb2.pyi +++ b/sdk/python/feast/protos/feast/core/FeatureService_pb2.pyi @@ -228,6 +228,27 @@ class LoggingConfig(google.protobuf.message.Message): ) -> None: ... def ClearField(self, field_name: typing_extensions.Literal["config", b"config", "kind", b"kind"]) -> None: ... + class CouchbaseColumnarDestination(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + DATABASE_FIELD_NUMBER: builtins.int + SCOPE_FIELD_NUMBER: builtins.int + COLLECTION_FIELD_NUMBER: builtins.int + database: builtins.str + """Destination database name""" + scope: builtins.str + """Destination scope name""" + collection: builtins.str + """Destination collection name""" + def __init__( + self, + *, + database: builtins.str = ..., + scope: builtins.str = ..., + collection: builtins.str = ..., + ) -> None: ... + def ClearField(self, field_name: typing_extensions.Literal["collection", b"collection", "database", b"database", "scope", b"scope"]) -> None: ... + SAMPLE_RATE_FIELD_NUMBER: builtins.int FILE_DESTINATION_FIELD_NUMBER: builtins.int BIGQUERY_DESTINATION_FIELD_NUMBER: builtins.int @@ -235,6 +256,7 @@ class LoggingConfig(google.protobuf.message.Message): SNOWFLAKE_DESTINATION_FIELD_NUMBER: builtins.int CUSTOM_DESTINATION_FIELD_NUMBER: builtins.int ATHENA_DESTINATION_FIELD_NUMBER: builtins.int + COUCHBASE_COLUMNAR_DESTINATION_FIELD_NUMBER: builtins.int sample_rate: builtins.float @property def file_destination(self) -> global___LoggingConfig.FileDestination: ... @@ -248,6 +270,8 @@ class LoggingConfig(google.protobuf.message.Message): def custom_destination(self) -> global___LoggingConfig.CustomDestination: ... @property def athena_destination(self) -> global___LoggingConfig.AthenaDestination: ... + @property + def couchbase_columnar_destination(self) -> global___LoggingConfig.CouchbaseColumnarDestination: ... def __init__( self, *, @@ -258,10 +282,11 @@ class LoggingConfig(google.protobuf.message.Message): snowflake_destination: global___LoggingConfig.SnowflakeDestination | None = ..., custom_destination: global___LoggingConfig.CustomDestination | None = ..., athena_destination: global___LoggingConfig.AthenaDestination | None = ..., + couchbase_columnar_destination: global___LoggingConfig.CouchbaseColumnarDestination | None = ..., ) -> None: ... - def HasField(self, field_name: typing_extensions.Literal["athena_destination", b"athena_destination", "bigquery_destination", b"bigquery_destination", "custom_destination", b"custom_destination", "destination", b"destination", "file_destination", b"file_destination", "redshift_destination", b"redshift_destination", "snowflake_destination", b"snowflake_destination"]) -> builtins.bool: ... - def ClearField(self, field_name: typing_extensions.Literal["athena_destination", b"athena_destination", "bigquery_destination", b"bigquery_destination", "custom_destination", b"custom_destination", "destination", b"destination", "file_destination", b"file_destination", "redshift_destination", b"redshift_destination", "sample_rate", b"sample_rate", "snowflake_destination", b"snowflake_destination"]) -> None: ... - def WhichOneof(self, oneof_group: typing_extensions.Literal["destination", b"destination"]) -> typing_extensions.Literal["file_destination", "bigquery_destination", "redshift_destination", "snowflake_destination", "custom_destination", "athena_destination"] | None: ... + def HasField(self, field_name: typing_extensions.Literal["athena_destination", b"athena_destination", "bigquery_destination", b"bigquery_destination", "couchbase_columnar_destination", b"couchbase_columnar_destination", "custom_destination", b"custom_destination", "destination", b"destination", "file_destination", b"file_destination", "redshift_destination", b"redshift_destination", "snowflake_destination", b"snowflake_destination"]) -> builtins.bool: ... + def ClearField(self, field_name: typing_extensions.Literal["athena_destination", b"athena_destination", "bigquery_destination", b"bigquery_destination", "couchbase_columnar_destination", b"couchbase_columnar_destination", "custom_destination", b"custom_destination", "destination", b"destination", "file_destination", b"file_destination", "redshift_destination", b"redshift_destination", "sample_rate", b"sample_rate", "snowflake_destination", b"snowflake_destination"]) -> None: ... + def WhichOneof(self, oneof_group: typing_extensions.Literal["destination", b"destination"]) -> typing_extensions.Literal["file_destination", "bigquery_destination", "redshift_destination", "snowflake_destination", "custom_destination", "athena_destination", "couchbase_columnar_destination"] | None: ... global___LoggingConfig = LoggingConfig diff --git a/sdk/python/feast/repo_config.py b/sdk/python/feast/repo_config.py index d943caa4c1a..66b3f201594 100644 --- a/sdk/python/feast/repo_config.py +++ b/sdk/python/feast/repo_config.py @@ -80,7 +80,7 @@ "remote": "feast.infra.online_stores.remote.RemoteOnlineStore", "singlestore": "feast.infra.online_stores.singlestore_online_store.singlestore.SingleStoreOnlineStore", "qdrant": "feast.infra.online_stores.cqdrant.QdrantOnlineStore", - "couchbase": "feast.infra.online_stores.couchbase_online_store.couchbase.CouchbaseOnlineStore", + "couchbase.online": "feast.infra.online_stores.couchbase_online_store.couchbase.CouchbaseOnlineStore", "milvus": "feast.infra.online_stores.milvus_online_store.milvus.MilvusOnlineStore", **LEGACY_ONLINE_STORE_CLASS_FOR_TYPE, } @@ -98,6 +98,7 @@ "mssql": "feast.infra.offline_stores.contrib.mssql_offline_store.mssql.MsSqlServerOfflineStore", "duckdb": "feast.infra.offline_stores.duckdb.DuckDBOfflineStore", "remote": "feast.infra.offline_stores.remote.RemoteOfflineStore", + "couchbase.offline": "feast.infra.offline_stores.contrib.couchbase_offline_store.couchbase.CouchbaseColumnarOfflineStore", } FEATURE_SERVER_CONFIG_CLASS_FOR_TYPE = { diff --git a/sdk/python/feast/static/chat/index.html b/sdk/python/feast/static/chat/index.html new file mode 100644 index 00000000000..302c3b55b6a --- /dev/null +++ b/sdk/python/feast/static/chat/index.html @@ -0,0 +1,129 @@ + + + + + + Feast Chat + + + +
+
Hello! How can I help you today?
+
+
+ + +
+ + + diff --git a/sdk/python/feast/templates/couchbase/bootstrap.py b/sdk/python/feast/templates/couchbase/bootstrap.py new file mode 100644 index 00000000000..1034e14e0de --- /dev/null +++ b/sdk/python/feast/templates/couchbase/bootstrap.py @@ -0,0 +1,108 @@ +import click +from couchbase_columnar.cluster import Cluster +from couchbase_columnar.common.errors import InvalidCredentialError +from couchbase_columnar.credential import Credential +from couchbase_columnar.options import ClusterOptions, QueryOptions, TimeoutOptions + +from feast.file_utils import replace_str_in_file +from feast.infra.offline_stores.contrib.couchbase_offline_store.couchbase import ( + CouchbaseColumnarOfflineStoreConfig, + df_to_columnar, +) + + +def bootstrap(): + # Bootstrap() will automatically be called from the init_repo() during `feast init` + + import pathlib + from datetime import datetime, timedelta + + from feast.driver_test_data import create_driver_hourly_stats_df + + repo_path = pathlib.Path(__file__).parent.absolute() / "feature_repo" + config_file = repo_path / "feature_store.yaml" + + if click.confirm("Configure Couchbase Online Store?", default=True): + connection_string = click.prompt( + "Couchbase Connection String", default="couchbase://127.0.0.1" + ) + user = click.prompt("Couchbase Username", default="Administrator") + password = click.prompt("Couchbase Password", hide_input=True) + bucket_name = click.prompt("Couchbase Bucket Name", default="feast") + kv_port = click.prompt("Couchbase KV Port", default=11210) + + replace_str_in_file( + config_file, "COUCHBASE_CONNECTION_STRING", connection_string + ) + replace_str_in_file(config_file, "COUCHBASE_USER", user) + replace_str_in_file(config_file, "COUCHBASE_PASSWORD", password) + replace_str_in_file(config_file, "COUCHBASE_BUCKET_NAME", bucket_name) + replace_str_in_file(config_file, "COUCHBASE_KV_PORT", str(kv_port)) + + if click.confirm( + "Configure Couchbase Columnar Offline Store? (Note: requires Couchbase Capella Columnar)", + default=True, + ): + end_date = datetime.now().replace(microsecond=0, second=0, minute=0) + start_date = end_date - timedelta(days=15) + + driver_entities = [1001, 1002, 1003, 1004, 1005] + driver_df = create_driver_hourly_stats_df(driver_entities, start_date, end_date) + + columnar_connection_string = click.prompt("Columnar Connection String") + columnar_user = click.prompt("Columnar Username") + columnar_password = click.prompt("Columnar Password", hide_input=True) + columnar_timeout = click.prompt("Couchbase Columnar Timeout", default=120) + + if click.confirm( + 'Should I upload example data to Couchbase Capella Columnar (overwriting "Default.Default.feast_driver_hourly_stats" table)?', + default=True, + ): + cred = Credential.from_username_and_password( + columnar_user, columnar_password + ) + timeout_opts = TimeoutOptions(dispatch_timeout=timedelta(seconds=120)) + cluster = Cluster.create_instance( + columnar_connection_string, + cred, + ClusterOptions(timeout_options=timeout_opts), + ) + + table_name = "Default.Default.feast_driver_hourly_stats" + try: + cluster.execute_query( + f"DROP COLLECTION {table_name} IF EXISTS", + QueryOptions(timeout=timedelta(seconds=columnar_timeout)), + ) + except InvalidCredentialError: + print("Error: Invalid Cluster Credentials.") + return + + offline_store = CouchbaseColumnarOfflineStoreConfig( + type="couchbase.offline", + connection_string=columnar_connection_string, + user=columnar_user, + password=columnar_password, + timeout=columnar_timeout, + ) + + df_to_columnar( + df=driver_df, table_name=table_name, offline_store=offline_store + ) + + replace_str_in_file( + config_file, + "COUCHBASE_COLUMNAR_CONNECTION_STRING", + columnar_connection_string, + ) + replace_str_in_file(config_file, "COUCHBASE_COLUMNAR_USER", columnar_user) + replace_str_in_file( + config_file, "COUCHBASE_COLUMNAR_PASSWORD", columnar_password + ) + replace_str_in_file( + config_file, "COUCHBASE_COLUMNAR_TIMEOUT", str(columnar_timeout) + ) + + +if __name__ == "__main__": + bootstrap() diff --git a/sdk/python/feast/templates/couchbase/feature_repo/example_repo.py b/sdk/python/feast/templates/couchbase/feature_repo/example_repo.py new file mode 100644 index 00000000000..363ba3c4664 --- /dev/null +++ b/sdk/python/feast/templates/couchbase/feature_repo/example_repo.py @@ -0,0 +1,134 @@ +# This is an example feature definition file + +from datetime import timedelta + +import pandas as pd + +from feast import Entity, FeatureService, FeatureView, Field, PushSource, RequestSource +from feast.infra.offline_stores.contrib.couchbase_offline_store.couchbase_source import ( + CouchbaseColumnarSource, +) +from feast.on_demand_feature_view import on_demand_feature_view +from feast.types import Float32, Float64, Int64 + +# Define an entity for the driver. You can think of an entity as a primary key used to +# fetch features. +driver = Entity(name="driver", join_keys=["driver_id"]) + +driver_stats_source = CouchbaseColumnarSource( + name="driver_hourly_stats_source", + query="SELECT * FROM Default.Default.`feast_driver_hourly_stats`", + database="Default", + scope="Default", + collection="feast_driver_hourly_stats", + timestamp_field="event_timestamp", + created_timestamp_column="created", +) + +# Our parquet files contain sample data that includes a driver_id column, timestamps and +# three feature column. Here we define a Feature View that will allow us to serve this +# data to our model online. +driver_stats_fv = FeatureView( + # The unique name of this feature view. Two feature views in a single + # project cannot have the same name + name="driver_hourly_stats", + entities=[driver], + ttl=timedelta(days=1), + # The list of features defined below act as a schema to both define features + # for both materialization of features into a store, and are used as references + # during retrieval for building a training dataset or serving features + schema=[ + Field(name="conv_rate", dtype=Float32), + Field(name="acc_rate", dtype=Float32), + Field(name="avg_daily_trips", dtype=Int64), + ], + online=True, + source=driver_stats_source, + # Tags are user defined key/value pairs that are attached to each + # feature view + tags={"team": "driver_performance"}, +) + +# Define a request data source which encodes features / information only +# available at request time (e.g. part of the user initiated HTTP request) +input_request = RequestSource( + name="vals_to_add", + schema=[ + Field(name="val_to_add", dtype=Int64), + Field(name="val_to_add_2", dtype=Int64), + ], +) + + +# Define an on demand feature view which can generate new features based on +# existing feature views and RequestSource features +@on_demand_feature_view( + sources=[driver_stats_fv, input_request], + schema=[ + Field(name="conv_rate_plus_val1", dtype=Float64), + Field(name="conv_rate_plus_val2", dtype=Float64), + ], +) +def transformed_conv_rate(inputs: pd.DataFrame) -> pd.DataFrame: + df = pd.DataFrame() + df["conv_rate_plus_val1"] = inputs["conv_rate"] + inputs["val_to_add"] + df["conv_rate_plus_val2"] = inputs["conv_rate"] + inputs["val_to_add_2"] + return df + + +# This groups features into a model version +driver_activity_v1 = FeatureService( + name="driver_activity_v1", + features=[ + driver_stats_fv[["conv_rate"]], # Sub-selects a feature from a feature view + transformed_conv_rate, # Selects all features from the feature view + ], +) +driver_activity_v2 = FeatureService( + name="driver_activity_v2", features=[driver_stats_fv, transformed_conv_rate] +) + +# Defines a way to push data (to be available offline, online or both) into Feast. +driver_stats_push_source = PushSource( + name="driver_stats_push_source", + batch_source=driver_stats_source, +) + +# Defines a slightly modified version of the feature view from above, where the source +# has been changed to the push source. This allows fresh features to be directly pushed +# to the online store for this feature view. +driver_stats_fresh_fv = FeatureView( + name="driver_hourly_stats_fresh", + entities=[driver], + ttl=timedelta(days=1), + schema=[ + Field(name="conv_rate", dtype=Float32), + Field(name="acc_rate", dtype=Float32), + Field(name="avg_daily_trips", dtype=Int64), + ], + online=True, + source=driver_stats_push_source, # Changed from above + tags={"team": "driver_performance"}, +) + + +# Define an on demand feature view which can generate new features based on +# existing feature views and RequestSource features +@on_demand_feature_view( + sources=[driver_stats_fresh_fv, input_request], # relies on fresh version of FV + schema=[ + Field(name="conv_rate_plus_val1", dtype=Float64), + Field(name="conv_rate_plus_val2", dtype=Float64), + ], +) +def transformed_conv_rate_fresh(inputs: pd.DataFrame) -> pd.DataFrame: + df = pd.DataFrame() + df["conv_rate_plus_val1"] = inputs["conv_rate"] + inputs["val_to_add"] + df["conv_rate_plus_val2"] = inputs["conv_rate"] + inputs["val_to_add_2"] + return df + + +driver_activity_v3 = FeatureService( + name="driver_activity_v3", + features=[driver_stats_fresh_fv, transformed_conv_rate_fresh], +) diff --git a/sdk/python/feast/templates/couchbase/feature_repo/feature_store.yaml b/sdk/python/feast/templates/couchbase/feature_repo/feature_store.yaml index bc21e44defd..96f45934eb5 100644 --- a/sdk/python/feast/templates/couchbase/feature_repo/feature_store.yaml +++ b/sdk/python/feast/templates/couchbase/feature_repo/feature_store.yaml @@ -1,11 +1,17 @@ project: my_project -registry: /path/to/registry.db +registry: data/registry.db provider: local online_store: - type: couchbase + type: couchbase.online connection_string: COUCHBASE_CONNECTION_STRING # Couchbase connection string, copied from 'Connect' page in Couchbase Capella console user: COUCHBASE_USER # Couchbase username from database access credentials password: COUCHBASE_PASSWORD # Couchbase password from database access credentials bucket_name: COUCHBASE_BUCKET_NAME # Couchbase bucket name, defaults to feast kv_port: COUCHBASE_KV_PORT # Couchbase key-value port, defaults to 11210. Required if custom ports are used. +offline_store: + type: couchbase.offline + connection_string: COUCHBASE_COLUMNAR_CONNECTION_STRING # Copied from Settings > Connection String page in Capella Columnar console, starts with couchbases:// + user: COUCHBASE_COLUMNAR_USER # Couchbase cluster access name from Settings > Access Control page in Capella Columnar console + password: COUCHBASE_COLUMNAR_PASSWORD # Couchbase password from Settings > Access Control page in Capella Columnar console + timeout: COUCHBASE_COLUMNAR_TIMEOUT # Timeout in seconds for Columnar operations, optional entity_key_serialization_version: 2 diff --git a/sdk/python/feast/templates/couchbase/feature_repo/test_workflow.py b/sdk/python/feast/templates/couchbase/feature_repo/test_workflow.py new file mode 100644 index 00000000000..192d575181c --- /dev/null +++ b/sdk/python/feast/templates/couchbase/feature_repo/test_workflow.py @@ -0,0 +1,112 @@ +import os.path +import subprocess +from datetime import datetime + +import pandas as pd + +from feast import FeatureStore + + +def run_demo(): + store = FeatureStore(repo_path=os.path.dirname(__file__)) + print("\n--- Run feast apply to setup feature store on Couchbase ---") + subprocess.run(["feast", "--chdir", os.path.dirname(__file__), "apply"]) + + print("\n--- Historical features for training ---") + fetch_historical_features_entity_df(store, for_batch_scoring=False) + + print("\n--- Historical features for batch scoring ---") + fetch_historical_features_entity_df(store, for_batch_scoring=True) + + print("\n--- Load features into online store ---") + store.materialize_incremental(end_date=datetime.now()) + + print("\n--- Online features ---") + fetch_online_features(store) + + print("\n--- Online features retrieved (instead) through a feature service---") + fetch_online_features(store, source="feature_service") + + print( + "\n--- Online features retrieved (using feature service v3, which uses a feature view with a push source---" + ) + fetch_online_features(store, source="push") + + print("\n--- Online features again with updated values from a stream push---") + fetch_online_features(store, source="push") + + print("\n--- Run feast teardown ---") + subprocess.run(["feast", "--chdir", os.path.dirname(__file__), "teardown"]) + + +def fetch_historical_features_entity_df(store: FeatureStore, for_batch_scoring: bool): + # Note: see https://docs.feast.dev/getting-started/concepts/feature-retrieval for more details on how to retrieve + # for all entities in the offline store instead + entity_df = pd.DataFrame.from_dict( + { + # entity's join key -> entity values + "driver_id": [1001, 1002, 1003], + # "event_timestamp" (reserved key) -> timestamps + "event_timestamp": [ + datetime(2021, 4, 12, 10, 59, 42), + datetime(2021, 4, 12, 8, 12, 10), + datetime(2021, 4, 12, 16, 40, 26), + ], + # (optional) label name -> label values. Feast does not process these + "label_driver_reported_satisfaction": [1, 5, 3], + # values we're using for an on-demand transformation + "val_to_add": [1, 2, 3], + "val_to_add_2": [10, 20, 30], + } + ) + # For batch scoring, we want the latest timestamps + if for_batch_scoring: + entity_df["event_timestamp"] = pd.to_datetime("now", utc=True) + + training_df = store.get_historical_features( + entity_df=entity_df, + features=[ + "driver_hourly_stats:conv_rate", + "driver_hourly_stats:acc_rate", + "driver_hourly_stats:avg_daily_trips", + "transformed_conv_rate:conv_rate_plus_val1", + "transformed_conv_rate:conv_rate_plus_val2", + ], + ).to_df() + print(training_df.head()) + + +def fetch_online_features(store, source: str = ""): + entity_rows = [ + # {join_key: entity_value} + { + "driver_id": 1001, + "val_to_add": 1000, + "val_to_add_2": 2000, + }, + { + "driver_id": 1002, + "val_to_add": 1001, + "val_to_add_2": 2002, + }, + ] + if source == "feature_service": + features_to_fetch = store.get_feature_service("driver_activity_v1") + elif source == "push": + features_to_fetch = store.get_feature_service("driver_activity_v3") + else: + features_to_fetch = [ + "driver_hourly_stats:acc_rate", + "transformed_conv_rate:conv_rate_plus_val1", + "transformed_conv_rate:conv_rate_plus_val2", + ] + returned_features = store.get_online_features( + features=features_to_fetch, + entity_rows=entity_rows, + ).to_dict() + for key, value in sorted(returned_features.items()): + print(key, " : ", value) + + +if __name__ == "__main__": + run_demo() diff --git a/sdk/python/feast/transformation/pandas_transformation.py b/sdk/python/feast/transformation/pandas_transformation.py index 35e786aac8f..66a5c65caf2 100644 --- a/sdk/python/feast/transformation/pandas_transformation.py +++ b/sdk/python/feast/transformation/pandas_transformation.py @@ -1,4 +1,4 @@ -from typing import Any, Callable +from typing import Any, Callable, Optional import dill import pandas as pd @@ -40,7 +40,9 @@ def transform_singleton(self, input_df: pd.DataFrame) -> pd.DataFrame: "PandasTransformation does not support singleton transformations." ) - def infer_features(self, random_input: dict[str, list[Any]]) -> list[Field]: + def infer_features( + self, random_input: dict[str, list[Any]], singleton: Optional[bool] + ) -> list[Field]: df = pd.DataFrame.from_dict(random_input) output_df: pd.DataFrame = self.transform(df) diff --git a/sdk/python/feast/transformation/python_transformation.py b/sdk/python/feast/transformation/python_transformation.py index ce2aaf2002d..20a9dd9ff6f 100644 --- a/sdk/python/feast/transformation/python_transformation.py +++ b/sdk/python/feast/transformation/python_transformation.py @@ -1,5 +1,5 @@ from types import FunctionType -from typing import Any +from typing import Any, Optional import dill import pyarrow @@ -45,7 +45,9 @@ def transform_singleton(self, input_dict: dict) -> dict: output_dict = self.udf.__call__(input_dict) return {**input_dict, **output_dict} - def infer_features(self, random_input: dict[str, Any]) -> list[Field]: + def infer_features( + self, random_input: dict[str, Any], singleton: Optional[bool] = False + ) -> list[Field]: output_dict: dict[str, Any] = self.transform(random_input) fields = [] @@ -58,6 +60,10 @@ def infer_features(self, random_input: dict[str, Any]) -> list[Field]: ) inferred_type = type(feature_value[0]) inferred_value = feature_value[0] + if singleton: + inferred_value = feature_value + inferred_type = None # type: ignore + else: inferred_type = type(feature_value) inferred_value = feature_value @@ -69,7 +75,7 @@ def infer_features(self, random_input: dict[str, Any]) -> list[Field]: python_type_to_feast_value_type( feature_name, value=inferred_value, - type_name=inferred_type.__name__, + type_name=inferred_type.__name__ if inferred_type else None, ) ), ) diff --git a/sdk/python/feast/transformation/substrait_transformation.py b/sdk/python/feast/transformation/substrait_transformation.py index 47e2ced9768..a6d9bfa18c0 100644 --- a/sdk/python/feast/transformation/substrait_transformation.py +++ b/sdk/python/feast/transformation/substrait_transformation.py @@ -1,5 +1,5 @@ from types import FunctionType -from typing import Any +from typing import Any, Optional import dill import pandas as pd @@ -61,7 +61,9 @@ def table_provider(names, schema: pyarrow.Schema): return table - def infer_features(self, random_input: dict[str, list[Any]]) -> list[Field]: + def infer_features( + self, random_input: dict[str, list[Any]], singleton: Optional[bool] + ) -> list[Field]: df = pd.DataFrame.from_dict(random_input) output_df: pd.DataFrame = self.transform(df) diff --git a/sdk/python/feast/type_map.py b/sdk/python/feast/type_map.py index 8e3941b05bf..edc9f0c66d8 100644 --- a/sdk/python/feast/type_map.py +++ b/sdk/python/feast/type_map.py @@ -580,6 +580,12 @@ def pa_to_feast_value_type(pa_type_as_str: str) -> ValueType: "bool": ValueType.BOOL, "null": ValueType.NULL, "list": ValueType.DOUBLE_LIST, + "list": ValueType.INT64_LIST, + "list": ValueType.INT32_LIST, + "list": ValueType.STRING_LIST, + "list": ValueType.BOOL_LIST, + "list": ValueType.BYTES_LIST, + "list": ValueType.FLOAT_LIST, } value_type = type_map[pa_type_as_str] @@ -1077,3 +1083,33 @@ def pa_to_athena_value_type(pa_type: "pyarrow.DataType") -> str: } return type_map[pa_type_as_str] + + +def cb_columnar_type_to_feast_value_type(type_str: str) -> ValueType: + """ + Convert a Couchbase Columnar type string to a Feast ValueType + """ + type_map: Dict[str, ValueType] = { + # primitive types + "boolean": ValueType.BOOL, + "string": ValueType.STRING, + "bigint": ValueType.INT64, + "double": ValueType.DOUBLE, + # special types + "null": ValueType.NULL, + "missing": ValueType.UNKNOWN, + # composite types + # todo: support for arrays of primitives + "object": ValueType.UNKNOWN, + "array": ValueType.UNKNOWN, + "multiset": ValueType.UNKNOWN, + "uuid": ValueType.STRING, + } + value = ( + type_map[type_str.lower()] + if type_str.lower() in type_map + else ValueType.UNKNOWN + ) + if value == ValueType.UNKNOWN: + print("unknown type:", type_str) + return value diff --git a/sdk/python/feast/types.py b/sdk/python/feast/types.py index 59980d816a8..4f13fbf2652 100644 --- a/sdk/python/feast/types.py +++ b/sdk/python/feast/types.py @@ -14,7 +14,7 @@ from abc import ABC, abstractmethod from datetime import datetime, timezone from enum import Enum -from typing import Dict, Union +from typing import Dict, List, Union import pyarrow @@ -196,6 +196,17 @@ def __str__(self): UnixTimestamp: pyarrow.timestamp("us", tz=_utc_now().tzname()), } +FEAST_VECTOR_TYPES: List[Union[ValueType, PrimitiveFeastType, ComplexFeastType]] = [ + ValueType.BYTES_LIST, + ValueType.INT32_LIST, + ValueType.INT64_LIST, + ValueType.FLOAT_LIST, + ValueType.BOOL_LIST, +] +for k in VALUE_TYPES_TO_FEAST_TYPES: + if k in FEAST_VECTOR_TYPES: + FEAST_VECTOR_TYPES.append(VALUE_TYPES_TO_FEAST_TYPES[k]) + def from_feast_to_pyarrow_type(feast_type: FeastType) -> pyarrow.DataType: """ diff --git a/sdk/python/feast/ui/package.json b/sdk/python/feast/ui/package.json index de74a03f2af..6c471c28708 100644 --- a/sdk/python/feast/ui/package.json +++ b/sdk/python/feast/ui/package.json @@ -6,7 +6,7 @@ "@elastic/datemath": "^5.0.3", "@elastic/eui": "^72.0.0", "@emotion/react": "^11.9.0", - "@feast-dev/feast-ui": "0.46.0", + "@feast-dev/feast-ui": "0.47.0", "@testing-library/jest-dom": "^5.16.4", "@testing-library/react": "^13.2.0", "@testing-library/user-event": "^13.5.0", diff --git a/sdk/python/feast/ui/yarn.lock b/sdk/python/feast/ui/yarn.lock index 896c9877f14..381e6079be0 100644 --- a/sdk/python/feast/ui/yarn.lock +++ b/sdk/python/feast/ui/yarn.lock @@ -1570,10 +1570,10 @@ minimatch "^3.1.2" strip-json-comments "^3.1.1" -"@feast-dev/feast-ui@0.46.0": - version "0.46.0" - resolved "https://registry.yarnpkg.com/@feast-dev/feast-ui/-/feast-ui-0.46.0.tgz#2ab5fa42b43c20829a6cbb44e66df8f4ee2597ae" - integrity sha512-d4EgsfhXH1nlpMGuD8M/D/2Z7OryUQkg4cUWvGadj06bwoUM60+ku0gGUZb2PnbfgdUdMrB5p7VS9di0jFurNA== +"@feast-dev/feast-ui@0.47.0": + version "0.47.0" + resolved "https://registry.yarnpkg.com/@feast-dev/feast-ui/-/feast-ui-0.47.0.tgz#f0d62ba96d3aec2593dab3a0a437250bb1176dcf" + integrity sha512-uszod/QFaR0GQauXa6KcV3TNtnZhYZFbS9+xuUYUqre+d/zY4GOR657c9KvAuuLHNXkyWuosvY/iDGq/6sfbbQ== dependencies: "@elastic/datemath" "^5.0.3" "@elastic/eui" "^95.12.0" diff --git a/sdk/python/feast/ui_server.py b/sdk/python/feast/ui_server.py index 1d115920c3a..d852bb279cc 100644 --- a/sdk/python/feast/ui_server.py +++ b/sdk/python/feast/ui_server.py @@ -69,6 +69,8 @@ def shutdown_event(): @app.get("/registry") def read_registry(): + if registry_proto is None: + return Response(status_code=503) # Service Unavailable return Response( content=registry_proto.SerializeToString(), media_type="application/octet-stream", diff --git a/sdk/python/feast/utils.py b/sdk/python/feast/utils.py index e64e38b143a..4cca1379ed3 100644 --- a/sdk/python/feast/utils.py +++ b/sdk/python/feast/utils.py @@ -343,6 +343,22 @@ def _convert_arrow_odfv_to_proto( for column, value_type in columns if column in table.column_names } + + # Ensure join keys are included in proto_values_by_column, but check if they exist first + for join_key, value_type in join_keys.items(): + if join_key not in proto_values_by_column: + # Check if the join key exists in the table before trying to access it + if join_key in table.column_names: + proto_values_by_column[join_key] = python_values_to_proto_values( + table.column(join_key).to_numpy(zero_copy_only=False), value_type + ) + else: + # Create null/default values if the join key isn't in the table + null_column = [None] * table.num_rows + proto_values_by_column[join_key] = python_values_to_proto_values( + null_column, value_type + ) + # Adding On Demand Features for feature in feature_view.features: if ( @@ -357,7 +373,7 @@ def _convert_arrow_odfv_to_proto( updated_table = pyarrow.RecordBatch.from_arrays( table.columns + [null_column], schema=table.schema.append( - pyarrow.field(feature.name, null_column.type) + pyarrow.field(feature.name, null_column.type) # type: ignore[attr-defined] ), ) proto_values_by_column[feature.name] = python_values_to_proto_values( @@ -368,7 +384,11 @@ def _convert_arrow_odfv_to_proto( entity_keys = [ EntityKeyProto( join_keys=join_keys, - entity_values=[proto_values_by_column[k][idx] for k in join_keys], + entity_values=[ + proto_values_by_column[k][idx] + for k in join_keys + if k in proto_values_by_column + ], ) for idx in range(table.num_rows) ] @@ -378,6 +398,12 @@ def _convert_arrow_odfv_to_proto( feature.name: proto_values_by_column[feature.name] for feature in feature_view.features } + if feature_view.write_to_online_store: + table_columns = [col.name for col in table.schema] + for feature in feature_view.schema: + if feature.name not in feature_dict and feature.name in table_columns: + feature_dict[feature.name] = proto_values_by_column[feature.name] + features = [dict(zip(feature_dict, vars)) for vars in zip(*feature_dict.values())] # We need to artificially add event_timestamps and created_timestamps @@ -441,19 +467,24 @@ def _group_feature_refs( all_feature_views: List["FeatureView"], all_on_demand_feature_views: List["OnDemandFeatureView"], ) -> Tuple[ - List[Tuple["FeatureView", List[str]]], List[Tuple["OnDemandFeatureView", List[str]]] + List[Tuple[Union["FeatureView", "OnDemandFeatureView"], List[str]]], + List[Tuple["OnDemandFeatureView", List[str]]], ]: """Get list of feature views and corresponding feature names based on feature references""" # view name to view proto - view_index = {view.projection.name_to_use(): view for view in all_feature_views} + view_index: Dict[str, Union["FeatureView", "OnDemandFeatureView"]] = { + view.projection.name_to_use(): view for view in all_feature_views + } # on demand view to on demand view proto - on_demand_view_index = { - view.projection.name_to_use(): view - for view in all_on_demand_feature_views - if view.projection - } + on_demand_view_index: Dict[str, "OnDemandFeatureView"] = {} + for view in all_on_demand_feature_views: + if view.projection and not view.write_to_online_store: + on_demand_view_index[view.projection.name_to_use()] = view + elif view.projection and view.write_to_online_store: + # we insert the ODFV view to FVs for ones that are written to the online store + view_index[view.projection.name_to_use()] = view # view name to feature names views_features = defaultdict(set) @@ -464,7 +495,16 @@ def _group_feature_refs( for ref in features: view_name, feat_name = ref.split(":") if view_name in view_index: - view_index[view_name].projection.get_feature(feat_name) # For validation + if hasattr(view_index[view_name], "write_to_online_store"): + tmp_feat_name = [ + f for f in view_index[view_name].schema if f.name == feat_name + ] + if len(tmp_feat_name) > 0: + feat_name = tmp_feat_name[0].name + else: + view_index[view_name].projection.get_feature( + feat_name + ) # For validation views_features[view_name].add(feat_name) elif view_name in on_demand_view_index: on_demand_view_index[view_name].projection.get_feature( @@ -480,7 +520,7 @@ def _group_feature_refs( else: raise FeatureViewNotFoundException(view_name) - fvs_result: List[Tuple["FeatureView", List[str]]] = [] + fvs_result: List[Tuple[Union["FeatureView", "OnDemandFeatureView"], List[str]]] = [] odfvs_result: List[Tuple["OnDemandFeatureView", List[str]]] = [] for view_name, feature_names in views_features.items(): @@ -557,73 +597,74 @@ def _augment_response_with_on_demand_transforms( odfv_result_names = set() for odfv_name, _feature_refs in odfv_feature_refs.items(): odfv = requested_odfv_map[odfv_name] - if odfv.mode == "python": - if initial_response_dict is None: - initial_response_dict = initial_response.to_dict() - transformed_features_dict: Dict[str, List[Any]] = odfv.transform_dict( - initial_response_dict - ) - elif odfv.mode in {"pandas", "substrait"}: - if initial_response_arrow is None: - initial_response_arrow = initial_response.to_arrow() - transformed_features_arrow = odfv.transform_arrow( - initial_response_arrow, full_feature_names + if not odfv.write_to_online_store: + if odfv.mode == "python": + if initial_response_dict is None: + initial_response_dict = initial_response.to_dict() + transformed_features_dict: Dict[str, List[Any]] = odfv.transform_dict( + initial_response_dict + ) + elif odfv.mode in {"pandas", "substrait"}: + if initial_response_arrow is None: + initial_response_arrow = initial_response.to_arrow() + transformed_features_arrow = odfv.transform_arrow( + initial_response_arrow, full_feature_names + ) + else: + raise Exception( + f"Invalid OnDemandFeatureMode: {odfv.mode}. Expected one of 'pandas', 'python', or 'substrait'." + ) + + transformed_features = ( + transformed_features_dict + if odfv.mode == "python" + else transformed_features_arrow ) - else: - raise Exception( - f"Invalid OnDemandFeatureMode: {odfv.mode}. Expected one of 'pandas', 'python', or 'substrait'." + transformed_columns = ( + transformed_features.column_names + if isinstance(transformed_features, pyarrow.Table) + else transformed_features ) - - transformed_features = ( - transformed_features_dict - if odfv.mode == "python" - else transformed_features_arrow - ) - transformed_columns = ( - transformed_features.column_names - if isinstance(transformed_features, pyarrow.Table) - else transformed_features - ) - selected_subset = [f for f in transformed_columns if f in _feature_refs] - - proto_values = [] - schema_dict = {k.name: k.dtype for k in odfv.schema} - for selected_feature in selected_subset: - feature_vector = transformed_features[selected_feature] - selected_feature_type = schema_dict.get(selected_feature, None) - feature_type: ValueType = ValueType.UNKNOWN - if selected_feature_type is not None: - if isinstance( - selected_feature_type, (ComplexFeastType, PrimitiveFeastType) - ): - feature_type = selected_feature_type.to_value_type() - elif not isinstance(selected_feature_type, ValueType): - raise TypeError( - f"Unexpected type for feature_type: {type(feature_type)}" + selected_subset = [f for f in transformed_columns if f in _feature_refs] + + proto_values = [] + schema_dict = {k.name: k.dtype for k in odfv.schema} + for selected_feature in selected_subset: + feature_vector = transformed_features[selected_feature] + selected_feature_type = schema_dict.get(selected_feature, None) + feature_type: ValueType = ValueType.UNKNOWN + if selected_feature_type is not None: + if isinstance( + selected_feature_type, (ComplexFeastType, PrimitiveFeastType) + ): + feature_type = selected_feature_type.to_value_type() + elif not isinstance(selected_feature_type, ValueType): + raise TypeError( + f"Unexpected type for feature_type: {type(feature_type)}" + ) + + proto_values.append( + python_values_to_proto_values( + feature_vector + if isinstance(feature_vector, list) + else [feature_vector] + if odfv.mode == "python" + else feature_vector.to_numpy(), + feature_type, ) - - proto_values.append( - python_values_to_proto_values( - feature_vector - if isinstance(feature_vector, list) - else [feature_vector] - if odfv.mode == "python" - else feature_vector.to_numpy(), - feature_type, ) - ) - odfv_result_names |= set(selected_subset) + odfv_result_names |= set(selected_subset) - online_features_response.metadata.feature_names.val.extend(selected_subset) - for feature_idx in range(len(selected_subset)): - online_features_response.results.append( - GetOnlineFeaturesResponse.FeatureVector( - values=proto_values[feature_idx], - statuses=[FieldStatus.PRESENT] * len(proto_values[feature_idx]), - event_timestamps=[Timestamp()] * len(proto_values[feature_idx]), + online_features_response.metadata.feature_names.val.extend(selected_subset) + for feature_idx in range(len(selected_subset)): + online_features_response.results.append( + GetOnlineFeaturesResponse.FeatureVector( + values=proto_values[feature_idx], + statuses=[FieldStatus.PRESENT] * len(proto_values[feature_idx]), + event_timestamps=[Timestamp()] * len(proto_values[feature_idx]), + ) ) - ) def _get_entity_maps( @@ -821,6 +862,7 @@ def get_needed_request_data( needed_request_data: Set[str] = set() for odfv, _ in grouped_odfv_refs: odfv_request_data_schema = odfv.get_request_data_schema() + # if odfv.write_to_online_store, we should not pass in the request data needed_request_data.update(odfv_request_data_schema.keys()) return needed_request_data @@ -1109,7 +1151,7 @@ def _get_online_request_context( entityless_case = DUMMY_ENTITY_NAME in [ entity_name for feature_view in feature_views - for entity_name in feature_view.entities + for entity_name in (feature_view.entities or []) ] return ( @@ -1172,7 +1214,13 @@ def _prepare_entities_to_read_from_online_store( odfv_entities: List[Entity] = [] request_source_keys: List[str] = [] for on_demand_feature_view in requested_on_demand_feature_views: - odfv_entities.append(*getattr(on_demand_feature_view, "entities", [])) + entities_for_odfv = getattr(on_demand_feature_view, "entities", []) + if len(entities_for_odfv) > 0 and isinstance(entities_for_odfv[0], str): + entities_for_odfv = [ + registry.get_entity(entity_name, project, allow_cache=True) + for entity_name in entities_for_odfv + ] + odfv_entities.extend(entities_for_odfv) for source in on_demand_feature_view.source_request_sources: source_schema = on_demand_feature_view.source_request_sources[source].schema for column in source_schema: diff --git a/sdk/python/requirements/py3.10-ci-requirements.txt b/sdk/python/requirements/py3.10-ci-requirements.txt index 96948e78e23..239d6b4de30 100644 --- a/sdk/python/requirements/py3.10-ci-requirements.txt +++ b/sdk/python/requirements/py3.10-ci-requirements.txt @@ -1,10 +1,10 @@ # This file was autogenerated by uv via the following command: # uv pip compile -p 3.10 --system --no-strip-extras setup.py --extra ci --output-file sdk/python/requirements/py3.10-ci-requirements.txt -aiobotocore==2.19.0 +aiobotocore==2.20.0 # via feast (setup.py) -aiohappyeyeballs==2.4.4 +aiohappyeyeballs==2.4.6 # via aiohttp -aiohttp==3.11.12 +aiohttp==3.11.13 # via aiobotocore aioitertools==0.12.0 # via aiobotocore @@ -51,13 +51,14 @@ atpublic==5.1 attrs==25.1.0 # via # aiohttp + # jsonlines # jsonschema # referencing azure-core==1.32.0 # via # azure-identity # azure-storage-blob -azure-identity==1.19.0 +azure-identity==1.20.0 # via feast (setup.py) azure-storage-blob==12.24.1 # via feast (setup.py) @@ -66,16 +67,19 @@ babel==2.17.0 # jupyterlab-server # sphinx beautifulsoup4==4.13.3 - # via nbconvert -bigtree==0.23.1 + # via + # docling + # nbconvert +bigtree==0.25.0 # via feast (setup.py) bleach[css]==6.2.0 # via nbconvert -boto3==1.36.3 +boto3==1.36.23 # via # feast (setup.py) + # ikvpy # moto -botocore==1.36.3 +botocore==1.36.23 # via # aiobotocore # boto3 @@ -86,12 +90,13 @@ build==1.2.2.post1 # feast (setup.py) # pip-tools # singlestoredb -cachetools==5.5.1 +cachetools==5.5.2 # via google-auth cassandra-driver==3.29.2 # via feast (setup.py) certifi==2025.1.31 # via + # docling # elastic-transport # httpcore # httpx @@ -101,8 +106,10 @@ certifi==2025.1.31 # snowflake-connector-python cffi==1.17.1 # via + # feast (setup.py) # argon2-cffi-bindings # cryptography + # ikvpy # snowflake-connector-python cfgv==3.4.0 # via pre-commit @@ -117,6 +124,7 @@ click==8.1.8 # geomet # great-expectations # pip-tools + # typer # uvicorn cloudpickle==3.1.1 # via dask @@ -130,7 +138,9 @@ comm==0.2.2 # ipywidgets couchbase==4.3.2 # via feast (setup.py) -coverage[toml]==7.6.10 +couchbase-columnar==1.0.0 + # via feast (setup.py) +coverage[toml]==7.6.12 # via pytest-cov cryptography==43.0.3 # via @@ -146,32 +156,47 @@ cryptography==43.0.3 # snowflake-connector-python # types-pyopenssl # types-redis -cython==3.0.11 +cython==3.0.12 # via thriftpy2 -dask[dataframe]==2025.1.0 +dask[dataframe]==2025.2.0 # via feast (setup.py) db-dtypes==1.4.1 # via google-cloud-bigquery debugpy==1.8.12 # via ipykernel -decorator==5.1.1 +decorator==5.2.1 # via ipython defusedxml==0.7.1 # via nbconvert -deltalake==0.24.0 +deltalake==0.25.1 # via feast (setup.py) deprecation==2.1.0 # via python-keycloak dill==0.3.9 - # via feast (setup.py) + # via + # feast (setup.py) + # multiprocess distlib==0.3.9 # via virtualenv docker==7.1.0 # via testcontainers +docling==2.24.0 + # via feast (setup.py) +docling-core[chunking]==2.20.0 + # via + # docling + # docling-ibm-models + # docling-parse +docling-ibm-models==3.4.0 + # via docling +docling-parse==3.4.0 + # via docling docutils==0.19 # via sphinx duckdb==1.1.3 # via ibis-framework +easyocr==1.7.2 + # via docling elastic-transport==8.17.0 # via elasticsearch elasticsearch==8.17.1 @@ -180,6 +205,8 @@ entrypoints==0.4 # via altair environs==9.5.0 # via pymilvus +et-xmlfile==2.0.0 + # via openpyxl exceptiongroup==1.2.2 # via # anyio @@ -197,8 +224,13 @@ fastjsonschema==2.21.1 # via nbformat filelock==3.17.0 # via + # huggingface-hub # snowflake-connector-python + # torch + # transformers # virtualenv +filetype==1.2.0 + # via docling fqdn==1.5.1 # via jsonschema frozenlist==1.5.0 @@ -209,6 +241,8 @@ fsspec==2024.9.0 # via # feast (setup.py) # dask + # huggingface-hub + # torch geomet==0.2.1.post1 # via cassandra-driver google-api-core[grpc]==2.24.1 @@ -236,7 +270,7 @@ google-cloud-bigquery-storage==2.28.0 # via feast (setup.py) google-cloud-bigtable==2.28.1 # via feast (setup.py) -google-cloud-core==2.4.1 +google-cloud-core==2.4.2 # via # google-cloud-bigquery # google-cloud-bigtable @@ -254,7 +288,7 @@ google-resumable-media==2.7.2 # via # google-cloud-bigquery # google-cloud-storage -googleapis-common-protos[grpc]==1.66.0 +googleapis-common-protos[grpc]==1.68.0 # via # feast (setup.py) # google-api-core @@ -262,6 +296,8 @@ googleapis-common-protos[grpc]==1.66.0 # grpcio-status great-expectations==0.18.22 # via feast (setup.py) +greenlet==3.1.1 + # via sqlalchemy grpc-google-iam-v1==0.14.0 # via google-cloud-bigtable grpcio==1.70.0 @@ -275,6 +311,7 @@ grpcio==1.70.0 # grpcio-status # grpcio-testing # grpcio-tools + # ikvpy # pymilvus # qdrant-client grpcio-health-checking==1.70.0 @@ -282,7 +319,9 @@ grpcio-health-checking==1.70.0 grpcio-reflection==1.70.0 # via feast (setup.py) grpcio-status==1.70.0 - # via google-api-core + # via + # google-api-core + # ikvpy grpcio-testing==1.70.0 # via feast (setup.py) grpcio-tools==1.70.0 @@ -317,15 +356,21 @@ httpx[http2]==0.27.2 # jupyterlab # python-keycloak # qdrant-client +huggingface-hub==0.29.1 + # via + # docling + # docling-ibm-models + # tokenizers + # transformers hyperframe==6.1.0 # via h2 -ibis-framework[duckdb]==9.5.0 +ibis-framework[duckdb, mssql]==9.5.0 # via # feast (setup.py) # ibis-substrait ibis-substrait==4.0.1 # via feast (setup.py) -identify==2.6.6 +identify==2.6.8 # via pre-commit idna==3.10 # via @@ -335,6 +380,10 @@ idna==3.10 # requests # snowflake-connector-python # yarl +ikvpy==0.0.36 + # via feast (setup.py) +imageio==2.37.0 + # via scikit-image imagesize==1.4.1 # via sphinx importlib-metadata==8.6.1 @@ -369,6 +418,7 @@ jinja2==3.1.5 # moto # nbconvert # sphinx + # torch jmespath==1.0.1 # via # aiobotocore @@ -376,16 +426,21 @@ jmespath==1.0.1 # botocore json5==0.10.0 # via jupyterlab-server +jsonlines==3.1.0 + # via docling-ibm-models jsonpatch==1.33 # via great-expectations jsonpointer==3.0.0 # via # jsonpatch # jsonschema +jsonref==1.1.0 + # via docling-core jsonschema[format-nongpl]==4.23.0 # via # feast (setup.py) # altair + # docling-core # great-expectations # jupyter-events # jupyterlab-server @@ -433,14 +488,25 @@ jwcrypto==1.5.6 # via python-keycloak kubernetes==20.13.0 # via feast (setup.py) +latex2mathml==3.77.0 + # via docling-core +lazy-loader==0.4 + # via scikit-image locket==1.0.0 # via partd +lxml==5.3.1 + # via + # docling + # python-docx + # python-pptx lz4==4.4.3 # via trino makefun==1.15.6 # via great-expectations markdown-it-py==3.0.0 # via rich +marko==2.1.2 + # via docling markupsafe==3.0.2 # via # jinja2 @@ -460,7 +526,7 @@ milvus-lite==2.4.11 # via pymilvus minio==7.2.11 # via feast (setup.py) -mistune==3.1.1 +mistune==3.1.2 # via # great-expectations # nbconvert @@ -470,6 +536,10 @@ mock==2.0.0 # via feast (setup.py) moto==4.2.14 # via feast (setup.py) +mpire[dill]==2.10.2 + # via semchunk +mpmath==1.3.0 + # via sympy msal==1.31.1 # via # azure-identity @@ -481,6 +551,8 @@ multidict==6.1.0 # aiobotocore # aiohttp # yarl +multiprocess==0.70.17 + # via mpire mypy==1.11.2 # via # feast (setup.py) @@ -501,6 +573,12 @@ nbformat==5.10.4 # nbconvert nest-asyncio==1.6.0 # via ipykernel +networkx==3.4.2 + # via + # scikit-image + # torch +ninja==1.11.1.3 + # via easyocr nodeenv==1.9.1 # via pre-commit notebook==7.3.2 @@ -515,15 +593,31 @@ numpy==1.26.4 # altair # dask # db-dtypes + # docling-ibm-models + # easyocr # faiss-cpu # great-expectations # ibis-framework + # imageio + # opencv-python-headless # pandas # pyarrow # qdrant-client + # safetensors + # scikit-image # scipy + # shapely + # tifffile + # torchvision + # transformers oauthlib==3.2.2 # via requests-oauthlib +opencv-python-headless==4.11.0.86 + # via + # docling-ibm-models + # easyocr +openpyxl==3.1.5 + # via docling overrides==7.7.0 # via jupyter-server packaging==24.2 @@ -536,6 +630,7 @@ packaging==24.2 # google-cloud-bigquery # great-expectations # gunicorn + # huggingface-hub # ibis-framework # ibis-substrait # ipykernel @@ -543,17 +638,22 @@ packaging==24.2 # jupyter-server # jupyterlab # jupyterlab-server + # lazy-loader # marshmallow # nbconvert # pytest + # scikit-image # snowflake-connector-python # sphinx + # transformers pandas==2.2.3 # via # feast (setup.py) # altair # dask # db-dtypes + # docling + # docling-core # google-cloud-bigquery # great-expectations # ibis-framework @@ -573,7 +673,18 @@ pbr==6.1.1 # via mock pexpect==4.9.0 # via ipython -pip==25.0 +pillow==11.1.0 + # via + # docling + # docling-core + # docling-ibm-models + # docling-parse + # easyocr + # imageio + # python-pptx + # scikit-image + # torchvision +pip==25.0.1 # via pip-tools pip-tools==7.4.1 # via feast (setup.py) @@ -598,7 +709,7 @@ prometheus-client==0.21.1 # jupyter-server prompt-toolkit==3.0.50 # via ipython -propcache==0.2.1 +propcache==0.3.0 # via # aiohttp # yarl @@ -622,6 +733,7 @@ protobuf==5.29.3 # grpcio-status # grpcio-testing # grpcio-tools + # ikvpy # mypy-protobuf # proto-plus # pymilvus @@ -630,11 +742,11 @@ psutil==5.9.0 # via # feast (setup.py) # ipykernel -psycopg[binary, pool]==3.2.4 +psycopg[binary, pool]==3.2.5 # via feast (setup.py) -psycopg-binary==3.2.4 +psycopg-binary==3.2.5 # via psycopg -psycopg-pool==3.2.4 +psycopg-pool==3.2.5 # via psycopg ptyprocess==0.7.0 # via @@ -667,6 +779,8 @@ pyasn1-modules==0.4.1 # via google-auth pybindgen==0.22.1 # via feast (setup.py) +pyclipper==1.3.0.post6 + # via easyocr pycparser==2.22 # via cffi pycryptodome==3.21.0 @@ -674,15 +788,23 @@ pycryptodome==3.21.0 pydantic==2.10.6 # via # feast (setup.py) + # docling + # docling-core + # docling-ibm-models + # docling-parse # fastapi # great-expectations + # pydantic-settings # qdrant-client pydantic-core==2.27.2 # via pydantic +pydantic-settings==2.8.0 + # via docling pygments==2.19.1 # via # feast (setup.py) # ipython + # mpire # nbconvert # rich # sphinx @@ -699,11 +821,15 @@ pymssql==2.3.2 pymysql==1.1.1 # via feast (setup.py) pyodbc==5.2.0 - # via feast (setup.py) + # via + # feast (setup.py) + # ibis-framework pyopenssl==24.3.0 # via snowflake-connector-python pyparsing==3.2.1 # via great-expectations +pypdfium2==4.30.1 + # via docling pyproject-hooks==1.2.0 # via # build @@ -740,6 +866,8 @@ pytest-timeout==1.4.2 # via feast (setup.py) pytest-xdist==3.6.1 # via feast (setup.py) +python-bidi==0.6.6 + # via easyocr python-dateutil==2.9.0.post0 # via # aiobotocore @@ -753,14 +881,19 @@ python-dateutil==2.9.0.post0 # moto # pandas # trino +python-docx==1.1.2 + # via docling python-dotenv==1.0.1 # via # environs + # pydantic-settings # uvicorn python-json-logger==3.2.1 # via jupyter-events python-keycloak==4.2.2 # via feast (setup.py) +python-pptx==1.0.2 + # via docling pytz==2025.1 # via # great-expectations @@ -772,11 +905,15 @@ pyyaml==6.0.2 # via # feast (setup.py) # dask + # docling-core + # easyocr + # huggingface-hub # ibis-substrait # jupyter-events # kubernetes # pre-commit # responses + # transformers # uvicorn pyzmq==26.2.1 # via @@ -796,15 +933,18 @@ regex==2024.11.6 # via # feast (setup.py) # parsimonious + # transformers requests==2.32.3 # via # feast (setup.py) # azure-core # docker + # docling # google-api-core # google-cloud-bigquery # google-cloud-storage # great-expectations + # huggingface-hub # jupyterlab-server # kubernetes # moto @@ -816,6 +956,7 @@ requests==2.32.3 # singlestoredb # snowflake-connector-python # sphinx + # transformers # trino requests-oauthlib==2.0.0 # via kubernetes @@ -832,23 +973,39 @@ rfc3986-validator==0.1.1 # jsonschema # jupyter-events rich==13.9.4 - # via ibis-framework -rpds-py==0.22.3 + # via + # ibis-framework + # typer +rpds-py==0.23.1 # via # jsonschema # referencing rsa==4.9 # via google-auth +rtree==1.3.0 + # via docling ruamel-yaml==0.17.40 # via great-expectations ruamel-yaml-clib==0.2.12 # via ruamel-yaml -ruff==0.9.5 +ruff==0.9.7 # via feast (setup.py) s3transfer==0.11.2 # via boto3 -scipy==1.15.1 - # via great-expectations +safetensors[torch]==0.5.2 + # via + # docling-ibm-models + # transformers +scikit-image==0.25.2 + # via easyocr +scipy==1.15.2 + # via + # docling + # easyocr + # great-expectations + # scikit-image +semchunk==2.2.2 + # via docling-core send2trash==1.8.3 # via jupyter-server setuptools==75.8.0 @@ -860,6 +1017,10 @@ setuptools==75.8.0 # pip-tools # pymilvus # singlestoredb +shapely==2.0.7 + # via easyocr +shellingham==1.5.4 + # via typer singlestoredb==1.7.2 # via feast (setup.py) six==1.17.0 @@ -912,8 +1073,13 @@ starlette==0.45.3 # via fastapi substrait==0.23.0 # via ibis-substrait +sympy==1.13.3 + # via torch tabulate==0.9.0 - # via feast (setup.py) + # via + # feast (setup.py) + # docling-core + # docling-parse tenacity==8.5.0 # via feast (setup.py) terminado==0.18.1 @@ -924,8 +1090,12 @@ testcontainers==4.8.2 # via feast (setup.py) thriftpy2==0.5.2 # via happybase +tifffile==2025.2.18 + # via scikit-image tinycss2==1.4.0 # via bleach +tokenizers==0.19.1 + # via transformers toml==0.10.2 # via feast (setup.py) tomli==2.2.1 @@ -946,6 +1116,18 @@ toolz==0.12.1 # dask # ibis-framework # partd +torch==2.2.2 + # via + # feast (setup.py) + # docling-ibm-models + # easyocr + # safetensors + # torchvision +torchvision==0.17.2 + # via + # feast (setup.py) + # docling-ibm-models + # easyocr tornado==6.4.2 # via # ipykernel @@ -957,8 +1139,14 @@ tornado==6.4.2 tqdm==4.67.1 # via # feast (setup.py) + # docling + # docling-ibm-models # great-expectations + # huggingface-hub # milvus-lite + # mpire + # semchunk + # transformers traitlets==5.14.3 # via # comm @@ -974,10 +1162,18 @@ traitlets==5.14.3 # nbclient # nbconvert # nbformat +transformers==4.42.4 + # via + # docling-core + # docling-ibm-models trino==0.333.0 # via feast (setup.py) -typeguard==4.4.1 +typeguard==4.4.2 # via feast (setup.py) +typer==0.12.5 + # via + # docling + # docling-core types-cffi==1.16.0.20241221 # via types-pyopenssl types-protobuf==3.19.22 @@ -1000,7 +1196,7 @@ types-redis==4.6.0.20241004 # via feast (setup.py) types-requests==2.30.0.0 # via feast (setup.py) -types-setuptools==75.8.0.20250110 +types-setuptools==75.8.0.20250225 # via # feast (setup.py) # types-cffi @@ -1016,8 +1212,10 @@ typing-extensions==4.12.2 # azure-identity # azure-storage-blob # beautifulsoup4 + # docling-core # fastapi # great-expectations + # huggingface-hub # ibis-framework # ipython # jwcrypto @@ -1029,16 +1227,20 @@ typing-extensions==4.12.2 # psycopg-pool # pydantic # pydantic-core + # python-docx + # python-pptx # referencing # rich # snowflake-connector-python # sqlalchemy # testcontainers + # torch # typeguard + # typer # uvicorn tzdata==2025.1 # via pandas -tzlocal==5.2 +tzlocal==5.3 # via # great-expectations # trino @@ -1086,7 +1288,7 @@ websocket-client==1.8.0 # via # jupyter-server # kubernetes -websockets==14.2 +websockets==15.0 # via uvicorn werkzeug==3.1.3 # via moto @@ -1100,6 +1302,8 @@ wrapt==1.17.2 # via # aiobotocore # testcontainers +xlsxwriter==3.2.2 + # via python-pptx xmltodict==0.14.2 # via moto yarl==1.18.3 diff --git a/sdk/python/requirements/py3.10-requirements.txt b/sdk/python/requirements/py3.10-requirements.txt index c25bda58b7e..ea4baadec05 100644 --- a/sdk/python/requirements/py3.10-requirements.txt +++ b/sdk/python/requirements/py3.10-requirements.txt @@ -10,7 +10,7 @@ attrs==25.1.0 # via # jsonschema # referencing -bigtree==0.23.1 +bigtree==0.25.0 # via feast (setup.py) certifi==2025.1.31 # via requests @@ -25,7 +25,7 @@ cloudpickle==3.1.1 # via dask colorama==0.4.6 # via feast (setup.py) -dask[dataframe]==2025.1.0 +dask[dataframe]==2025.2.0 # via feast (setup.py) dill==0.3.9 # via feast (setup.py) @@ -35,6 +35,8 @@ fastapi==0.115.8 # via feast (setup.py) fsspec==2025.2.0 # via dask +greenlet==3.1.1 + # via sqlalchemy gunicorn==23.0.0 # via # feast (setup.py) @@ -84,7 +86,7 @@ prometheus-client==0.21.1 # via feast (setup.py) protobuf==5.29.3 # via feast (setup.py) -psutil==6.1.1 +psutil==7.0.0 # via feast (setup.py) pyarrow==18.0.0 # via @@ -117,7 +119,7 @@ referencing==0.36.2 # jsonschema-specifications requests==2.32.3 # via feast (setup.py) -rpds-py==0.22.3 +rpds-py==0.23.1 # via # jsonschema # referencing @@ -143,7 +145,7 @@ toolz==1.0.0 # partd tqdm==4.67.1 # via feast (setup.py) -typeguard==4.4.1 +typeguard==4.4.2 # via feast (setup.py) typing-extensions==4.12.2 # via @@ -170,7 +172,7 @@ uvloop==0.21.0 # via uvicorn watchfiles==1.0.4 # via uvicorn -websockets==14.2 +websockets==15.0 # via uvicorn zipp==3.21.0 # via importlib-metadata diff --git a/sdk/python/requirements/py3.11-ci-requirements.txt b/sdk/python/requirements/py3.11-ci-requirements.txt index 20976b02045..af6b5b469c9 100644 --- a/sdk/python/requirements/py3.11-ci-requirements.txt +++ b/sdk/python/requirements/py3.11-ci-requirements.txt @@ -1,10 +1,10 @@ # This file was autogenerated by uv via the following command: # uv pip compile -p 3.11 --system --no-strip-extras setup.py --extra ci --output-file sdk/python/requirements/py3.11-ci-requirements.txt -aiobotocore==2.19.0 +aiobotocore==2.20.0 # via feast (setup.py) -aiohappyeyeballs==2.4.4 +aiohappyeyeballs==2.4.6 # via aiohttp -aiohttp==3.11.12 +aiohttp==3.11.13 # via aiobotocore aioitertools==0.12.0 # via aiobotocore @@ -49,13 +49,14 @@ atpublic==5.1 attrs==25.1.0 # via # aiohttp + # jsonlines # jsonschema # referencing azure-core==1.32.0 # via # azure-identity # azure-storage-blob -azure-identity==1.19.0 +azure-identity==1.20.0 # via feast (setup.py) azure-storage-blob==12.24.1 # via feast (setup.py) @@ -64,16 +65,19 @@ babel==2.17.0 # jupyterlab-server # sphinx beautifulsoup4==4.13.3 - # via nbconvert -bigtree==0.23.1 + # via + # docling + # nbconvert +bigtree==0.25.0 # via feast (setup.py) bleach[css]==6.2.0 # via nbconvert -boto3==1.36.3 +boto3==1.36.23 # via # feast (setup.py) + # ikvpy # moto -botocore==1.36.3 +botocore==1.36.23 # via # aiobotocore # boto3 @@ -84,12 +88,13 @@ build==1.2.2.post1 # feast (setup.py) # pip-tools # singlestoredb -cachetools==5.5.1 +cachetools==5.5.2 # via google-auth cassandra-driver==3.29.2 # via feast (setup.py) certifi==2025.1.31 # via + # docling # elastic-transport # httpcore # httpx @@ -99,8 +104,10 @@ certifi==2025.1.31 # snowflake-connector-python cffi==1.17.1 # via + # feast (setup.py) # argon2-cffi-bindings # cryptography + # ikvpy # snowflake-connector-python cfgv==3.4.0 # via pre-commit @@ -115,6 +122,7 @@ click==8.1.8 # geomet # great-expectations # pip-tools + # typer # uvicorn cloudpickle==3.1.1 # via dask @@ -128,7 +136,9 @@ comm==0.2.2 # ipywidgets couchbase==4.3.2 # via feast (setup.py) -coverage[toml]==7.6.10 +couchbase-columnar==1.0.0 + # via feast (setup.py) +coverage[toml]==7.6.12 # via pytest-cov cryptography==43.0.3 # via @@ -144,32 +154,47 @@ cryptography==43.0.3 # snowflake-connector-python # types-pyopenssl # types-redis -cython==3.0.11 +cython==3.0.12 # via thriftpy2 -dask[dataframe]==2025.1.0 +dask[dataframe]==2025.2.0 # via feast (setup.py) db-dtypes==1.4.1 # via google-cloud-bigquery debugpy==1.8.12 # via ipykernel -decorator==5.1.1 +decorator==5.2.1 # via ipython defusedxml==0.7.1 # via nbconvert -deltalake==0.24.0 +deltalake==0.25.1 # via feast (setup.py) deprecation==2.1.0 # via python-keycloak dill==0.3.9 - # via feast (setup.py) + # via + # feast (setup.py) + # multiprocess distlib==0.3.9 # via virtualenv docker==7.1.0 # via testcontainers +docling==2.24.0 + # via feast (setup.py) +docling-core[chunking]==2.20.0 + # via + # docling + # docling-ibm-models + # docling-parse +docling-ibm-models==3.4.0 + # via docling +docling-parse==3.4.0 + # via docling docutils==0.19 # via sphinx duckdb==1.1.3 # via ibis-framework +easyocr==1.7.2 + # via docling elastic-transport==8.17.0 # via elasticsearch elasticsearch==8.17.1 @@ -178,6 +203,8 @@ entrypoints==0.4 # via altair environs==9.5.0 # via pymilvus +et-xmlfile==2.0.0 + # via openpyxl execnet==2.1.1 # via pytest-xdist executing==2.2.0 @@ -190,8 +217,13 @@ fastjsonschema==2.21.1 # via nbformat filelock==3.17.0 # via + # huggingface-hub # snowflake-connector-python + # torch + # transformers # virtualenv +filetype==1.2.0 + # via docling fqdn==1.5.1 # via jsonschema frozenlist==1.5.0 @@ -202,6 +234,8 @@ fsspec==2024.9.0 # via # feast (setup.py) # dask + # huggingface-hub + # torch geomet==0.2.1.post1 # via cassandra-driver google-api-core[grpc]==2.24.1 @@ -229,7 +263,7 @@ google-cloud-bigquery-storage==2.28.0 # via feast (setup.py) google-cloud-bigtable==2.28.1 # via feast (setup.py) -google-cloud-core==2.4.1 +google-cloud-core==2.4.2 # via # google-cloud-bigquery # google-cloud-bigtable @@ -247,7 +281,7 @@ google-resumable-media==2.7.2 # via # google-cloud-bigquery # google-cloud-storage -googleapis-common-protos[grpc]==1.66.0 +googleapis-common-protos[grpc]==1.68.0 # via # feast (setup.py) # google-api-core @@ -255,6 +289,8 @@ googleapis-common-protos[grpc]==1.66.0 # grpcio-status great-expectations==0.18.22 # via feast (setup.py) +greenlet==3.1.1 + # via sqlalchemy grpc-google-iam-v1==0.14.0 # via google-cloud-bigtable grpcio==1.70.0 @@ -268,6 +304,7 @@ grpcio==1.70.0 # grpcio-status # grpcio-testing # grpcio-tools + # ikvpy # pymilvus # qdrant-client grpcio-health-checking==1.70.0 @@ -275,7 +312,9 @@ grpcio-health-checking==1.70.0 grpcio-reflection==1.70.0 # via feast (setup.py) grpcio-status==1.70.0 - # via google-api-core + # via + # google-api-core + # ikvpy grpcio-testing==1.70.0 # via feast (setup.py) grpcio-tools==1.70.0 @@ -310,15 +349,21 @@ httpx[http2]==0.27.2 # jupyterlab # python-keycloak # qdrant-client +huggingface-hub==0.29.1 + # via + # docling + # docling-ibm-models + # tokenizers + # transformers hyperframe==6.1.0 # via h2 -ibis-framework[duckdb]==9.5.0 +ibis-framework[duckdb, mssql]==9.5.0 # via # feast (setup.py) # ibis-substrait ibis-substrait==4.0.1 # via feast (setup.py) -identify==2.6.6 +identify==2.6.8 # via pre-commit idna==3.10 # via @@ -328,6 +373,10 @@ idna==3.10 # requests # snowflake-connector-python # yarl +ikvpy==0.0.36 + # via feast (setup.py) +imageio==2.37.0 + # via scikit-image imagesize==1.4.1 # via sphinx importlib-metadata==8.6.1 @@ -360,6 +409,7 @@ jinja2==3.1.5 # moto # nbconvert # sphinx + # torch jmespath==1.0.1 # via # aiobotocore @@ -367,16 +417,21 @@ jmespath==1.0.1 # botocore json5==0.10.0 # via jupyterlab-server +jsonlines==3.1.0 + # via docling-ibm-models jsonpatch==1.33 # via great-expectations jsonpointer==3.0.0 # via # jsonpatch # jsonschema +jsonref==1.1.0 + # via docling-core jsonschema[format-nongpl]==4.23.0 # via # feast (setup.py) # altair + # docling-core # great-expectations # jupyter-events # jupyterlab-server @@ -424,14 +479,25 @@ jwcrypto==1.5.6 # via python-keycloak kubernetes==20.13.0 # via feast (setup.py) +latex2mathml==3.77.0 + # via docling-core +lazy-loader==0.4 + # via scikit-image locket==1.0.0 # via partd +lxml==5.3.1 + # via + # docling + # python-docx + # python-pptx lz4==4.4.3 # via trino makefun==1.15.6 # via great-expectations markdown-it-py==3.0.0 # via rich +marko==2.1.2 + # via docling markupsafe==3.0.2 # via # jinja2 @@ -451,7 +517,7 @@ milvus-lite==2.4.11 # via pymilvus minio==7.2.11 # via feast (setup.py) -mistune==3.1.1 +mistune==3.1.2 # via # great-expectations # nbconvert @@ -461,6 +527,10 @@ mock==2.0.0 # via feast (setup.py) moto==4.2.14 # via feast (setup.py) +mpire[dill]==2.10.2 + # via semchunk +mpmath==1.3.0 + # via sympy msal==1.31.1 # via # azure-identity @@ -472,6 +542,8 @@ multidict==6.1.0 # aiobotocore # aiohttp # yarl +multiprocess==0.70.17 + # via mpire mypy==1.11.2 # via # feast (setup.py) @@ -492,6 +564,12 @@ nbformat==5.10.4 # nbconvert nest-asyncio==1.6.0 # via ipykernel +networkx==3.4.2 + # via + # scikit-image + # torch +ninja==1.11.1.3 + # via easyocr nodeenv==1.9.1 # via pre-commit notebook==7.3.2 @@ -506,15 +584,31 @@ numpy==1.26.4 # altair # dask # db-dtypes + # docling-ibm-models + # easyocr # faiss-cpu # great-expectations # ibis-framework + # imageio + # opencv-python-headless # pandas # pyarrow # qdrant-client + # safetensors + # scikit-image # scipy + # shapely + # tifffile + # torchvision + # transformers oauthlib==3.2.2 # via requests-oauthlib +opencv-python-headless==4.11.0.86 + # via + # docling-ibm-models + # easyocr +openpyxl==3.1.5 + # via docling overrides==7.7.0 # via jupyter-server packaging==24.2 @@ -527,6 +621,7 @@ packaging==24.2 # google-cloud-bigquery # great-expectations # gunicorn + # huggingface-hub # ibis-framework # ibis-substrait # ipykernel @@ -534,17 +629,22 @@ packaging==24.2 # jupyter-server # jupyterlab # jupyterlab-server + # lazy-loader # marshmallow # nbconvert # pytest + # scikit-image # snowflake-connector-python # sphinx + # transformers pandas==2.2.3 # via # feast (setup.py) # altair # dask # db-dtypes + # docling + # docling-core # google-cloud-bigquery # great-expectations # ibis-framework @@ -564,7 +664,18 @@ pbr==6.1.1 # via mock pexpect==4.9.0 # via ipython -pip==25.0 +pillow==11.1.0 + # via + # docling + # docling-core + # docling-ibm-models + # docling-parse + # easyocr + # imageio + # python-pptx + # scikit-image + # torchvision +pip==25.0.1 # via pip-tools pip-tools==7.4.1 # via feast (setup.py) @@ -589,7 +700,7 @@ prometheus-client==0.21.1 # jupyter-server prompt-toolkit==3.0.50 # via ipython -propcache==0.2.1 +propcache==0.3.0 # via # aiohttp # yarl @@ -613,6 +724,7 @@ protobuf==5.29.3 # grpcio-status # grpcio-testing # grpcio-tools + # ikvpy # mypy-protobuf # proto-plus # pymilvus @@ -621,11 +733,11 @@ psutil==5.9.0 # via # feast (setup.py) # ipykernel -psycopg[binary, pool]==3.2.4 +psycopg[binary, pool]==3.2.5 # via feast (setup.py) -psycopg-binary==3.2.4 +psycopg-binary==3.2.5 # via psycopg -psycopg-pool==3.2.4 +psycopg-pool==3.2.5 # via psycopg ptyprocess==0.7.0 # via @@ -658,6 +770,8 @@ pyasn1-modules==0.4.1 # via google-auth pybindgen==0.22.1 # via feast (setup.py) +pyclipper==1.3.0.post6 + # via easyocr pycparser==2.22 # via cffi pycryptodome==3.21.0 @@ -665,15 +779,23 @@ pycryptodome==3.21.0 pydantic==2.10.6 # via # feast (setup.py) + # docling + # docling-core + # docling-ibm-models + # docling-parse # fastapi # great-expectations + # pydantic-settings # qdrant-client pydantic-core==2.27.2 # via pydantic +pydantic-settings==2.8.0 + # via docling pygments==2.19.1 # via # feast (setup.py) # ipython + # mpire # nbconvert # rich # sphinx @@ -690,11 +812,15 @@ pymssql==2.3.2 pymysql==1.1.1 # via feast (setup.py) pyodbc==5.2.0 - # via feast (setup.py) + # via + # feast (setup.py) + # ibis-framework pyopenssl==24.3.0 # via snowflake-connector-python pyparsing==3.2.1 # via great-expectations +pypdfium2==4.30.1 + # via docling pyproject-hooks==1.2.0 # via # build @@ -731,6 +857,8 @@ pytest-timeout==1.4.2 # via feast (setup.py) pytest-xdist==3.6.1 # via feast (setup.py) +python-bidi==0.6.6 + # via easyocr python-dateutil==2.9.0.post0 # via # aiobotocore @@ -744,14 +872,19 @@ python-dateutil==2.9.0.post0 # moto # pandas # trino +python-docx==1.1.2 + # via docling python-dotenv==1.0.1 # via # environs + # pydantic-settings # uvicorn python-json-logger==3.2.1 # via jupyter-events python-keycloak==4.2.2 # via feast (setup.py) +python-pptx==1.0.2 + # via docling pytz==2025.1 # via # great-expectations @@ -763,11 +896,15 @@ pyyaml==6.0.2 # via # feast (setup.py) # dask + # docling-core + # easyocr + # huggingface-hub # ibis-substrait # jupyter-events # kubernetes # pre-commit # responses + # transformers # uvicorn pyzmq==26.2.1 # via @@ -787,15 +924,18 @@ regex==2024.11.6 # via # feast (setup.py) # parsimonious + # transformers requests==2.32.3 # via # feast (setup.py) # azure-core # docker + # docling # google-api-core # google-cloud-bigquery # google-cloud-storage # great-expectations + # huggingface-hub # jupyterlab-server # kubernetes # moto @@ -807,6 +947,7 @@ requests==2.32.3 # singlestoredb # snowflake-connector-python # sphinx + # transformers # trino requests-oauthlib==2.0.0 # via kubernetes @@ -823,23 +964,39 @@ rfc3986-validator==0.1.1 # jsonschema # jupyter-events rich==13.9.4 - # via ibis-framework -rpds-py==0.22.3 + # via + # ibis-framework + # typer +rpds-py==0.23.1 # via # jsonschema # referencing rsa==4.9 # via google-auth +rtree==1.3.0 + # via docling ruamel-yaml==0.17.40 # via great-expectations ruamel-yaml-clib==0.2.12 # via ruamel-yaml -ruff==0.9.5 +ruff==0.9.7 # via feast (setup.py) s3transfer==0.11.2 # via boto3 -scipy==1.15.1 - # via great-expectations +safetensors[torch]==0.5.2 + # via + # docling-ibm-models + # transformers +scikit-image==0.25.2 + # via easyocr +scipy==1.15.2 + # via + # docling + # easyocr + # great-expectations + # scikit-image +semchunk==2.2.2 + # via docling-core send2trash==1.8.3 # via jupyter-server setuptools==75.8.0 @@ -851,6 +1008,10 @@ setuptools==75.8.0 # pip-tools # pymilvus # singlestoredb +shapely==2.0.7 + # via easyocr +shellingham==1.5.4 + # via typer singlestoredb==1.7.2 # via feast (setup.py) six==1.17.0 @@ -903,8 +1064,13 @@ starlette==0.45.3 # via fastapi substrait==0.23.0 # via ibis-substrait +sympy==1.13.3 + # via torch tabulate==0.9.0 - # via feast (setup.py) + # via + # feast (setup.py) + # docling-core + # docling-parse tenacity==8.5.0 # via feast (setup.py) terminado==0.18.1 @@ -915,10 +1081,16 @@ testcontainers==4.8.2 # via feast (setup.py) thriftpy2==0.5.2 # via happybase +tifffile==2025.2.18 + # via scikit-image tinycss2==1.4.0 # via bleach +tokenizers==0.19.1 + # via transformers toml==0.10.2 # via feast (setup.py) +tomli==2.2.1 + # via coverage tomlkit==0.13.2 # via snowflake-connector-python toolz==0.12.1 @@ -927,6 +1099,18 @@ toolz==0.12.1 # dask # ibis-framework # partd +torch==2.2.2 + # via + # feast (setup.py) + # docling-ibm-models + # easyocr + # safetensors + # torchvision +torchvision==0.17.2 + # via + # feast (setup.py) + # docling-ibm-models + # easyocr tornado==6.4.2 # via # ipykernel @@ -938,8 +1122,14 @@ tornado==6.4.2 tqdm==4.67.1 # via # feast (setup.py) + # docling + # docling-ibm-models # great-expectations + # huggingface-hub # milvus-lite + # mpire + # semchunk + # transformers traitlets==5.14.3 # via # comm @@ -955,10 +1145,18 @@ traitlets==5.14.3 # nbclient # nbconvert # nbformat +transformers==4.42.4 + # via + # docling-core + # docling-ibm-models trino==0.333.0 # via feast (setup.py) -typeguard==4.4.1 +typeguard==4.4.2 # via feast (setup.py) +typer==0.12.5 + # via + # docling + # docling-core types-cffi==1.16.0.20241221 # via types-pyopenssl types-protobuf==3.19.22 @@ -981,7 +1179,7 @@ types-redis==4.6.0.20241004 # via feast (setup.py) types-requests==2.30.0.0 # via feast (setup.py) -types-setuptools==75.8.0.20250110 +types-setuptools==75.8.0.20250225 # via # feast (setup.py) # types-cffi @@ -996,8 +1194,10 @@ typing-extensions==4.12.2 # azure-identity # azure-storage-blob # beautifulsoup4 + # docling-core # fastapi # great-expectations + # huggingface-hub # ibis-framework # ipython # jwcrypto @@ -1007,14 +1207,18 @@ typing-extensions==4.12.2 # psycopg-pool # pydantic # pydantic-core + # python-docx + # python-pptx # referencing # snowflake-connector-python # sqlalchemy # testcontainers + # torch # typeguard + # typer tzdata==2025.1 # via pandas -tzlocal==5.2 +tzlocal==5.3 # via # great-expectations # trino @@ -1062,7 +1266,7 @@ websocket-client==1.8.0 # via # jupyter-server # kubernetes -websockets==14.2 +websockets==15.0 # via uvicorn werkzeug==3.1.3 # via moto @@ -1076,6 +1280,8 @@ wrapt==1.17.2 # via # aiobotocore # testcontainers +xlsxwriter==3.2.2 + # via python-pptx xmltodict==0.14.2 # via moto yarl==1.18.3 diff --git a/sdk/python/requirements/py3.11-requirements.txt b/sdk/python/requirements/py3.11-requirements.txt index f79776a9147..d33da6d75c2 100644 --- a/sdk/python/requirements/py3.11-requirements.txt +++ b/sdk/python/requirements/py3.11-requirements.txt @@ -10,7 +10,7 @@ attrs==25.1.0 # via # jsonschema # referencing -bigtree==0.23.1 +bigtree==0.25.0 # via feast (setup.py) certifi==2025.1.31 # via requests @@ -25,7 +25,7 @@ cloudpickle==3.1.1 # via dask colorama==0.4.6 # via feast (setup.py) -dask[dataframe]==2025.1.0 +dask[dataframe]==2025.2.0 # via feast (setup.py) dill==0.3.9 # via feast (setup.py) @@ -33,6 +33,8 @@ fastapi==0.115.8 # via feast (setup.py) fsspec==2025.2.0 # via dask +greenlet==3.1.1 + # via sqlalchemy gunicorn==23.0.0 # via # feast (setup.py) @@ -82,7 +84,7 @@ prometheus-client==0.21.1 # via feast (setup.py) protobuf==5.29.3 # via feast (setup.py) -psutil==6.1.1 +psutil==7.0.0 # via feast (setup.py) pyarrow==18.0.0 # via @@ -115,7 +117,7 @@ referencing==0.36.2 # jsonschema-specifications requests==2.32.3 # via feast (setup.py) -rpds-py==0.22.3 +rpds-py==0.23.1 # via # jsonschema # referencing @@ -139,7 +141,7 @@ toolz==1.0.0 # partd tqdm==4.67.1 # via feast (setup.py) -typeguard==4.4.1 +typeguard==4.4.2 # via feast (setup.py) typing-extensions==4.12.2 # via @@ -165,7 +167,7 @@ uvloop==0.21.0 # via uvicorn watchfiles==1.0.4 # via uvicorn -websockets==14.2 +websockets==15.0 # via uvicorn zipp==3.21.0 # via importlib-metadata diff --git a/sdk/python/requirements/py3.9-ci-requirements.txt b/sdk/python/requirements/py3.9-ci-requirements.txt index 00eb59c93d2..4473d933258 100644 --- a/sdk/python/requirements/py3.9-ci-requirements.txt +++ b/sdk/python/requirements/py3.9-ci-requirements.txt @@ -1,10 +1,10 @@ # This file was autogenerated by uv via the following command: # uv pip compile -p 3.9 --system --no-strip-extras setup.py --extra ci --output-file sdk/python/requirements/py3.9-ci-requirements.txt -aiobotocore==2.19.0 +aiobotocore==2.20.0 # via feast (setup.py) -aiohappyeyeballs==2.4.4 +aiohappyeyeballs==2.4.6 # via aiohttp -aiohttp==3.11.12 +aiohttp==3.11.13 # via aiobotocore aioitertools==0.12.0 # via aiobotocore @@ -51,13 +51,14 @@ atpublic==4.1.0 attrs==25.1.0 # via # aiohttp + # jsonlines # jsonschema # referencing azure-core==1.32.0 # via # azure-identity # azure-storage-blob -azure-identity==1.19.0 +azure-identity==1.20.0 # via feast (setup.py) azure-storage-blob==12.24.1 # via feast (setup.py) @@ -66,18 +67,21 @@ babel==2.17.0 # jupyterlab-server # sphinx beautifulsoup4==4.13.3 - # via nbconvert + # via + # docling + # nbconvert bidict==0.23.1 # via ibis-framework -bigtree==0.23.1 +bigtree==0.25.0 # via feast (setup.py) bleach[css]==6.2.0 # via nbconvert -boto3==1.36.3 +boto3==1.36.23 # via # feast (setup.py) + # ikvpy # moto -botocore==1.36.3 +botocore==1.36.23 # via # aiobotocore # boto3 @@ -88,12 +92,13 @@ build==1.2.2.post1 # feast (setup.py) # pip-tools # singlestoredb -cachetools==5.5.1 +cachetools==5.5.2 # via google-auth cassandra-driver==3.29.2 # via feast (setup.py) certifi==2025.1.31 # via + # docling # elastic-transport # httpcore # httpx @@ -103,8 +108,10 @@ certifi==2025.1.31 # snowflake-connector-python cffi==1.17.1 # via + # feast (setup.py) # argon2-cffi-bindings # cryptography + # ikvpy # snowflake-connector-python cfgv==3.4.0 # via pre-commit @@ -119,6 +126,7 @@ click==8.1.8 # geomet # great-expectations # pip-tools + # typer # uvicorn cloudpickle==3.1.1 # via dask @@ -132,7 +140,9 @@ comm==0.2.2 # ipywidgets couchbase==4.3.2 # via feast (setup.py) -coverage[toml]==7.6.10 +couchbase-columnar==1.0.0 + # via feast (setup.py) +coverage[toml]==7.6.12 # via pytest-cov cryptography==43.0.3 # via @@ -148,7 +158,7 @@ cryptography==43.0.3 # snowflake-connector-python # types-pyopenssl # types-redis -cython==3.0.11 +cython==3.0.12 # via thriftpy2 dask[dataframe]==2024.8.0 # via @@ -160,32 +170,47 @@ db-dtypes==1.4.1 # via google-cloud-bigquery debugpy==1.8.12 # via ipykernel -decorator==5.1.1 +decorator==5.2.1 # via ipython defusedxml==0.7.1 # via nbconvert -deltalake==0.24.0 +deltalake==0.25.1 # via feast (setup.py) deprecation==2.1.0 # via python-keycloak dill==0.3.9 - # via feast (setup.py) + # via + # feast (setup.py) + # multiprocess distlib==0.3.9 # via virtualenv docker==7.1.0 # via testcontainers +docling==2.24.0 + # via feast (setup.py) +docling-core[chunking]==2.20.0 + # via + # docling + # docling-ibm-models + # docling-parse +docling-ibm-models==3.4.0 + # via docling +docling-parse==3.4.0 + # via docling docutils==0.19 # via sphinx duckdb==0.10.3 # via ibis-framework +easyocr==1.7.2 + # via docling elastic-transport==8.17.0 # via elasticsearch elasticsearch==8.17.1 # via feast (setup.py) entrypoints==0.4 # via altair -environs==9.5.0 - # via pymilvus +et-xmlfile==2.0.0 + # via openpyxl exceptiongroup==1.2.2 # via # anyio @@ -203,8 +228,13 @@ fastjsonschema==2.21.1 # via nbformat filelock==3.17.0 # via + # huggingface-hub # snowflake-connector-python + # torch + # transformers # virtualenv +filetype==1.2.0 + # via docling fqdn==1.5.1 # via jsonschema frozenlist==1.5.0 @@ -215,6 +245,8 @@ fsspec==2024.9.0 # via # feast (setup.py) # dask + # huggingface-hub + # torch geomet==0.2.1.post1 # via cassandra-driver google-api-core[grpc]==2.24.1 @@ -242,7 +274,7 @@ google-cloud-bigquery-storage==2.28.0 # via feast (setup.py) google-cloud-bigtable==2.28.1 # via feast (setup.py) -google-cloud-core==2.4.1 +google-cloud-core==2.4.2 # via # google-cloud-bigquery # google-cloud-bigtable @@ -260,7 +292,7 @@ google-resumable-media==2.7.2 # via # google-cloud-bigquery # google-cloud-storage -googleapis-common-protos[grpc]==1.66.0 +googleapis-common-protos[grpc]==1.68.0 # via # feast (setup.py) # google-api-core @@ -268,9 +300,11 @@ googleapis-common-protos[grpc]==1.66.0 # grpcio-status great-expectations==0.18.22 # via feast (setup.py) +greenlet==3.1.1 + # via sqlalchemy grpc-google-iam-v1==0.14.0 # via google-cloud-bigtable -grpcio==1.70.0 +grpcio==1.67.1 # via # feast (setup.py) # google-api-core @@ -281,17 +315,20 @@ grpcio==1.70.0 # grpcio-status # grpcio-testing # grpcio-tools + # ikvpy # pymilvus # qdrant-client -grpcio-health-checking==1.70.0 +grpcio-health-checking==1.67.1 # via feast (setup.py) -grpcio-reflection==1.70.0 +grpcio-reflection==1.67.1 # via feast (setup.py) -grpcio-status==1.70.0 - # via google-api-core -grpcio-testing==1.70.0 +grpcio-status==1.67.1 + # via + # google-api-core + # ikvpy +grpcio-testing==1.67.1 # via feast (setup.py) -grpcio-tools==1.70.0 +grpcio-tools==1.67.1 # via # feast (setup.py) # qdrant-client @@ -323,15 +360,21 @@ httpx[http2]==0.27.2 # jupyterlab # python-keycloak # qdrant-client +huggingface-hub==0.29.1 + # via + # docling + # docling-ibm-models + # tokenizers + # transformers hyperframe==6.1.0 # via h2 -ibis-framework[duckdb]==9.0.0 +ibis-framework[duckdb, mssql]==9.0.0 # via # feast (setup.py) # ibis-substrait ibis-substrait==4.0.1 # via feast (setup.py) -identify==2.6.6 +identify==2.6.8 # via pre-commit idna==3.10 # via @@ -341,6 +384,10 @@ idna==3.10 # requests # snowflake-connector-python # yarl +ikvpy==0.0.36 + # via feast (setup.py) +imageio==2.37.0 + # via scikit-image imagesize==1.4.1 # via sphinx importlib-metadata==8.6.1 @@ -382,6 +429,7 @@ jinja2==3.1.5 # moto # nbconvert # sphinx + # torch jmespath==1.0.1 # via # aiobotocore @@ -389,16 +437,21 @@ jmespath==1.0.1 # botocore json5==0.10.0 # via jupyterlab-server +jsonlines==3.1.0 + # via docling-ibm-models jsonpatch==1.33 # via great-expectations jsonpointer==3.0.0 # via # jsonpatch # jsonschema +jsonref==1.1.0 + # via docling-core jsonschema[format-nongpl]==4.23.0 # via # feast (setup.py) # altair + # docling-core # great-expectations # jupyter-events # jupyterlab-server @@ -446,23 +499,32 @@ jwcrypto==1.5.6 # via python-keycloak kubernetes==20.13.0 # via feast (setup.py) +latex2mathml==3.77.0 + # via docling-core +lazy-loader==0.4 + # via scikit-image locket==1.0.0 # via partd +lxml==5.3.1 + # via + # docling + # python-docx + # python-pptx lz4==4.4.3 # via trino makefun==1.15.6 # via great-expectations markdown-it-py==3.0.0 # via rich +marko==2.1.2 + # via docling markupsafe==3.0.2 # via # jinja2 # nbconvert # werkzeug marshmallow==3.26.1 - # via - # environs - # great-expectations + # via great-expectations matplotlib-inline==0.1.7 # via # ipykernel @@ -473,7 +535,7 @@ milvus-lite==2.4.11 # via pymilvus minio==7.2.11 # via feast (setup.py) -mistune==3.1.1 +mistune==3.1.2 # via # great-expectations # nbconvert @@ -483,6 +545,10 @@ mock==2.0.0 # via feast (setup.py) moto==4.2.14 # via feast (setup.py) +mpire[dill]==2.10.2 + # via semchunk +mpmath==1.3.0 + # via sympy msal==1.31.1 # via # azure-identity @@ -494,6 +560,8 @@ multidict==6.1.0 # aiobotocore # aiohttp # yarl +multiprocess==0.70.17 + # via mpire mypy==1.11.2 # via # feast (setup.py) @@ -514,6 +582,12 @@ nbformat==5.10.4 # nbconvert nest-asyncio==1.6.0 # via ipykernel +networkx==3.2.1 + # via + # scikit-image + # torch +ninja==1.11.1.3 + # via easyocr nodeenv==1.9.1 # via pre-commit notebook==7.3.2 @@ -528,15 +602,31 @@ numpy==1.26.4 # altair # dask # db-dtypes + # docling-ibm-models + # easyocr # faiss-cpu # great-expectations # ibis-framework + # imageio + # opencv-python-headless # pandas # pyarrow # qdrant-client + # safetensors + # scikit-image # scipy + # shapely + # tifffile + # torchvision + # transformers oauthlib==3.2.2 # via requests-oauthlib +opencv-python-headless==4.11.0.86 + # via + # docling-ibm-models + # easyocr +openpyxl==3.1.5 + # via docling overrides==7.7.0 # via jupyter-server packaging==24.2 @@ -549,17 +639,21 @@ packaging==24.2 # google-cloud-bigquery # great-expectations # gunicorn + # huggingface-hub # ibis-substrait # ipykernel # jupyter-events # jupyter-server # jupyterlab # jupyterlab-server + # lazy-loader # marshmallow # nbconvert # pytest + # scikit-image # snowflake-connector-python # sphinx + # transformers pandas==2.2.3 # via # feast (setup.py) @@ -567,6 +661,8 @@ pandas==2.2.3 # dask # dask-expr # db-dtypes + # docling + # docling-core # google-cloud-bigquery # great-expectations # ibis-framework @@ -586,7 +682,18 @@ pbr==6.1.1 # via mock pexpect==4.9.0 # via ipython -pip==25.0 +pillow==11.1.0 + # via + # docling + # docling-core + # docling-ibm-models + # docling-parse + # easyocr + # imageio + # python-pptx + # scikit-image + # torchvision +pip==25.0.1 # via pip-tools pip-tools==7.4.1 # via feast (setup.py) @@ -611,7 +718,7 @@ prometheus-client==0.21.1 # jupyter-server prompt-toolkit==3.0.50 # via ipython -propcache==0.2.1 +propcache==0.3.0 # via # aiohttp # yarl @@ -635,6 +742,7 @@ protobuf==5.29.3 # grpcio-status # grpcio-testing # grpcio-tools + # ikvpy # mypy-protobuf # proto-plus # pymilvus @@ -643,11 +751,11 @@ psutil==5.9.0 # via # feast (setup.py) # ipykernel -psycopg[binary, pool]==3.2.4 +psycopg[binary, pool]==3.2.5 # via feast (setup.py) -psycopg-binary==3.2.4 +psycopg-binary==3.2.5 # via psycopg -psycopg-pool==3.2.4 +psycopg-pool==3.2.5 # via psycopg ptyprocess==0.7.0 # via @@ -680,6 +788,8 @@ pyasn1-modules==0.4.1 # via google-auth pybindgen==0.22.1 # via feast (setup.py) +pyclipper==1.3.0.post6 + # via easyocr pycparser==2.22 # via cffi pycryptodome==3.21.0 @@ -687,15 +797,23 @@ pycryptodome==3.21.0 pydantic==2.10.6 # via # feast (setup.py) + # docling + # docling-core + # docling-ibm-models + # docling-parse # fastapi # great-expectations + # pydantic-settings # qdrant-client pydantic-core==2.27.2 # via pydantic +pydantic-settings==2.8.0 + # via docling pygments==2.19.1 # via # feast (setup.py) # ipython + # mpire # nbconvert # rich # sphinx @@ -705,18 +823,22 @@ pyjwt[crypto]==2.10.1 # msal # singlestoredb # snowflake-connector-python -pymilvus==2.4.9 +pymilvus==2.5.4 # via feast (setup.py) pymssql==2.3.2 # via feast (setup.py) pymysql==1.1.1 # via feast (setup.py) pyodbc==5.2.0 - # via feast (setup.py) + # via + # feast (setup.py) + # ibis-framework pyopenssl==24.3.0 # via snowflake-connector-python pyparsing==3.2.1 # via great-expectations +pypdfium2==4.30.1 + # via docling pyproject-hooks==1.2.0 # via # build @@ -753,6 +875,8 @@ pytest-timeout==1.4.2 # via feast (setup.py) pytest-xdist==3.6.1 # via feast (setup.py) +python-bidi==0.6.6 + # via easyocr python-dateutil==2.9.0.post0 # via # aiobotocore @@ -766,14 +890,19 @@ python-dateutil==2.9.0.post0 # moto # pandas # trino +python-docx==1.1.2 + # via docling python-dotenv==1.0.1 # via - # environs + # pydantic-settings + # pymilvus # uvicorn python-json-logger==3.2.1 # via jupyter-events python-keycloak==4.2.2 # via feast (setup.py) +python-pptx==1.0.2 + # via docling pytz==2025.1 # via # great-expectations @@ -785,11 +914,15 @@ pyyaml==6.0.2 # via # feast (setup.py) # dask + # docling-core + # easyocr + # huggingface-hub # ibis-substrait # jupyter-events # kubernetes # pre-commit # responses + # transformers # uvicorn pyzmq==26.2.1 # via @@ -809,15 +942,18 @@ regex==2024.11.6 # via # feast (setup.py) # parsimonious + # transformers requests==2.32.3 # via # feast (setup.py) # azure-core # docker + # docling # google-api-core # google-cloud-bigquery # google-cloud-storage # great-expectations + # huggingface-hub # jupyterlab-server # kubernetes # moto @@ -829,6 +965,7 @@ requests==2.32.3 # singlestoredb # snowflake-connector-python # sphinx + # transformers # trino requests-oauthlib==2.0.0 # via kubernetes @@ -845,23 +982,39 @@ rfc3986-validator==0.1.1 # jsonschema # jupyter-events rich==13.9.4 - # via ibis-framework -rpds-py==0.22.3 + # via + # ibis-framework + # typer +rpds-py==0.23.1 # via # jsonschema # referencing rsa==4.9 # via google-auth +rtree==1.3.0 + # via docling ruamel-yaml==0.17.40 # via great-expectations ruamel-yaml-clib==0.2.12 # via ruamel-yaml -ruff==0.9.5 +ruff==0.9.7 # via feast (setup.py) s3transfer==0.11.2 # via boto3 +safetensors[torch]==0.5.2 + # via + # docling-ibm-models + # transformers +scikit-image==0.24.0 + # via easyocr scipy==1.13.1 - # via great-expectations + # via + # docling + # easyocr + # great-expectations + # scikit-image +semchunk==2.2.2 + # via docling-core send2trash==1.8.3 # via jupyter-server setuptools==75.8.0 @@ -873,6 +1026,10 @@ setuptools==75.8.0 # pip-tools # pymilvus # singlestoredb +shapely==2.0.7 + # via easyocr +shellingham==1.5.4 + # via typer singlestoredb==1.7.2 # via feast (setup.py) six==1.17.0 @@ -925,8 +1082,13 @@ starlette==0.45.3 # via fastapi substrait==0.23.0 # via ibis-substrait +sympy==1.13.3 + # via torch tabulate==0.9.0 - # via feast (setup.py) + # via + # feast (setup.py) + # docling-core + # docling-parse tenacity==8.5.0 # via feast (setup.py) terminado==0.18.1 @@ -937,8 +1099,12 @@ testcontainers==4.8.2 # via feast (setup.py) thriftpy2==0.5.2 # via happybase +tifffile==2024.8.30 + # via scikit-image tinycss2==1.4.0 # via bleach +tokenizers==0.19.1 + # via transformers toml==0.10.2 # via feast (setup.py) tomli==2.2.1 @@ -959,6 +1125,18 @@ toolz==0.12.1 # dask # ibis-framework # partd +torch==2.2.2 + # via + # feast (setup.py) + # docling-ibm-models + # easyocr + # safetensors + # torchvision +torchvision==0.17.2 + # via + # feast (setup.py) + # docling-ibm-models + # easyocr tornado==6.4.2 # via # ipykernel @@ -970,8 +1148,14 @@ tornado==6.4.2 tqdm==4.67.1 # via # feast (setup.py) + # docling + # docling-ibm-models # great-expectations + # huggingface-hub # milvus-lite + # mpire + # semchunk + # transformers traitlets==5.14.3 # via # comm @@ -987,10 +1171,18 @@ traitlets==5.14.3 # nbclient # nbconvert # nbformat +transformers==4.42.4 + # via + # docling-core + # docling-ibm-models trino==0.333.0 # via feast (setup.py) -typeguard==4.4.1 +typeguard==4.4.2 # via feast (setup.py) +typer==0.12.5 + # via + # docling + # docling-core types-cffi==1.16.0.20241221 # via types-pyopenssl types-protobuf==3.19.22 @@ -1013,7 +1205,7 @@ types-redis==4.6.0.20241004 # via feast (setup.py) types-requests==2.30.0.0 # via feast (setup.py) -types-setuptools==75.8.0.20250110 +types-setuptools==75.8.0.20250225 # via # feast (setup.py) # types-cffi @@ -1030,8 +1222,10 @@ typing-extensions==4.12.2 # azure-identity # azure-storage-blob # beautifulsoup4 + # docling-core # fastapi # great-expectations + # huggingface-hub # ibis-framework # ipython # jwcrypto @@ -1043,18 +1237,22 @@ typing-extensions==4.12.2 # psycopg-pool # pydantic # pydantic-core + # python-docx # python-json-logger + # python-pptx # referencing # rich # snowflake-connector-python # sqlalchemy # starlette # testcontainers + # torch # typeguard + # typer # uvicorn tzdata==2025.1 # via pandas -tzlocal==5.2 +tzlocal==5.3 # via # great-expectations # trino @@ -1103,7 +1301,7 @@ websocket-client==1.8.0 # via # jupyter-server # kubernetes -websockets==14.2 +websockets==15.0 # via uvicorn werkzeug==3.1.3 # via moto @@ -1117,6 +1315,8 @@ wrapt==1.17.2 # via # aiobotocore # testcontainers +xlsxwriter==3.2.2 + # via python-pptx xmltodict==0.14.2 # via moto yarl==1.18.3 diff --git a/sdk/python/requirements/py3.9-requirements.txt b/sdk/python/requirements/py3.9-requirements.txt index 1b82c994a1c..e7aa5a42409 100644 --- a/sdk/python/requirements/py3.9-requirements.txt +++ b/sdk/python/requirements/py3.9-requirements.txt @@ -10,7 +10,7 @@ attrs==25.1.0 # via # jsonschema # referencing -bigtree==0.23.1 +bigtree==0.25.0 # via feast (setup.py) certifi==2025.1.31 # via requests @@ -39,6 +39,8 @@ fastapi==0.115.8 # via feast (setup.py) fsspec==2025.2.0 # via dask +greenlet==3.1.1 + # via sqlalchemy gunicorn==23.0.0 # via # feast (setup.py) @@ -91,7 +93,7 @@ prometheus-client==0.21.1 # via feast (setup.py) protobuf==5.29.3 # via feast (setup.py) -psutil==6.1.1 +psutil==7.0.0 # via feast (setup.py) pyarrow==18.0.0 # via @@ -124,7 +126,7 @@ referencing==0.36.2 # jsonschema-specifications requests==2.32.3 # via feast (setup.py) -rpds-py==0.22.3 +rpds-py==0.23.1 # via # jsonschema # referencing @@ -150,7 +152,7 @@ toolz==1.0.0 # partd tqdm==4.67.1 # via feast (setup.py) -typeguard==4.4.1 +typeguard==4.4.2 # via feast (setup.py) typing-extensions==4.12.2 # via @@ -178,7 +180,7 @@ uvloop==0.21.0 # via uvicorn watchfiles==1.0.4 # via uvicorn -websockets==14.2 +websockets==15.0 # via uvicorn zipp==3.21.0 # via importlib-metadata diff --git a/sdk/python/tests/doctest/test_all.py b/sdk/python/tests/doctest/test_all.py index d1b2161252f..de032264e6d 100644 --- a/sdk/python/tests/doctest/test_all.py +++ b/sdk/python/tests/doctest/test_all.py @@ -77,9 +77,11 @@ def test_docstrings(): full_name = package.__name__ + "." + name try: - temp_module = importlib.import_module(full_name) - if is_pkg: - next_packages.append(temp_module) + # https://github.com/feast-dev/feast/issues/5088 + if "ikv" not in full_name and "milvus" not in full_name: + temp_module = importlib.import_module(full_name) + if is_pkg: + next_packages.append(temp_module) except ModuleNotFoundError: pass diff --git a/sdk/python/tests/example_repos/example_feature_repo_1.py b/sdk/python/tests/example_repos/example_feature_repo_1.py index ea33859f4de..1671bd0ae3a 100644 --- a/sdk/python/tests/example_repos/example_feature_repo_1.py +++ b/sdk/python/tests/example_repos/example_feature_repo_1.py @@ -125,6 +125,8 @@ vector_search_metric="L2", ), Field(name="item_id", dtype=String), + Field(name="content", dtype=String), + Field(name="title", dtype=String), ], source=rag_documents_source, ttl=timedelta(hours=24), diff --git a/sdk/python/tests/foo_provider.py b/sdk/python/tests/foo_provider.py index ca6a02c4bd0..2aa674c0aa5 100644 --- a/sdk/python/tests/foo_provider.py +++ b/sdk/python/tests/foo_provider.py @@ -169,9 +169,10 @@ def retrieve_online_documents_v2( config: RepoConfig, table: FeatureView, requested_features: List[str], - query: List[float], + query: Optional[List[float]], top_k: int, distance_metric: Optional[str] = None, + query_string: Optional[str] = None, ) -> List[ Tuple[ Optional[datetime], diff --git a/sdk/python/tests/integration/feature_repos/universal/online_store/couchbase.py b/sdk/python/tests/integration/feature_repos/universal/online_store/couchbase.py index f2ba12da8da..2723ff13a30 100644 --- a/sdk/python/tests/integration/feature_repos/universal/online_store/couchbase.py +++ b/sdk/python/tests/integration/feature_repos/universal/online_store/couchbase.py @@ -66,7 +66,7 @@ def create_online_store(self) -> Dict[str, object]: # Return the configuration for Feast return { - "type": "couchbase", + "type": "couchbase.online", "connection_string": "couchbase://127.0.0.1", "user": self.username, "password": self.password, diff --git a/sdk/python/tests/unit/infra/offline_stores/test_offline_store.py b/sdk/python/tests/unit/infra/offline_stores/test_offline_store.py index afc0e4e5c8f..fe2c437617a 100644 --- a/sdk/python/tests/unit/infra/offline_stores/test_offline_store.py +++ b/sdk/python/tests/unit/infra/offline_stores/test_offline_store.py @@ -21,6 +21,7 @@ TrinoRetrievalJob, ) from feast.infra.offline_stores.dask import DaskRetrievalJob +from feast.infra.offline_stores.file_source import FileSource from feast.infra.offline_stores.offline_store import RetrievalJob, RetrievalMetadata from feast.infra.offline_stores.redshift import ( RedshiftOfflineStoreConfig, @@ -246,3 +247,28 @@ def test_to_arrow_timeout(retrieval_job, timeout: Optional[int]): with patch.object(retrieval_job, "_to_arrow_internal") as mock_to_arrow_internal: retrieval_job.to_arrow(timeout=timeout) mock_to_arrow_internal.assert_called_once_with(timeout=timeout) + + +@pytest.mark.parametrize( + "repo_path, uri, expected", + [ + # Remote URI - Should return as-is + ( + "/some/repo", + "s3://bucket-name/file.parquet", + "s3://bucket-name/file.parquet", + ), + # Absolute Path - Should return as-is + ("/some/repo", "/abs/path/file.parquet", "/abs/path/file.parquet"), + # Relative Path with repo_path - Should combine + ("/some/repo", "data/output.parquet", "/some/repo/data/output.parquet"), + # Relative Path without repo_path - Should return absolute path + (None, "C:/path/to/file.parquet", "C:/path/to/file.parquet"), + ], + ids=["s3_uri", "absolute_path", "relative_path", "windows_path"], +) +def test_get_uri_for_file_path( + repo_path: Optional[str], uri: str, expected: str +) -> None: + result = FileSource.get_uri_for_file_path(repo_path=repo_path, uri=uri) + assert result == expected, f"Expected {expected}, but got {result}" diff --git a/sdk/python/tests/unit/infra/registry/__init__.py b/sdk/python/tests/unit/infra/registry/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/sdk/python/tests/unit/infra/registry/test_registry.py b/sdk/python/tests/unit/infra/registry/test_registry.py new file mode 100644 index 00000000000..65dea2ff680 --- /dev/null +++ b/sdk/python/tests/unit/infra/registry/test_registry.py @@ -0,0 +1,197 @@ +from datetime import datetime, timedelta, timezone +from unittest.mock import patch + +import pytest + +from feast.infra.registry.caching_registry import CachingRegistry + + +class TestCachingRegistry(CachingRegistry): + """Test subclass that implements abstract methods as no-ops""" + + def _get_any_feature_view(self, *args, **kwargs): + pass + + def _get_data_source(self, *args, **kwargs): + pass + + def _get_entity(self, *args, **kwargs): + pass + + def _get_feature_service(self, *args, **kwargs): + pass + + def _get_feature_view(self, *args, **kwargs): + pass + + def _get_infra(self, *args, **kwargs): + pass + + def _get_on_demand_feature_view(self, *args, **kwargs): + pass + + def _get_permission(self, *args, **kwargs): + pass + + def _get_project(self, *args, **kwargs): + pass + + def _get_saved_dataset(self, *args, **kwargs): + pass + + def _get_stream_feature_view(self, *args, **kwargs): + pass + + def _get_validation_reference(self, *args, **kwargs): + pass + + def _list_all_feature_views(self, *args, **kwargs): + pass + + def _list_data_sources(self, *args, **kwargs): + pass + + def _list_entities(self, *args, **kwargs): + pass + + def _list_feature_services(self, *args, **kwargs): + pass + + def _list_feature_views(self, *args, **kwargs): + pass + + def _list_on_demand_feature_views(self, *args, **kwargs): + pass + + def _list_permissions(self, *args, **kwargs): + pass + + def _list_project_metadata(self, *args, **kwargs): + pass + + def _list_projects(self, *args, **kwargs): + pass + + def _list_saved_datasets(self, *args, **kwargs): + pass + + def _list_stream_feature_views(self, *args, **kwargs): + pass + + def _list_validation_references(self, *args, **kwargs): + pass + + def apply_data_source(self, *args, **kwargs): + pass + + def apply_entity(self, *args, **kwargs): + pass + + def apply_feature_service(self, *args, **kwargs): + pass + + def apply_feature_view(self, *args, **kwargs): + pass + + def apply_materialization(self, *args, **kwargs): + pass + + def apply_permission(self, *args, **kwargs): + pass + + def apply_project(self, *args, **kwargs): + pass + + def apply_saved_dataset(self, *args, **kwargs): + pass + + def apply_user_metadata(self, *args, **kwargs): + pass + + def apply_validation_reference(self, *args, **kwargs): + pass + + def commit(self, *args, **kwargs): + pass + + def delete_data_source(self, *args, **kwargs): + pass + + def delete_entity(self, *args, **kwargs): + pass + + def delete_feature_service(self, *args, **kwargs): + pass + + def delete_feature_view(self, *args, **kwargs): + pass + + def delete_permission(self, *args, **kwargs): + pass + + def delete_project(self, *args, **kwargs): + pass + + def delete_validation_reference(self, *args, **kwargs): + pass + + def get_user_metadata(self, *args, **kwargs): + pass + + def proto(self, *args, **kwargs): + pass + + def update_infra(self, *args, **kwargs): + pass + + +@pytest.fixture +def registry(): + """Fixture to create a real instance of CachingRegistry""" + return TestCachingRegistry( + project="test_example", cache_ttl_seconds=2, cache_mode="sync" + ) + + +def test_cache_expiry_triggers_refresh(registry): + """Test that an expired cache triggers a refresh""" + # Set cache creation time to a value that is expired + registry.cached_registry_proto = "some_cached_data" + registry.cached_registry_proto_created = datetime.now(timezone.utc) - timedelta( + seconds=5 + ) + + # Mock _refresh_cached_registry_if_necessary to check if it is called + with patch.object( + CachingRegistry, + "_refresh_cached_registry_if_necessary", + wraps=registry._refresh_cached_registry_if_necessary, + ) as mock_refresh_check: + registry._refresh_cached_registry_if_necessary() + mock_refresh_check.assert_called_once() + + # Now check if the refresh was actually triggered + with patch.object( + CachingRegistry, "refresh", wraps=registry.refresh + ) as mock_refresh: + registry._refresh_cached_registry_if_necessary() + mock_refresh.assert_called_once() + + +def test_skip_refresh_if_lock_held(registry): + """Test that refresh is skipped if the lock is already held by another thread""" + registry.cached_registry_proto = "some_cached_data" + registry.cached_registry_proto_created = datetime.now(timezone.utc) - timedelta( + seconds=5 + ) + + # Acquire the lock manually to simulate another thread holding it + registry._refresh_lock.acquire() + with patch.object( + CachingRegistry, "refresh", wraps=registry.refresh + ) as mock_refresh: + registry._refresh_cached_registry_if_necessary() + + # Since the lock was already held, refresh should NOT be called + mock_refresh.assert_not_called() + registry._refresh_lock.release() diff --git a/sdk/python/tests/unit/infra/test_inference_unit_tests.py b/sdk/python/tests/unit/infra/test_inference_unit_tests.py index 54488d43212..951f7033d23 100644 --- a/sdk/python/tests/unit/infra/test_inference_unit_tests.py +++ b/sdk/python/tests/unit/infra/test_inference_unit_tests.py @@ -154,23 +154,6 @@ def python_native_test_invalid_pandas_view( } return output_dict - with pytest.raises(TypeError): - - @on_demand_feature_view( - sources=[date_request], - schema=[ - Field(name="output", dtype=UnixTimestamp), - Field(name="object_output", dtype=String), - ], - mode="python", - ) - def python_native_test_invalid_dict_view( - features_df: pd.DataFrame, - ) -> pd.DataFrame: - data = pd.DataFrame() - data["output"] = features_df["some_date"] - return data - def test_datasource_inference(): # Create Feature Views diff --git a/sdk/python/tests/unit/infra/utils/snowflake/test_snowflake_utils.py b/sdk/python/tests/unit/infra/utils/snowflake/test_snowflake_utils.py new file mode 100644 index 00000000000..8ae6ec63ba5 --- /dev/null +++ b/sdk/python/tests/unit/infra/utils/snowflake/test_snowflake_utils.py @@ -0,0 +1,71 @@ +import tempfile +from typing import Optional + +import pytest +from cryptography.hazmat.primitives import serialization +from cryptography.hazmat.primitives.asymmetric import rsa + +from feast.infra.utils.snowflake.snowflake_utils import parse_private_key_path + +PRIVATE_KEY_PASSPHRASE = "test" + + +def _pem_private_key(passphrase: Optional[str]): + private_key = rsa.generate_private_key(public_exponent=65537, key_size=2048) + return private_key.private_bytes( + encoding=serialization.Encoding.PEM, + format=serialization.PrivateFormat.TraditionalOpenSSL, + encryption_algorithm=( + serialization.BestAvailableEncryption(passphrase.encode()) + if passphrase + else serialization.NoEncryption() + ), + ) + + +@pytest.fixture +def unencrypted_private_key(): + return _pem_private_key(None) + + +@pytest.fixture +def encrypted_private_key(): + return _pem_private_key(PRIVATE_KEY_PASSPHRASE) + + +def test_parse_private_key_path_key_content_unencrypted(unencrypted_private_key): + parse_private_key_path( + None, + None, + unencrypted_private_key, + ) + + +def test_parse_private_key_path_key_content_encrypted(encrypted_private_key): + parse_private_key_path( + PRIVATE_KEY_PASSPHRASE, + None, + encrypted_private_key, + ) + + +def test_parse_private_key_path_key_path_unencrypted(unencrypted_private_key): + with tempfile.NamedTemporaryFile(mode="wb") as f: + f.write(unencrypted_private_key) + f.flush() + parse_private_key_path( + None, + f.name, + None, + ) + + +def test_parse_private_key_path_key_path_encrypted(encrypted_private_key): + with tempfile.NamedTemporaryFile(mode="wb") as f: + f.write(encrypted_private_key) + f.flush() + parse_private_key_path( + PRIVATE_KEY_PASSPHRASE, + f.name, + None, + ) diff --git a/sdk/python/tests/unit/online_store/test_online_retrieval.py b/sdk/python/tests/unit/online_store/test_online_retrieval.py index 5e5a3104c13..ea76ed6f544 100644 --- a/sdk/python/tests/unit/online_store/test_online_retrieval.py +++ b/sdk/python/tests/unit/online_store/test_online_retrieval.py @@ -640,12 +640,12 @@ def test_sqlite_get_online_documents() -> None: item_keys = [ EntityKeyProto( - join_keys=["item_id"], entity_values=[ValueProto(int64_val=i)] + join_keys=["item_id"], entity_values=[ValueProto(string_val=str(i))] ) for i in range(n) ] data = [] - for item_key in item_keys: + for i, item_key in enumerate(item_keys): data.append( ( item_key, @@ -656,19 +656,17 @@ def test_sqlite_get_online_documents() -> None: vector_length, ) ) - ) + ), + "content": ValueProto( + string_val=f"the {i}th sentence with some text" + ), + "title": ValueProto(string_val=f"Title {i}"), }, _utc_now(), _utc_now(), ) ) - provider.online_write_batch( - config=store.config, - table=document_embeddings_fv, - data=data, - progress=None, - ) documents_df = pd.DataFrame( { "item_id": [str(i) for i in range(n)], @@ -678,26 +676,42 @@ def test_sqlite_get_online_documents() -> None: ) for i in range(n) ], + "content": [f"the {i}th sentence with some text" for i in range(n)], + "title": [f"Title {i}" for i in range(n)], "event_timestamp": [_utc_now() for _ in range(n)], } ) - store.write_to_online_store( - feature_view_name="document_embeddings", - df=documents_df, + print(len(data), documents_df.shape[0]) + provider.online_write_batch( + config=store.config, + table=document_embeddings_fv, + data=data, + progress=None, ) - document_table = store._provider._online_store._conn.execute( "SELECT name FROM sqlite_master WHERE type='table' and name like '%_document_embeddings';" ).fetchall() + assert len(document_table) == 1 document_table_name = document_table[0][0] + record_count = len( store._provider._online_store._conn.execute( f"select * from {document_table_name}" ).fetchall() ) - assert record_count == len(data) + documents_df.shape[0] + assert record_count == len(data) * len(document_embeddings_fv.features) + store.write_to_online_store( + feature_view_name="document_embeddings", + df=documents_df, + ) + record_count = len( + store._provider._online_store._conn.execute( + f"select * from {document_table_name}" + ).fetchall() + ) + assert record_count == len(data) * len(document_embeddings_fv.features) query_embedding = np.random.random( vector_length, @@ -753,6 +767,93 @@ def test_sqlite_vec_import() -> None: assert result == [(2, 2.39), (1, 2.39)] +def test_sqlite_hybrid_search() -> None: + imdb_sample_data = { + "Rank": {0: 1, 1: 2, 2: 3, 3: 4, 4: 5}, + "Title": { + 0: "Guardians of the Galaxy", + 1: "Prometheus", + 2: "Split", + 3: "Sing", + 4: "Suicide Squad", + }, + "Genre": { + 0: "Action,Adventure,Sci-Fi", + 1: "Adventure,Mystery,Sci-Fi", + 2: "Horror,Thriller", + 3: "Animation,Comedy,Family", + 4: "Action,Adventure,Fantasy", + }, + "Description": { + 0: "A group of intergalactic criminals are forced to work together to stop a fanatical warrior from taking control of the universe.", + 1: "Following clues to the origin of mankind, a team finds a structure on a distant moon, but they soon realize they are not alone.", + 2: "Three girls are kidnapped by a man with a diagnosed 23 distinct personalities. They must try to escape before the apparent emergence of a frightful new 24th.", + 3: "In a city of humanoid animals, a hustling theater impresario's attempt to save his theater with a singing competition becomes grander than he anticipates even as its finalists' find that their lives will never be the same.", + 4: "A secret government agency recruits some of the most dangerous incarcerated super-villains to form a defensive task force. Their first mission: save the world from the apocalypse.", + }, + "Director": { + 0: "James Gunn", + 1: "Ridley Scott", + 2: "M. Night Shyamalan", + 3: "Christophe Lourdelet", + 4: "David Ayer", + }, + "Actors": { + 0: "Chris Pratt, Vin Diesel, Bradley Cooper, Zoe Saldana", + 1: "Noomi Rapace, Logan Marshall-Green, Michael Fassbender, Charlize Theron", + 2: "James McAvoy, Anya Taylor-Joy, Haley Lu Richardson, Jessica Sula", + 3: "Matthew McConaughey,Reese Witherspoon, Seth MacFarlane, Scarlett Johansson", + 4: "Will Smith, Jared Leto, Margot Robbie, Viola Davis", + }, + "Year": {0: 2014, 1: 2012, 2: 2016, 3: 2016, 4: 2016}, + "Runtime (Minutes)": {0: 121, 1: 124, 2: 117, 3: 108, 4: 123}, + "Rating": {0: 8.1, 1: 7.0, 2: 7.3, 3: 7.2, 4: 6.2}, + "Votes": {0: 757074, 1: 485820, 2: 157606, 3: 60545, 4: 393727}, + "Revenue (Millions)": {0: 333.13, 1: 126.46, 2: 138.12, 3: 270.32, 4: 325.02}, + "Metascore": {0: 76.0, 1: 65.0, 2: 62.0, 3: 59.0, 4: 40.0}, + } + df = pd.DataFrame(imdb_sample_data) + db = sqlite3.connect(":memory:") + + cur = db.cursor() + + cur.execute( + 'create virtual table imdb using fts5(title, description, genre, rating, tokenize="porter unicode61");' + ) + cur.executemany( + "insert into imdb (title, description, genre, rating) values (?,?,?,?);", + df[["Title", "Description", "Genre", "Rating"]].to_records(index=False), + ) + db.commit() + + query = "Prom" + res = cur.execute(f"""select title, description, genre, rating, rank + from imdb + where title MATCH "{query}*" + ORDER BY rank + limit 5""").fetchall() + assert len(res) == 1 + assert res[0][0] == "Prometheus" + + q = "(title : the OR of) AND (genre: Action OR Comedy)" + res_df = pd.read_sql_query( + f""" + select + rowid, + title, + description, + bm25(imdb, 10.0, 5.0) + from imdb + where imdb MATCH "{q}" + ORDER BY bm25(imdb, 10.0, 5.0) + limit 5 + """, + db, + ) + res_df["rowid"].tolist() == [1, 4, 5] + res_df["title"].tolist() == ["Guardians of the Galaxy", "Sing", "Suicide Squad"] + + @pytest.mark.skipif( sys.version_info[0:2] != (3, 10), reason="Only works on Python 3.10", @@ -780,7 +881,7 @@ def test_sqlite_get_online_documents_v2() -> None: for i in range(n) ] data = [] - for item_key in item_keys: + for i, item_key in enumerate(item_keys): data.append( ( item_key, @@ -789,7 +890,11 @@ def test_sqlite_get_online_documents_v2() -> None: float_list_val=FloatListProto( val=[float(x) for x in np.random.random(vector_length)] ) - ) + ), + "content": ValueProto( + string_val=f"the {i}th sentence with some text" + ), + "title": ValueProto(string_val=f"Title {i}"), }, _utc_now(), _utc_now(), @@ -806,16 +911,95 @@ def test_sqlite_get_online_documents_v2() -> None: # Test vector similarity search query_embedding = [float(x) for x in np.random.random(vector_length)] result = store.retrieve_online_documents_v2( - features=["document_embeddings:Embeddings"], + features=[ + "document_embeddings:Embeddings", + "document_embeddings:content", + "document_embeddings:title", + ], query=query_embedding, top_k=3, ).to_dict() assert "Embeddings" in result + assert "content" in result + assert "title" in result assert "distance" in result + assert ["1th sentence with some text" in r for r in result["content"]] + assert ["Title " in r for r in result["title"]] assert len(result["distance"]) == 3 +def test_sqlite_get_online_documents_v2_search() -> None: + """Test retrieving documents using v2 method with key word search""" + n = 10 + vector_length = 8 + runner = CliRunner() + with runner.local_repo( + get_example_repo("example_feature_repo_1.py"), "file" + ) as store: + store.config.online_store.text_search_enabled = True + store.config.entity_key_serialization_version = 3 + document_embeddings_fv = store.get_feature_view(name="document_embeddings") + + provider = store._get_provider() + + # Create test data + item_keys = [ + EntityKeyProto( + join_keys=["item_id"], entity_values=[ValueProto(int64_val=i)] + ) + for i in range(n) + ] + data = [] + for i, item_key in enumerate(item_keys): + data.append( + ( + item_key, + { + "Embeddings": ValueProto( + float_list_val=FloatListProto( + val=[float(x) for x in np.random.random(vector_length)] + ) + ), + "content": ValueProto( + string_val=f"the {i}th sentence with some text" + ), + "title": ValueProto(string_val=f"Title {i}"), + }, + _utc_now(), + _utc_now(), + ) + ) + + provider.online_write_batch( + config=store.config, + table=document_embeddings_fv, + data=data, + progress=None, + ) + + # Test vector similarity search + # query_embedding = [float(x) for x in np.random.random(vector_length)] + result = store.retrieve_online_documents_v2( + features=[ + "document_embeddings:Embeddings", + "document_embeddings:content", + "document_embeddings:title", + ], + query_string="(content: 5) OR (title: 1) OR (title: 3)", + top_k=3, + ).to_dict() + + assert "Embeddings" in result + assert "content" in result + assert "title" in result + assert "distance" in result + assert ["1th sentence with some text" in r for r in result["content"]] + assert ["Title " in r for r in result["title"]] + assert len(result["distance"]) == 2 + assert result["distance"] == [-1.8458267450332642, -1.8458267450332642] + + @pytest.mark.skip(reason="Skipping this test as CI struggles with it") def test_local_milvus() -> None: import random @@ -1094,12 +1278,12 @@ def test_milvus_native_from_feast_data() -> None: search_res = client.search( collection_name=COLLECTION_NAME, data=[query_embedding], - limit=3, # Top 3 results + limit=5, # Top 3 results output_fields=["item_id", "author_id", "sentence_chunks"], ) # Validate the search results - assert len(search_res[0]) == 3 + assert len(search_res[0]) == 5 print("Search Results:", search_res[0]) # Clean up the collection diff --git a/sdk/python/tests/unit/test_on_demand_feature_view.py b/sdk/python/tests/unit/test_on_demand_feature_view.py index 4b30bd6be99..69724779cdd 100644 --- a/sdk/python/tests/unit/test_on_demand_feature_view.py +++ b/sdk/python/tests/unit/test_on_demand_feature_view.py @@ -24,6 +24,7 @@ OnDemandFeatureView, PandasTransformation, PythonTransformation, + on_demand_feature_view, ) from feast.types import Float32 @@ -356,3 +357,65 @@ def test_on_demand_feature_view_stored_writes(): assert transformed_output["output3"] is not None and isinstance( transformed_output["output3"], datetime.datetime ) + + +def test_function_call_syntax(): + CUSTOM_FUNCTION_NAME = "custom-function-name" + file_source = FileSource(name="my-file-source", path="test.parquet") + feature_view = FeatureView( + name="my-feature-view", + entities=[], + schema=[ + Field(name="feature1", dtype=Float32), + Field(name="feature2", dtype=Float32), + ], + source=file_source, + ) + sources = [feature_view] + + def transform_features(features_df: pd.DataFrame) -> pd.DataFrame: + df = pd.DataFrame() + df["output1"] = features_df["feature1"] + df["output2"] = features_df["feature2"] + return df + + odfv = on_demand_feature_view( + sources=sources, + schema=[ + Field(name="output1", dtype=Float32), + Field(name="output2", dtype=Float32), + ], + )(transform_features) + + assert odfv.name == transform_features.__name__ + assert isinstance(odfv, OnDemandFeatureView) + + proto = odfv.to_proto() + assert proto.spec.name == transform_features.__name__ + + deserialized = OnDemandFeatureView.from_proto(proto) + assert deserialized.name == transform_features.__name__ + + def another_transform(features_df: pd.DataFrame) -> pd.DataFrame: + df = pd.DataFrame() + df["output1"] = features_df["feature1"] + df["output2"] = features_df["feature2"] + return df + + odfv_custom = on_demand_feature_view( + name=CUSTOM_FUNCTION_NAME, + sources=sources, + schema=[ + Field(name="output1", dtype=Float32), + Field(name="output2", dtype=Float32), + ], + )(another_transform) + + assert odfv_custom.name == CUSTOM_FUNCTION_NAME + assert isinstance(odfv_custom, OnDemandFeatureView) + + proto = odfv_custom.to_proto() + assert proto.spec.name == CUSTOM_FUNCTION_NAME + + deserialized = OnDemandFeatureView.from_proto(proto) + assert deserialized.name == CUSTOM_FUNCTION_NAME diff --git a/sdk/python/tests/unit/test_on_demand_python_transformation.py b/sdk/python/tests/unit/test_on_demand_python_transformation.py index a0c33fadfda..7ae9f1c70e6 100644 --- a/sdk/python/tests/unit/test_on_demand_python_transformation.py +++ b/sdk/python/tests/unit/test_on_demand_python_transformation.py @@ -1,5 +1,7 @@ import os import re +import sqlite3 +import sys import tempfile import unittest from datetime import datetime, timedelta @@ -20,10 +22,12 @@ from feast.feature_view import DUMMY_ENTITY_FIELD from feast.field import Field from feast.infra.online_stores.sqlite import SqliteOnlineStoreConfig +from feast.nlp_test_data import create_document_chunks_df from feast.on_demand_feature_view import on_demand_feature_view from feast.types import ( Array, Bool, + Bytes, Float32, Float64, Int64, @@ -161,7 +165,11 @@ def python_demo_view(inputs: dict[str, Any]) -> dict[str, Any]: @on_demand_feature_view( sources=[driver_stats_fv[["conv_rate", "acc_rate"]]], schema=[ - Field(name="conv_rate_plus_acc_python_singleton", dtype=Float64) + Field(name="conv_rate_plus_acc_python_singleton", dtype=Float64), + Field( + name="conv_rate_plus_acc_python_singleton_array", + dtype=Array(Float64), + ), ], mode="python", singleton=True, @@ -171,6 +179,7 @@ def python_singleton_view(inputs: dict[str, Any]) -> dict[str, Any]: output["conv_rate_plus_acc_python_singleton"] = ( inputs["conv_rate"] + inputs["acc_rate"] ) + output["conv_rate_plus_acc_python_singleton_array"] = [0.1, 0.2, 0.3] return output @on_demand_feature_view( @@ -852,6 +861,9 @@ def test_stored_writes(self): assert driver_stats_fv.entities == [driver.name] assert driver_stats_fv.entity_columns == [] + ODFV_STRING_CONSTANT = "guaranteed constant" + ODFV_OTHER_STRING_CONSTANT = "somethign else" + @on_demand_feature_view( entities=[driver], sources=[ @@ -863,6 +875,7 @@ def test_stored_writes(self): Field(name="current_datetime", dtype=UnixTimestamp), Field(name="counter", dtype=Int64), Field(name="input_datetime", dtype=UnixTimestamp), + Field(name="string_constant", dtype=String), ], mode="python", write_to_online_store=True, @@ -880,6 +893,7 @@ def python_stored_writes_feature_view( "current_datetime": [datetime.now() for _ in inputs["conv_rate"]], "counter": [c + 1 for c in inputs["counter"]], "input_datetime": [d for d in inputs["input_datetime"]], + "string_constant": [ODFV_STRING_CONSTANT], } return output @@ -933,30 +947,13 @@ def python_stored_writes_feature_view( "created": current_datetime, } ] - odfv_entity_rows_to_write = [ - { - "driver_id": 1001, - "counter": 0, - "input_datetime": current_datetime, - } - ] fv_entity_rows_to_read = [ { "driver_id": 1001, } ] - # Note that here we shouldn't have to pass the request source features for reading - # because they should have already been written to the online store - odfv_entity_rows_to_read = [ - { - "driver_id": 1001, - "conv_rate": 0.25, - "acc_rate": 0.50, - "counter": 0, - "input_datetime": current_datetime, - } - ] - print("storing fv features") + print("") + print("storing FV features") self.store.write_to_online_store( feature_view_name="driver_hourly_stats", df=fv_entity_rows_to_write, @@ -978,11 +975,58 @@ def python_stored_writes_feature_view( "acc_rate": [0.25], } - print("storing odfv features") + # Note that here we shouldn't have to pass the request source features for reading + # because they should have already been written to the online store + odfv_entity_rows_to_write = [ + { + "driver_id": 1002, + "counter": 0, + "conv_rate": 0.25, + "acc_rate": 0.50, + "input_datetime": current_datetime, + "string_constant": ODFV_OTHER_STRING_CONSTANT, + } + ] + odfv_entity_rows_to_read = [ + { + "driver_id": 1002, + "conv_rate_plus_acc": 7, # note how this is not the correct value and would be calculate on demand + "conv_rate": 0.25, + "acc_rate": 0.50, + "counter": 0, + "input_datetime": current_datetime, + "string_constant": ODFV_STRING_CONSTANT, + } + ] + print("storing ODFV features") self.store.write_to_online_store( feature_view_name="python_stored_writes_feature_view", df=odfv_entity_rows_to_write, ) + _conn = sqlite3.connect(self.store.config.online_store.path) + _table_name = ( + self.store.project + + "_" + + self.store.get_on_demand_feature_view( + "python_stored_writes_feature_view" + ).name + ) + sample = pd.read_sql( + f""" + select + feature_name, + value + from {_table_name} + """, + _conn, + ) + assert ( + sample[sample["feature_name"] == "string_constant"]["value"] + .astype(str) + .str.contains("guaranteed constant") + .values[0] + ) + print("reading odfv features") online_odfv_python_response = self.store.get_online_features( entity_rows=odfv_entity_rows_to_read, @@ -991,6 +1035,7 @@ def python_stored_writes_feature_view( "python_stored_writes_feature_view:current_datetime", "python_stored_writes_feature_view:counter", "python_stored_writes_feature_view:input_datetime", + "python_stored_writes_feature_view:string_constant", ], ).to_dict() print(online_odfv_python_response) @@ -1001,5 +1046,248 @@ def python_stored_writes_feature_view( "counter", "current_datetime", "input_datetime", + "string_constant", + ] + ) + # This should be 1 because we write the value of 0 and during the write, the counter is incremented + assert online_odfv_python_response["counter"] == [1] + assert online_odfv_python_response["string_constant"] == [ + ODFV_STRING_CONSTANT + ] + assert online_odfv_python_response["string_constant"] != [ + ODFV_OTHER_STRING_CONSTANT + ] + + def test_stored_writes_with_explode(self): + with tempfile.TemporaryDirectory() as data_dir: + self.store = FeatureStore( + config=RepoConfig( + project="test_on_demand_python_transformation_explode", + registry=os.path.join(data_dir, "registry.db"), + provider="local", + entity_key_serialization_version=3, + online_store=SqliteOnlineStoreConfig( + path=os.path.join(data_dir, "online.db"), + vector_enabled=True, + vector_len=5, + ), + ) + ) + + documents = { + "doc_1": "Hello world. How are you?", + "doc_2": "This is a test. Document chunking example.", + } + start_date = datetime.now() - timedelta(days=15) + end_date = datetime.now() + + documents_df = create_document_chunks_df( + documents, + start_date, + end_date, + embedding_size=60, + ) + corpus_path = os.path.join(data_dir, "documents.parquet") + documents_df.to_parquet(path=corpus_path, allow_truncated_timestamps=True) + + chunk = Entity( + name="chunk", join_keys=["chunk_id"], value_type=ValueType.STRING + ) + document = Entity( + name="document", join_keys=["document_id"], value_type=ValueType.STRING + ) + + input_explode_request_source = RequestSource( + name="counter_source", + schema=[ + Field(name="document_id", dtype=String), + Field(name="document_text", dtype=String), + Field(name="document_bytes", dtype=Bytes), + ], + ) + + @on_demand_feature_view( + entities=[chunk, document], + sources=[ + input_explode_request_source, + ], + schema=[ + Field(name="document_id", dtype=String), + Field(name="chunk_id", dtype=String), + Field(name="chunk_text", dtype=String), + Field( + name="vector", + dtype=Array(Float32), + vector_index=True, + vector_search_metric="L2", + ), + ], + mode="python", + write_to_online_store=True, + ) + def python_stored_writes_feature_view_explode_singleton( + inputs: dict[str, Any], + ): + output: dict[str, Any] = { + "document_id": ["doc_1", "doc_1", "doc_2", "doc_2"], + "chunk_id": ["chunk-1", "chunk-2", "chunk-1", "chunk-2"], + "chunk_text": [ + "hello friends", + "how are you?", + "This is a test.", + "Document chunking example.", + ], + "vector": [ + [0.1] * 5, + [0.2] * 5, + [0.3] * 5, + [0.4] * 5, + ], + } + return output + + assert python_stored_writes_feature_view_explode_singleton.entities == [ + chunk.name, + document.name, + ] + assert ( + python_stored_writes_feature_view_explode_singleton.entity_columns[ + 0 + ].name + == document.join_key + ) + assert ( + python_stored_writes_feature_view_explode_singleton.entity_columns[ + 1 + ].name + == chunk.join_key + ) + + self.store.apply( + [ + chunk, + document, + input_explode_request_source, + python_stored_writes_feature_view_explode_singleton, + ] + ) + odfv_applied = self.store.get_on_demand_feature_view( + "python_stored_writes_feature_view_explode_singleton" + ) + + assert odfv_applied.features[1].vector_index + + assert odfv_applied.entities == [chunk.name, document.name] + + # Note here that after apply() is called, the entity_columns are populated with the join_key + assert odfv_applied.entity_columns[1].name == chunk.join_key + assert odfv_applied.entity_columns[0].name == document.join_key + + assert len(self.store.list_all_feature_views()) == 1 + assert len(self.store.list_feature_views()) == 0 + assert len(self.store.list_on_demand_feature_views()) == 1 + assert len(self.store.list_stream_feature_views()) == 0 + assert ( + python_stored_writes_feature_view_explode_singleton.entity_columns + == self.store.get_on_demand_feature_view( + "python_stored_writes_feature_view_explode_singleton" + ).entity_columns + ) + + odfv_entity_rows_to_write = [ + { + "document_id": "document_1", + "document_text": "Hello world. How are you?", + }, + { + "document_id": "document_2", + "document_text": "This is a test. Document chunking example.", + }, + ] + fv_entity_rows_to_read = [ + { + "document_id": "doc_1", + "chunk_id": "chunk-2", + }, + { + "document_id": "doc_2", + "chunk_id": "chunk-1", + }, + ] + + self.store.write_to_online_store( + feature_view_name="python_stored_writes_feature_view_explode_singleton", + df=odfv_entity_rows_to_write, + ) + _table_name = ( + self.store.project + + "_" + + self.store.get_on_demand_feature_view( + "python_stored_writes_feature_view_explode_singleton" + ).name + ) + _conn = sqlite3.connect(self.store.config.online_store.path) + sample = pd.read_sql( + f""" + select + entity_key, + feature_name, + value + from {_table_name} + """, + _conn, + ) + print(f"\nsample from {_table_name}:\n{sample}") + + # verifying we retrieve doc_1 chunk-2 + filt = (sample["feature_name"] == "chunk_text") & ( + sample["value"] + .apply(lambda x: x.decode("latin1")) + .str.contains("how are") + ) + assert ( + sample[filt]["entity_key"].astype(str).str.contains("doc_1") + & sample[filt]["entity_key"].astype(str).str.contains("chunk-2") + ).values[0] + + print("reading fv features") + online_python_response = self.store.get_online_features( + entity_rows=fv_entity_rows_to_read, + features=[ + "python_stored_writes_feature_view_explode_singleton:document_id", + "python_stored_writes_feature_view_explode_singleton:chunk_id", + "python_stored_writes_feature_view_explode_singleton:chunk_text", + ], + ).to_dict() + assert sorted(list(online_python_response.keys())) == sorted( + [ + "chunk_id", + "chunk_text", + "document_id", ] ) + assert online_python_response == { + "document_id": ["doc_1", "doc_2"], + "chunk_id": ["chunk-2", "chunk-1"], + "chunk_text": ["how are you?", "This is a test."], + } + + if sys.version_info[0:2] == (3, 10): + query_embedding = [0.05] * 5 + online_python_vec_response = self.store.retrieve_online_documents_v2( + features=[ + "python_stored_writes_feature_view_explode_singleton:document_id", + "python_stored_writes_feature_view_explode_singleton:chunk_id", + "python_stored_writes_feature_view_explode_singleton:chunk_text", + ], + query=query_embedding, + top_k=2, + ).to_dict() + + assert online_python_vec_response is not None + assert online_python_vec_response == { + "document_id": ["doc_1", "doc_1"], + "chunk_id": ["chunk-1", "chunk-2"], + "chunk_text": ["hello friends", "how are you?"], + "distance": [0.11180340498685837, 0.3354102075099945], + } diff --git a/setup.py b/setup.py index f20c94d5511..91af19d6a0f 100644 --- a/setup.py +++ b/setup.py @@ -110,7 +110,7 @@ "cassandra-driver>=3.24.0,<4", ] -GE_REQUIRED = ["great_expectations>=0.15.41"] +GE_REQUIRED = ["great_expectations>=0.15.41,<1"] AZURE_REQUIRED = [ "azure-storage-blob>=0.37.0", @@ -143,21 +143,31 @@ DELTA_REQUIRED = ["deltalake"] +DOCLING_REQUIRED = ["docling>=2.23.0"] + ELASTICSEARCH_REQUIRED = ["elasticsearch>=8.13.0"] SINGLESTORE_REQUIRED = ["singlestoredb<1.8.0"] -COUCHBASE_REQUIRED = ["couchbase==4.3.2"] +COUCHBASE_REQUIRED = [ + "couchbase==4.3.2", + "couchbase-columnar==1.0.0" +] MSSQL_REQUIRED = ["ibis-framework[mssql]>=9.0.0,<10"] FAISS_REQUIRED = ["faiss-cpu>=1.7.0,<2"] QDRANT_REQUIRED = ["qdrant-client>=1.12.0"] -GO_REQUIRED = ["cffi~=1.15.0"] +GO_REQUIRED = ["cffi>=1.15.0"] MILVUS_REQUIRED = ["pymilvus"] +TORCH_REQUIRED = [ + "torch>=2.2.2", + "torchvision>=0.17.2", +] + CI_REQUIRED = ( [ "build", @@ -229,8 +239,14 @@ + FAISS_REQUIRED + QDRANT_REQUIRED + MILVUS_REQUIRED + + DOCLING_REQUIRED + + TORCH_REQUIRED +) +NLP_REQUIRED = ( + DOCLING_REQUIRED + + MILVUS_REQUIRED + + TORCH_REQUIRED ) - DOCS_REQUIRED = CI_REQUIRED DEV_REQUIRED = CI_REQUIRED @@ -304,6 +320,9 @@ "qdrant": QDRANT_REQUIRED, "go": GO_REQUIRED, "milvus": MILVUS_REQUIRED, + "docling": DOCLING_REQUIRED, + "pytorch": TORCH_REQUIRED, + "nlp": NLP_REQUIRED, }, include_package_data=True, license="Apache", diff --git a/ui/README.md b/ui/README.md index a2326e1a9ef..bf9ccd367d9 100644 --- a/ui/README.md +++ b/ui/README.md @@ -77,7 +77,7 @@ The advantage of importing Feast UI as a module is in the ease of customization. ##### Fetching the Project List -You can use `projectListPromise` to provide a promise that overrides where the Feast UI fetches the project list from. +By default, the Feast UI fetches the project list from the app root path. You can use `projectListPromise` to provide a promise that overrides where it's fetched from. ```jsx { const queryClient = reactQueryClient || defaultQueryClient; + const basename = process.env.PUBLIC_URL ?? ''; return ( // Disable v7_relativeSplatPath: custom tab routes don't currently work with it - + - + diff --git a/ui/src/FeastUISansProviders.tsx b/ui/src/FeastUISansProviders.tsx index 8a12abdc39f..52676c5d0b5 100644 --- a/ui/src/FeastUISansProviders.tsx +++ b/ui/src/FeastUISansProviders.tsx @@ -40,8 +40,8 @@ interface FeastUIConfigs { projectListPromise?: Promise; } -const defaultProjectListPromise = () => { - return fetch("/projects-list.json", { +const defaultProjectListPromise = (basename: string) => { + return fetch(`${basename}/projects-list.json`, { headers: { "Content-Type": "application/json", }, @@ -51,8 +51,10 @@ const defaultProjectListPromise = () => { }; const FeastUISansProviders = ({ + basename = "", feastUIConfigs, }: { + basename?: string; feastUIConfigs?: FeastUIConfigs; }) => { const projectListContext: ProjectsListContextInterface = @@ -61,9 +63,7 @@ const FeastUISansProviders = ({ projectsListPromise: feastUIConfigs?.projectListPromise, isCustom: true, } - : { projectsListPromise: defaultProjectListPromise(), isCustom: false }; - - const BASE_URL = process.env.PUBLIC_URL || "" + : { projectsListPromise: defaultProjectListPromise(basename), isCustom: false }; return ( @@ -76,9 +76,9 @@ const FeastUISansProviders = ({ > - }> + }> } /> - }> + }> } /> } /> +interface EuiCustomLinkProps extends Omit { + to: To; +} + +const isModifiedEvent = (event: React.MouseEvent) => !!(event.metaKey || event.altKey || event.ctrlKey || event.shiftKey); -const isLeftClickEvent = (event) => event.button === 0; +const isLeftClickEvent = (event: React.MouseEvent) => event.button === 0; -const isTargetBlank = (event) => { - const target = event.target.getAttribute("target"); +const isTargetBlank = (event: React.MouseEvent) => { + const target = (event.target as Element).getAttribute("target"); return target && target !== "_self"; }; -export default function EuiCustomLink({ to, ...rest }) { +export default function EuiCustomLink({ to, ...rest }: EuiCustomLinkProps) { // This is the key! const navigate = useNavigate(); - function onClick(event) { + const onClick: React.MouseEventHandler = (event) => { if (event.defaultPrevented) { return; } @@ -41,6 +44,5 @@ export default function EuiCustomLink({ to, ...rest }) { // Generate the correct link href (with basename accounted for) const href = useHref(to); - const props = { ...rest, href, onClick }; - return ; + return ; } diff --git a/ui/src/components/FeaturesInServiceDisplay.tsx b/ui/src/components/FeaturesInServiceDisplay.tsx index bec2550a5d3..63dd447c6f3 100644 --- a/ui/src/components/FeaturesInServiceDisplay.tsx +++ b/ui/src/components/FeaturesInServiceDisplay.tsx @@ -28,10 +28,7 @@ const FeaturesInServiceList = ({ featureViews }: FeatureViewsListInterace) => { field: "featureViewName", render: (name: string) => { return ( - + {name} ); diff --git a/ui/src/components/FeaturesListDisplay.tsx b/ui/src/components/FeaturesListDisplay.tsx index 2a0628b0f56..61f57b478d6 100644 --- a/ui/src/components/FeaturesListDisplay.tsx +++ b/ui/src/components/FeaturesListDisplay.tsx @@ -21,8 +21,7 @@ const FeaturesList = ({ field: "name", render: (item: string) => ( {item} diff --git a/ui/src/components/ObjectsCountStats.tsx b/ui/src/components/ObjectsCountStats.tsx index eff3f8a2ca7..bf1dd2dc9dd 100644 --- a/ui/src/components/ObjectsCountStats.tsx +++ b/ui/src/components/ObjectsCountStats.tsx @@ -55,7 +55,7 @@ const ObjectsCountStats = () => { navigate(`${process.env.PUBLIC_URL || ""}/p/${projectName}/feature-service`)} + onClick={() => navigate(`/p/${projectName}/feature-service`)} description="Feature Services→" title={data.featureServices} reverse @@ -65,7 +65,7 @@ const ObjectsCountStats = () => { navigate(`${process.env.PUBLIC_URL || ""}/p/${projectName}/feature-view`)} + onClick={() => navigate(`/p/${projectName}/feature-view`)} title={data.featureViews} reverse /> @@ -74,7 +74,7 @@ const ObjectsCountStats = () => { navigate(`${process.env.PUBLIC_URL || ""}/p/${projectName}/entity`)} + onClick={() => navigate(`/p/${projectName}/entity`)} title={data.entities} reverse /> @@ -83,7 +83,7 @@ const ObjectsCountStats = () => { navigate(`${process.env.PUBLIC_URL || ""}/p/${projectName}/data-source`)} + onClick={() => navigate(`/p/${projectName}/data-source`)} title={data.dataSources} reverse /> diff --git a/ui/src/components/ProjectSelector.tsx b/ui/src/components/ProjectSelector.tsx index edbcf9d98fe..1bb7ebf85a7 100644 --- a/ui/src/components/ProjectSelector.tsx +++ b/ui/src/components/ProjectSelector.tsx @@ -22,7 +22,7 @@ const ProjectSelector = () => { const basicSelectId = useGeneratedHtmlId({ prefix: "basicSelect" }); const onChange = (e: React.ChangeEvent) => { - navigate(`${process.env.PUBLIC_URL || ""}/p/${e.target.value}`); + navigate(`/p/${e.target.value}`); }; return ( diff --git a/ui/src/index.tsx b/ui/src/index.tsx index 04eda8a1ba4..9cca508fcae 100644 --- a/ui/src/index.tsx +++ b/ui/src/index.tsx @@ -96,16 +96,7 @@ root.render( { - return res.json(); - }) - }} + feastUIConfigs={{ tabsRegistry }} /> ); diff --git a/ui/src/pages/RootProjectSelectionPage.tsx b/ui/src/pages/RootProjectSelectionPage.tsx index 5e19b6606b8..fb488e714bc 100644 --- a/ui/src/pages/RootProjectSelectionPage.tsx +++ b/ui/src/pages/RootProjectSelectionPage.tsx @@ -21,12 +21,12 @@ const RootProjectSelectionPage = () => { useEffect(() => { if (data && data.default) { // If a default is set, redirect there. - navigate(`${process.env.PUBLIC_URL || ""}/p/${data.default}`); + navigate(`/p/${data.default}`); } if (data && data.projects.length === 1) { // If there is only one project, redirect there. - navigate(`${process.env.PUBLIC_URL || ""}/p/${data.projects[0].id}`); + navigate(`/p/${data.projects[0].id}`); } }, [data, navigate]); @@ -38,7 +38,7 @@ const RootProjectSelectionPage = () => { title={`${item.name}`} description={item?.description || ""} onClick={() => { - navigate(`${process.env.PUBLIC_URL || ""}/p/${item.id}`); + navigate(`/p/${item.id}`); }} /> diff --git a/ui/src/pages/Sidebar.tsx b/ui/src/pages/Sidebar.tsx index de98b213242..44cde07e79d 100644 --- a/ui/src/pages/Sidebar.tsx +++ b/ui/src/pages/Sidebar.tsx @@ -53,7 +53,7 @@ const SideNav = () => { : "" }`; - const baseUrl = `${process.env.PUBLIC_URL || ""}/p/${projectName}`; + const baseUrl = `/p/${projectName}`; const sideNav: React.ComponentProps['items'] = [ { diff --git a/ui/src/pages/data-sources/DataSourcesListingTable.tsx b/ui/src/pages/data-sources/DataSourcesListingTable.tsx index e4f06d6bd0a..fd1ff73deb7 100644 --- a/ui/src/pages/data-sources/DataSourcesListingTable.tsx +++ b/ui/src/pages/data-sources/DataSourcesListingTable.tsx @@ -20,10 +20,7 @@ const DatasourcesListingTable = ({ sortable: true, render: (name: string) => { return ( - + {name} ); diff --git a/ui/src/pages/entities/EntitiesListingTable.tsx b/ui/src/pages/entities/EntitiesListingTable.tsx index baf4ddb8e47..06190409b04 100644 --- a/ui/src/pages/entities/EntitiesListingTable.tsx +++ b/ui/src/pages/entities/EntitiesListingTable.tsx @@ -20,10 +20,7 @@ const EntitiesListingTable = ({ entities }: EntitiesListingTableProps) => { sortable: true, render: (name: string) => { return ( - + {name} ); diff --git a/ui/src/pages/entities/FeatureViewEdgesList.tsx b/ui/src/pages/entities/FeatureViewEdgesList.tsx index 8a0b6164b49..3419bfcb4b7 100644 --- a/ui/src/pages/entities/FeatureViewEdgesList.tsx +++ b/ui/src/pages/entities/FeatureViewEdgesList.tsx @@ -53,10 +53,7 @@ const FeatureViewEdgesList = ({ fvNames }: FeatureViewEdgesListInterace) => { field: "", render: ({ name }: { name: string }) => { return ( - + {name} ); diff --git a/ui/src/pages/feature-services/FeatureServiceListingTable.tsx b/ui/src/pages/feature-services/FeatureServiceListingTable.tsx index 13ffa764092..69d4d1f969d 100644 --- a/ui/src/pages/feature-services/FeatureServiceListingTable.tsx +++ b/ui/src/pages/feature-services/FeatureServiceListingTable.tsx @@ -30,10 +30,7 @@ const FeatureServiceListingTable = ({ field: "spec.name", render: (name: string) => { return ( - + {name} ); diff --git a/ui/src/pages/feature-services/FeatureServiceOverviewTab.tsx b/ui/src/pages/feature-services/FeatureServiceOverviewTab.tsx index 4d3d350f084..fcb1dc018b3 100644 --- a/ui/src/pages/feature-services/FeatureServiceOverviewTab.tsx +++ b/ui/src/pages/feature-services/FeatureServiceOverviewTab.tsx @@ -109,7 +109,7 @@ const FeatureServiceOverviewTab = () => { tags={data.spec.tags} createLink={(key, value) => { return ( - `${process.env.PUBLIC_URL || ""}/p/${projectName}/feature-service?` + + `/p/${projectName}/feature-service?` + encodeSearchQueryString(`${key}:${value}`) ); }} @@ -133,7 +133,7 @@ const FeatureServiceOverviewTab = () => { color="primary" onClick={() => { navigate( - `${process.env.PUBLIC_URL || ""}/p/${projectName}/entity/${entity.name}` + `/p/${projectName}/entity/${entity.name}` ); }} onClickAriaLabel={entity.name} diff --git a/ui/src/pages/feature-views/ConsumingFeatureServicesList.tsx b/ui/src/pages/feature-views/ConsumingFeatureServicesList.tsx index bb9961c19ca..603a4d96ba4 100644 --- a/ui/src/pages/feature-views/ConsumingFeatureServicesList.tsx +++ b/ui/src/pages/feature-views/ConsumingFeatureServicesList.tsx @@ -18,10 +18,7 @@ const ConsumingFeatureServicesList = ({ field: "", render: ({ name }: { name: string }) => { return ( - + {name} ); diff --git a/ui/src/pages/feature-views/FeatureViewListingTable.tsx b/ui/src/pages/feature-views/FeatureViewListingTable.tsx index ff1a31c4162..02756492c91 100644 --- a/ui/src/pages/feature-views/FeatureViewListingTable.tsx +++ b/ui/src/pages/feature-views/FeatureViewListingTable.tsx @@ -31,10 +31,7 @@ const FeatureViewListingTable = ({ sortable: true, render: (name: string, item: genericFVType) => { return ( - + {name} {(item.type === "ondemand" && ondemand) || (item.type === "stream" && stream)} ); diff --git a/ui/src/pages/feature-views/RegularFeatureViewOverviewTab.tsx b/ui/src/pages/feature-views/RegularFeatureViewOverviewTab.tsx index cde4f46d4ed..3bbb906e05b 100644 --- a/ui/src/pages/feature-views/RegularFeatureViewOverviewTab.tsx +++ b/ui/src/pages/feature-views/RegularFeatureViewOverviewTab.tsx @@ -96,7 +96,7 @@ const RegularFeatureViewOverviewTab = ({ { - navigate(`${process.env.PUBLIC_URL || ""}/p/${projectName}/entity/${entity}`); + navigate(`/p/${projectName}/entity/${entity}`); }} onClickAriaLabel={entity} data-test-sub="testExample1" @@ -134,7 +134,7 @@ const RegularFeatureViewOverviewTab = ({ tags={data.spec.tags} createLink={(key, value) => { return ( - `${process.env.PUBLIC_URL || ""}/p/${projectName}/feature-view?` + + `/p/${projectName}/feature-view?` + encodeSearchQueryString(`${key}:${value}`) ); }} diff --git a/ui/src/pages/feature-views/StreamFeatureViewOverviewTab.tsx b/ui/src/pages/feature-views/StreamFeatureViewOverviewTab.tsx index b4514a5edd5..9aff3d59f3f 100644 --- a/ui/src/pages/feature-views/StreamFeatureViewOverviewTab.tsx +++ b/ui/src/pages/feature-views/StreamFeatureViewOverviewTab.tsx @@ -96,8 +96,7 @@ const StreamFeatureViewOverviewTab = ({ {inputGroup?.name} diff --git a/ui/src/pages/feature-views/components/FeatureViewProjectionDisplayPanel.tsx b/ui/src/pages/feature-views/components/FeatureViewProjectionDisplayPanel.tsx index 2a68cc49b51..104ef0f93be 100644 --- a/ui/src/pages/feature-views/components/FeatureViewProjectionDisplayPanel.tsx +++ b/ui/src/pages/feature-views/components/FeatureViewProjectionDisplayPanel.tsx @@ -31,8 +31,7 @@ const FeatureViewProjectionDisplayPanel = (featureViewProjection: RequestDataDis {featureViewProjection?.featureViewName} diff --git a/ui/src/pages/feature-views/components/RequestDataDisplayPanel.tsx b/ui/src/pages/feature-views/components/RequestDataDisplayPanel.tsx index 8ec973c3dad..6893dfd6a32 100644 --- a/ui/src/pages/feature-views/components/RequestDataDisplayPanel.tsx +++ b/ui/src/pages/feature-views/components/RequestDataDisplayPanel.tsx @@ -39,8 +39,7 @@ const RequestDataDisplayPanel = ({ {requestDataSource?.name} diff --git a/ui/src/pages/features/FeatureOverviewTab.tsx b/ui/src/pages/features/FeatureOverviewTab.tsx index cc7879b0383..eb101fe3955 100644 --- a/ui/src/pages/features/FeatureOverviewTab.tsx +++ b/ui/src/pages/features/FeatureOverviewTab.tsx @@ -63,8 +63,8 @@ const FeatureOverviewTab = () => { FeatureView + to={`/p/${projectName}/feature-view/${FeatureViewName}`} + > {FeatureViewName} diff --git a/ui/src/pages/saved-data-sets/DatasetsListingTable.tsx b/ui/src/pages/saved-data-sets/DatasetsListingTable.tsx index af794a35f98..7b73e9cd6dc 100644 --- a/ui/src/pages/saved-data-sets/DatasetsListingTable.tsx +++ b/ui/src/pages/saved-data-sets/DatasetsListingTable.tsx @@ -19,10 +19,7 @@ const DatasetsListingTable = ({ datasets }: DatasetsListingTableProps) => { sortable: true, render: (name: string) => { return ( - + {name} );