diff --git a/.github/workflows/pr_local_integration_tests.yml b/.github/workflows/pr_local_integration_tests.yml index 0787b8a3f2c..143cfe40973 100644 --- a/.github/workflows/pr_local_integration_tests.yml +++ b/.github/workflows/pr_local_integration_tests.yml @@ -42,6 +42,13 @@ jobs: enable-cache: true - name: Install dependencies run: make install-python-dependencies-ci + - name: Cache Hadoop tarball + uses: actions/cache@v4 + with: + path: ~/hadoop-3.4.2.tar.gz + key: hadoop-3.4.2 + - name: Install Hadoop dependencies + run: make install-hadoop-dependencies-ci - name: Test local integration tests if: ${{ always() }} # this will guarantee that step won't be canceled and resources won't leak run: make test-python-integration-local diff --git a/CHANGELOG.md b/CHANGELOG.md index 7b2cf7a551c..374e436b882 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,33 @@ # Changelog +# [0.56.0](https://github.com/feast-dev/feast/compare/v0.55.0...v0.56.0) (2025-10-27) + + +### Bug Fixes + +* Add mode field to Transformation proto for proper serialization ([2390d2e](https://github.com/feast-dev/feast/commit/2390d2ea654e299fc74f697212542b755f3b4938)) +* Date wise remote offline store historical data retrieval ([#5686](https://github.com/feast-dev/feast/issues/5686)) ([949ba3d](https://github.com/feast-dev/feast/commit/949ba3dae420f82f493018113d1fd6de9e130a56)) +* Fix STRING type handling in on-demand feature views ([#5669](https://github.com/feast-dev/feast/issues/5669)) ([dfbb743](https://github.com/feast-dev/feast/commit/dfbb7433f059e6f0d1d4ef6c0ef65b63dac1c1ff)) +* Fixed torch install issue in CI ([366e5a8](https://github.com/feast-dev/feast/commit/366e5a8c8f8093eda840b667849c6d2e45fa56bb)) +* ODFV not getting counted in resource count ([1d640b6](https://github.com/feast-dev/feast/commit/1d640b6c8136c47a78887e4490a5b7ae677b7c99)) +* Skip tag updates if user do not have permissions ([#5673](https://github.com/feast-dev/feast/issues/5673)) ([0a951ce](https://github.com/feast-dev/feast/commit/0a951ce8d7f9b31490fa279339eacd444d2d2434)) + + +### Features + +* Add document of Go feature server. ([#5697](https://github.com/feast-dev/feast/issues/5697)) ([cbd1dde](https://github.com/feast-dev/feast/commit/cbd1dde9a0a6e5a3ec7e3520b6613d3818bcd842)) +* Add flexible commandArgs support for complete Feast CLI control ([#5678](https://github.com/feast-dev/feast/issues/5678)) ([6414924](https://github.com/feast-dev/feast/commit/64149246c1925e9f3dcac60d9ab629225c232261)) +* Add HDFS as a feature registry ([#5655](https://github.com/feast-dev/feast/issues/5655)) ([4c65872](https://github.com/feast-dev/feast/commit/4c65872ee6cf7e14ed14c8a8a7e141126027e575)) +* Add nodeSelector to service config ([#5675](https://github.com/feast-dev/feast/issues/5675)) ([9728cde](https://github.com/feast-dev/feast/commit/9728cde4d3cf4d22a790d3a3af2eba705b7a56d3)) +* Add OTEL based observability to the Go Feature Server ([#5685](https://github.com/feast-dev/feast/issues/5685)) ([f4afdad](https://github.com/feast-dev/feast/commit/f4afdad27c7fe92e9778e29ad08e4b227a3c17a4)) +* Added health endpoint for the UI ([#5665](https://github.com/feast-dev/feast/issues/5665)) ([3aec5d5](https://github.com/feast-dev/feast/commit/3aec5d5fd24540d10f07e79c081c2658ca35678c)) +* Added kuberay support ([e0b698d](https://github.com/feast-dev/feast/commit/e0b698d7b8733c8177ca053bc89defb01ebeb538)) +* Added support for filtering multi-projects ([#5688](https://github.com/feast-dev/feast/issues/5688)) ([eb0a86e](https://github.com/feast-dev/feast/commit/eb0a86eb81defb5ccb2407a0d5f2b2425bcb61c1)) +* Batch Embedding at scale for RAG with Ray ([cc2a46d](https://github.com/feast-dev/feast/commit/cc2a46d54c413ed52a9bf568588dd06096592c1f)) +* Optimize SQL entity handling without creating temporary tables ([#5695](https://github.com/feast-dev/feast/issues/5695)) ([aa2c838](https://github.com/feast-dev/feast/commit/aa2c8386253181145c4f314187e0873b96b2be59)) +* Support aggregation in odfv ([#5666](https://github.com/feast-dev/feast/issues/5666)) ([564e965](https://github.com/feast-dev/feast/commit/564e9651dabea5458a77a8889920749cb1a6a5ed)) +* Support cache_mode for registries ([021e9ea](https://github.com/feast-dev/feast/commit/021e9ea759bfee0292c5f7c804119ed9a15d6a58)) + # [0.55.0](https://github.com/feast-dev/feast/compare/v0.54.0...v0.55.0) (2025-10-14) diff --git a/Makefile b/Makefile index e1d10404ded..6dc410b83b5 100644 --- a/Makefile +++ b/Makefile @@ -83,11 +83,28 @@ install-python-dependencies-minimal: ## Install minimal Python dependencies usin install-python-dependencies-ci: ## Install Python CI dependencies in system environment using uv # Install CPU-only torch first to prevent CUDA dependency issues pip uninstall torch torchvision -y || true - pip install torch torchvision --index-url https://download.pytorch.org/whl/cpu --force-reinstall - uv pip sync --system sdk/python/requirements/py$(PYTHON_VERSION)-ci-requirements.txt + @if [ "$$(uname -s)" = "Linux" ]; then \ + echo "Installing dependencies with torch CPU index for Linux..."; \ + uv pip sync --system --extra-index-url https://download.pytorch.org/whl/cpu --index-strategy unsafe-best-match sdk/python/requirements/py$(PYTHON_VERSION)-ci-requirements.txt; \ + else \ + echo "Installing dependencies from PyPI for macOS..."; \ + uv pip sync --system sdk/python/requirements/py$(PYTHON_VERSION)-ci-requirements.txt; \ + fi uv pip install --system --no-deps -e . -# Used by multicloud/Dockerfile.dev +# Used in github actions/ci +install-hadoop-dependencies-ci: ## Install Hadoop dependencies + @if [ ! -f $$HOME/hadoop-3.4.2.tar.gz ]; then \ + echo "Downloading Hadoop tarball..."; \ + wget -q https://dlcdn.apache.org/hadoop/common/hadoop-3.4.2/hadoop-3.4.2.tar.gz -O $$HOME/hadoop-3.4.2.tar.gz; \ + else \ + echo "Using cached Hadoop tarball"; \ + fi + @if [ ! -d $$HOME/hadoop ]; then \ + echo "Extracting Hadoop tarball..."; \ + tar -xzf $$HOME/hadoop-3.4.2.tar.gz -C $$HOME; \ + mv $$HOME/hadoop-3.4.2 $$HOME/hadoop; \ + fi install-python-ci-dependencies: ## Install Python CI dependencies in system environment using piptools python -m piptools sync sdk/python/requirements/py$(PYTHON_VERSION)-ci-requirements.txt pip install --no-deps -e . @@ -146,6 +163,9 @@ test-python-integration: ## Run Python integration tests (CI) test-python-integration-local: ## Run Python integration tests (local dev mode) FEAST_IS_LOCAL_TEST=True \ FEAST_LOCAL_ONLINE_CONTAINER=True \ + HADOOP_HOME=$$HOME/hadoop \ + CLASSPATH="$$( $$HADOOP_HOME/bin/hadoop classpath --glob ):$$CLASSPATH" \ + HADOOP_USER_NAME=root \ python -m pytest --tb=short -v -n 8 --color=yes --integration --durations=10 --timeout=1200 --timeout_method=thread --dist loadgroup \ -k "not test_lambda_materialization and not test_snowflake_materialization" \ -m "not rbac_remote_integration_test" \ diff --git a/docs/README.md b/docs/README.md index 02ecaefa10c..172111964b7 100644 --- a/docs/README.md +++ b/docs/README.md @@ -52,6 +52,11 @@ Feast helps ML platform/MLOps teams with DevOps experience productionize real-ti * *For AI Engineers*: Feast provides a platform designed to scale your AI applications by enabling seamless integration of richer data and facilitating fine-tuning. With Feast, you can optimize the performance of your AI models while ensuring a scalable and efficient data pipeline. + +![](assets/feast_persona_diagram.png) + + + ## What Feast is not? ### Feast is not diff --git a/docs/assets/feast_persona_diagram.png b/docs/assets/feast_persona_diagram.png new file mode 100644 index 00000000000..005902a68c2 Binary files /dev/null and b/docs/assets/feast_persona_diagram.png differ diff --git a/docs/reference/beta-on-demand-feature-view.md b/docs/reference/beta-on-demand-feature-view.md index f45620a1cee..2482bbc1c8f 100644 --- a/docs/reference/beta-on-demand-feature-view.md +++ b/docs/reference/beta-on-demand-feature-view.md @@ -35,10 +35,40 @@ When defining an ODFV, you can specify the transformation mode using the `mode` ### Singleton Transformations in Native Python Mode -Native Python mode supports transformations on singleton dictionaries by setting `singleton=True`. This allows you to -write transformation functions that operate on a single row at a time, making the code more intuitive and aligning with +Native Python mode supports transformations on singleton dictionaries by setting `singleton=True`. This allows you to +write transformation functions that operate on a single row at a time, making the code more intuitive and aligning with how data scientists typically think about data transformations. +## Aggregations + +On Demand Feature Views support aggregations that compute aggregate statistics over groups of rows. When using aggregations, data is grouped by entity columns (e.g., `driver_id`) and aggregated before being passed to the transformation function. + +**Important**: Aggregations and transformations are mutually exclusive. When aggregations are specified, they replace the transformation function. + +### Usage + +```python +from feast import Aggregation +from datetime import timedelta + +@on_demand_feature_view( + sources=[driver_hourly_stats_view], + schema=[ + Field(name="total_trips", dtype=Int64), + Field(name="avg_rating", dtype=Float64), + ], + aggregations=[ + Aggregation(column="trips", function="sum"), + Aggregation(column="rating", function="mean"), + ], +) +def driver_aggregated_stats(inputs): + # No transformation function needed when using aggregations + pass +``` + +Aggregated columns are automatically named using the pattern `{function}_{column}` (e.g., `sum_trips`, `mean_rating`). + ## Example See [https://github.com/feast-dev/on-demand-feature-views-demo](https://github.com/feast-dev/on-demand-feature-views-demo) for an example on how to use on demand feature views. diff --git a/docs/reference/compute-engine/ray.md b/docs/reference/compute-engine/ray.md index 4ecc449e40b..5547901b873 100644 --- a/docs/reference/compute-engine/ray.md +++ b/docs/reference/compute-engine/ray.md @@ -62,11 +62,22 @@ batch_engine: | `max_parallelism_multiplier` | int | 2 | Parallelism as multiple of CPU cores | | `target_partition_size_mb` | int | 64 | Target partition size (MB) | | `window_size_for_joins` | string | "1H" | Time window for distributed joins | -| `ray_address` | string | None | Ray cluster address (None = local Ray) | +| `ray_address` | string | None | Ray cluster address (triggers REMOTE mode) | +| `use_kuberay` | boolean | None | Enable KubeRay mode (overrides ray_address) | +| `kuberay_conf` | dict | None | **KubeRay configuration dict** with keys: `cluster_name` (required), `namespace` (default: "default"), `auth_token`, `auth_server`, `skip_tls` (default: false) | +| `enable_ray_logging` | boolean | false | Enable Ray progress bars and logging | | `enable_distributed_joins` | boolean | true | Enable distributed joins for large datasets | | `staging_location` | string | None | Remote path for batch materialization jobs | -| `ray_conf` | dict | None | Ray configuration parameters | -| `execution_timeout_seconds` | int | None | Timeout for job execution in seconds | +| `ray_conf` | dict | None | Ray configuration parameters (memory, CPU limits) | + +### Mode Detection Precedence + +The Ray compute engine automatically detects the execution mode: + +1. **Environment Variables** → KubeRay mode (if `FEAST_RAY_USE_KUBERAY=true`) +2. **Config `kuberay_conf`** → KubeRay mode +3. **Config `ray_address`** → Remote mode +4. **Default** → Local mode ## Usage Examples diff --git a/docs/reference/data-sources/trino.md b/docs/reference/data-sources/trino.md index 74c8c3cc956..5448726533c 100644 --- a/docs/reference/data-sources/trino.md +++ b/docs/reference/data-sources/trino.md @@ -20,8 +20,8 @@ from feast.infra.offline_stores.contrib.trino_offline_store.trino_source import ) driver_hourly_stats = TrinoSource( - event_timestamp_column="event_timestamp", - table_ref="feast.driver_stats", + timestamp_field="event_timestamp", + table="feast.driver_stats", created_timestamp_column="created", ) ``` diff --git a/docs/reference/feature-servers/go-feature-server.md b/docs/reference/feature-servers/go-feature-server.md new file mode 100644 index 00000000000..fafe3fa917f --- /dev/null +++ b/docs/reference/feature-servers/go-feature-server.md @@ -0,0 +1,54 @@ +# Go feature server (Alpha) + +## Overview +The Go feature server is an HTTP/gRPC endpoint that serves features. It is written in Go. + +## Configuration of `feature_store.yaml` +The current Go feature server needs a Python based feature Transformation service support. Please refer to the following code as an example: +``` +# -*- coding: utf-8 -*- +from feast.feature_store import FeatureStore + + +def main(): + # Init the Feature Store + store = FeatureStore(repo_path="./feature_repo/") + + # Start the feature transformation server + # default port is 6569 + store.serve_transformations(6569) + +if __name__ == "__main__": + main() +``` +At the same time, we need to configure the `feature_store.yaml` as following: + +``` +... +entity_key_serialization_version: 3 +feature_server: + type: local + transformation_service_endpoint: "localhost:6569" +... +``` +## Supported APIs +Here is the list of supported APIs: +| Method | API | Comment | +|:---: | :---: | :---: | +| POST | /get-online-features | Retrieve features of one or many entities | +| GET | /health | Status of the Go Feature Server | + +## OTEL based Observability +The Go feature server support [OTEL](https://opentelemetry.io/) based Observabilities. +To enable it, we need to set the global env `ENABLE_OTEL_TRACING` to `"true"` (as a string type!) in the container or your local OS. +``` +export ENABLE_OTEL_TRACING='true' +``` +There are example OTEL infra setup under the `/go/infra/docker/otel` folder. + +## Demo +Please check the Reference[2] for a local demo of Go feature server. If you want to see a real world example of applying Go feature server in Production, please check Reference[1]. + +## Reference +1. [Expedia Group's Go Feature Server Implementation (in Production)](https://github.com/EXPEbdodla/feast) +2. [A Go Feature server demo from Feast](https://github.com/feast-dev/feast-credit-score-local-tutorial) \ No newline at end of file diff --git a/docs/reference/offline-stores/ray.md b/docs/reference/offline-stores/ray.md index 58f62c34ece..a46102ee132 100644 --- a/docs/reference/offline-stores/ray.md +++ b/docs/reference/offline-stores/ray.md @@ -9,10 +9,11 @@ The Ray offline store is a data I/O implementation that leverages [Ray](https:// The Ray offline store provides: - Ray-based data reading from file sources (Parquet, CSV, etc.) -- Support for both local and distributed Ray clusters +- Support for local, remote, and KubeRay (Kubernetes-managed) clusters - Integration with various storage backends (local files, S3, GCS, HDFS) - Efficient data filtering and column selection - Timestamp-based data processing with timezone awareness +- Enterprise-ready KubeRay cluster support via CodeFlare SDK ## Functionality Matrix @@ -59,9 +60,15 @@ For complex feature processing, historical feature retrieval, and distributed jo ## Configuration -The Ray offline store can be configured in your `feature_store.yaml` file. Below are two main configuration patterns: +The Ray offline store can be configured in your `feature_store.yaml` file. It supports **three execution modes**: -### Basic Ray Offline Store +1. **LOCAL**: Ray runs locally on the same machine (default) +2. **REMOTE**: Connects to a remote Ray cluster via `ray_address` +3. **KUBERAY**: Connects to Ray clusters on Kubernetes via CodeFlare SDK + +### Execution Modes + +#### Local Mode (Default) For simple data I/O operations without distributed processing: @@ -72,7 +79,44 @@ provider: local offline_store: type: ray storage_path: data/ray_storage # Optional: Path for storing datasets - ray_address: localhost:10001 # Optional: Ray cluster address +``` + +#### Remote Ray Cluster + +Connect to an existing Ray cluster: + +```yaml +offline_store: + type: ray + storage_path: s3://my-bucket/feast-data + ray_address: "ray://my-cluster.example.com:10001" +``` + +#### KubeRay Cluster (Kubernetes) + +Connect to Ray clusters on Kubernetes using CodeFlare SDK: + +```yaml +offline_store: + type: ray + storage_path: s3://my-bucket/feast-data + use_kuberay: true + kuberay_conf: + cluster_name: "feast-ray-cluster" + namespace: "feast-system" + auth_token: "${RAY_AUTH_TOKEN}" + auth_server: "https://api.openshift.com:6443" + skip_tls: false + enable_ray_logging: false +``` + +**Environment Variables** (alternative to config file): +```bash +export FEAST_RAY_USE_KUBERAY=true +export FEAST_RAY_CLUSTER_NAME=feast-ray-cluster +export FEAST_RAY_AUTH_TOKEN=your-token +export FEAST_RAY_AUTH_SERVER=https://api.openshift.com:6443 +export FEAST_RAY_NAMESPACE=feast-system ``` ### Ray Offline Store + Compute Engine @@ -175,8 +219,29 @@ batch_engine: |--------|------|---------|-------------| | `type` | string | Required | Must be `feast.offline_stores.contrib.ray_offline_store.ray.RayOfflineStore` or `ray` | | `storage_path` | string | None | Path for storing temporary files and datasets | -| `ray_address` | string | None | Address of the Ray cluster (e.g., "localhost:10001") | +| `ray_address` | string | None | Ray cluster address (triggers REMOTE mode, e.g., "ray://host:10001") | +| `use_kuberay` | boolean | None | Enable KubeRay mode (overrides ray_address) | +| `kuberay_conf` | dict | None | **KubeRay configuration dict** with keys: `cluster_name` (required), `namespace` (default: "default"), `auth_token`, `auth_server`, `skip_tls` (default: false) | +| `enable_ray_logging` | boolean | false | Enable Ray progress bars and verbose logging | | `ray_conf` | dict | None | Ray initialization parameters for resource management (e.g., memory, CPU limits) | +| `broadcast_join_threshold_mb` | int | 100 | Size threshold for broadcast joins (MB) | +| `enable_distributed_joins` | boolean | true | Enable distributed joins for large datasets | +| `max_parallelism_multiplier` | int | 2 | Parallelism as multiple of CPU cores | +| `target_partition_size_mb` | int | 64 | Target partition size (MB) | +| `window_size_for_joins` | string | "1H" | Time window for distributed joins | + +#### Mode Detection Precedence + +The Ray offline store automatically detects the execution mode using the following precedence: + +1. **Environment Variables** (highest priority) + - `FEAST_RAY_USE_KUBERAY`, `FEAST_RAY_CLUSTER_NAME`, etc. +2. **Config `kuberay_conf`** + - If present → KubeRay mode +3. **Config `ray_address`** + - If present → Remote mode +4. **Default** + - Local mode (lowest priority) #### Ray Compute Engine Options @@ -385,6 +450,8 @@ job.persist(hdfs_storage, allow_overwrite=True) ### Using Ray Cluster +#### Standard Ray Cluster + To use Ray in cluster mode for distributed data access: 1. Start a Ray cluster: @@ -406,6 +473,53 @@ offline_store: ray start --address='head-node-ip:10001' ``` +#### KubeRay Cluster (Kubernetes) + +To use Feast with Ray clusters on Kubernetes via CodeFlare SDK: + +**Prerequisites:** +- KubeRay cluster deployed on Kubernetes +- CodeFlare SDK installed: `pip install codeflare-sdk` +- Access credentials for the Kubernetes cluster + +**Configuration:** + +1. Using configuration file: +```yaml +offline_store: + type: ray + use_kuberay: true + storage_path: s3://my-bucket/feast-data + kuberay_conf: + cluster_name: "feast-ray-cluster" + namespace: "feast-system" + auth_token: "${RAY_AUTH_TOKEN}" + auth_server: "https://api.openshift.com:6443" + skip_tls: false + enable_ray_logging: false +``` + +2. Using environment variables: +```bash +export FEAST_RAY_USE_KUBERAY=true +export FEAST_RAY_CLUSTER_NAME=feast-ray-cluster +export FEAST_RAY_AUTH_TOKEN=your-k8s-token +export FEAST_RAY_AUTH_SERVER=https://api.openshift.com:6443 +export FEAST_RAY_NAMESPACE=feast-system +export FEAST_RAY_SKIP_TLS=false + +# Then use standard Feast code +python your_feast_script.py +``` + +**Features:** +- The CodeFlare SDK handles cluster connection and authentication +- Automatic TLS certificate management +- Authentication with Kubernetes clusters +- Namespace isolation +- Secure communication between client and Ray cluster +- Automatic cluster discovery + ### Data Source Validation The Ray offline store validates data sources to ensure compatibility: diff --git a/docs/reference/registries/README.md b/docs/reference/registries/README.md index ac0f58e6135..01671cf2212 100644 --- a/docs/reference/registries/README.md +++ b/docs/reference/registries/README.md @@ -26,6 +26,10 @@ Please see [Registry](../../getting-started/components/registry.md) for a concep [snowflake.md](snowflake.md) {% endcontent-ref %} +{% content-ref url="hdfs.md" %} +[hdfs.md](hdfs.md) +{% endcontent-ref %} + {% content-ref url="remote.md" %} [remote.md](remote.md) {% endcontent-ref %} diff --git a/docs/reference/registries/hdfs.md b/docs/reference/registries/hdfs.md new file mode 100644 index 00000000000..c6f6b641aff --- /dev/null +++ b/docs/reference/registries/hdfs.md @@ -0,0 +1,42 @@ +# HDFS Registry + +## Description + +HDFS registry provides support for storing the protobuf representation of your feature store objects (data sources, feature views, feature services, etc.) in Hadoop Distributed File System (HDFS). + +While it can be used in production, there are still inherent limitations with a file-based registries, since changing a single field in the registry requires re-writing the whole registry file. With multiple concurrent writers, this presents a risk of data loss, or bottlenecks writes to the registry since all changes have to be serialized (e.g. when running materialization for multiple feature views or time ranges concurrently). + +### Pre-requisites + +The HDFS registry requires Hadoop 3.3+ to be installed and the `HADOOP_HOME` environment variable set. + +### Authentication and User Configuration + +The HDFS registry is using `pyarrow.fs.HadoopFileSystem` and **does not** support specifying HDFS users or Kerberos credentials directly in the `feature_store.yaml` configuration. It relies entirely on the Hadoop and system environment configuration available to the process running Feast. + +By default, `pyarrow.fs.HadoopFileSystem` inherits authentication from the underlying Hadoop client libraries and environment variables, such as: + +- `HADOOP_USER_NAME` +- `KRB5CCNAME` +- `hadoop.security.authentication` +- Any other relevant properties in `core-site.xml` and `hdfs-site.xml` + +For more information, refer to: +- [pyarrow.fs.HadoopFileSystem API Reference](https://arrow.apache.org/docs/python/generated/pyarrow.fs.HadoopFileSystem.html) +- [Hadoop Security: Simple & Kerberos Authentication](https://hadoop.apache.org/docs/stable/hadoop-project-dist/hadoop-common/SecureMode.html) + +## Example + +An example of how to configure this would be: + +{% code title="feature_store.yaml" %} +```yaml +project: feast_hdfs +registry: + path: hdfs://[YOUR NAMENODE HOST]:[YOUR NAMENODE PORT]/[PATH TO REGISTRY]/registry.pb + cache_ttl_seconds: 60 +online_store: null +offline_store: null +``` +{% endcode %} + diff --git a/go.mod b/go.mod index 91ab0c05559..a097aa67719 100644 --- a/go.mod +++ b/go.mod @@ -1,8 +1,8 @@ module github.com/feast-dev/feast -go 1.23 +go 1.24.0 -toolchain go1.23.12 +toolchain go1.24.4 require ( github.com/apache/arrow/go/v17 v17.0.0 @@ -19,10 +19,16 @@ require ( github.com/roberson-io/mmh3 v0.0.0-20190729202758-fdfce3ba6225 github.com/rs/zerolog v1.33.0 github.com/spaolacci/murmur3 v1.1.0 - github.com/stretchr/testify v1.9.0 - google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 - google.golang.org/grpc v1.67.0 - google.golang.org/protobuf v1.34.2 + github.com/stretchr/testify v1.11.1 + go.opentelemetry.io/otel v1.38.0 + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.38.0 + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.38.0 + go.opentelemetry.io/otel/sdk v1.38.0 + go.opentelemetry.io/otel/trace v1.38.0 + golang.org/x/sync v0.17.0 + google.golang.org/genproto/googleapis/rpc v0.0.0-20250825161204-c5933d9347a5 + google.golang.org/grpc v1.75.0 + google.golang.org/protobuf v1.36.8 ) require ( @@ -45,12 +51,16 @@ require ( github.com/aws/aws-sdk-go-v2/service/ssooidc v1.30.1 // indirect github.com/aws/aws-sdk-go-v2/service/sts v1.33.19 // indirect github.com/aws/smithy-go v1.22.2 // indirect + github.com/cenkalti/backoff/v5 v5.0.3 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect + github.com/go-logr/logr v1.4.3 // indirect + github.com/go-logr/stdr v1.2.2 // indirect github.com/goccy/go-json v0.10.3 // indirect github.com/golang/snappy v0.0.4 // indirect github.com/google/flatbuffers v24.3.25+incompatible // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.2 // indirect github.com/klauspost/asmfmt v1.3.2 // indirect github.com/klauspost/compress v1.17.9 // indirect github.com/klauspost/cpuid/v2 v2.2.8 // indirect @@ -62,14 +72,17 @@ require ( github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/stretchr/objx v0.5.2 // indirect github.com/zeebo/xxh3 v1.0.2 // indirect + go.opentelemetry.io/auto/sdk v1.1.0 // indirect + go.opentelemetry.io/otel/metric v1.38.0 // indirect + go.opentelemetry.io/proto/otlp v1.7.1 // indirect golang.org/x/exp v0.0.0-20240909161429-701f63a606c0 // indirect - golang.org/x/mod v0.21.0 // indirect - golang.org/x/net v0.29.0 // indirect - golang.org/x/sync v0.8.0 // indirect - golang.org/x/sys v0.25.0 // indirect - golang.org/x/text v0.18.0 // indirect - golang.org/x/tools v0.25.0 // indirect + golang.org/x/mod v0.26.0 // indirect + golang.org/x/net v0.43.0 // indirect + golang.org/x/sys v0.35.0 // indirect + golang.org/x/text v0.28.0 // indirect + golang.org/x/tools v0.35.0 // indirect golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20250825161204-c5933d9347a5 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index 7778a906ecd..f2530758915 100644 --- a/go.sum +++ b/go.sum @@ -6,8 +6,6 @@ github.com/apache/arrow/go/v17 v17.0.0 h1:RRR2bdqKcdbss9Gxy2NS/hK8i4LDMh23L6BbkN github.com/apache/arrow/go/v17 v17.0.0/go.mod h1:jR7QHkODl15PfYyjM2nU+yTLScZ/qfj7OSUZmJ8putc= github.com/apache/thrift v0.21.0 h1:tdPmh/ptjE1IJnhbhrcl2++TauVjy242rkV/UzJChnE= github.com/apache/thrift v0.21.0/go.mod h1:W1H8aR/QRtYNvrPeFXBtobyRkd0/YVhTc6i07XIAgDw= -github.com/aws/aws-sdk-go-v2 v1.36.3 h1:mJoei2CxPutQVxaATCzDUjcZEjVRdpsiiXi2o38yqWM= -github.com/aws/aws-sdk-go-v2 v1.36.3/go.mod h1:LLXuLpgzEbD766Z5ECcRmi8AzSwfZItDtmABVkRLGzg= github.com/aws/aws-sdk-go-v2 v1.36.4 h1:GySzjhVvx0ERP6eyfAbAuAXLtAda5TEy19E5q5W8I9E= github.com/aws/aws-sdk-go-v2 v1.36.4/go.mod h1:LLXuLpgzEbD766Z5ECcRmi8AzSwfZItDtmABVkRLGzg= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.10 h1:zAybnyUQXIZ5mok5Jqwlf58/TFE7uvd3IAsa1aF9cXs= @@ -18,12 +16,8 @@ github.com/aws/aws-sdk-go-v2/credentials v1.17.67 h1:9KxtdcIA/5xPNQyZRgUSpYOE6j9 github.com/aws/aws-sdk-go-v2/credentials v1.17.67/go.mod h1:p3C44m+cfnbv763s52gCqrjaqyPikj9Sg47kUVaNZQQ= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.30 h1:x793wxmUWVDhshP8WW2mlnXuFrO4cOd3HLBroh1paFw= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.30/go.mod h1:Jpne2tDnYiFascUEs2AWHJL9Yp7A5ZVy3TNyxaAjD6M= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.34 h1:ZK5jHhnrioRkUNOc+hOgQKlUL5JeC3S6JgLxtQ+Rm0Q= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.34/go.mod h1:p4VfIceZokChbA9FzMbRGz5OV+lekcVtHlPKEO0gSZY= github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.35 h1:o1v1VFfPcDVlK3ll1L5xHsaQAFdNtZ5GXnNR7SwueC4= github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.35/go.mod h1:rZUQNYMNG+8uZxz9FOerQJ+FceCiodXvixpeRtdESrU= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.34 h1:SZwFm17ZUNNg5Np0ioo/gq8Mn6u9w19Mri8DnJ15Jf0= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.34/go.mod h1:dFZsC0BLo346mvKQLWmoJxT+Sjp+qcVR1tRVHQGOH9Q= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.35 h1:R5b82ubO2NntENm3SAm0ADME+H630HomNJdgv+yZ3xw= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.35/go.mod h1:FuA+nmgMRfkzVKYDNEqQadvEMxtxl9+RLT9ribCwEMs= github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3 h1:bIqFDwgGXXN1Kpp99pDOdKMTTb5d2KyU5X/BZxjOkRo= @@ -56,6 +50,8 @@ github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs= github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c= github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA= github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0= +github.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1xcsSM= +github.com/cenkalti/backoff/v5 v5.0.3/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= @@ -65,6 +61,11 @@ github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/r github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= +github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/goccy/go-json v0.10.3 h1:KZ5WoDbxAIgm2HNbYckL0se1fHD6rz5j4ywS6ebzDqA= github.com/goccy/go-json v0.10.3/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= @@ -74,16 +75,22 @@ github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/flatbuffers v24.3.25+incompatible h1:CX395cjN9Kke9mmalRoL3d81AtFUxJM+yDthflgJGkI= github.com/google/flatbuffers v24.3.25+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= -github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= -github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.2 h1:8Tjv8EJ+pM1xP8mK6egEbD1OgnVTyacbefKhmbLhIhU= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.2/go.mod h1:pkJQ2tZHJ0aFOVEEot6oZmaVEZcRme73eIFmhiVuRWs= github.com/klauspost/asmfmt v1.3.2 h1:4Ri7ox3EwapiOjCki+hw14RyKk201CN4rzyCJRFLpK4= github.com/klauspost/asmfmt v1.3.2/go.mod h1:AG8TuvYojzulgDAMCnYn50l/5QV3Bs/tp6j0HLHbNSE= github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA= github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= github.com/klauspost/cpuid/v2 v2.2.8 h1:+StwCXwm9PdpiEkPyzBXIy+M9KUb4ODm0Zarf1kS5BM= github.com/klauspost/cpuid/v2 v2.2.8/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= @@ -106,6 +113,8 @@ github.com/redis/go-redis/v9 v9.6.1 h1:HHDteefn6ZkTtY5fGUE8tj8uy85AHk6zP7CpzIAM0 github.com/redis/go-redis/v9 v9.6.1/go.mod h1:0C0c6ycQsdpVNQpxb1njEQIqkx5UcsM8FJCQLgE9+RA= github.com/roberson-io/mmh3 v0.0.0-20190729202758-fdfce3ba6225 h1:ZMsPCp7oYgjoIFt1c+sM2qojxZXotSYcMF8Ur9/LJlM= github.com/roberson-io/mmh3 v0.0.0-20190729202758-fdfce3ba6225/go.mod h1:XEESr+X1SY8ZSuc3jqsTlb3clCkqQJ4DcF3Qxv1N3PM= +github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII= +github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o= github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= github.com/rs/zerolog v1.33.0 h1:1cU2KZkvPxNyfgEmhHAz/1A9Bz+llsdYzklWFzgp0r8= github.com/rs/zerolog v1.33.0/go.mod h1:/7mN4D5sKwJLZQ2b/znpjC3/GQWY/xaDXUM0kKWRHss= @@ -113,42 +122,65 @@ github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0b github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= -github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= -github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ= github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0= github.com/zeebo/xxh3 v1.0.2 h1:xZmwmqxHZA8AI603jOQ0tMqmBr9lPeFwGg6d+xy9DC0= github.com/zeebo/xxh3 v1.0.2/go.mod h1:5NWz9Sef7zIDm2JHfFlcQvNekmcEl9ekUZQQKCYaDcA= +go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= +go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= +go.opentelemetry.io/otel v1.38.0 h1:RkfdswUDRimDg0m2Az18RKOsnI8UDzppJAtj01/Ymk8= +go.opentelemetry.io/otel v1.38.0/go.mod h1:zcmtmQ1+YmQM9wrNsTGV/q/uyusom3P8RxwExxkZhjM= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.38.0 h1:GqRJVj7UmLjCVyVJ3ZFLdPRmhDUp2zFmQe3RHIOsw24= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.38.0/go.mod h1:ri3aaHSmCTVYu2AWv44YMauwAQc0aqI9gHKIcSbI1pU= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.38.0 h1:aTL7F04bJHUlztTsNGJ2l+6he8c+y/b//eR0jjjemT4= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.38.0/go.mod h1:kldtb7jDTeol0l3ewcmd8SDvx3EmIE7lyvqbasU3QC4= +go.opentelemetry.io/otel/metric v1.38.0 h1:Kl6lzIYGAh5M159u9NgiRkmoMKjvbsKtYRwgfrA6WpA= +go.opentelemetry.io/otel/metric v1.38.0/go.mod h1:kB5n/QoRM8YwmUahxvI3bO34eVtQf2i4utNVLr9gEmI= +go.opentelemetry.io/otel/sdk v1.38.0 h1:l48sr5YbNf2hpCUj/FoGhW9yDkl+Ma+LrVl8qaM5b+E= +go.opentelemetry.io/otel/sdk v1.38.0/go.mod h1:ghmNdGlVemJI3+ZB5iDEuk4bWA3GkTpW+DOoZMYBVVg= +go.opentelemetry.io/otel/sdk/metric v1.38.0 h1:aSH66iL0aZqo//xXzQLYozmWrXxyFkBJ6qT5wthqPoM= +go.opentelemetry.io/otel/sdk/metric v1.38.0/go.mod h1:dg9PBnW9XdQ1Hd6ZnRz689CbtrUp0wMMs9iPcgT9EZA= +go.opentelemetry.io/otel/trace v1.38.0 h1:Fxk5bKrDZJUH+AMyyIXGcFAPah0oRcT+LuNtJrmcNLE= +go.opentelemetry.io/otel/trace v1.38.0/go.mod h1:j1P9ivuFsTceSWe1oY+EeW3sc+Pp42sO++GHkg4wwhs= +go.opentelemetry.io/proto/otlp v1.7.1 h1:gTOMpGDb0WTBOP8JaO72iL3auEZhVmAQg4ipjOVAtj4= +go.opentelemetry.io/proto/otlp v1.7.1/go.mod h1:b2rVh6rfI/s2pHWNlB7ILJcRALpcNDzKhACevjI+ZnE= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= golang.org/x/exp v0.0.0-20240909161429-701f63a606c0 h1:e66Fs6Z+fZTbFBAxKfP3PALWBtpfqks2bwGcexMxgtk= golang.org/x/exp v0.0.0-20240909161429-701f63a606c0/go.mod h1:2TbTHSBQa924w8M6Xs1QcRcFwyucIwBGpK1p2f1YFFY= -golang.org/x/mod v0.21.0 h1:vvrHzRwRfVKSiLrG+d4FMl/Qi4ukBCE6kZlTUkDYRT0= -golang.org/x/mod v0.21.0/go.mod h1:6SkKJ3Xj0I0BrPOZoBy3bdMptDDU9oJrpohJ3eWZ1fY= -golang.org/x/net v0.29.0 h1:5ORfpBpCs4HzDYoodCDBbwHzdR5UrLBZ3sOnUJmFoHo= -golang.org/x/net v0.29.0/go.mod h1:gLkgy8jTGERgjzMic6DS9+SP0ajcu6Xu3Orq/SpETg0= -golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ= -golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/mod v0.26.0 h1:EGMPT//Ezu+ylkCijjPc+f4Aih7sZvaAr+O3EHBxvZg= +golang.org/x/mod v0.26.0/go.mod h1:/j6NAhSk8iQ723BGAUyoAcn7SlD7s15Dp9Nd/SfeaFQ= +golang.org/x/net v0.43.0 h1:lat02VYK2j4aLzMzecihNvTlJNQUq316m2Mr9rnM6YE= +golang.org/x/net v0.43.0/go.mod h1:vhO1fvI4dGsIjh73sWfUVjj3N7CA9WkKJNQm2svM6Jg= +golang.org/x/sync v0.17.0 h1:l60nONMj9l5drqw6jlhIELNv9I0A4OFgRsG9k2oT9Ug= +golang.org/x/sync v0.17.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.25.0 h1:r+8e+loiHxRqhXVl6ML1nO3l1+oFoWbnlu2Ehimmi34= -golang.org/x/sys v0.25.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/text v0.18.0 h1:XvMDiNzPAl0jr17s6W9lcaIhGUfUORdGCNsuLmPG224= -golang.org/x/text v0.18.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= -golang.org/x/tools v0.25.0 h1:oFU9pkj/iJgs+0DT+VMHrx+oBKs/LJMV+Uvg78sl+fE= -golang.org/x/tools v0.25.0/go.mod h1:/vtpO8WL1N9cQC3FN5zPqb//fRXskFHbLKk4OW1Q7rg= +golang.org/x/sys v0.35.0 h1:vz1N37gP5bs89s7He8XuIYXpyY0+QlsKmzipCbUtyxI= +golang.org/x/sys v0.35.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/text v0.28.0 h1:rhazDwis8INMIwQ4tpjLDzUhx6RlXqZNPEM0huQojng= +golang.org/x/text v0.28.0/go.mod h1:U8nCwOR8jO/marOQ0QbDiOngZVEBB7MAiitBuMjXiNU= +golang.org/x/tools v0.35.0 h1:mBffYraMEf7aa0sB+NuKnuCy8qI/9Bughn8dC2Gu5r0= +golang.org/x/tools v0.35.0/go.mod h1:NKdj5HkL/73byiZSJjqJgKn3ep7KjFkBOkR/Hps3VPw= golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da h1:noIWHXmPHxILtqtCOPIhSt0ABwskkZKjD3bXGnZGpNY= golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90= -gonum.org/v1/gonum v0.15.0 h1:2lYxjRbTYyxkJxlhC+LvJIx3SsANPdRybu1tGj9/OrQ= -gonum.org/v1/gonum v0.15.0/go.mod h1:xzZVBJBtS+Mz4q0Yl2LJTk+OxOg4jiXZ7qBoM0uISGo= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 h1:pPJltXNxVzT4pK9yD8vR9X75DaWYYmLGMsEvBfFQZzQ= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1/go.mod h1:UqMtugtsSgubUsoxbuAoiCXvqvErP7Gf0so0mK9tHxU= -google.golang.org/grpc v1.67.0 h1:IdH9y6PF5MPSdAntIcpjQ+tXO41pcQsfZV2RxtQgVcw= -google.golang.org/grpc v1.67.0/go.mod h1:1gLDyUQU7CTLJI90u3nXZ9ekeghjeM7pTDZlqFNg2AA= -google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= -google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk= +gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E= +google.golang.org/genproto/googleapis/api v0.0.0-20250825161204-c5933d9347a5 h1:BIRfGDEjiHRrk0QKZe3Xv2ieMhtgRGeLcZQ0mIVn4EY= +google.golang.org/genproto/googleapis/api v0.0.0-20250825161204-c5933d9347a5/go.mod h1:j3QtIyytwqGr1JUDtYXwtMXWPKsEa5LtzIFN1Wn5WvE= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250825161204-c5933d9347a5 h1:eaY8u2EuxbRv7c3NiGK0/NedzVsCcV6hDuU5qPX5EGE= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250825161204-c5933d9347a5/go.mod h1:M4/wBTSeyLxupu3W3tJtOgB14jILAS/XWPSSa3TAlJc= +google.golang.org/grpc v1.75.0 h1:+TW+dqTd2Biwe6KKfhE5JpiYIBWq865PhKGSXiivqt4= +google.golang.org/grpc v1.75.0/go.mod h1:JtPAzKiq4v1xcAB2hydNlWI2RnF85XXcV0mhKXr2ecQ= +google.golang.org/protobuf v1.36.8 h1:xHScyCOEuuwZEc6UtSOvPbAT4zRh0xcNRYekJwfqyMc= +google.golang.org/protobuf v1.36.8/go.mod h1:fuxRtAxBytpl4zzqUh6/eyUujkJdNiuEkXntxiD/uRU= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= diff --git a/go/README.md b/go/README.md index eebbfb7d62d..a8e381519a4 100644 --- a/go/README.md +++ b/go/README.md @@ -1,12 +1,113 @@ -[Update 10/31/2024] This Go feature server code is updated from the Expedia Group's forked Feast branch (https://github.com/ExpediaGroup/feast.git) on 10/22/2024. Thanks the engineers of the Expedia Groups who contributed and improved the Go feature server. +[Update 10/31/2024] This Go feature server code is updated from the Expedia Group's forked Feast branch (https://github.com/ExpediaGroup/feast.git) on 10/22/2024. Thanks the engineers of the Expedia Groups who contributed and improved the Go Feature Server. -This directory contains the Go logic that's executed by the `EmbeddedOnlineFeatureServer` from Python. - ## Build and Run To build and run the Go Feature Server locally, create a feature_store.yaml file with necessary configurations and run below commands: ```bash - go build -o feast ./go/main.go - ./feast --type=http --port=8080 -``` \ No newline at end of file + go build -o feast-go ./go/main.go + # start the http server + ./feast-go --type=http --port=8080 + # or start the gRPC server + #./feast-go --type=grpc --port=[your-choice] +``` + +## OTEL based observability +The OS level env variable `ENABLE_OTEL_TRACING=="true"/"false"` (string type) is used to enable/disable this service (with Tracing only). + +The default exporter URL is "http://localhost:4318". The default schema of sending data to collector is **HTTP**. Please refer the following two docs about the configuration of the OTEL exporter: +1. https://opentelemetry.io/docs/languages/sdk-configuration/otlp-exporter/ +2. https://pkg.go.dev/go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp#WithEndpoint + +## List of files have OTEL observability code +1. internal/feast/transformation/transformation.go +2. internal/feast/onlinestore/redisonlinestore.go +3. internal/feast/server/grpc_server.go +4. internal/feast/server/http_server.go +5. internal/feast/server/server_commons.go +6. internal/feast/featurestore.go + +## Example monitoring infra setup +1. docker compose file to setup Prometheus, Jaeger, and OTEL-collector. +```yaml +services: + prometheus: + image: prom/prometheus + volumes: + - ./prometheus.yaml:/etc/prometheus/prometheus.yaml + ports: + - 9090:9090 # web UI http://localhost:9090 + jaeger: + image: jaegertracing/all-in-one:latest + ports: + - 16686:16686 # Web UI: http://localhost:16686 + - 14268:14268 # http based receiver + - 14250:14250 # gRPC based receiver + otel-collector: + image: otel/opentelemetry-collector-contrib + volumes: + - ./otel-collector-config.yaml:/etc/otelcol-contrib/config.yaml + ports: + #- 1888:1888 # pprof extension + - 8888:8888 # Prometheus metrics exposed by the Collector + - 8889:8889 # Prometheus exporter metrics + #- 13133:13133 # health_check extension + #- 4317:4317 # OTLP gRPC receiver + - 4318:4318 # OTLP http receiver + - 55679:55679 # zpages extension. check http://localhost:55679/debug/tracez + depends_on: + - jaeger + - prometheus +``` +2. OTEL collector configure file. +```yaml +receivers: + otlp: + protocols: + http: + endpoint: 0.0.0.0:4318 + +exporters: + prometheus: + endpoint: 0.0.0.0:8889 + namespace: feast-go + otlp/jaeger: # this is a gRPC based exporter. use "otelhttp" for http based exporter. + endpoint: jaeger:4317 + tls: + insecure: true + +extensions: + zpages: + endpoint: 0.0.0.0:55679 + +processors: + batch: + +service: + extensions: [zpages] + pipelines: + metrics: + receivers: [otlp] + exporters: [prometheus] + traces: + receivers: [otlp] + exporters: [otlp/jaeger] +``` +3. Prometheus config. +```yaml + #https://github.com/prometheus/prometheus/blob/release-3.6/config/testdata/conf.good.yml + scrape_configs: + - job_name: 'otel-collector' + scrape_interval: 1m + scrape_timeout: 30s # Increase this if needed + static_configs: + # Check the IP address of or Docker host network. + # Refer: https://stackoverflow.com/questions/48546124/what-is-the-linux-equivalent-of-host-docker-internal + - targets: ['172.17.0.1:8888'] # Replace with the Collector's IP and port + - job_name: 'otel-collected' + scrape_interval: 1m + scrape_timeout: 30s # Increase this if needed + static_configs: + - targets: ['172.17.0.1:8889'] # Replace with the Collector's IP and port +``` +4. Jaeger config file is not used in this setup. \ No newline at end of file diff --git a/go/infra/docker/otel/compose.yaml b/go/infra/docker/otel/compose.yaml new file mode 100644 index 00000000000..7872797d205 --- /dev/null +++ b/go/infra/docker/otel/compose.yaml @@ -0,0 +1,29 @@ +services: + prometheus: + image: prom/prometheus + volumes: + - ./prometheus.yaml:/etc/prometheus/prometheus.yaml + ports: + - 9090:9090 + jaeger: + image: jaegertracing/all-in-one:latest + #volumes: + # - ./jaeger.yaml:/jaeger/config.yaml + ports: + - 16686:16686 # Web UI: http://localhost:16686 + - 14268:14268 # http based receiver + - 14250:14250 # gRPC based receiver + otel-collector: + image: otel/opentelemetry-collector-contrib + volumes: + - ./otel-collector-config.yaml:/etc/otelcol-contrib/config.yaml + ports: + #- 1888:1888 # pprof extension + - 8888:8888 # Prometheus metrics exposed by the Collector + - 8889:8889 # Prometheus exporter metrics + #- 13133:13133 # health_check extension + #- 4317:4317 # OTLP gRPC receiver + - 4318:4318 # OTLP http receiver + - 55679:55679 # zpages extension + depends_on: + - jaeger \ No newline at end of file diff --git a/go/infra/docker/otel/otel-collector-config.yaml b/go/infra/docker/otel/otel-collector-config.yaml new file mode 100644 index 00000000000..92ecfe017ac --- /dev/null +++ b/go/infra/docker/otel/otel-collector-config.yaml @@ -0,0 +1,31 @@ +receivers: + otlp: + protocols: + http: + endpoint: 0.0.0.0:4318 + +exporters: + prometheus: + endpoint: 0.0.0.0:8889 + namespace: feast-go + otlp/jaeger: + endpoint: jaeger:4317 + tls: + insecure: true + +extensions: + zpages: + endpoint: 0.0.0.0:55679 + +processors: + batch: + +service: + extensions: [zpages] + pipelines: + metrics: + receivers: [otlp] + exporters: [prometheus] + traces: + receivers: [otlp] + exporters: [otlp/jaeger] \ No newline at end of file diff --git a/go/infra/docker/otel/prometheus.yaml b/go/infra/docker/otel/prometheus.yaml new file mode 100644 index 00000000000..8ab32f38870 --- /dev/null +++ b/go/infra/docker/otel/prometheus.yaml @@ -0,0 +1,14 @@ + #https://github.com/prometheus/prometheus/blob/release-3.6/config/testdata/conf.good.yml + scrape_configs: + - job_name: 'otel-collector' + scrape_interval: 1m + scrape_timeout: 30s # Increase this if needed + static_configs: + # Check the IP address of or Docker host network. + # Refer: https://stackoverflow.com/questions/48546124/what-is-the-linux-equivalent-of-host-docker-internal + - targets: ['172.17.0.1:8888'] # Replace with the Collector's IP and port + - job_name: 'otel-collected' + scrape_interval: 1m + scrape_timeout: 30s # Increase this if needed + static_configs: + - targets: ['172.17.0.1:8889'] # Replace with the Collector's IP and port \ No newline at end of file diff --git a/go/internal/feast/featurestore.go b/go/internal/feast/featurestore.go index abe1d195def..f6abd50e3d1 100644 --- a/go/internal/feast/featurestore.go +++ b/go/internal/feast/featurestore.go @@ -7,6 +7,7 @@ import ( "github.com/apache/arrow/go/v17/arrow/memory" //"gopkg.in/DataDog/dd-trace-go.v1/ddtrace/tracer" + "go.opentelemetry.io/otel" "github.com/feast-dev/feast/go/internal/feast/model" "github.com/feast-dev/feast/go/internal/feast/onlineserving" @@ -17,6 +18,8 @@ import ( prototypes "github.com/feast-dev/feast/go/protos/feast/types" ) +var tracer = otel.Tracer("github.com/feast-dev/feast/go/feast") + type FeatureStore struct { config *registry.RepoConfig registry *registry.Registry @@ -322,8 +325,8 @@ func (fs *FeatureStore) readFromOnlineStore(ctx context.Context, entityRows []*p requestedFeatureNames []string, ) ([][]onlinestore.FeatureData, error) { // Create a Datadog span from context - //span, _ := tracer.StartSpanFromContext(ctx, "fs.readFromOnlineStore") - //defer span.Finish() + ctx, span := tracer.Start(ctx, "fs.readFromOnlineStore") + defer span.End() numRows := len(entityRows) entityRowsValue := make([]*prototypes.EntityKey, numRows) diff --git a/go/internal/feast/onlinestore/redisonlinestore.go b/go/internal/feast/onlinestore/redisonlinestore.go index 3fa6cf580c7..e39b3505710 100644 --- a/go/internal/feast/onlinestore/redisonlinestore.go +++ b/go/internal/feast/onlinestore/redisonlinestore.go @@ -11,7 +11,7 @@ import ( "strings" "github.com/feast-dev/feast/go/internal/feast/registry" - //"gopkg.in/DataDog/dd-trace-go.v1/ddtrace/tracer" + "go.opentelemetry.io/otel" "github.com/redis/go-redis/v9" "github.com/spaolacci/murmur3" @@ -24,6 +24,8 @@ import ( //redistrace "gopkg.in/DataDog/dd-trace-go.v1/contrib/redis/go-redis.v9" ) +var tracer = otel.Tracer("github.com/feast-dev/feast/go/onlinestore") + type redisType int const ( @@ -211,8 +213,8 @@ func (r *RedisOnlineStore) buildRedisKeys(entityKeys []*types.EntityKey) ([]*[]b } func (r *RedisOnlineStore) OnlineRead(ctx context.Context, entityKeys []*types.EntityKey, featureViewNames []string, featureNames []string) ([][]FeatureData, error) { - //span, _ := tracer.StartSpanFromContext(ctx, "redis.OnlineRead") - //defer span.Finish() + ctx, span := tracer.Start(ctx, "redis.OnlineRead") + defer span.End() featureCount := len(featureNames) featureViewIndices, indicesFeatureView, index := r.buildFeatureViewIndices(featureViewNames, featureNames) diff --git a/go/internal/feast/server/grpc_server.go b/go/internal/feast/server/grpc_server.go index d5e18b1c9ef..ab76aa554ca 100644 --- a/go/internal/feast/server/grpc_server.go +++ b/go/internal/feast/server/grpc_server.go @@ -9,7 +9,7 @@ import ( prototypes "github.com/feast-dev/feast/go/protos/feast/types" "github.com/feast-dev/feast/go/types" "github.com/google/uuid" - //"gopkg.in/DataDog/dd-trace-go.v1/ddtrace/tracer" + ) const feastServerVersion = "0.0.1" @@ -34,16 +34,16 @@ func (s *grpcServingServiceServer) GetFeastServingInfo(ctx context.Context, requ // Metadata contains feature names that corresponds to the number of rows in response.Results. // Results contains values including the value of the feature, the event timestamp, and feature status in a columnar format. func (s *grpcServingServiceServer) GetOnlineFeatures(ctx context.Context, request *serving.GetOnlineFeaturesRequest) (*serving.GetOnlineFeaturesResponse, error) { - //span, ctx := tracer.StartSpanFromContext(ctx, "getOnlineFeatures", tracer.ResourceName("ServingService/GetOnlineFeatures")) - //defer span.Finish() + ctx, span := tracer.Start(ctx, "server.getOnlineFeatures") + defer span.End() - //logSpanContext := LogWithSpanContext(span) + logSpanContext := LogWithSpanContext(span) requestId := GenerateRequestId() featuresOrService, err := s.fs.ParseFeatures(request.GetKind()) if err != nil { - //logSpanContext.Error().Err(err).Msg("Error parsing feature service or feature list from request") + logSpanContext.Error().Err(err).Msg("Error parsing feature service or feature list from request") return nil, err } @@ -56,7 +56,7 @@ func (s *grpcServingServiceServer) GetOnlineFeatures(ctx context.Context, reques request.GetFullFeatureNames()) if err != nil { - //logSpanContext.Error().Err(err).Msg("Error getting online features") + logSpanContext.Error().Err(err).Msg("Error getting online features") return nil, err } @@ -75,7 +75,7 @@ func (s *grpcServingServiceServer) GetOnlineFeatures(ctx context.Context, reques featureNames[idx] = vector.Name values, err := types.ArrowValuesToProtoValues(vector.Values) if err != nil { - //logSpanContext.Error().Err(err).Msg("Error converting Arrow values to proto values") + logSpanContext.Error().Err(err).Msg("Error converting Arrow values to proto values") return nil, err } if _, ok := request.Entities[vector.Name]; ok { @@ -93,13 +93,13 @@ func (s *grpcServingServiceServer) GetOnlineFeatures(ctx context.Context, reques if featureService != nil && featureService.LoggingConfig != nil && s.loggingService != nil { logger, err := s.loggingService.GetOrCreateLogger(featureService) if err != nil { - //logSpanContext.Error().Err(err).Msg("Error to instantiating logger for feature service: " + featuresOrService.FeatureService.Name) + logSpanContext.Error().Err(err).Msg("Error to instantiating logger for feature service: " + featuresOrService.FeatureService.Name) fmt.Printf("Couldn't instantiate logger for feature service %s: %+v", featuresOrService.FeatureService.Name, err) } err = logger.Log(request.Entities, resp.Results[len(request.Entities):], resp.Metadata.FeatureNames.Val[len(request.Entities):], request.RequestContext, requestId) if err != nil { - //logSpanContext.Error().Err(err).Msg("Error to logging to feature service: " + featuresOrService.FeatureService.Name) + logSpanContext.Error().Err(err).Msg("Error to logging to feature service: " + featuresOrService.FeatureService.Name) fmt.Printf("LoggerImpl error[%s]: %+v", featuresOrService.FeatureService.Name, err) } } diff --git a/go/internal/feast/server/http_server.go b/go/internal/feast/server/http_server.go index def58aedb88..312a0a6352e 100644 --- a/go/internal/feast/server/http_server.go +++ b/go/internal/feast/server/http_server.go @@ -5,10 +5,8 @@ import ( "encoding/json" "fmt" "net/http" - //"os" "runtime" "strconv" - //"strings" "time" "github.com/feast-dev/feast/go/internal/feast" @@ -19,8 +17,6 @@ import ( prototypes "github.com/feast-dev/feast/go/protos/feast/types" "github.com/feast-dev/feast/go/types" "github.com/rs/zerolog/log" - //httptrace "gopkg.in/DataDog/dd-trace-go.v1/contrib/net/http" - //"gopkg.in/DataDog/dd-trace-go.v1/ddtrace/tracer" ) type httpServer struct { @@ -150,10 +146,10 @@ func (s *httpServer) getOnlineFeatures(w http.ResponseWriter, r *http.Request) { var err error ctx := r.Context() - //span, ctx := tracer.StartSpanFromContext(r.Context(), "getOnlineFeatures", tracer.ResourceName("/get-online-features")) - //defer span.Finish(tracer.WithError(err)) + ctx, span := tracer.Start(r.Context(), "server.getOnlineFeatures") + defer span.End() - //logSpanContext := LogWithSpanContext(span) + logSpanContext := LogWithSpanContext(span) if r.Method != "POST" { http.NotFound(w, r) @@ -166,7 +162,7 @@ func (s *httpServer) getOnlineFeatures(w http.ResponseWriter, r *http.Request) { if statusQuery != "" { status, err = strconv.ParseBool(statusQuery) if err != nil { - //logSpanContext.Error().Err(err).Msg("Error parsing status query parameter") + logSpanContext.Error().Err(err).Msg("Error parsing status query parameter") writeJSONError(w, fmt.Errorf("Error parsing status query parameter: %+v", err), http.StatusBadRequest) return } @@ -176,7 +172,7 @@ func (s *httpServer) getOnlineFeatures(w http.ResponseWriter, r *http.Request) { var request getOnlineFeaturesRequest err = decoder.Decode(&request) if err != nil { - //logSpanContext.Error().Err(err).Msg("Error decoding JSON request data") + logSpanContext.Error().Err(err).Msg("Error decoding JSON request data") writeJSONError(w, fmt.Errorf("Error decoding JSON request data: %+v", err), http.StatusInternalServerError) return } @@ -184,7 +180,7 @@ func (s *httpServer) getOnlineFeatures(w http.ResponseWriter, r *http.Request) { if request.FeatureService != nil { featureService, err = s.fs.GetFeatureService(*request.FeatureService) if err != nil { - //logSpanContext.Error().Err(err).Msg("Error getting feature service from registry") + logSpanContext.Error().Err(err).Msg("Error getting feature service from registry") writeJSONError(w, fmt.Errorf("Error getting feature service from registry: %+v", err), http.StatusInternalServerError) return } @@ -207,7 +203,7 @@ func (s *httpServer) getOnlineFeatures(w http.ResponseWriter, r *http.Request) { request.FullFeatureNames) if err != nil { - //logSpanContext.Error().Err(err).Msg("Error getting feature vector") + logSpanContext.Error().Err(err).Msg("Error getting feature vector") writeJSONError(w, fmt.Errorf("Error getting feature vector: %+v", err), http.StatusInternalServerError) return } @@ -249,7 +245,7 @@ func (s *httpServer) getOnlineFeatures(w http.ResponseWriter, r *http.Request) { err = json.NewEncoder(w).Encode(response) if err != nil { - //logSpanContext.Error().Err(err).Msg("Error encoding response") + logSpanContext.Error().Err(err).Msg("Error encoding response") writeJSONError(w, fmt.Errorf("Error encoding response: %+v", err), http.StatusInternalServerError) return } @@ -257,7 +253,7 @@ func (s *httpServer) getOnlineFeatures(w http.ResponseWriter, r *http.Request) { if featureService != nil && featureService.LoggingConfig != nil && s.loggingService != nil { logger, err := s.loggingService.GetOrCreateLogger(featureService) if err != nil { - //logSpanContext.Error().Err(err).Msgf("Couldn't instantiate logger for feature service %s", featureService.Name) + logSpanContext.Error().Err(err).Msgf("Couldn't instantiate logger for feature service %s", featureService.Name) writeJSONError(w, fmt.Errorf("Couldn't instantiate logger for feature service %s: %+v", featureService.Name, err), http.StatusInternalServerError) return } @@ -270,7 +266,7 @@ func (s *httpServer) getOnlineFeatures(w http.ResponseWriter, r *http.Request) { for _, vector := range featureVectors[len(request.Entities):] { values, err := types.ArrowValuesToProtoValues(vector.Values) if err != nil { - //logSpanContext.Error().Err(err).Msg("Couldn't convert arrow values into protobuf") + logSpanContext.Error().Err(err).Msg("Couldn't convert arrow values into protobuf") writeJSONError(w, fmt.Errorf("Couldn't convert arrow values into protobuf: %+v", err), http.StatusInternalServerError) return } @@ -340,11 +336,6 @@ func recoverMiddleware(next http.Handler) http.Handler { } func (s *httpServer) Serve(host string, port int) error { - // DD - //if strings.ToLower(os.Getenv("ENABLE_DATADOG_TRACING")) == "true" { - // tracer.Start(tracer.WithRuntimeMetrics()) - // defer tracer.Stop() - //} mux := http.NewServeMux() mux.Handle("/get-online-features", recoverMiddleware(http.HandlerFunc(s.getOnlineFeatures))) mux.HandleFunc("/health", healthCheckHandler) diff --git a/go/internal/feast/server/server_commons.go b/go/internal/feast/server/server_commons.go index 140269d5c1c..a6959076c4a 100644 --- a/go/internal/feast/server/server_commons.go +++ b/go/internal/feast/server/server_commons.go @@ -1,31 +1,23 @@ package server import ( - "github.com/rs/zerolog" - //"gopkg.in/DataDog/dd-trace-go.v1/ddtrace/tracer" "os" -) -func LogWiwithSpanContext() zerolog.Logger { - var logger = zerolog.New(os.Stderr).With(). - Timestamp(). - Logger() + "github.com/rs/zerolog" + "go.opentelemetry.io/otel/trace" + "go.opentelemetry.io/otel" +) - return logger -} +var tracer = otel.Tracer("github.com/feast-dev/feast/go/server") -/* -func LogWithSpanContext(span tracer.Span) zerolog.Logger { - spanContext := span.Context() +func LogWithSpanContext(span trace.Span) zerolog.Logger { + spanContext := span.SpanContext() var logger = zerolog.New(os.Stderr).With(). + Str("trace_id", spanContext.TraceID().String()). + Str("span_id", spanContext.SpanID().String()). Timestamp(). Logger() - //Int64("trace_id", int64(spanContext.TraceID())). - //Int64("span_id", int64(spanContext.SpanID())). - //Timestamp(). - //Logger() return logger } -*/ diff --git a/go/internal/feast/transformation/transformation.go b/go/internal/feast/transformation/transformation.go index d6df03039d7..1080967b664 100644 --- a/go/internal/feast/transformation/transformation.go +++ b/go/internal/feast/transformation/transformation.go @@ -9,7 +9,7 @@ import ( "github.com/apache/arrow/go/v17/arrow" "github.com/apache/arrow/go/v17/arrow/memory" "github.com/rs/zerolog/log" - //"gopkg.in/DataDog/dd-trace-go.v1/ddtrace/tracer" + "go.opentelemetry.io/otel" "github.com/feast-dev/feast/go/internal/feast/model" "github.com/feast-dev/feast/go/internal/feast/onlineserving" @@ -17,6 +17,8 @@ import ( "github.com/feast-dev/feast/go/types" ) +var tracer = otel.Tracer("github.com/feast-dev/feast/go/transformation") + /* TransformationCallback is a Python callback function's expected signature. The function should accept name of the on demand feature view and pointers to input & output record batches. @@ -40,8 +42,8 @@ func AugmentResponseWithOnDemandTransforms( fullFeatureNames bool, ) ([]*onlineserving.FeatureVector, error) { - //span, _ := tracer.StartSpanFromContext(ctx, "transformation.AugmentResponseWithOnDemandTransforms") - //defer span.Finish() + ctx, span := tracer.Start(ctx, "transformation.AugmentResponseWithOnDemandTransforms") + defer span.End() result := make([]*onlineserving.FeatureVector, 0) var err error diff --git a/go/main.go b/go/main.go index feb54faa2e0..77999671e07 100644 --- a/go/main.go +++ b/go/main.go @@ -1,12 +1,13 @@ package main import ( + "context" "flag" "fmt" "net" "os" "os/signal" - //"strings" + "strings" "syscall" "github.com/feast-dev/feast/go/internal/feast" @@ -18,10 +19,18 @@ import ( "google.golang.org/grpc" "google.golang.org/grpc/health" "google.golang.org/grpc/health/grpc_health_v1" - //grpctrace "gopkg.in/DataDog/dd-trace-go.v1/contrib/google.golang.org/grpc" - //"gopkg.in/DataDog/dd-trace-go.v1/ddtrace/tracer" + + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/exporters/otlp/otlptrace" + "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp" + "go.opentelemetry.io/otel/sdk/resource" + sdktrace "go.opentelemetry.io/otel/sdk/trace" + semconv "go.opentelemetry.io/otel/semconv/v1.37.0" + "go.opentelemetry.io/otel/trace" ) +var tracer trace.Tracer + type ServerStarter interface { StartHttpServer(fs *feast.FeatureStore, host string, port int, writeLoggedFeaturesCallback logging.OfflineStoreWriteCallback, loggingOpts *logging.LoggingOptions) error StartGrpcServer(fs *feast.FeatureStore, host string, port int, writeLoggedFeaturesCallback logging.OfflineStoreWriteCallback, loggingOpts *logging.LoggingOptions) error @@ -56,6 +65,32 @@ func main() { flag.IntVar(&port, "port", port, "Specify a port for the server") flag.Parse() + // Initialize tracer + if OTELTracingEnabled() { + ctx := context.Background() + + exp, err := newExporter(ctx) + if err != nil { + log.Fatal().Stack().Err(err).Msg("Failed to initialize exporter.") + } + + // Create a new tracer provider with a batch span processor and the given exporter. + tp, err := newTracerProvider(exp) + if err != nil { + log.Fatal().Stack().Err(err).Msg("Failed to initialize tracer provider.") + } + + // Handle shutdown properly so nothing leaks. + defer func() { _ = tp.Shutdown(ctx) }() + + otel.SetTracerProvider(tp) + + // Finally, set the tracer that can be used for this package. + tracer = tp.Tracer("github.com/feast-dev/feast/go") + + log.Info().Msg("OTEL based tracing started.") + } + repoConfig, err := registry.NewRepoConfigFromFile(repoPath) if err != nil { log.Fatal().Stack().Err(err).Msg("Failed to convert to RepoConfig") @@ -110,11 +145,6 @@ func constructLoggingService(fs *feast.FeatureStore, writeLoggedFeaturesCallback // StartGprcServerWithLogging starts gRPC server with enabled feature logging func StartGrpcServer(fs *feast.FeatureStore, host string, port int, writeLoggedFeaturesCallback logging.OfflineStoreWriteCallback, loggingOpts *logging.LoggingOptions) error { - // #DD - //if strings.ToLower(os.Getenv("ENABLE_DATADOG_TRACING")) == "true" { - // tracer.Start(tracer.WithRuntimeMetrics()) - // defer tracer.Stop() - //} loggingService, err := constructLoggingService(fs, writeLoggedFeaturesCallback, loggingOpts) if err != nil { return err @@ -178,3 +208,35 @@ func StartHttpServer(fs *feast.FeatureStore, host string, port int, writeLoggedF return ser.Serve(host, port) } + +func OTELTracingEnabled() bool { + return strings.ToLower(os.Getenv("ENABLE_OTEL_TRACING")) == "true" +} + +func newExporter(ctx context.Context) (*otlptrace.Exporter, error) { + exp, err := otlptracehttp.New(ctx, + otlptracehttp.WithInsecure()) + if err != nil { + return nil, err + } + return exp, nil +} + +func newTracerProvider(exp sdktrace.SpanExporter) (*sdktrace.TracerProvider, error) { + r, err := resource.Merge( + resource.Default(), + resource.NewWithAttributes( + semconv.SchemaURL, + semconv.ServiceName("FeastGoFeatureServer"), + ), + ) + + if err != nil { + return nil, err + } + + return sdktrace.NewTracerProvider( + sdktrace.WithBatcher(exp), + sdktrace.WithResource(r), + ), nil +} diff --git a/infra/charts/feast-feature-server/Chart.yaml b/infra/charts/feast-feature-server/Chart.yaml index 5d1cc97c4c5..eaf14f21fe6 100644 --- a/infra/charts/feast-feature-server/Chart.yaml +++ b/infra/charts/feast-feature-server/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: feast-feature-server description: Feast Feature Server in Go or Python type: application -version: 0.55.0 +version: 0.56.0 keywords: - machine learning - big data diff --git a/infra/charts/feast-feature-server/README.md b/infra/charts/feast-feature-server/README.md index c334acf24e6..e73be95a909 100644 --- a/infra/charts/feast-feature-server/README.md +++ b/infra/charts/feast-feature-server/README.md @@ -1,6 +1,6 @@ # Feast Python / Go Feature Server Helm Charts -Current chart version is `0.55.0` +Current chart version is `0.56.0` ## Installation @@ -35,13 +35,14 @@ See [here](https://github.com/feast-dev/feast/tree/master/examples/python-helm-d | Key | Type | Default | Description | |-----|------|---------|-------------| | affinity | object | `{}` | | +| commandArgs | list | `[]` | Override the default command arguments for complete control over CLI options If not specified, falls back to legacy behavior based on feast_mode Example for UI mode with custom options: commandArgs: - "feast" - "--log-level" - "INFO" - "ui" - "--root_path" - "/feast" - "--registry_ttl_sec" - "300" - "-h" - "0.0.0.0" - "-p" - "8888" | | extraEnvs | list | `[]` | Additional environment variables to be set in the container | | feast_mode | string | `"online"` | Feast supported deployment modes - online (default), offline, ui and registry | | feature_store_yaml_base64 | string | `""` | [required] a base64 encoded version of feature_store.yaml | | fullnameOverride | string | `""` | | | image.pullPolicy | string | `"IfNotPresent"` | | | image.repository | string | `"quay.io/feastdev/feature-server"` | Docker image for Feature Server repository | -| image.tag | string | `"0.55.0"` | The Docker image tag (can be overwritten if custom feature server deps are needed for on demand transforms) | +| image.tag | string | `"0.56.0"` | The Docker image tag (can be overwritten if custom feature server deps are needed for on demand transforms) | | imagePullSecrets | list | `[]` | | | livenessProbe.initialDelaySeconds | int | `30` | | | livenessProbe.periodSeconds | int | `30` | | diff --git a/infra/charts/feast-feature-server/templates/deployment.yaml b/infra/charts/feast-feature-server/templates/deployment.yaml index 8af21aae80b..6cd17d4a4d1 100644 --- a/infra/charts/feast-feature-server/templates/deployment.yaml +++ b/infra/charts/feast-feature-server/templates/deployment.yaml @@ -47,6 +47,10 @@ spec: {{- toYaml . | nindent 12 }} {{- end}} command: + {{- if .Values.commandArgs }} + {{- toYaml .Values.commandArgs | nindent 12 }} + {{- else }} + {{- /* Fallback to legacy behavior for backward compatibility */}} {{- if eq .Values.feast_mode "offline" }} - "feast" - "--log-level" @@ -82,6 +86,7 @@ spec: - "0.0.0.0" {{- end }} {{- end }} + {{- end }} ports: - name: {{ .Values.feast_mode }} {{- if eq .Values.feast_mode "offline" }} diff --git a/infra/charts/feast-feature-server/values.yaml b/infra/charts/feast-feature-server/values.yaml index 719e8f9cb7a..0f94b5bc579 100644 --- a/infra/charts/feast-feature-server/values.yaml +++ b/infra/charts/feast-feature-server/values.yaml @@ -9,7 +9,7 @@ image: repository: quay.io/feastdev/feature-server pullPolicy: IfNotPresent # image.tag -- The Docker image tag (can be overwritten if custom feature server deps are needed for on demand transforms) - tag: 0.55.0 + tag: 0.56.0 logLevel: "WARNING" # Set log level DEBUG, INFO, WARNING, ERROR, and CRITICAL (case-insensitive) @@ -29,6 +29,24 @@ feature_store_yaml_base64: "" # feast_mode -- Feast supported deployment modes - online (default), offline, ui and registry feast_mode: "online" +# commandArgs -- Override the default command arguments for complete control over CLI options +# If not specified, falls back to legacy behavior based on feast_mode +# Example for UI mode with custom options: +# commandArgs: +# - "feast" +# - "--log-level" +# - "INFO" +# - "ui" +# - "--root_path" +# - "/feast" +# - "--registry_ttl_sec" +# - "300" +# - "-h" +# - "0.0.0.0" +# - "-p" +# - "8888" +commandArgs: [] + podAnnotations: {} podSecurityContext: {} diff --git a/infra/charts/feast/Chart.yaml b/infra/charts/feast/Chart.yaml index b3c62a8afca..307e01563cd 100644 --- a/infra/charts/feast/Chart.yaml +++ b/infra/charts/feast/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v1 description: Feature store for machine learning name: feast -version: 0.55.0 +version: 0.56.0 keywords: - machine learning - big data diff --git a/infra/charts/feast/README.md b/infra/charts/feast/README.md index a9912769b0a..dca35b67065 100644 --- a/infra/charts/feast/README.md +++ b/infra/charts/feast/README.md @@ -8,7 +8,7 @@ This repo contains Helm charts for Feast Java components that are being installe ## Chart: Feast -Feature store for machine learning Current chart version is `0.55.0` +Feature store for machine learning Current chart version is `0.56.0` ## Installation @@ -65,8 +65,8 @@ See [here](https://github.com/feast-dev/feast/tree/master/examples/java-demo) fo | Repository | Name | Version | |------------|------|---------| | https://charts.helm.sh/stable | redis | 10.5.6 | -| https://feast-helm-charts.storage.googleapis.com | feature-server(feature-server) | 0.55.0 | -| https://feast-helm-charts.storage.googleapis.com | transformation-service(transformation-service) | 0.55.0 | +| https://feast-helm-charts.storage.googleapis.com | feature-server(feature-server) | 0.56.0 | +| https://feast-helm-charts.storage.googleapis.com | transformation-service(transformation-service) | 0.56.0 | ## Values diff --git a/infra/charts/feast/charts/feature-server/Chart.yaml b/infra/charts/feast/charts/feature-server/Chart.yaml index 9e45c70ef87..43a7e41c3ab 100644 --- a/infra/charts/feast/charts/feature-server/Chart.yaml +++ b/infra/charts/feast/charts/feature-server/Chart.yaml @@ -1,8 +1,8 @@ apiVersion: v1 description: "Feast Feature Server: Online feature serving service for Feast" name: feature-server -version: 0.55.0 -appVersion: v0.55.0 +version: 0.56.0 +appVersion: v0.56.0 keywords: - machine learning - big data diff --git a/infra/charts/feast/charts/feature-server/README.md b/infra/charts/feast/charts/feature-server/README.md index c84b8df1805..46159a543c5 100644 --- a/infra/charts/feast/charts/feature-server/README.md +++ b/infra/charts/feast/charts/feature-server/README.md @@ -1,6 +1,6 @@ # feature-server -![Version: 0.55.0](https://img.shields.io/badge/Version-0.55.0-informational?style=flat-square) ![AppVersion: v0.55.0](https://img.shields.io/badge/AppVersion-v0.55.0-informational?style=flat-square) +![Version: 0.56.0](https://img.shields.io/badge/Version-0.56.0-informational?style=flat-square) ![AppVersion: v0.56.0](https://img.shields.io/badge/AppVersion-v0.56.0-informational?style=flat-square) Feast Feature Server: Online feature serving service for Feast @@ -17,7 +17,7 @@ Feast Feature Server: Online feature serving service for Feast | envOverrides | object | `{}` | Extra environment variables to set | | image.pullPolicy | string | `"IfNotPresent"` | Image pull policy | | image.repository | string | `"quay.io/feastdev/feature-server-java"` | Docker image for Feature Server repository | -| image.tag | string | `"0.55.0"` | Image tag | +| image.tag | string | `"0.56.0"` | Image tag | | ingress.grpc.annotations | object | `{}` | Extra annotations for the ingress | | ingress.grpc.auth.enabled | bool | `false` | Flag to enable auth | | ingress.grpc.class | string | `"nginx"` | Which ingress controller to use | diff --git a/infra/charts/feast/charts/feature-server/values.yaml b/infra/charts/feast/charts/feature-server/values.yaml index 12e89fc1b7c..c054158be62 100644 --- a/infra/charts/feast/charts/feature-server/values.yaml +++ b/infra/charts/feast/charts/feature-server/values.yaml @@ -5,7 +5,7 @@ image: # image.repository -- Docker image for Feature Server repository repository: quay.io/feastdev/feature-server-java # image.tag -- Image tag - tag: 0.55.0 + tag: 0.56.0 # image.pullPolicy -- Image pull policy pullPolicy: IfNotPresent diff --git a/infra/charts/feast/charts/transformation-service/Chart.yaml b/infra/charts/feast/charts/transformation-service/Chart.yaml index 62659b31705..10c15403d9d 100644 --- a/infra/charts/feast/charts/transformation-service/Chart.yaml +++ b/infra/charts/feast/charts/transformation-service/Chart.yaml @@ -1,8 +1,8 @@ apiVersion: v1 description: "Transformation service: to compute on-demand features" name: transformation-service -version: 0.55.0 -appVersion: v0.55.0 +version: 0.56.0 +appVersion: v0.56.0 keywords: - machine learning - big data diff --git a/infra/charts/feast/charts/transformation-service/README.md b/infra/charts/feast/charts/transformation-service/README.md index 4cb9619c65b..716d5f9332b 100644 --- a/infra/charts/feast/charts/transformation-service/README.md +++ b/infra/charts/feast/charts/transformation-service/README.md @@ -1,6 +1,6 @@ # transformation-service -![Version: 0.55.0](https://img.shields.io/badge/Version-0.55.0-informational?style=flat-square) ![AppVersion: v0.55.0](https://img.shields.io/badge/AppVersion-v0.55.0-informational?style=flat-square) +![Version: 0.56.0](https://img.shields.io/badge/Version-0.56.0-informational?style=flat-square) ![AppVersion: v0.56.0](https://img.shields.io/badge/AppVersion-v0.56.0-informational?style=flat-square) Transformation service: to compute on-demand features @@ -13,7 +13,7 @@ Transformation service: to compute on-demand features | envOverrides | object | `{}` | Extra environment variables to set | | image.pullPolicy | string | `"IfNotPresent"` | Image pull policy | | image.repository | string | `"quay.io/feastdev/feature-transformation-server"` | Docker image for Transformation Server repository | -| image.tag | string | `"0.55.0"` | Image tag | +| image.tag | string | `"0.56.0"` | Image tag | | nodeSelector | object | `{}` | Node labels for pod assignment | | podLabels | object | `{}` | Labels to be added to Feast Serving pods | | replicaCount | int | `1` | Number of pods that will be created | diff --git a/infra/charts/feast/charts/transformation-service/values.yaml b/infra/charts/feast/charts/transformation-service/values.yaml index af5b3705a26..fe805f0bed1 100644 --- a/infra/charts/feast/charts/transformation-service/values.yaml +++ b/infra/charts/feast/charts/transformation-service/values.yaml @@ -5,7 +5,7 @@ image: # image.repository -- Docker image for Transformation Server repository repository: quay.io/feastdev/feature-transformation-server # image.tag -- Image tag - tag: 0.55.0 + tag: 0.56.0 # image.pullPolicy -- Image pull policy pullPolicy: IfNotPresent diff --git a/infra/charts/feast/requirements.yaml b/infra/charts/feast/requirements.yaml index adcf4bf9d5f..2ef611c9a0e 100644 --- a/infra/charts/feast/requirements.yaml +++ b/infra/charts/feast/requirements.yaml @@ -1,12 +1,12 @@ dependencies: - name: feature-server alias: feature-server - version: 0.55.0 + version: 0.56.0 condition: feature-server.enabled repository: https://feast-helm-charts.storage.googleapis.com - name: transformation-service alias: transformation-service - version: 0.55.0 + version: 0.56.0 condition: transformation-service.enabled repository: https://feast-helm-charts.storage.googleapis.com - name: redis diff --git a/infra/feast-operator/Makefile b/infra/feast-operator/Makefile index 9b6abd8bd90..9541ed99656 100644 --- a/infra/feast-operator/Makefile +++ b/infra/feast-operator/Makefile @@ -3,7 +3,7 @@ # To re-generate a bundle for another specific version without changing the standard setup, you can: # - use the VERSION as arg of the bundle target (e.g make bundle VERSION=0.0.2) # - use environment variables to overwrite this value (e.g export VERSION=0.0.2) -VERSION ?= 0.55.0 +VERSION ?= 0.56.0 # CHANNELS define the bundle channels used in the bundle. # Add a new line here if you would like to change its default config. (E.g CHANNELS = "candidate,fast,stable") diff --git a/infra/feast-operator/api/feastversion/version.go b/infra/feast-operator/api/feastversion/version.go index 3c89617400f..9279475e53d 100644 --- a/infra/feast-operator/api/feastversion/version.go +++ b/infra/feast-operator/api/feastversion/version.go @@ -17,4 +17,4 @@ limitations under the License. package feastversion // Feast release version. Keep on line #20, this is critical to release CI -const FeastVersion = "0.55.0" +const FeastVersion = "0.56.0" diff --git a/infra/feast-operator/api/v1alpha1/featurestore_types.go b/infra/feast-operator/api/v1alpha1/featurestore_types.go index 191505cc6a6..9250309b1cc 100644 --- a/infra/feast-operator/api/v1alpha1/featurestore_types.go +++ b/infra/feast-operator/api/v1alpha1/featurestore_types.go @@ -541,6 +541,7 @@ type OptionalCtrConfigs struct { EnvFrom *[]corev1.EnvFromSource `json:"envFrom,omitempty"` ImagePullPolicy *corev1.PullPolicy `json:"imagePullPolicy,omitempty"` Resources *corev1.ResourceRequirements `json:"resources,omitempty"` + NodeSelector *map[string]string `json:"nodeSelector,omitempty"` } // AuthzConfig defines the authorization settings for the deployed Feast services. diff --git a/infra/feast-operator/api/v1alpha1/zz_generated.deepcopy.go b/infra/feast-operator/api/v1alpha1/zz_generated.deepcopy.go index 1a893c82cf8..7ea04929b3d 100644 --- a/infra/feast-operator/api/v1alpha1/zz_generated.deepcopy.go +++ b/infra/feast-operator/api/v1alpha1/zz_generated.deepcopy.go @@ -768,6 +768,17 @@ func (in *OptionalCtrConfigs) DeepCopyInto(out *OptionalCtrConfigs) { *out = new(v1.ResourceRequirements) (*in).DeepCopyInto(*out) } + if in.NodeSelector != nil { + in, out := &in.NodeSelector, &out.NodeSelector + *out = new(map[string]string) + if **in != nil { + in, out := *in, *out + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new OptionalCtrConfigs. diff --git a/infra/feast-operator/bundle/manifests/feast-operator.clusterserviceversion.yaml b/infra/feast-operator/bundle/manifests/feast-operator.clusterserviceversion.yaml index 9dc7e58cb4e..78205683183 100644 --- a/infra/feast-operator/bundle/manifests/feast-operator.clusterserviceversion.yaml +++ b/infra/feast-operator/bundle/manifests/feast-operator.clusterserviceversion.yaml @@ -50,10 +50,10 @@ metadata: } ] capabilities: Basic Install - createdAt: "2025-10-14T14:37:31Z" + createdAt: "2025-10-27T20:28:39Z" operators.operatorframework.io/builder: operator-sdk-v1.38.0 operators.operatorframework.io/project_layout: go.kubebuilder.io/v4 - name: feast-operator.v0.55.0 + name: feast-operator.v0.56.0 namespace: placeholder spec: apiservicedefinitions: {} @@ -225,10 +225,10 @@ spec: - /manager env: - name: RELATED_IMAGE_FEATURE_SERVER - value: quay.io/feastdev/feature-server:0.55.0 + value: quay.io/feastdev/feature-server:0.56.0 - name: RELATED_IMAGE_CRON_JOB value: quay.io/openshift/origin-cli:4.17 - image: quay.io/feastdev/feast-operator:0.55.0 + image: quay.io/feastdev/feast-operator:0.56.0 livenessProbe: httpGet: path: /healthz @@ -318,8 +318,8 @@ spec: name: Feast Community url: https://lf-aidata.atlassian.net/wiki/spaces/FEAST/ relatedImages: - - image: quay.io/feastdev/feature-server:0.55.0 + - image: quay.io/feastdev/feature-server:0.56.0 name: feature-server - image: quay.io/openshift/origin-cli:4.17 name: cron-job - version: 0.55.0 + version: 0.56.0 diff --git a/infra/feast-operator/bundle/manifests/feast.dev_featurestores.yaml b/infra/feast-operator/bundle/manifests/feast.dev_featurestores.yaml index 5afa2ea3704..9c0c09d141a 100644 --- a/infra/feast-operator/bundle/manifests/feast.dev_featurestores.yaml +++ b/infra/feast-operator/bundle/manifests/feast.dev_featurestores.yaml @@ -259,6 +259,10 @@ spec: description: PullPolicy describes a policy for if/when to pull a container image type: string + nodeSelector: + additionalProperties: + type: string + type: object resources: description: ResourceRequirements describes the compute resource requirements. @@ -1025,6 +1029,10 @@ spec: - error - critical type: string + nodeSelector: + additionalProperties: + type: string + type: object resources: description: ResourceRequirements describes the compute resource requirements. @@ -1480,6 +1488,10 @@ spec: - error - critical type: string + nodeSelector: + additionalProperties: + type: string + type: object resources: description: ResourceRequirements describes the compute resource requirements. @@ -1946,6 +1958,10 @@ spec: - error - critical type: string + nodeSelector: + additionalProperties: + type: string + type: object resources: description: ResourceRequirements describes the compute resource requirements. @@ -2441,6 +2457,10 @@ spec: - error - critical type: string + nodeSelector: + additionalProperties: + type: string + type: object resources: description: ResourceRequirements describes the compute resource requirements. @@ -4216,6 +4236,10 @@ spec: description: PullPolicy describes a policy for if/when to pull a container image type: string + nodeSelector: + additionalProperties: + type: string + type: object resources: description: ResourceRequirements describes the compute resource requirements. @@ -4994,6 +5018,10 @@ spec: - error - critical type: string + nodeSelector: + additionalProperties: + type: string + type: object resources: description: ResourceRequirements describes the compute resource requirements. @@ -5457,6 +5485,10 @@ spec: - error - critical type: string + nodeSelector: + additionalProperties: + type: string + type: object resources: description: ResourceRequirements describes the compute resource requirements. @@ -5935,6 +5967,10 @@ spec: - error - critical type: string + nodeSelector: + additionalProperties: + type: string + type: object resources: description: ResourceRequirements describes the compute resource requirements. @@ -6440,6 +6476,10 @@ spec: - error - critical type: string + nodeSelector: + additionalProperties: + type: string + type: object resources: description: ResourceRequirements describes the compute resource requirements. diff --git a/infra/feast-operator/config/component_metadata.yaml b/infra/feast-operator/config/component_metadata.yaml index 395624fb21a..cda672ead9a 100644 --- a/infra/feast-operator/config/component_metadata.yaml +++ b/infra/feast-operator/config/component_metadata.yaml @@ -1,5 +1,5 @@ # This file is required to configure Feast release information for ODH/RHOAI Operator releases: - name: Feast - version: 0.55.0 + version: 0.56.0 repoUrl: https://github.com/feast-dev/feast diff --git a/infra/feast-operator/config/crd/bases/feast.dev_featurestores.yaml b/infra/feast-operator/config/crd/bases/feast.dev_featurestores.yaml index 8debe3639f9..c964d46c27d 100644 --- a/infra/feast-operator/config/crd/bases/feast.dev_featurestores.yaml +++ b/infra/feast-operator/config/crd/bases/feast.dev_featurestores.yaml @@ -259,6 +259,10 @@ spec: description: PullPolicy describes a policy for if/when to pull a container image type: string + nodeSelector: + additionalProperties: + type: string + type: object resources: description: ResourceRequirements describes the compute resource requirements. @@ -1025,6 +1029,10 @@ spec: - error - critical type: string + nodeSelector: + additionalProperties: + type: string + type: object resources: description: ResourceRequirements describes the compute resource requirements. @@ -1480,6 +1488,10 @@ spec: - error - critical type: string + nodeSelector: + additionalProperties: + type: string + type: object resources: description: ResourceRequirements describes the compute resource requirements. @@ -1946,6 +1958,10 @@ spec: - error - critical type: string + nodeSelector: + additionalProperties: + type: string + type: object resources: description: ResourceRequirements describes the compute resource requirements. @@ -2441,6 +2457,10 @@ spec: - error - critical type: string + nodeSelector: + additionalProperties: + type: string + type: object resources: description: ResourceRequirements describes the compute resource requirements. @@ -4216,6 +4236,10 @@ spec: description: PullPolicy describes a policy for if/when to pull a container image type: string + nodeSelector: + additionalProperties: + type: string + type: object resources: description: ResourceRequirements describes the compute resource requirements. @@ -4994,6 +5018,10 @@ spec: - error - critical type: string + nodeSelector: + additionalProperties: + type: string + type: object resources: description: ResourceRequirements describes the compute resource requirements. @@ -5457,6 +5485,10 @@ spec: - error - critical type: string + nodeSelector: + additionalProperties: + type: string + type: object resources: description: ResourceRequirements describes the compute resource requirements. @@ -5935,6 +5967,10 @@ spec: - error - critical type: string + nodeSelector: + additionalProperties: + type: string + type: object resources: description: ResourceRequirements describes the compute resource requirements. @@ -6440,6 +6476,10 @@ spec: - error - critical type: string + nodeSelector: + additionalProperties: + type: string + type: object resources: description: ResourceRequirements describes the compute resource requirements. diff --git a/infra/feast-operator/config/default/related_image_fs_patch.yaml b/infra/feast-operator/config/default/related_image_fs_patch.yaml index b7ed2b05900..30cd7d3616f 100644 --- a/infra/feast-operator/config/default/related_image_fs_patch.yaml +++ b/infra/feast-operator/config/default/related_image_fs_patch.yaml @@ -2,7 +2,7 @@ path: "/spec/template/spec/containers/0/env/0" value: name: RELATED_IMAGE_FEATURE_SERVER - value: quay.io/feastdev/feature-server:0.55.0 + value: quay.io/feastdev/feature-server:0.56.0 - op: replace path: "/spec/template/spec/containers/0/env/1" value: diff --git a/infra/feast-operator/config/manager/kustomization.yaml b/infra/feast-operator/config/manager/kustomization.yaml index be172b92a24..80aaa3faf28 100644 --- a/infra/feast-operator/config/manager/kustomization.yaml +++ b/infra/feast-operator/config/manager/kustomization.yaml @@ -5,4 +5,4 @@ kind: Kustomization images: - name: controller newName: quay.io/feastdev/feast-operator - newTag: 0.55.0 + newTag: 0.56.0 diff --git a/infra/feast-operator/config/overlays/odh/params.env b/infra/feast-operator/config/overlays/odh/params.env index c048b0151db..b112bb2d854 100644 --- a/infra/feast-operator/config/overlays/odh/params.env +++ b/infra/feast-operator/config/overlays/odh/params.env @@ -1,3 +1,3 @@ -RELATED_IMAGE_FEAST_OPERATOR=quay.io/feastdev/feast-operator:0.55.0 -RELATED_IMAGE_FEATURE_SERVER=quay.io/feastdev/feature-server:0.55.0 +RELATED_IMAGE_FEAST_OPERATOR=quay.io/feastdev/feast-operator:0.56.0 +RELATED_IMAGE_FEATURE_SERVER=quay.io/feastdev/feature-server:0.56.0 RELATED_IMAGE_CRON_JOB=quay.io/openshift/origin-cli:4.17 diff --git a/infra/feast-operator/config/overlays/rhoai/params.env b/infra/feast-operator/config/overlays/rhoai/params.env index e39233f168c..f548227235d 100644 --- a/infra/feast-operator/config/overlays/rhoai/params.env +++ b/infra/feast-operator/config/overlays/rhoai/params.env @@ -1,3 +1,3 @@ -RELATED_IMAGE_FEAST_OPERATOR=quay.io/feastdev/feast-operator:0.55.0 -RELATED_IMAGE_FEATURE_SERVER=quay.io/feastdev/feature-server:0.55.0 +RELATED_IMAGE_FEAST_OPERATOR=quay.io/feastdev/feast-operator:0.56.0 +RELATED_IMAGE_FEATURE_SERVER=quay.io/feastdev/feature-server:0.56.0 RELATED_IMAGE_CRON_JOB=registry.redhat.io/openshift4/ose-cli@sha256:bc35a9fc663baf0d6493cc57e89e77a240a36c43cf38fb78d8e61d3b87cf5cc5 \ No newline at end of file diff --git a/infra/feast-operator/dist/install.yaml b/infra/feast-operator/dist/install.yaml index 9efa8044a29..58886675ec1 100644 --- a/infra/feast-operator/dist/install.yaml +++ b/infra/feast-operator/dist/install.yaml @@ -267,6 +267,10 @@ spec: description: PullPolicy describes a policy for if/when to pull a container image type: string + nodeSelector: + additionalProperties: + type: string + type: object resources: description: ResourceRequirements describes the compute resource requirements. @@ -1033,6 +1037,10 @@ spec: - error - critical type: string + nodeSelector: + additionalProperties: + type: string + type: object resources: description: ResourceRequirements describes the compute resource requirements. @@ -1488,6 +1496,10 @@ spec: - error - critical type: string + nodeSelector: + additionalProperties: + type: string + type: object resources: description: ResourceRequirements describes the compute resource requirements. @@ -1954,6 +1966,10 @@ spec: - error - critical type: string + nodeSelector: + additionalProperties: + type: string + type: object resources: description: ResourceRequirements describes the compute resource requirements. @@ -2449,6 +2465,10 @@ spec: - error - critical type: string + nodeSelector: + additionalProperties: + type: string + type: object resources: description: ResourceRequirements describes the compute resource requirements. @@ -4224,6 +4244,10 @@ spec: description: PullPolicy describes a policy for if/when to pull a container image type: string + nodeSelector: + additionalProperties: + type: string + type: object resources: description: ResourceRequirements describes the compute resource requirements. @@ -5002,6 +5026,10 @@ spec: - error - critical type: string + nodeSelector: + additionalProperties: + type: string + type: object resources: description: ResourceRequirements describes the compute resource requirements. @@ -5465,6 +5493,10 @@ spec: - error - critical type: string + nodeSelector: + additionalProperties: + type: string + type: object resources: description: ResourceRequirements describes the compute resource requirements. @@ -5943,6 +5975,10 @@ spec: - error - critical type: string + nodeSelector: + additionalProperties: + type: string + type: object resources: description: ResourceRequirements describes the compute resource requirements. @@ -6448,6 +6484,10 @@ spec: - error - critical type: string + nodeSelector: + additionalProperties: + type: string + type: object resources: description: ResourceRequirements describes the compute resource requirements. @@ -8443,10 +8483,10 @@ spec: - /manager env: - name: RELATED_IMAGE_FEATURE_SERVER - value: quay.io/feastdev/feature-server:0.55.0 + value: quay.io/feastdev/feature-server:0.56.0 - name: RELATED_IMAGE_CRON_JOB value: quay.io/openshift/origin-cli:4.17 - image: quay.io/feastdev/feast-operator:0.55.0 + image: quay.io/feastdev/feast-operator:0.56.0 livenessProbe: httpGet: path: /healthz diff --git a/infra/feast-operator/dist/operator-e2e-tests b/infra/feast-operator/dist/operator-e2e-tests index 5eb52e2a90a..2a8e2a02352 100755 Binary files a/infra/feast-operator/dist/operator-e2e-tests and b/infra/feast-operator/dist/operator-e2e-tests differ diff --git a/infra/feast-operator/docs/api/markdown/ref.md b/infra/feast-operator/docs/api/markdown/ref.md index 68978a08cf0..fac7ebfa784 100644 --- a/infra/feast-operator/docs/api/markdown/ref.md +++ b/infra/feast-operator/docs/api/markdown/ref.md @@ -46,6 +46,7 @@ _Appears in:_ | `envFrom` _[EnvFromSource](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.30/#envfromsource-v1-core)_ | | | `imagePullPolicy` _[PullPolicy](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.30/#pullpolicy-v1-core)_ | | | `resources` _[ResourceRequirements](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.30/#resourcerequirements-v1-core)_ | | +| `nodeSelector` _map[string]string_ | | #### CronJobContainerConfigs @@ -64,6 +65,7 @@ _Appears in:_ | `envFrom` _[EnvFromSource](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.30/#envfromsource-v1-core)_ | | | `imagePullPolicy` _[PullPolicy](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.30/#pullpolicy-v1-core)_ | | | `resources` _[ResourceRequirements](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.30/#resourcerequirements-v1-core)_ | | +| `nodeSelector` _map[string]string_ | | | `commands` _string array_ | Array of commands to be executed (in order) against a Feature Store deployment. Defaults to "feast apply" & "feast materialize-incremental $(date -u +'%Y-%m-%dT%H:%M:%S')" | @@ -566,6 +568,7 @@ _Appears in:_ | `envFrom` _[EnvFromSource](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.30/#envfromsource-v1-core)_ | | | `imagePullPolicy` _[PullPolicy](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.30/#pullpolicy-v1-core)_ | | | `resources` _[ResourceRequirements](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.30/#resourcerequirements-v1-core)_ | | +| `nodeSelector` _map[string]string_ | | #### PvcConfig @@ -688,6 +691,7 @@ _Appears in:_ | `envFrom` _[EnvFromSource](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.30/#envfromsource-v1-core)_ | | | `imagePullPolicy` _[PullPolicy](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.30/#pullpolicy-v1-core)_ | | | `resources` _[ResourceRequirements](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.30/#resourcerequirements-v1-core)_ | | +| `nodeSelector` _map[string]string_ | | | `tls` _[TlsConfigs](#tlsconfigs)_ | | | `logLevel` _string_ | LogLevel sets the logging level for the server Allowed values: "debug", "info", "warning", "error", "critical". | @@ -750,6 +754,7 @@ _Appears in:_ | `envFrom` _[EnvFromSource](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.30/#envfromsource-v1-core)_ | | | `imagePullPolicy` _[PullPolicy](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.30/#pullpolicy-v1-core)_ | | | `resources` _[ResourceRequirements](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.30/#resourcerequirements-v1-core)_ | | +| `nodeSelector` _map[string]string_ | | | `tls` _[TlsConfigs](#tlsconfigs)_ | | | `logLevel` _string_ | LogLevel sets the logging level for the server Allowed values: "debug", "info", "warning", "error", "critical". | diff --git a/infra/feast-operator/internal/controller/services/services.go b/infra/feast-operator/internal/controller/services/services.go index 5b70d6e1911..3b14f9c49db 100644 --- a/infra/feast-operator/internal/controller/services/services.go +++ b/infra/feast-operator/internal/controller/services/services.go @@ -412,6 +412,7 @@ func (feast *FeastServices) setPod(podSpec *corev1.PodSpec) error { feast.mountPvcConfigs(podSpec) feast.mountEmptyDirVolumes(podSpec) feast.mountUserDefinedVolumes(podSpec) + feast.applyNodeSelector(podSpec) return nil } @@ -793,6 +794,56 @@ func (feast *FeastServices) getLogLevelForType(feastType FeastServiceType) *stri return nil } +func (feast *FeastServices) getNodeSelectorForType(feastType FeastServiceType) *map[string]string { + if serviceConfigs := feast.getServerConfigs(feastType); serviceConfigs != nil { + return serviceConfigs.ContainerConfigs.OptionalCtrConfigs.NodeSelector + } + return nil +} + +func (feast *FeastServices) applyNodeSelector(podSpec *corev1.PodSpec) { + // Merge node selectors from all services + mergedNodeSelector := make(map[string]string) + + // Check all service types for node selector configuration + allServiceTypes := append(feastServerTypes, UIFeastType) + for _, feastType := range allServiceTypes { + if selector := feast.getNodeSelectorForType(feastType); selector != nil && len(*selector) > 0 { + for k, v := range *selector { + mergedNodeSelector[k] = v + } + } + } + + // If no service has node selector configured, we're done + if len(mergedNodeSelector) == 0 { + return + } + + // Merge with any existing node selectors (from ops team or other sources) + // This preserves pre-existing selectors while adding operator requirements + finalNodeSelector := feast.mergeNodeSelectors(podSpec.NodeSelector, mergedNodeSelector) + podSpec.NodeSelector = finalNodeSelector +} + +// mergeNodeSelectors merges existing and operator node selectors +// Existing selectors are preserved, operator selectors can override existing keys +func (feast *FeastServices) mergeNodeSelectors(existing, operator map[string]string) map[string]string { + merged := make(map[string]string) + + // Start with existing selectors (from ops team or other sources) + for k, v := range existing { + merged[k] = v + } + + // Add/override with operator selectors + for k, v := range operator { + merged[k] = v + } + + return merged +} + // GetObjectMeta returns the feast k8s object metadata with type func (feast *FeastServices) GetObjectMeta() metav1.ObjectMeta { return metav1.ObjectMeta{Name: GetFeastName(feast.Handler.FeatureStore), Namespace: feast.Handler.FeatureStore.Namespace} diff --git a/infra/feast-operator/internal/controller/services/services_test.go b/infra/feast-operator/internal/controller/services/services_test.go index 0c43aff5954..14509fe9933 100644 --- a/infra/feast-operator/internal/controller/services/services_test.go +++ b/infra/feast-operator/internal/controller/services/services_test.go @@ -203,4 +203,193 @@ var _ = Describe("Registry Service", func() { Expect(ports[1].Name).To(Equal(string(RegistryFeastType) + "-rest")) }) }) + + Describe("NodeSelector Configuration", func() { + It("should apply NodeSelector to pod spec when configured", func() { + // Set NodeSelector for registry service + nodeSelector := map[string]string{ + "kubernetes.io/os": "linux", + "node-type": "compute", + } + featureStore.Spec.Services.Registry.Local.Server.ContainerConfigs.OptionalCtrConfigs.NodeSelector = &nodeSelector + Expect(k8sClient.Update(ctx, featureStore)).To(Succeed()) + Expect(feast.ApplyDefaults()).To(Succeed()) + applySpecToStatus(featureStore) + feast.refreshFeatureStore(ctx, typeNamespacedName) + + // Create deployment and verify NodeSelector is applied + deployment := feast.initFeastDeploy() + Expect(deployment).NotTo(BeNil()) + Expect(feast.setDeployment(deployment)).To(Succeed()) + + // Verify NodeSelector is applied to pod spec + expectedNodeSelector := map[string]string{ + "kubernetes.io/os": "linux", + "node-type": "compute", + } + Expect(deployment.Spec.Template.Spec.NodeSelector).To(Equal(expectedNodeSelector)) + }) + + It("should merge NodeSelectors from multiple services", func() { + // Set NodeSelector for registry service + registryNodeSelector := map[string]string{ + "kubernetes.io/os": "linux", + "node-type": "compute", + } + featureStore.Spec.Services.Registry.Local.Server.ContainerConfigs.OptionalCtrConfigs.NodeSelector = ®istryNodeSelector + + // Set NodeSelector for online store service + onlineNodeSelector := map[string]string{ + "node-type": "online", + "zone": "us-west-1a", + } + featureStore.Spec.Services.OnlineStore = &feastdevv1alpha1.OnlineStore{ + Server: &feastdevv1alpha1.ServerConfigs{ + ContainerConfigs: feastdevv1alpha1.ContainerConfigs{ + DefaultCtrConfigs: feastdevv1alpha1.DefaultCtrConfigs{ + Image: ptr("test-image"), + }, + OptionalCtrConfigs: feastdevv1alpha1.OptionalCtrConfigs{ + NodeSelector: &onlineNodeSelector, + }, + }, + }, + } + + Expect(k8sClient.Update(ctx, featureStore)).To(Succeed()) + Expect(feast.ApplyDefaults()).To(Succeed()) + applySpecToStatus(featureStore) + feast.refreshFeatureStore(ctx, typeNamespacedName) + + // Create deployment and verify merged NodeSelector is applied + deployment := feast.initFeastDeploy() + Expect(deployment).NotTo(BeNil()) + Expect(feast.setDeployment(deployment)).To(Succeed()) + + // Verify NodeSelector merges all service selectors (online overrides registry for node-type) + expectedNodeSelector := map[string]string{ + "kubernetes.io/os": "linux", + "node-type": "online", + "zone": "us-west-1a", + } + Expect(deployment.Spec.Template.Spec.NodeSelector).To(Equal(expectedNodeSelector)) + }) + + It("should merge operator NodeSelector with existing selectors (mutating webhook scenario)", func() { + // Set NodeSelector for UI service + uiNodeSelector := map[string]string{ + "node-type": "ui", + } + featureStore.Spec.Services.UI = &feastdevv1alpha1.ServerConfigs{ + ContainerConfigs: feastdevv1alpha1.ContainerConfigs{ + DefaultCtrConfigs: feastdevv1alpha1.DefaultCtrConfigs{ + Image: ptr("test-image"), + }, + OptionalCtrConfigs: feastdevv1alpha1.OptionalCtrConfigs{ + NodeSelector: &uiNodeSelector, + }, + }, + } + + Expect(k8sClient.Update(ctx, featureStore)).To(Succeed()) + Expect(feast.ApplyDefaults()).To(Succeed()) + applySpecToStatus(featureStore) + feast.refreshFeatureStore(ctx, typeNamespacedName) + + // Create deployment first + deployment := feast.initFeastDeploy() + Expect(deployment).NotTo(BeNil()) + Expect(feast.setDeployment(deployment)).To(Succeed()) + + // Simulate a mutating webhook or admission controller adding node selectors + // This would happen after the operator creates the pod spec but before scheduling + existingNodeSelector := map[string]string{ + "team": "ml", + "environment": "prod", + } + deployment.Spec.Template.Spec.NodeSelector = existingNodeSelector + + // Apply the node selector logic again to test merging + // This simulates the operator reconciling and re-applying node selectors + feast.applyNodeSelector(&deployment.Spec.Template.Spec) + + // Verify NodeSelector merges existing and operator selectors + expectedNodeSelector := map[string]string{ + "team": "ml", + "environment": "prod", + "node-type": "ui", + } + Expect(deployment.Spec.Template.Spec.NodeSelector).To(Equal(expectedNodeSelector)) + }) + + It("should apply UI service NodeSelector when UI has highest precedence", func() { + // Set NodeSelector for online service + onlineNodeSelector := map[string]string{ + "node-type": "online", + } + featureStore.Spec.Services.OnlineStore = &feastdevv1alpha1.OnlineStore{ + Server: &feastdevv1alpha1.ServerConfigs{ + ContainerConfigs: feastdevv1alpha1.ContainerConfigs{ + DefaultCtrConfigs: feastdevv1alpha1.DefaultCtrConfigs{ + Image: ptr("test-image"), + }, + OptionalCtrConfigs: feastdevv1alpha1.OptionalCtrConfigs{ + NodeSelector: &onlineNodeSelector, + }, + }, + }, + } + + // Set NodeSelector for UI service (should win) + uiNodeSelector := map[string]string{ + "node-type": "ui", + "zone": "us-east-1", + } + featureStore.Spec.Services.UI = &feastdevv1alpha1.ServerConfigs{ + ContainerConfigs: feastdevv1alpha1.ContainerConfigs{ + DefaultCtrConfigs: feastdevv1alpha1.DefaultCtrConfigs{ + Image: ptr("test-image"), + }, + OptionalCtrConfigs: feastdevv1alpha1.OptionalCtrConfigs{ + NodeSelector: &uiNodeSelector, + }, + }, + } + + Expect(k8sClient.Update(ctx, featureStore)).To(Succeed()) + Expect(feast.ApplyDefaults()).To(Succeed()) + applySpecToStatus(featureStore) + feast.refreshFeatureStore(ctx, typeNamespacedName) + + // Create deployment and verify UI service selector is applied + deployment := feast.initFeastDeploy() + Expect(deployment).NotTo(BeNil()) + Expect(feast.setDeployment(deployment)).To(Succeed()) + + // Verify NodeSelector is applied with UI service's selector (UI wins) + expectedNodeSelector := map[string]string{ + "node-type": "ui", + "zone": "us-east-1", + } + Expect(deployment.Spec.Template.Spec.NodeSelector).To(Equal(expectedNodeSelector)) + }) + + It("should handle empty NodeSelector gracefully", func() { + // Set empty NodeSelector + emptyNodeSelector := map[string]string{} + featureStore.Spec.Services.Registry.Local.Server.ContainerConfigs.OptionalCtrConfigs.NodeSelector = &emptyNodeSelector + Expect(k8sClient.Update(ctx, featureStore)).To(Succeed()) + Expect(feast.ApplyDefaults()).To(Succeed()) + applySpecToStatus(featureStore) + feast.refreshFeatureStore(ctx, typeNamespacedName) + + // Create deployment and verify no NodeSelector is applied (empty selector) + deployment := feast.initFeastDeploy() + Expect(deployment).NotTo(BeNil()) + Expect(feast.setDeployment(deployment)).To(Succeed()) + + // Verify no NodeSelector is applied (empty selector) + Expect(deployment.Spec.Template.Spec.NodeSelector).To(BeEmpty()) + }) + }) }) diff --git a/java/pom.xml b/java/pom.xml index 59793c2cabf..d9625cecf49 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -35,7 +35,7 @@ - 0.55.0 + 0.56.0 https://github.com/feast-dev/feast UTF-8 diff --git a/protos/feast/core/FeatureView.proto b/protos/feast/core/FeatureView.proto index 9ea33b33fb7..6306d425be3 100644 --- a/protos/feast/core/FeatureView.proto +++ b/protos/feast/core/FeatureView.proto @@ -36,7 +36,7 @@ message FeatureView { FeatureViewMeta meta = 2; } -// Next available id: 16 +// Next available id: 17 // TODO(adchia): refactor common fields from this and ODFV into separate metadata proto message FeatureViewSpec { // Name of the feature view. Must be unique. Not updated. @@ -51,18 +51,9 @@ message FeatureViewSpec { // List of specifications for each feature defined as part of this feature view. repeated FeatureSpecV2 features = 4; - // List of specifications for each entity defined as part of this feature view. - repeated FeatureSpecV2 entity_columns = 12; - - // Description of the feature view. - string description = 10; - // User defined metadata map tags = 5; - // Owner of the feature view. - string owner = 11; - // Features in this feature view can only be retrieved from online serving // younger than ttl. Ttl is measured as the duration of time between // the feature's event timestamp and when the feature is retrieved @@ -71,13 +62,23 @@ message FeatureViewSpec { // Batch/Offline DataSource where this view can retrieve offline feature data. DataSource batch_source = 7; - // Streaming DataSource from where this view can consume "online" feature data. - DataSource stream_source = 9; // Whether these features should be served online or not // This is also used to determine whether the features should be written to the online store bool online = 8; + // Streaming DataSource from where this view can consume "online" feature data. + DataSource stream_source = 9; + + // Description of the feature view. + string description = 10; + + // Owner of the feature view. + string owner = 11; + + // List of specifications for each entity defined as part of this feature view. + repeated FeatureSpecV2 entity_columns = 12; + // Whether these features should be written to the offline store bool offline = 13; @@ -85,6 +86,9 @@ message FeatureViewSpec { // Feature transformation for batch feature views FeatureTransformationV2 feature_transformation = 15; + + // The transformation mode (e.g., "python", "pandas", "spark", "sql", "ray") + string mode = 16; } message FeatureViewMeta { diff --git a/protos/feast/core/Transformation.proto b/protos/feast/core/Transformation.proto index 7033f553f16..68a8b48229e 100644 --- a/protos/feast/core/Transformation.proto +++ b/protos/feast/core/Transformation.proto @@ -15,6 +15,9 @@ message UserDefinedFunctionV2 { // The string representation of the udf string body_text = 3; + + // The transformation mode (e.g., "python", "pandas", "ray", "spark", "sql") + string mode = 4; } // A feature transformation executed as a user-defined function diff --git a/sdk/python/docs/source/feast.infra.compute_engines.local.backends.rst b/sdk/python/docs/source/feast.infra.compute_engines.local.backends.rst index 39205f3c4df..69eeed70bb5 100644 --- a/sdk/python/docs/source/feast.infra.compute_engines.local.backends.rst +++ b/sdk/python/docs/source/feast.infra.compute_engines.local.backends.rst @@ -7,7 +7,7 @@ Submodules feast.infra.compute\_engines.local.backends.base module ------------------------------------------------------- -.. automodule:: feast.infra.compute_engines.local.backends.base +.. automodule:: feast.infra.compute_engines.backends.base :members: :undoc-members: :show-inheritance: @@ -15,7 +15,7 @@ feast.infra.compute\_engines.local.backends.base module feast.infra.compute\_engines.local.backends.factory module ---------------------------------------------------------- -.. automodule:: feast.infra.compute_engines.local.backends.factory +.. automodule:: feast.infra.compute_engines.backends.factory :members: :undoc-members: :show-inheritance: @@ -23,7 +23,7 @@ feast.infra.compute\_engines.local.backends.factory module feast.infra.compute\_engines.local.backends.pandas\_backend module ------------------------------------------------------------------ -.. automodule:: feast.infra.compute_engines.local.backends.pandas_backend +.. automodule:: feast.infra.compute_engines.backends.pandas_backend :members: :undoc-members: :show-inheritance: @@ -31,7 +31,7 @@ feast.infra.compute\_engines.local.backends.pandas\_backend module feast.infra.compute\_engines.local.backends.polars\_backend module ------------------------------------------------------------------ -.. automodule:: feast.infra.compute_engines.local.backends.polars_backend +.. automodule:: feast.infra.compute_engines.backends.polars_backend :members: :undoc-members: :show-inheritance: @@ -39,7 +39,7 @@ feast.infra.compute\_engines.local.backends.polars\_backend module Module contents --------------- -.. automodule:: feast.infra.compute_engines.local.backends +.. automodule:: feast.infra.compute_engines.backends :members: :undoc-members: :show-inheritance: diff --git a/sdk/python/docs/source/feast.infra.compute_engines.local.rst b/sdk/python/docs/source/feast.infra.compute_engines.local.rst index 6525199e6dc..d6ce47cb140 100644 --- a/sdk/python/docs/source/feast.infra.compute_engines.local.rst +++ b/sdk/python/docs/source/feast.infra.compute_engines.local.rst @@ -7,7 +7,7 @@ Subpackages .. toctree:: :maxdepth: 4 - feast.infra.compute_engines.local.backends + feast.infra.compute_engines.backends Submodules ---------- diff --git a/sdk/python/feast/api/registry/rest/metrics.py b/sdk/python/feast/api/registry/rest/metrics.py index 32253e85aca..58dc7156144 100644 --- a/sdk/python/feast/api/registry/rest/metrics.py +++ b/sdk/python/feast/api/registry/rest/metrics.py @@ -98,8 +98,8 @@ def count_resources_for_project(project_name: str): features = {"features": []} try: feature_views = grpc_call( - grpc_handler.ListFeatureViews, - RegistryServer_pb2.ListFeatureViewsRequest( + grpc_handler.ListAllFeatureViews, + RegistryServer_pb2.ListAllFeatureViewsRequest( project=project_name, allow_cache=allow_cache ), ) diff --git a/sdk/python/feast/batch_feature_view.py b/sdk/python/feast/batch_feature_view.py index c168aaeec10..3f3e1bf20ec 100644 --- a/sdk/python/feast/batch_feature_view.py +++ b/sdk/python/feast/batch_feature_view.py @@ -57,7 +57,6 @@ class BatchFeatureView(FeatureView): """ name: str - mode: Union[TransformationMode, str] entities: List[str] ttl: Optional[timedelta] source: DataSource @@ -136,6 +135,7 @@ def __init__( schema=schema, source=source, # type: ignore[arg-type] sink_source=sink_source, + mode=mode, ) def get_feature_transformation(self) -> Optional[Transformation]: @@ -145,7 +145,8 @@ def get_feature_transformation(self) -> Optional[Transformation]: TransformationMode.PANDAS, TransformationMode.PYTHON, TransformationMode.SQL, - ) or self.mode in ("pandas", "python", "sql"): + TransformationMode.RAY, + ) or self.mode in ("pandas", "python", "sql", "ray"): return Transformation( mode=self.mode, udf=self.udf, udf_string=self.udf_string or "" ) diff --git a/sdk/python/feast/cli/cli.py b/sdk/python/feast/cli/cli.py index 1bdf1f5ff85..91fa2a92606 100644 --- a/sdk/python/feast/cli/cli.py +++ b/sdk/python/feast/cli/cli.py @@ -411,6 +411,7 @@ def materialize_incremental_command(ctx: click.Context, end_ts: str, views: List "couchbase", "milvus", "ray", + "ray_rag", ], case_sensitive=False, ), diff --git a/sdk/python/feast/feature_server.py b/sdk/python/feast/feature_server.py index 0cc90b294d8..fee7e56e9c1 100644 --- a/sdk/python/feast/feature_server.py +++ b/sdk/python/feast/feature_server.py @@ -120,27 +120,26 @@ class SaveDocumentRequest(BaseModel): data: dict -def _get_features( +async def _get_features( request: Union[GetOnlineFeaturesRequest, GetOnlineDocumentsRequest], store: "feast.FeatureStore", ): if request.feature_service: - feature_service = store.get_feature_service( - request.feature_service, allow_cache=True + feature_service = await run_in_threadpool( + store.get_feature_service, request.feature_service, allow_cache=True ) assert_permissions( resource=feature_service, actions=[AuthzedAction.READ_ONLINE] ) features = feature_service # type: ignore else: - all_feature_views, all_on_demand_feature_views = ( - utils._get_feature_views_to_use( - store.registry, - store.project, - request.features, - allow_cache=True, - hide_dummy_entity=False, - ) + all_feature_views, all_on_demand_feature_views = await run_in_threadpool( + utils._get_feature_views_to_use, + store.registry, + store.project, + request.features, + allow_cache=True, + hide_dummy_entity=False, ) for feature_view in all_feature_views: assert_permissions( @@ -230,7 +229,7 @@ async def lifespan(app: FastAPI): ) async def get_online_features(request: GetOnlineFeaturesRequest) -> Dict[str, Any]: # Initialize parameters for FeatureStore.get_online_features(...) call - features = await run_in_threadpool(_get_features, request, store) + features = await _get_features(request, store) read_params = dict( features=features, @@ -265,7 +264,7 @@ async def retrieve_online_documents( "This endpoint is in alpha and will be moved to /get-online-features when stable." ) # Initialize parameters for FeatureStore.retrieve_online_documents_v2(...) call - features = await run_in_threadpool(_get_features, request, store) + features = await _get_features(request, store) read_params = dict(features=features, query=request.query, top_k=request.top_k) if request.api_version == 2 and request.query_string is not None: @@ -342,26 +341,31 @@ async def push(request: PushFeaturesRequest) -> None: else: store.push(**push_params) - def _get_feast_object( + async def _get_feast_object( feature_view_name: str, allow_registry_cache: bool ) -> FeastObject: try: - return store.get_stream_feature_view( # type: ignore - feature_view_name, allow_registry_cache=allow_registry_cache + return await run_in_threadpool( + store.get_stream_feature_view, + feature_view_name, + allow_registry_cache=allow_registry_cache, ) except FeatureViewNotFoundException: - return store.get_feature_view( # type: ignore - feature_view_name, allow_registry_cache=allow_registry_cache + return await run_in_threadpool( + store.get_feature_view, + feature_view_name, + allow_registry_cache=allow_registry_cache, ) @app.post("/write-to-online-store", dependencies=[Depends(inject_user_details)]) - def write_to_online_store(request: WriteToFeatureStoreRequest) -> None: + async def write_to_online_store(request: WriteToFeatureStoreRequest) -> None: df = pd.DataFrame(request.df) feature_view_name = request.feature_view_name allow_registry_cache = request.allow_registry_cache - resource = _get_feast_object(feature_view_name, allow_registry_cache) + resource = await _get_feast_object(feature_view_name, allow_registry_cache) assert_permissions(resource=resource, actions=[AuthzedAction.WRITE_ONLINE]) - store.write_to_online_store( + await run_in_threadpool( + store.write_to_online_store, feature_view_name=feature_view_name, df=df, allow_registry_cache=allow_registry_cache, @@ -428,10 +432,11 @@ async def chat_ui(): return Response(content=content, media_type="text/html") @app.post("/materialize", dependencies=[Depends(inject_user_details)]) - def materialize(request: MaterializeRequest) -> None: + async def materialize(request: MaterializeRequest) -> None: for feature_view in request.feature_views or []: + resource = await _get_feast_object(feature_view, True) assert_permissions( - resource=_get_feast_object(feature_view, True), + resource=resource, actions=[AuthzedAction.WRITE_ONLINE], ) @@ -450,7 +455,8 @@ def materialize(request: MaterializeRequest) -> None: start_date = utils.make_tzaware(parser.parse(request.start_ts)) end_date = utils.make_tzaware(parser.parse(request.end_ts)) - store.materialize( + await run_in_threadpool( + store.materialize, start_date, end_date, request.feature_views, @@ -458,14 +464,17 @@ def materialize(request: MaterializeRequest) -> None: ) @app.post("/materialize-incremental", dependencies=[Depends(inject_user_details)]) - def materialize_incremental(request: MaterializeIncrementalRequest) -> None: + async def materialize_incremental(request: MaterializeIncrementalRequest) -> None: for feature_view in request.feature_views or []: + resource = await _get_feast_object(feature_view, True) assert_permissions( - resource=_get_feast_object(feature_view, True), + resource=resource, actions=[AuthzedAction.WRITE_ONLINE], ) - store.materialize_incremental( - utils.make_tzaware(parser.parse(request.end_ts)), request.feature_views + await run_in_threadpool( + store.materialize_incremental, + utils.make_tzaware(parser.parse(request.end_ts)), + request.feature_views, ) @app.exception_handler(Exception) diff --git a/sdk/python/feast/feature_store.py b/sdk/python/feast/feature_store.py index 611ad6dde85..7a0f362c8c7 100644 --- a/sdk/python/feast/feature_store.py +++ b/sdk/python/feast/feature_store.py @@ -1073,6 +1073,17 @@ def apply( self._registry.commit() + # Refresh the registry cache to ensure that changes are immediately visible + # This is especially important for UI and other clients that may be reading + # from the registry, as it ensures they see the updated state without waiting + # for the cache TTL to expire. + # + # Behavior by cache_mode: + # - sync mode: Immediate consistency - refresh after apply + # - thread mode: Eventual consistency - skip refresh, background thread handles it + if self.config.registry.cache_mode == "sync": + self.refresh_registry() + def teardown(self): """Tears down all local and cloud resources for the feature store.""" tables: List[FeatureView] = [] diff --git a/sdk/python/feast/feature_view.py b/sdk/python/feast/feature_view.py index 1e57b56b8c5..a9406657a51 100644 --- a/sdk/python/feast/feature_view.py +++ b/sdk/python/feast/feature_view.py @@ -39,6 +39,7 @@ from feast.protos.feast.core.Transformation_pb2 import ( FeatureTransformationV2 as FeatureTransformationProto, ) +from feast.transformation.mode import TransformationMode from feast.types import from_value_type from feast.value_type import ValueType @@ -86,6 +87,9 @@ class FeatureView(BaseFeatureView): tags: A dictionary of key-value pairs to store arbitrary metadata. owner: The owner of the feature view, typically the email of the primary maintainer. + mode: The transformation mode for feature transformations. Only meaningful when + transformations are applied. Choose from TransformationMode enum values + (e.g., PYTHON, PANDAS, RAY, SQL, SPARK, SUBSTRAIT). """ name: str @@ -102,6 +106,7 @@ class FeatureView(BaseFeatureView): tags: Dict[str, str] owner: str materialization_intervals: List[Tuple[datetime, datetime]] + mode: Optional[Union["TransformationMode", str]] def __init__( self, @@ -117,6 +122,7 @@ def __init__( description: str = "", tags: Optional[Dict[str, str]] = None, owner: str = "", + mode: Optional[Union["TransformationMode", str]] = None, ): """ Creates a FeatureView object. @@ -140,6 +146,8 @@ def __init__( tags (optional): A dictionary of key-value pairs to store arbitrary metadata. owner (optional): The owner of the feature view, typically the email of the primary maintainer. + mode (optional): The transformation mode for feature transformations. Only meaningful + when transformations are applied. Choose from TransformationMode enum values. Raises: ValueError: A field mapping conflicts with an Entity or a Feature. @@ -148,6 +156,7 @@ def __init__( self.entities = [e.name for e in entities] if entities else [DUMMY_ENTITY_NAME] self.ttl = ttl schema = schema or [] + self.mode = mode # Normalize source self.stream_source = None @@ -252,6 +261,7 @@ def __init__( ) self.online = online self.offline = offline + self.mode = mode self.materialization_intervals = [] def __hash__(self): @@ -439,6 +449,14 @@ def to_proto_spec( substrait_transformation=transformation_proto, ) + mode_str = "" + if self.mode: + mode_str = ( + self.mode.value + if isinstance(self.mode, TransformationMode) + else self.mode + ) + return FeatureViewSpecProto( name=self.name, entities=self.entities, @@ -454,6 +472,7 @@ def to_proto_spec( stream_source=stream_source_proto, source_views=source_view_protos, feature_transformation=feature_transformation_proto, + mode=mode_str, ) def to_proto_meta(self): @@ -527,6 +546,7 @@ def _from_proto_internal( if has_transformation and cls == FeatureView: from feast.batch_feature_view import BatchFeatureView + from feast.transformation.factory import get_transformation_class_from_type from feast.transformation.python_transformation import PythonTransformation from feast.transformation.substrait_transformation import ( SubstraitTransformation, @@ -538,14 +558,30 @@ def _from_proto_internal( transformation = None if feature_transformation_proto.HasField("user_defined_function"): - transformation = PythonTransformation.from_proto( - feature_transformation_proto.user_defined_function - ) + udf_proto = feature_transformation_proto.user_defined_function + if udf_proto.mode: + try: + transformation_class = get_transformation_class_from_type( + udf_proto.mode + ) + transformation = transformation_class.from_proto(udf_proto) + except (ValueError, KeyError): + transformation = PythonTransformation.from_proto(udf_proto) + else: + transformation = PythonTransformation.from_proto(udf_proto) elif feature_transformation_proto.HasField("substrait_transformation"): transformation = SubstraitTransformation.from_proto( feature_transformation_proto.substrait_transformation ) + mode: Union[TransformationMode, str] + if feature_view_proto.spec.mode: + mode = feature_view_proto.spec.mode + elif transformation and hasattr(transformation, "mode"): + mode = transformation.mode + else: + mode = TransformationMode.PYTHON + feature_view: FeatureView = BatchFeatureView( # type: ignore[assignment] name=feature_view_proto.spec.name, description=feature_view_proto.spec.description, @@ -560,9 +596,14 @@ def _from_proto_internal( ), source=source_views if source_views else batch_source, # type: ignore[arg-type] sink_source=batch_source if source_views else None, + mode=mode, feature_transformation=transformation, ) else: + mode_from_spec = ( + feature_view_proto.spec.mode if feature_view_proto.spec.mode else None + ) + feature_view = cls( # type: ignore[assignment] name=feature_view_proto.spec.name, description=feature_view_proto.spec.description, @@ -577,6 +618,7 @@ def _from_proto_internal( ), source=source_views if source_views else batch_source, sink_source=batch_source if source_views else None, + mode=mode_from_spec, ) if stream_source: feature_view.stream_source = stream_source diff --git a/sdk/python/feast/infra/compute_engines/local/backends/__init__.py b/sdk/python/feast/infra/compute_engines/backends/__init__.py similarity index 100% rename from sdk/python/feast/infra/compute_engines/local/backends/__init__.py rename to sdk/python/feast/infra/compute_engines/backends/__init__.py diff --git a/sdk/python/feast/infra/compute_engines/local/backends/base.py b/sdk/python/feast/infra/compute_engines/backends/base.py similarity index 100% rename from sdk/python/feast/infra/compute_engines/local/backends/base.py rename to sdk/python/feast/infra/compute_engines/backends/base.py diff --git a/sdk/python/feast/infra/compute_engines/local/backends/factory.py b/sdk/python/feast/infra/compute_engines/backends/factory.py similarity index 84% rename from sdk/python/feast/infra/compute_engines/local/backends/factory.py rename to sdk/python/feast/infra/compute_engines/backends/factory.py index 6d3774f6393..ffe969f3003 100644 --- a/sdk/python/feast/infra/compute_engines/local/backends/factory.py +++ b/sdk/python/feast/infra/compute_engines/backends/factory.py @@ -3,8 +3,8 @@ import pandas as pd import pyarrow -from feast.infra.compute_engines.local.backends.base import DataFrameBackend -from feast.infra.compute_engines.local.backends.pandas_backend import PandasBackend +from feast.infra.compute_engines.backends.base import DataFrameBackend +from feast.infra.compute_engines.backends.pandas_backend import PandasBackend class BackendFactory: @@ -46,7 +46,7 @@ def _is_polars(entity_df) -> bool: @staticmethod def _get_polars_backend(): - from feast.infra.compute_engines.local.backends.polars_backend import ( + from feast.infra.compute_engines.backends.polars_backend import ( PolarsBackend, ) diff --git a/sdk/python/feast/infra/compute_engines/local/backends/pandas_backend.py b/sdk/python/feast/infra/compute_engines/backends/pandas_backend.py similarity index 93% rename from sdk/python/feast/infra/compute_engines/local/backends/pandas_backend.py rename to sdk/python/feast/infra/compute_engines/backends/pandas_backend.py index 76ddd688424..8ea5a4a9213 100644 --- a/sdk/python/feast/infra/compute_engines/local/backends/pandas_backend.py +++ b/sdk/python/feast/infra/compute_engines/backends/pandas_backend.py @@ -3,7 +3,7 @@ import pandas as pd import pyarrow as pa -from feast.infra.compute_engines.local.backends.base import DataFrameBackend +from feast.infra.compute_engines.backends.base import DataFrameBackend class PandasBackend(DataFrameBackend): diff --git a/sdk/python/feast/infra/compute_engines/local/backends/polars_backend.py b/sdk/python/feast/infra/compute_engines/backends/polars_backend.py similarity index 94% rename from sdk/python/feast/infra/compute_engines/local/backends/polars_backend.py rename to sdk/python/feast/infra/compute_engines/backends/polars_backend.py index 352ffecdab8..92e348c3652 100644 --- a/sdk/python/feast/infra/compute_engines/local/backends/polars_backend.py +++ b/sdk/python/feast/infra/compute_engines/backends/polars_backend.py @@ -3,7 +3,7 @@ import polars as pl import pyarrow as pa -from feast.infra.compute_engines.local.backends.base import DataFrameBackend +from feast.infra.compute_engines.backends.base import DataFrameBackend class PolarsBackend(DataFrameBackend): diff --git a/sdk/python/feast/infra/compute_engines/feature_builder.py b/sdk/python/feast/infra/compute_engines/feature_builder.py index fbea3a85d26..8510b676c07 100644 --- a/sdk/python/feast/infra/compute_engines/feature_builder.py +++ b/sdk/python/feast/infra/compute_engines/feature_builder.py @@ -154,6 +154,15 @@ def get_column_info( ) field_mapping = self.get_field_mapping(self.task.feature_view) + # For feature views with transformations that need access to all source columns, + # we need to read ALL source columns, not just the output feature columns. + # This is specifically for transformations that create new columns or need raw data. + mode = getattr(getattr(view, "feature_transformation", None), "mode", None) + if mode == "ray" or getattr(mode, "value", None) == "ray": + # Signal to read all columns by passing empty list for feature_cols + # The transformation will produce the output columns defined in the schema + feature_cols = [] + return ColumnInfo( join_keys=join_keys, feature_cols=feature_cols, diff --git a/sdk/python/feast/infra/compute_engines/local/compute.py b/sdk/python/feast/infra/compute_engines/local/compute.py index 556468f5e1d..26f537da7cf 100644 --- a/sdk/python/feast/infra/compute_engines/local/compute.py +++ b/sdk/python/feast/infra/compute_engines/local/compute.py @@ -12,10 +12,10 @@ MaterializationTask, ) from feast.infra.common.retrieval_task import HistoricalRetrievalTask +from feast.infra.compute_engines.backends.base import DataFrameBackend +from feast.infra.compute_engines.backends.factory import BackendFactory from feast.infra.compute_engines.base import ComputeEngine from feast.infra.compute_engines.dag.context import ExecutionContext -from feast.infra.compute_engines.local.backends.base import DataFrameBackend -from feast.infra.compute_engines.local.backends.factory import BackendFactory from feast.infra.compute_engines.local.feature_builder import LocalFeatureBuilder from feast.infra.compute_engines.local.job import ( LocalMaterializationJob, diff --git a/sdk/python/feast/infra/compute_engines/local/feature_builder.py b/sdk/python/feast/infra/compute_engines/local/feature_builder.py index 9b2306c0f01..a98573621fb 100644 --- a/sdk/python/feast/infra/compute_engines/local/feature_builder.py +++ b/sdk/python/feast/infra/compute_engines/local/feature_builder.py @@ -2,8 +2,8 @@ from feast.infra.common.materialization_job import MaterializationTask from feast.infra.common.retrieval_task import HistoricalRetrievalTask +from feast.infra.compute_engines.backends.base import DataFrameBackend from feast.infra.compute_engines.feature_builder import FeatureBuilder -from feast.infra.compute_engines.local.backends.base import DataFrameBackend from feast.infra.compute_engines.local.nodes import ( LocalAggregationNode, LocalDedupNode, diff --git a/sdk/python/feast/infra/compute_engines/local/nodes.py b/sdk/python/feast/infra/compute_engines/local/nodes.py index 870a098261d..985a089daae 100644 --- a/sdk/python/feast/infra/compute_engines/local/nodes.py +++ b/sdk/python/feast/infra/compute_engines/local/nodes.py @@ -5,11 +5,11 @@ from feast import BatchFeatureView, StreamFeatureView from feast.data_source import DataSource +from feast.infra.compute_engines.backends.base import DataFrameBackend from feast.infra.compute_engines.dag.context import ColumnInfo, ExecutionContext from feast.infra.compute_engines.dag.model import DAGFormat from feast.infra.compute_engines.dag.node import DAGNode from feast.infra.compute_engines.local.arrow_table_value import ArrowTableValue -from feast.infra.compute_engines.local.backends.base import DataFrameBackend from feast.infra.compute_engines.local.local_node import LocalNode from feast.infra.compute_engines.utils import ( create_offline_store_retrieval_job, diff --git a/sdk/python/feast/infra/compute_engines/ray/compute.py b/sdk/python/feast/infra/compute_engines/ray/compute.py index 24d98cae7fb..a5c1b3caab5 100644 --- a/sdk/python/feast/infra/compute_engines/ray/compute.py +++ b/sdk/python/feast/infra/compute_engines/ray/compute.py @@ -2,8 +2,6 @@ from datetime import datetime from typing import Sequence, Union -import ray - from feast import ( BatchFeatureView, Entity, @@ -26,6 +24,10 @@ ) from feast.infra.compute_engines.ray.utils import write_to_online_store from feast.infra.offline_stores.offline_store import RetrievalJob +from feast.infra.ray_initializer import ( + ensure_ray_initialized, + get_ray_wrapper, +) from feast.infra.registry.base_registry import BaseRegistry logger = logging.getLogger(__name__) @@ -58,37 +60,7 @@ def __init__( def _ensure_ray_initialized(self): """Ensure Ray is initialized with proper configuration.""" - if not ray.is_initialized(): - if self.config.ray_address: - ray.init( - address=self.config.ray_address, - ignore_reinit_error=True, - include_dashboard=False, - ) - else: - ray_init_args = { - "ignore_reinit_error": True, - "include_dashboard": False, - } - - # Add configuration from ray_conf if provided - if self.config.ray_conf: - ray_init_args.update(self.config.ray_conf) - - ray.init(**ray_init_args) - - # Configure Ray context for optimal performance - from ray.data.context import DatasetContext - - ctx = DatasetContext.get_current() - ctx.enable_tensor_extension_casting = False - - # Log Ray cluster information - cluster_resources = ray.cluster_resources() - logger.info( - f"Ray cluster initialized with {cluster_resources.get('CPU', 0)} CPUs, " - f"{cluster_resources.get('memory', 0) / (1024**3):.1f}GB memory" - ) + ensure_ray_initialized(self.config) def update( self, @@ -230,7 +202,8 @@ def _materialize_from_offline_store( # Write to sink_source using Ray data try: - ray_dataset = ray.data.from_arrow(arrow_table) + ray_wrapper = get_ray_wrapper() + ray_dataset = ray_wrapper.from_arrow(arrow_table) ray_dataset.write_parquet(sink_source.path) except Exception as e: logger.error( diff --git a/sdk/python/feast/infra/compute_engines/ray/config.py b/sdk/python/feast/infra/compute_engines/ray/config.py index c6d74d262dd..bb6b63a05c5 100644 --- a/sdk/python/feast/infra/compute_engines/ray/config.py +++ b/sdk/python/feast/infra/compute_engines/ray/config.py @@ -46,9 +46,6 @@ class RayComputeEngineConfig(FeastConfigBaseModel): enable_optimization: bool = True """Enable automatic performance optimizations.""" - execution_timeout_seconds: Optional[int] = None - """Timeout for job execution in seconds.""" - @property def window_size_timedelta(self) -> timedelta: """Convert window size string to timedelta.""" @@ -64,3 +61,19 @@ def window_size_timedelta(self) -> timedelta: else: # Default to 1 hour return timedelta(hours=1) + + # KubeRay/CodeFlare SDK configurations + use_kuberay: Optional[bool] = None + """Whether to use KubeRay/CodeFlare SDK for Ray cluster management""" + + cluster_name: Optional[str] = None + """Name of the KubeRay cluster to connect to (required for KubeRay mode)""" + + auth_token: Optional[str] = None + """Authentication token for Ray cluster connection (for secure clusters)""" + + kuberay_conf: Optional[Dict[str, Any]] = None + """KubeRay/CodeFlare configuration parameters (passed to CodeFlare SDK)""" + + enable_ray_logging: bool = False + """Enable Ray progress bars and verbose logging""" diff --git a/sdk/python/feast/infra/compute_engines/ray/feature_builder.py b/sdk/python/feast/infra/compute_engines/ray/feature_builder.py index 07c5c6f1113..49a957da183 100644 --- a/sdk/python/feast/infra/compute_engines/ray/feature_builder.py +++ b/sdk/python/feast/infra/compute_engines/ray/feature_builder.py @@ -161,6 +161,7 @@ def build_output_nodes(self, view, final_node): name="output", feature_view=view, inputs=[final_node], + config=self.config, ) self.nodes.append(node) @@ -275,6 +276,7 @@ def _build_materialization_plan(self) -> ExecutionPlan: name=f"{view.name}:write", feature_view=view, inputs=[processing_node], + config=self.config, ) view_to_write_node[view.name] = write_node diff --git a/sdk/python/feast/infra/compute_engines/ray/job.py b/sdk/python/feast/infra/compute_engines/ray/job.py index b2e88f1d5c5..06eea4e5d88 100644 --- a/sdk/python/feast/infra/compute_engines/ray/job.py +++ b/sdk/python/feast/infra/compute_engines/ray/job.py @@ -5,7 +5,6 @@ import pandas as pd import pyarrow as pa -import ray from ray.data import Dataset from feast import OnDemandFeatureView @@ -21,6 +20,7 @@ from feast.infra.compute_engines.dag.value import DAGValue from feast.infra.offline_stores.file_source import SavedDatasetFileStorage from feast.infra.offline_stores.offline_store import RetrievalJob, RetrievalMetadata +from feast.infra.ray_initializer import get_ray_wrapper from feast.repo_config import RepoConfig from feast.saved_dataset import SavedDatasetStorage @@ -69,10 +69,11 @@ def _ensure_executed(self) -> DAGValue: self._result_dataset = result.data else: # If result is not a Ray Dataset, convert it + ray_wrapper = get_ray_wrapper() if isinstance(result.data, pd.DataFrame): - self._result_dataset = ray.data.from_pandas(result.data) + self._result_dataset = ray_wrapper.from_pandas(result.data) elif isinstance(result.data, pa.Table): - self._result_dataset = ray.data.from_arrow(result.data) + self._result_dataset = ray_wrapper.from_arrow(result.data) else: raise ValueError( f"Unsupported result type: {type(result.data)}" diff --git a/sdk/python/feast/infra/compute_engines/ray/nodes.py b/sdk/python/feast/infra/compute_engines/ray/nodes.py index eaf48847113..32126a9e42f 100644 --- a/sdk/python/feast/infra/compute_engines/ray/nodes.py +++ b/sdk/python/feast/infra/compute_engines/ray/nodes.py @@ -23,6 +23,7 @@ write_to_online_store, ) from feast.infra.compute_engines.utils import create_offline_store_retrieval_job +from feast.infra.ray_initializer import get_ray_wrapper from feast.infra.ray_shared_utils import ( apply_field_mapping, broadcast_join, @@ -72,10 +73,12 @@ def execute(self, context: ExecutionContext) -> DAGValue: else: try: arrow_table = retrieval_job.to_arrow() - ray_dataset = ray.data.from_arrow(arrow_table) + ray_wrapper = get_ray_wrapper() + ray_dataset = ray_wrapper.from_arrow(arrow_table) except Exception: df = retrieval_job.to_df() - ray_dataset = ray.data.from_pandas(df) + ray_wrapper = get_ray_wrapper() + ray_dataset = ray_wrapper.from_pandas(df) field_mapping = getattr(self.source, "field_mapping", None) if field_mapping: @@ -130,7 +133,8 @@ def execute(self, context: ExecutionContext) -> DAGValue: entity_df = context.entity_df if isinstance(entity_df, pd.DataFrame): - entity_dataset = ray.data.from_pandas(entity_df) + ray_wrapper = get_ray_wrapper() + entity_dataset = ray_wrapper.from_pandas(entity_df) else: entity_dataset = entity_df @@ -169,7 +173,9 @@ def join_with_aggregated_features(batch: pd.DataFrame) -> pd.DataFrame: return result joined_dataset = entity_dataset.map_batches( - join_with_aggregated_features, batch_format="pandas" + join_with_aggregated_features, + batch_format="pandas", + concurrency=self.config.max_workers or 12, ) else: if feature_size <= self.config.broadcast_join_threshold_mb * 1024 * 1024: @@ -270,8 +276,8 @@ def apply_filters(batch: pd.DataFrame) -> pd.DataFrame: else: # Use current time for TTL calculation (real-time retrieval) # Check if timestamp column is timezone-aware - if pd.api.types.is_datetime64tz_dtype( - filtered_batch[timestamp_col] + if isinstance( + filtered_batch[timestamp_col].dtype, pd.DatetimeTZDtype ): # Use timezone-aware current time current_time = datetime.now(timezone.utc) @@ -423,7 +429,8 @@ def _fallback_pandas_aggregation(self, dataset: Dataset, agg_dict: dict) -> Data result_df = result_df.reset_index() # Convert back to Ray Dataset - return ray.data.from_pandas(result_df) + ray_wrapper = get_ray_wrapper() + return ray_wrapper.from_pandas(result_df) else: return dataset @@ -512,31 +519,59 @@ def execute(self, context: ExecutionContext) -> DAGValue: input_value.assert_format(DAGFormat.RAY) dataset: Dataset = input_value.data - transformation_serialized = None - if hasattr(self.transformation, "udf") and callable(self.transformation.udf): - transformation_serialized = dill.dumps(self.transformation.udf) - elif callable(self.transformation): - transformation_serialized = dill.dumps(self.transformation) + # Check transformation mode + from feast.transformation.mode import TransformationMode - @safe_batch_processor - def apply_transformation_with_serialized_udf( - batch: pd.DataFrame, - ) -> pd.DataFrame: - """Apply the transformation using pre-serialized UDF.""" - if transformation_serialized: - transformation_func = dill.loads(transformation_serialized) - transformed_batch = transformation_func(batch) + transformation_mode = getattr( + self.transformation, "mode", TransformationMode.PYTHON + ) + is_ray_native = transformation_mode in (TransformationMode.RAY, "ray") + if is_ray_native: + transformation_func = None + if hasattr(self.transformation, "udf") and callable( + self.transformation.udf + ): + transformation_func = self.transformation.udf + elif callable(self.transformation): + transformation_func = self.transformation + + if transformation_func: + transformed_dataset = transformation_func(dataset) else: logger.warning( - "No serialized transformation available, returning original batch" + "No transformation function available in RAY mode, returning original dataset" ) - transformed_batch = batch + transformed_dataset = dataset + else: + transformation_serialized = None + if hasattr(self.transformation, "udf") and callable( + self.transformation.udf + ): + transformation_serialized = dill.dumps(self.transformation.udf) + elif callable(self.transformation): + transformation_serialized = dill.dumps(self.transformation) - return transformed_batch + @safe_batch_processor + def apply_transformation_with_serialized_udf( + batch: pd.DataFrame, + ) -> pd.DataFrame: + """Apply the transformation using pre-serialized UDF.""" + if transformation_serialized: + transformation_func = dill.loads(transformation_serialized) + transformed_batch = transformation_func(batch) + else: + logger.warning( + "No serialized transformation available, returning original batch" + ) + transformed_batch = batch - transformed_dataset = dataset.map_batches( - apply_transformation_with_serialized_udf, batch_format="pandas" - ) + return transformed_batch + + transformed_dataset = dataset.map_batches( + apply_transformation_with_serialized_udf, + batch_format="pandas", + concurrency=self.config.max_workers or 12, + ) return DAGValue( data=transformed_dataset, @@ -593,7 +628,9 @@ def apply_transformation(batch: pd.DataFrame) -> pd.DataFrame: return transformation_func(batch) transformed_dataset = parent_value.data.map_batches( - apply_transformation + apply_transformation, + batch_format="pandas", + concurrency=self.config.max_workers or 12, ) return DAGValue( data=transformed_dataset, @@ -625,9 +662,11 @@ def __init__( name: str, feature_view: Union[BatchFeatureView, StreamFeatureView, FeatureView], inputs=None, + config: Optional[RayComputeEngineConfig] = None, ): super().__init__(name, inputs=inputs) self.feature_view = feature_view + self.config = config def execute(self, context: ExecutionContext) -> DAGValue: """Execute the write operation.""" @@ -671,7 +710,9 @@ def write_batch_with_serialized_artifacts(batch: pd.DataFrame) -> pd.DataFrame: return batch written_dataset = dataset.map_batches( - write_batch_with_serialized_artifacts, batch_format="pandas" + write_batch_with_serialized_artifacts, + batch_format="pandas", + concurrency=self.config.max_workers if self.config else 12, ) written_dataset = written_dataset.materialize() diff --git a/sdk/python/feast/infra/feature_servers/multicloud/requirements.txt b/sdk/python/feast/infra/feature_servers/multicloud/requirements.txt index 27bf1536bc4..4b3c6f959f0 100644 --- a/sdk/python/feast/infra/feature_servers/multicloud/requirements.txt +++ b/sdk/python/feast/infra/feature_servers/multicloud/requirements.txt @@ -1,2 +1,2 @@ # keep VERSION on line #2, this is critical to release CI -feast[minimal] == 0.55.0 +feast[minimal] == 0.56.0 diff --git a/sdk/python/feast/infra/offline_stores/contrib/clickhouse_offline_store/clickhouse.py b/sdk/python/feast/infra/offline_stores/contrib/clickhouse_offline_store/clickhouse.py index bca6339fb15..5e8cf3d9053 100644 --- a/sdk/python/feast/infra/offline_stores/contrib/clickhouse_offline_store/clickhouse.py +++ b/sdk/python/feast/infra/offline_stores/contrib/clickhouse_offline_store/clickhouse.py @@ -191,6 +191,43 @@ def pull_latest_from_table_or_query( on_demand_feature_views=None, ) + @staticmethod + def pull_all_from_table_or_query( + config: RepoConfig, + data_source: DataSource, + join_key_columns: List[str], + feature_name_columns: List[str], + timestamp_field: str, + created_timestamp_column: Optional[str] = None, + start_date: Optional[datetime] = None, + end_date: Optional[datetime] = None, + ) -> RetrievalJob: + assert isinstance(config.offline_store, ClickhouseOfflineStoreConfig) + assert isinstance(data_source, ClickhouseSource) + + from_expression = data_source.get_table_query_string() + + timestamp_fields = [timestamp_field] + + if created_timestamp_column: + timestamp_fields.append(created_timestamp_column) + + field_string = ", ".join( + join_key_columns + feature_name_columns + timestamp_fields + ) + + query = f""" + SELECT {field_string} + FROM {from_expression} + WHERE {timestamp_field} BETWEEN parseDateTimeBestEffort('{start_date}') AND parseDateTimeBestEffort('{end_date}') + """ + + return ClickhouseRetrievalJob( + query=query, + config=config, + full_feature_names=False, + ) + class ClickhouseRetrievalJob(PostgreSQLRetrievalJob): def _to_df_internal(self, timeout: Optional[int] = None) -> pd.DataFrame: diff --git a/sdk/python/feast/infra/offline_stores/contrib/ray_offline_store/ray.py b/sdk/python/feast/infra/offline_stores/contrib/ray_offline_store/ray.py index 1ec2853cf95..98247c6c0e0 100644 --- a/sdk/python/feast/infra/offline_stores/contrib/ray_offline_store/ray.py +++ b/sdk/python/feast/infra/offline_stores/contrib/ray_offline_store/ray.py @@ -11,7 +11,6 @@ import pandas as pd import pyarrow as pa import ray -import ray.data from ray.data import Dataset from ray.data.context import DatasetContext @@ -39,6 +38,10 @@ get_pyarrow_schema_from_batch_source, infer_event_timestamp_from_entity_df, ) +from feast.infra.ray_initializer import ( + ensure_ray_initialized, + get_ray_wrapper, +) from feast.infra.ray_shared_utils import ( _build_required_columns, apply_field_mapping, @@ -338,6 +341,19 @@ class RayOfflineStoreConfig(FeastConfigBaseModel): # Ray configuration for resource management (memory, CPU limits) ray_conf: Optional[Dict[str, Any]] = None + # KubeRay/CodeFlare SDK configurations + use_kuberay: Optional[bool] = None + """Whether to use KubeRay/CodeFlare SDK for Ray cluster management""" + + cluster_name: Optional[str] = None + """Name of the KubeRay cluster to connect to (required for KubeRay mode)""" + + auth_token: Optional[str] = None + """Authentication token for Ray cluster connection (for secure clusters)""" + + kuberay_conf: Optional[Dict[str, Any]] = None + """KubeRay/CodeFlare configuration parameters (passed to CodeFlare SDK)""" + class RayResourceManager: """ @@ -350,10 +366,18 @@ def __init__(self, config: Optional[RayOfflineStoreConfig] = None) -> None: Initialize the resource manager with cluster resource information. """ self.config = config or RayOfflineStoreConfig() + + if not ray.is_initialized(): + self.cluster_resources = {"CPU": 4, "memory": 8 * 1024**3} + self.available_memory = 8 * 1024**3 + self.available_cpus = 4 + self.num_nodes = 1 + return + self.cluster_resources = ray.cluster_resources() self.available_memory = self.cluster_resources.get("memory", 8 * 1024**3) self.available_cpus = int(self.cluster_resources.get("CPU", 4)) - self.num_nodes = len(ray.nodes()) if ray.is_initialized() else 1 + self.num_nodes = len(ray.nodes()) def configure_ray_context(self) -> None: """ @@ -367,7 +391,8 @@ def configure_ray_context(self) -> None: else: ctx.target_shuffle_buffer_size = 512 * 1024**2 ctx.target_max_block_size = 128 * 1024**2 - ctx.min_parallelism = self.available_cpus + + ctx.read_op_min_num_blocks = self.available_cpus multiplier = ( self.config.max_parallelism_multiplier if self.config.max_parallelism_multiplier is not None @@ -457,8 +482,8 @@ def optimize_dataset_for_join(self, ds: Dataset, join_keys: List[str]) -> Datase if not join_keys: # For datasets without join keys, use simple repartitioning return ds.repartition(num_blocks=optimal_partitions) - # For datasets with join keys, use shuffle for better distribution - return ds.random_shuffle(num_blocks=optimal_partitions) + # For datasets with join keys, repartition then shuffle for better distribution + return ds.repartition(num_blocks=optimal_partitions).random_shuffle() def _manual_point_in_time_join( self, @@ -919,7 +944,7 @@ def _create_metadata(self) -> RetrievalMetadata: else: try: result = self._resolve() - if isinstance(result, Dataset): + if is_ray_data(result): timestamp_col = _safe_infer_event_timestamp_column( result, "event_timestamp" ) @@ -964,11 +989,12 @@ def _get_ray_dataset(self) -> Dataset: return self._cached_dataset result = self._resolve() - if isinstance(result, Dataset): + if is_ray_data(result): self._cached_dataset = result return result elif isinstance(result, pd.DataFrame): - self._cached_dataset = ray.data.from_pandas(result) + ray_wrapper = get_ray_wrapper() + self._cached_dataset = ray_wrapper.from_pandas(result) return self._cached_dataset else: raise ValueError(f"Unsupported result type: {type(result)}") @@ -1225,82 +1251,13 @@ def _suppress_ray_logging() -> None: @staticmethod def _ensure_ray_initialized(config: Optional[RepoConfig] = None) -> None: """Ensure Ray is initialized with proper configuration.""" - ray_config = None - if config and hasattr(config, "offline_store"): - ray_config = config.offline_store - if isinstance(ray_config, RayOfflineStoreConfig): - if not ray_config.enable_ray_logging: - RayOfflineStore._suppress_ray_logging() - - if not ray.is_initialized(): - ray_init_kwargs: Dict[str, Any] = { - "ignore_reinit_error": True, - "include_dashboard": False, - } - - if ( - ray_config - and isinstance(ray_config, RayOfflineStoreConfig) - and not ray_config.enable_ray_logging - ): - ray_init_kwargs.update( - { - "log_to_driver": False, - "logging_level": "ERROR", - } - ) - - if config and hasattr(config, "offline_store"): - if isinstance(ray_config, RayOfflineStoreConfig): - if ray_config.ray_address: - ray_init_kwargs["address"] = ray_config.ray_address - else: - ray_init_kwargs.update( - { - "_node_ip_address": os.getenv( - "RAY_NODE_IP", "127.0.0.1" - ), - "num_cpus": os.cpu_count() or 4, - } - ) - - if ray_config.ray_conf: - ray_init_kwargs.update(ray_config.ray_conf) - else: - pass # Use default initialization - - ray.init(**ray_init_kwargs) - - ctx = DatasetContext.get_current() - ctx.shuffle_strategy = "sort" # type: ignore - ctx.enable_tensor_extension_casting = False - - if ( - ray_config - and isinstance(ray_config, RayOfflineStoreConfig) - and not ray_config.enable_ray_logging - ): - RayOfflineStore._suppress_ray_logging() - - if ray.is_initialized(): - cluster_resources = ray.cluster_resources() - if ( - not ray_config - or not isinstance(ray_config, RayOfflineStoreConfig) - or ray_config.enable_ray_logging - ): - logger.info( - f"Ray cluster initialized with {cluster_resources.get('CPU', 0)} CPUs, " - f"{cluster_resources.get('memory', 0) / (1024**3):.1f}GB memory" - ) + ensure_ray_initialized(config) def _init_ray(self, config: RepoConfig) -> None: ray_config = config.offline_store assert isinstance(ray_config, RayOfflineStoreConfig) - RayOfflineStore._ensure_ray_initialized(config) - if not ray_config.enable_ray_logging: - RayOfflineStore._suppress_ray_logging() + RayOfflineStore._ensure_ray_initialized(config) if self._resource_manager is None: self._resource_manager = RayResourceManager(ray_config) @@ -1378,12 +1335,13 @@ def offline_write_batch( batch_source_path = feature_view.batch_source.file_options.uri feature_path = FileSource.get_uri_for_file_path(repo_path, batch_source_path) - ds = ray.data.from_arrow(table) + ray_wrapper = get_ray_wrapper() + ds = ray_wrapper.from_arrow(table) try: if feature_path.endswith(".parquet"): if os.path.exists(feature_path): - existing_ds = ray.data.read_parquet(feature_path) + existing_ds = ray_wrapper.read_parquet(feature_path) combined_ds = existing_ds.union(ds) combined_ds.write_parquet(feature_path) else: @@ -1408,7 +1366,7 @@ def offline_write_batch( df.to_parquet(feature_path, index=False) else: os.makedirs(feature_path, exist_ok=True) - ds_fallback = ray.data.from_pandas(df) + ds_fallback = ray_wrapper.from_pandas(df) ds_fallback.write_parquet(feature_path) if progress: @@ -1443,15 +1401,21 @@ def _process_filtered_batch( return _handle_empty_dataframe_case( join_key_columns, feature_name_columns, timestamp_columns ) - all_required_columns = _build_required_columns( - join_key_columns, feature_name_columns, timestamp_columns - ) + if not join_key_columns: batch[DUMMY_ENTITY_ID] = DUMMY_ENTITY_VAL - available_columns = [ - col for col in all_required_columns if col in batch.columns - ] - batch = batch[available_columns] + + # If feature_name_columns is empty, it means "keep all columns" (for transformations) + # Otherwise, filter to only the requested columns + if feature_name_columns: + all_required_columns = _build_required_columns( + join_key_columns, feature_name_columns, timestamp_columns + ) + available_columns = [ + col for col in all_required_columns if col in batch.columns + ] + batch = batch[available_columns] + if ( "event_timestamp" not in batch.columns and timestamp_field_mapped != "event_timestamp" @@ -1473,8 +1437,22 @@ def _load_and_filter_dataset( ) -> pd.DataFrame: try: field_mapping = getattr(data_source, "field_mapping", None) + + if not feature_name_columns: + columns_to_read = None + else: + columns_to_read = list( + set(join_key_columns + feature_name_columns + [timestamp_field]) + ) + if created_timestamp_column: + columns_to_read.append(created_timestamp_column) + ds = RayOfflineStore._create_filtered_dataset( - source_path, timestamp_field, start_date, end_date + source_path, + timestamp_field, + start_date, + end_date, + columns=columns_to_read, ) df = ds.to_pandas() if field_mapping: @@ -1523,8 +1501,22 @@ def _load_and_filter_dataset_ray( ) -> Dataset: try: field_mapping = getattr(data_source, "field_mapping", None) + + if not feature_name_columns: + columns_to_read = None + else: + columns_to_read = list( + set(join_key_columns + feature_name_columns + [timestamp_field]) + ) + if created_timestamp_column: + columns_to_read.append(created_timestamp_column) + ds = RayOfflineStore._create_filtered_dataset( - source_path, timestamp_field, start_date, end_date + source_path, + timestamp_field, + start_date, + end_date, + columns=columns_to_read, ) if field_mapping: ds = apply_field_mapping(ds, field_mapping) @@ -1776,10 +1768,11 @@ def write_logged_features( absolute_path = FileSource.get_uri_for_file_path(repo_path, destination.path) try: + ray_wrapper = get_ray_wrapper() if isinstance(data, Path): - ds = ray.data.read_parquet(str(data)) + ds = ray_wrapper.read_parquet(str(data)) else: - ds = ray.data.from_arrow(data) + ds = ray_wrapper.from_arrow(data) # Normalize feature timestamp precision to seconds to match test expectations during write # Note: Don't normalize __log_timestamp as it's used for time range filtering @@ -1829,9 +1822,11 @@ def _create_filtered_dataset( timestamp_field: str, start_date: Optional[datetime] = None, end_date: Optional[datetime] = None, + columns: Optional[List[str]] = None, ) -> Dataset: """Helper method to create a filtered dataset based on timestamp range.""" - ds = ray.data.read_parquet(source_path) + ray_wrapper = get_ray_wrapper() + ds = ray_wrapper.read_parquet(source_path, columns=columns) try: col_names = ds.schema().names @@ -1888,11 +1883,12 @@ def get_historical_features( store._init_ray(config) # Load entity_df as Ray dataset for distributed processing + ray_wrapper = get_ray_wrapper() if isinstance(entity_df, str): - entity_ds = ray.data.read_csv(entity_df) + entity_ds = ray_wrapper.read_csv(entity_df) entity_df_sample = entity_ds.limit(1000).to_pandas() else: - entity_ds = ray.data.from_pandas(entity_df) + entity_ds = ray_wrapper.from_pandas(entity_df) entity_df_sample = entity_df.copy() entity_ds = ensure_timestamp_compatibility(entity_ds, ["event_timestamp"]) @@ -1940,9 +1936,12 @@ def get_historical_features( entities = fv.entities or [] entity_objs = [registry.get_entity(e, project) for e in entities] - original_join_keys, _, timestamp_field, created_col = _get_column_names( - fv, entity_objs - ) + ( + original_join_keys, + reverse_mapped_feature_names, + timestamp_field, + created_col, + ) = _get_column_names(fv, entity_objs) if fv.projection.join_key_map: join_keys = [ @@ -1952,11 +1951,12 @@ def get_historical_features( else: join_keys = original_join_keys - requested_feats = [ref.split(":", 1)[1] for ref in fv_feature_refs] + # Get the logical feature names from refs + logical_requested_feats = [ref.split(":", 1)[1] for ref in fv_feature_refs] available_feature_names = [f.name for f in fv.features] missing_feats = [ - f for f in requested_feats if f not in available_feature_names + f for f in logical_requested_feats if f not in available_feature_names ] if missing_feats: raise KeyError( @@ -1964,16 +1964,37 @@ def get_historical_features( f"(available: {available_feature_names})" ) + # Build reverse field mapping to get actual source column names + reverse_field_mapping = {} + if fv.batch_source.field_mapping: + reverse_field_mapping = { + v: k for k, v in fv.batch_source.field_mapping.items() + } + + # Map logical feature names to actual source column names + requested_feats = [ + reverse_field_mapping.get(feat, feat) + for feat in logical_requested_feats + ] + source_info = resolve_feature_view_source_with_fallback( fv, config, is_materialization=False ) # Read from the resolved data source source_path = store._get_source_path(source_info.data_source, config) - feature_ds = ray.data.read_parquet(source_path) - logger.info( - f"Reading feature view {fv.name}: {source_info.source_description}" - ) + + if not source_info.has_transformation: + required_feature_columns = set( + original_join_keys + requested_feats + [timestamp_field] + ) + if created_col: + required_feature_columns.add(created_col) + feature_ds = ray_wrapper.read_parquet( + source_path, columns=list(required_feature_columns) + ) + else: + feature_ds = ray_wrapper.read_parquet(source_path) # Apply transformation if available if source_info.has_transformation and source_info.transformation_func: @@ -2012,10 +2033,23 @@ def apply_transformation_with_serialized_func( field_mapping = getattr(fv.batch_source, "field_mapping", None) if field_mapping: feature_ds = apply_field_mapping(feature_ds, field_mapping) - join_keys = [field_mapping.get(k, k) for k in join_keys] + # Update original_join_keys to logical names after forward mapping + original_join_keys = [ + field_mapping.get(k, k) for k in original_join_keys + ] + # Recompute join_keys from updated original_join_keys + if fv.projection.join_key_map: + join_keys = [ + fv.projection.join_key_map.get(key, key) + for key in original_join_keys + ] + else: + join_keys = original_join_keys timestamp_field = field_mapping.get(timestamp_field, timestamp_field) if created_col: created_col = field_mapping.get(created_col, created_col) + # Also map requested_feats back to logical names after forward mapping + requested_feats = [field_mapping.get(f, f) for f in requested_feats] if ( timestamp_field != "event_timestamp" diff --git a/sdk/python/feast/infra/offline_stores/contrib/trino_offline_store/trino.py b/sdk/python/feast/infra/offline_stores/contrib/trino_offline_store/trino.py index fd126e87db6..aa4cb2c8a60 100644 --- a/sdk/python/feast/infra/offline_stores/contrib/trino_offline_store/trino.py +++ b/sdk/python/feast/infra/offline_stores/contrib/trino_offline_store/trino.py @@ -372,6 +372,8 @@ def get_historical_features( ) # Generate the Trino SQL query from the query context + if type(entity_df) is str: + table_reference = f"({entity_df})" query = offline_utils.build_point_in_time_query( query_context, left_table_query_string=table_reference, @@ -454,9 +456,7 @@ def _upload_entity_df_and_get_entity_schema( ) -> Dict[str, np.dtype]: """Uploads a Pandas entity dataframe into a Trino table and returns the resulting table""" if type(entity_df) is str: - client.execute_query(f"CREATE TABLE {table_name} AS ({entity_df})") - - results = client.execute_query(f"SELECT * FROM {table_name} LIMIT 1") + results = client.execute_query(f"SELECT * FROM ({entity_df}) LIMIT 1") limited_entity_df = pd.DataFrame( data=results.data, columns=results.columns_names diff --git a/sdk/python/feast/infra/offline_stores/remote.py b/sdk/python/feast/infra/offline_stores/remote.py index a5f50c7b45c..abe75ca57e5 100644 --- a/sdk/python/feast/infra/offline_stores/remote.py +++ b/sdk/python/feast/infra/offline_stores/remote.py @@ -197,6 +197,7 @@ def get_historical_features( registry: BaseRegistry, project: str, full_feature_names: bool = False, + **kwargs, ) -> RemoteRetrievalJob: assert isinstance(config.offline_store, RemoteOfflineStoreConfig) @@ -219,6 +220,15 @@ def get_historical_features( "name_aliases": name_aliases, } + # Extract and serialize start_date/end_date for remote transmission + start_date = kwargs.get("start_date", None) + end_date = kwargs.get("end_date", None) + + if start_date is not None: + api_parameters["start_date"] = start_date.isoformat() + if end_date is not None: + api_parameters["end_date"] = end_date.isoformat() + return RemoteRetrievalJob( client=client, api=OfflineStore.get_historical_features.__name__, diff --git a/sdk/python/feast/infra/online_stores/dynamodb.py b/sdk/python/feast/infra/online_stores/dynamodb.py index e60796b2963..c577159884d 100644 --- a/sdk/python/feast/infra/online_stores/dynamodb.py +++ b/sdk/python/feast/infra/online_stores/dynamodb.py @@ -108,15 +108,24 @@ class DynamoDBOnlineStore(OnlineStore): Attributes: _dynamodb_client: Boto3 DynamoDB client. _dynamodb_resource: Boto3 DynamoDB resource. + _aioboto_session: Async boto session. + _aioboto_client: Async boto client. + _aioboto_context_stack: Async context stack. """ _dynamodb_client = None _dynamodb_resource = None + def __init__(self): + super().__init__() + self._aioboto_session = None + self._aioboto_client = None + self._aioboto_context_stack = None + async def initialize(self, config: RepoConfig): online_config = config.online_store - await _get_aiodynamodb_client( + await self._get_aiodynamodb_client( online_config.region, online_config.max_pool_connections, online_config.keepalive_timeout, @@ -127,7 +136,59 @@ async def initialize(self, config: RepoConfig): ) async def close(self): - await _aiodynamodb_close() + await self._aiodynamodb_close() + + def _get_aioboto_session(self): + if self._aioboto_session is None: + logger.debug("initializing the aiobotocore session") + self._aioboto_session = session.get_session() + return self._aioboto_session + + async def _get_aiodynamodb_client( + self, + region: str, + max_pool_connections: int, + keepalive_timeout: float, + connect_timeout: Union[int, float], + read_timeout: Union[int, float], + total_max_retry_attempts: Union[int, None], + retry_mode: Union[Literal["legacy", "standard", "adaptive"], None], + ): + if self._aioboto_client is None: + logger.debug("initializing the aiobotocore dynamodb client") + + retries: Dict[str, Any] = {} + if total_max_retry_attempts is not None: + retries["total_max_attempts"] = total_max_retry_attempts + if retry_mode is not None: + retries["mode"] = retry_mode + + client_context = self._get_aioboto_session().create_client( + "dynamodb", + region_name=region, + config=AioConfig( + max_pool_connections=max_pool_connections, + connect_timeout=connect_timeout, + read_timeout=read_timeout, + retries=retries if retries else None, + connector_args={"keepalive_timeout": keepalive_timeout}, + ), + ) + self._aioboto_context_stack = contextlib.AsyncExitStack() + self._aioboto_client = ( + await self._aioboto_context_stack.enter_async_context(client_context) + ) + return self._aioboto_client + + async def _aiodynamodb_close(self): + if self._aioboto_client: + await self._aioboto_client.close() + self._aioboto_client = None + if self._aioboto_context_stack: + await self._aioboto_context_stack.aclose() + self._aioboto_context_stack = None + if self._aioboto_session: + self._aioboto_session = None @property def async_supported(self) -> SupportedAsyncMethods: @@ -249,7 +310,18 @@ def update( # tags won't be updated in the create_table call if the table already exists if do_tag_updates[table_name]: tags = self._table_tags(online_config, table_instance) - self._update_tags(dynamodb_client, table_name, tags) + try: + self._update_tags(dynamodb_client, table_name, tags) + except ClientError as ce: + # If tag update fails with AccessDeniedException, log warning and continue + # This allows Feast to work in environments where IAM roles don't have + # dynamodb:TagResource and dynamodb:UntagResource permissions + if ce.response["Error"]["Code"] == "AccessDeniedException": + logger.warning( + f"Unable to update tags for table {table_name} due to insufficient permissions." + ) + else: + raise for table_to_delete in tables_to_delete: _delete_table_idempotent( @@ -351,7 +423,7 @@ async def online_write_batch_async( _to_client_write_item(config, entity_key, features, timestamp) for entity_key, features, timestamp, _ in _latest_data_to_write(data) ] - client = await _get_aiodynamodb_client( + client = await self._get_aiodynamodb_client( online_config.region, online_config.max_pool_connections, online_config.keepalive_timeout, @@ -462,7 +534,7 @@ def to_tbl_resp(raw_client_response): batches.append(batch) entity_id_batches.append(entity_id_batch) - client = await _get_aiodynamodb_client( + client = await self._get_aiodynamodb_client( online_config.region, online_config.max_pool_connections, online_config.keepalive_timeout, @@ -616,66 +688,7 @@ def _to_client_batch_get_payload(online_config, table_name, batch): } -_aioboto_session = None -_aioboto_client = None -_aioboto_context_stack = None - - -def _get_aioboto_session(): - global _aioboto_session - if _aioboto_session is None: - logger.debug("initializing the aiobotocore session") - _aioboto_session = session.get_session() - return _aioboto_session - - -async def _get_aiodynamodb_client( - region: str, - max_pool_connections: int, - keepalive_timeout: float, - connect_timeout: Union[int, float], - read_timeout: Union[int, float], - total_max_retry_attempts: Union[int, None], - retry_mode: Union[Literal["legacy", "standard", "adaptive"], None], -): - global _aioboto_client, _aioboto_context_stack - if _aioboto_client is None: - logger.debug("initializing the aiobotocore dynamodb client") - - retries: Dict[str, Any] = {} - if total_max_retry_attempts is not None: - retries["total_max_attempts"] = total_max_retry_attempts - if retry_mode is not None: - retries["mode"] = retry_mode - - client_context = _get_aioboto_session().create_client( - "dynamodb", - region_name=region, - config=AioConfig( - max_pool_connections=max_pool_connections, - connect_timeout=connect_timeout, - read_timeout=read_timeout, - retries=retries if retries else None, - connector_args={"keepalive_timeout": keepalive_timeout}, - ), - ) - _aioboto_context_stack = contextlib.AsyncExitStack() - _aioboto_client = await _aioboto_context_stack.enter_async_context( - client_context - ) - return _aioboto_client - - -async def _aiodynamodb_close(): - global _aioboto_client, _aioboto_session, _aioboto_context_stack - if _aioboto_client: - await _aioboto_client.close() - _aioboto_client = None - if _aioboto_context_stack: - await _aioboto_context_stack.aclose() - _aioboto_context_stack = None - if _aioboto_session: - _aioboto_session = None +# Global async client functions removed - now using instance methods def _initialize_dynamodb_client( diff --git a/sdk/python/feast/infra/ray_initializer.py b/sdk/python/feast/infra/ray_initializer.py new file mode 100644 index 00000000000..eea21eaa321 --- /dev/null +++ b/sdk/python/feast/infra/ray_initializer.py @@ -0,0 +1,660 @@ +# Copyright 2025 The Feast Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Centralized Ray Initialization Module for Feast. + +This module combines configuration management and initialization logic for a +complete, self-contained Ray setup system. +""" + +import logging +import os +from enum import Enum +from typing import Any, Dict, List, Optional, Union + +import ray +from ray.data.context import DatasetContext + +logger = logging.getLogger(__name__) + + +class RayExecutionMode(Enum): + """Ray execution modes supported by Feast.""" + + LOCAL = "local" + REMOTE = "remote" + KUBERAY = "kuberay" + + +class RayConfigManager: + """ + Manages Ray configuration and execution mode determination. + + Supports three main scenarios: + 1. Local Ray: Single-machine development and testing + 2. Remote Ray: Connect to existing Ray standalone cluster + 3. KubeRay: Ray on Kubernetes with CodeFlare SDK + + The manager determines execution mode based on configuration precedence: + 1. Environment variable FEAST_RAY_EXECUTION_MODE (highest) + 2. KubeRay mode (use_kuberay=True or cluster_name specified) + 3. Remote mode (ray_address specified) + 4. Local mode (default fallback) + """ + + def __init__(self, config: Optional[Union[Dict[str, Any], object]] = None): + """ + Initialize Ray configuration manager. + + Args: + config: Ray configuration (RayOfflineStoreConfig, RayComputeEngineConfig, or dict) + """ + self.config = config or {} + self._execution_mode: Optional[RayExecutionMode] = None + self._codeflare_config: Optional[Dict[str, Any]] = None + + def determine_execution_mode(self) -> RayExecutionMode: + """ + Determine the appropriate Ray execution mode based on configuration. + + Precedence (highest to lowest): + 1. Environment variable FEAST_RAY_EXECUTION_MODE (explicit override) + 2. KubeRay mode (use_kuberay=True or cluster_name specified) + 3. Remote mode (ray_address specified) + 4. Local mode (default fallback) + + Returns: + RayExecutionMode enum value + """ + if self._execution_mode is not None: + return self._execution_mode + + # 1. Check environment variable override first (highest precedence) + env_mode = os.getenv("FEAST_RAY_EXECUTION_MODE", "").lower() + if env_mode in ["local", "remote", "kuberay"]: + self._execution_mode = RayExecutionMode(env_mode) + logger.info( + f"Ray execution mode set via FEAST_RAY_EXECUTION_MODE: {env_mode}" + ) + return self._execution_mode + + # 2. Check for KubeRay configuration (second highest precedence) + use_kuberay = self._get_config_value("use_kuberay") + + # Check for cluster_name in kuberay_conf + kuberay_conf = self._get_config_value("kuberay_conf", {}) or {} + cluster_name = kuberay_conf.get("cluster_name") + + # Environment variables can enable KubeRay + if os.getenv("FEAST_USE_KUBERAY", "").lower() == "true": + use_kuberay = True + if os.getenv("FEAST_RAY_CLUSTER_NAME"): + cluster_name = os.getenv("FEAST_RAY_CLUSTER_NAME") + + # KubeRay takes precedence over remote/local if configured + if use_kuberay or cluster_name: + self._execution_mode = RayExecutionMode.KUBERAY + reason = [] + if use_kuberay: + reason.append("use_kuberay=True") + if cluster_name: + reason.append(f"cluster_name='{cluster_name}'") + logger.info(f"Ray execution mode: KubeRay ({', '.join(reason)})") + return self._execution_mode + + # 3. Check for remote Ray configuration (third precedence) + ray_address = self._get_config_value("ray_address") or os.getenv("RAY_ADDRESS") + + if ray_address: + self._execution_mode = RayExecutionMode.REMOTE + logger.info(f"Ray execution mode: Remote (ray_address='{ray_address}')") + return self._execution_mode + + # 4. Default to local Ray (lowest precedence - fallback) + self._execution_mode = RayExecutionMode.LOCAL + logger.info( + "Ray execution mode: Local (default - no KubeRay or remote configuration found)" + ) + return self._execution_mode + + def get_kuberay_config(self) -> Dict[str, Any]: + """ + Get KubeRay/CodeFlare SDK configuration. + + Returns: + Dictionary of KubeRay configuration with passthrough settings + """ + if self._codeflare_config is not None: + return self._codeflare_config + + # Get passthrough configuration from kuberay_conf first + kuberay_conf = self._get_config_value("kuberay_conf", {}) or {} + + config = { + "use_kuberay": ( + os.getenv("FEAST_USE_KUBERAY", "").lower() == "true" + or self._get_config_value("use_kuberay", False) + ), + # Get values from kuberay_conf or environment variables + "cluster_name": ( + os.getenv("FEAST_RAY_CLUSTER_NAME") or kuberay_conf.get("cluster_name") + ), + "namespace": ( + os.getenv("FEAST_RAY_NAMESPACE") + or kuberay_conf.get("namespace", "default") + ), + } + + # Add authentication configuration from kuberay_conf or environment variables + auth_token = ( + os.getenv("FEAST_RAY_AUTH_TOKEN") + or os.getenv("RAY_AUTH_TOKEN") + or kuberay_conf.get("auth_token") + ) + if auth_token: + config["auth_token"] = auth_token + + # Add authentication server URL + auth_server = ( + os.getenv("FEAST_RAY_AUTH_SERVER") + or os.getenv("RAY_AUTH_SERVER") + or kuberay_conf.get("auth_server") + ) + if auth_server: + config["auth_server"] = auth_server + + # Add skip TLS verification setting + skip_tls = os.getenv( + "FEAST_RAY_SKIP_TLS", "" + ).lower() == "true" or kuberay_conf.get("skip_tls", False) + config["skip_tls"] = skip_tls + + # Add any additional configuration from kuberay_conf + for key, value in kuberay_conf.items(): + if key not in config: # Don't override already processed keys + config[key] = value + + self._codeflare_config = config + return config + + def _get_config_value(self, key: str, default: Any = None) -> Any: + """ + Get configuration value from config object or dictionary. + + Args: + key: Configuration key + default: Default value if key not found + + Returns: + Configuration value + """ + if hasattr(self.config, key): + return getattr(self.config, key) + elif isinstance(self.config, dict): + return self.config.get(key, default) + else: + return default + + +class StandardRayWrapper: + """Wrapper for Ray Native operations.""" + + def read_parquet(self, path: Union[str, List[str]], **kwargs) -> Any: + """Read parquet files using standard Ray.""" + return ray.data.read_parquet(path, **kwargs) + + def read_csv(self, path: Union[str, List[str]], **kwargs) -> Any: + """Read CSV files using standard Ray.""" + return ray.data.read_csv(path, **kwargs) + + def from_pandas(self, df: Any) -> Any: + """Create dataset from pandas DataFrame using standard Ray.""" + return ray.data.from_pandas(df) + + def from_arrow(self, table: Any) -> Any: + """Create dataset from Arrow table using standard Ray.""" + return ray.data.from_arrow(table) + + +class CodeFlareRayWrapper: + """Wrapper for Ray operations on KubeRay clusters using CodeFlare SDK.""" + + def __init__( + self, + cluster_name: str, + namespace: str, + auth_token: str, + auth_server: str, + skip_tls: bool = False, + enable_logging: bool = False, + ): + """Initialize CodeFlare Ray wrapper with cluster connection parameters.""" + self.cluster_name = cluster_name + self.namespace = namespace + self.auth_token = auth_token + self.auth_server = auth_server + self.skip_tls = skip_tls + self.enable_logging = enable_logging + self.cluster = None + + # Authenticate and setup Ray connection + self._authenticate_codeflare() + self._setup_ray_connection() + + def _authenticate_codeflare(self): + """Authenticate with CodeFlare SDK.""" + try: + from codeflare_sdk import TokenAuthentication + + auth = TokenAuthentication( + token=self.auth_token, + server=self.auth_server, + skip_tls=self.skip_tls, + ) + auth.login() + except Exception as e: + logger.error(f"CodeFlare authentication failed: {e}") + raise + + def _setup_ray_connection(self): + """Setup Ray connection to KubeRay cluster using TLS certificates.""" + try: + from codeflare_sdk import generate_cert, get_cluster + + self.cluster = get_cluster( + cluster_name=self.cluster_name, namespace=self.namespace + ) + if self.cluster is None: + raise RuntimeError( + f"Failed to find KubeRay cluster '{self.cluster_name}' in namespace '{self.namespace}'" + ) + generate_cert.generate_tls_cert(self.cluster_name, self.namespace) + generate_cert.export_env(self.cluster_name, self.namespace) + + cluster_uri = self.cluster.cluster_uri() + runtime_env = { + "pip": ["feast"], + "env_vars": {"RAY_DISABLE_IMPORT_WARNING": "1"}, + } + + ray.shutdown() + + logging_level = "INFO" if self.enable_logging else "ERROR" + + ray.init( + address=cluster_uri, + ignore_reinit_error=True, + logging_level=logging_level, + log_to_driver=self.enable_logging, + runtime_env=runtime_env, + ) + + logger.info(f"Ray connected successfully to cluster: {self.cluster_name}") + + except Exception as e: + logger.error(f"Ray connection failed: {e}") + raise + + # Ray Data API methods - wrapped in @ray.remote to execute on cluster workers + def read_parquet(self, path: Union[str, List[str]], **kwargs) -> Any: + """Read parquet files - runs remotely on KubeRay cluster workers.""" + from feast.infra.ray_shared_utils import RemoteDatasetProxy + + @ray.remote + def _remote_read_parquet(file_path, read_kwargs): + import ray + + return ray.data.read_parquet(file_path, **read_kwargs) + + return RemoteDatasetProxy(_remote_read_parquet.remote(path, kwargs)) + + def read_csv(self, path: Union[str, List[str]], **kwargs) -> Any: + """Read CSV files - runs remotely on KubeRay cluster workers.""" + from feast.infra.ray_shared_utils import RemoteDatasetProxy + + @ray.remote + def _remote_read_csv(file_path, read_kwargs): + import ray + + return ray.data.read_csv(file_path, **read_kwargs) + + return RemoteDatasetProxy(_remote_read_csv.remote(path, kwargs)) + + def from_pandas(self, df: Any) -> Any: + """Create dataset from pandas DataFrame - runs remotely on KubeRay cluster workers.""" + from feast.infra.ray_shared_utils import RemoteDatasetProxy + + @ray.remote + def _remote_from_pandas(dataframe): + import ray + + return ray.data.from_pandas(dataframe) + + return RemoteDatasetProxy(_remote_from_pandas.remote(df)) + + def from_arrow(self, table: Any) -> Any: + """Create dataset from Arrow table - runs remotely on KubeRay cluster workers.""" + from feast.infra.ray_shared_utils import RemoteDatasetProxy + + @ray.remote + def _remote_from_arrow(arrow_table): + import ray + + return ray.data.from_arrow(arrow_table) + + return RemoteDatasetProxy(_remote_from_arrow.remote(table)) + + +# Global state tracking +_ray_initialized = False +_ray_wrapper: Optional[Union[StandardRayWrapper, CodeFlareRayWrapper]] = None + + +def _suppress_ray_logging() -> None: + """Suppress Ray and Ray Data logging completely.""" + import warnings + + # Suppress Ray warnings + warnings.filterwarnings("ignore", category=DeprecationWarning, module="ray") + warnings.filterwarnings("ignore", category=UserWarning, module="ray") + + # Set environment variables to suppress Ray output + os.environ["RAY_DISABLE_IMPORT_WARNING"] = "1" + os.environ["RAY_SUPPRESS_UNVERIFIED_TLS_WARNING"] = "1" + os.environ["RAY_LOG_LEVEL"] = "ERROR" + os.environ["RAY_DATA_LOG_LEVEL"] = "ERROR" + os.environ["RAY_DISABLE_PROGRESS_BARS"] = "1" + + # Suppress all Ray-related loggers + ray_loggers = [ + "ray", + "ray.data", + "ray.data.dataset", + "ray.data.context", + "ray.data._internal.streaming_executor", + "ray.data._internal.execution", + "ray.data._internal", + "ray.tune", + "ray.serve", + "ray.util", + "ray._private", + ] + for logger_name in ray_loggers: + logging.getLogger(logger_name).setLevel(logging.ERROR) + + # Configure DatasetContext to disable progress bars + try: + ctx = DatasetContext.get_current() + ctx.enable_progress_bars = False + if hasattr(ctx, "verbose_progress"): + ctx.verbose_progress = False + except Exception: + pass # Ignore if Ray Data is not available + + +def _initialize_local_ray(config: Any, enable_logging: bool = False) -> None: + """ + Initialize Ray in local mode. + + Args: + config: Configuration object (RayOfflineStoreConfig or RayComputeEngineConfig) + enable_logging: Whether to enable Ray logging + """ + logger.info("Initializing Ray in LOCAL mode") + + ray_init_kwargs: Dict[str, Any] = { + "ignore_reinit_error": True, + "include_dashboard": False, + } + + if enable_logging: + ray_init_kwargs.update( + { + "log_to_driver": True, + "logging_level": "INFO", + } + ) + else: + ray_init_kwargs.update( + { + "log_to_driver": False, + "logging_level": "ERROR", + } + ) + _suppress_ray_logging() + + # Add local configuration + ray_init_kwargs.update( + { + "_node_ip_address": os.getenv("RAY_NODE_IP", "127.0.0.1"), + "num_cpus": os.cpu_count() or 4, + } + ) + + # Merge with user-provided ray_conf if available + if hasattr(config, "ray_conf") and config.ray_conf: + ray_init_kwargs.update(config.ray_conf) + + # Initialize Ray + ray.init(**ray_init_kwargs) + + # Configure DatasetContext + ctx = DatasetContext.get_current() + ctx.shuffle_strategy = "sort" # type: ignore + ctx.enable_tensor_extension_casting = False + + # Log cluster info + if enable_logging: + cluster_resources = ray.cluster_resources() + logger.info( + f"Ray local cluster initialized with {cluster_resources.get('CPU', 0)} CPUs, " + f"{cluster_resources.get('memory', 0) / (1024**3):.1f}GB memory" + ) + + +def _initialize_remote_ray(config: Any, enable_logging: bool = False) -> None: + """ + Initialize Ray in remote mode (connect to existing Ray cluster). + + Args: + config: Configuration object with ray_address + enable_logging: Whether to enable Ray logging + """ + ray_address = getattr(config, "ray_address", None) + if not ray_address: + ray_address = os.getenv("RAY_ADDRESS") + + if not ray_address: + raise ValueError("ray_address must be specified for remote Ray mode") + + logger.info(f"Initializing Ray in REMOTE mode, connecting to: {ray_address}") + + ray_init_kwargs: Dict[str, Any] = { + "address": ray_address, + "ignore_reinit_error": True, + "include_dashboard": False, + } + + if enable_logging: + ray_init_kwargs.update( + { + "log_to_driver": True, + "logging_level": "INFO", + } + ) + else: + ray_init_kwargs.update( + { + "log_to_driver": False, + "logging_level": "ERROR", + } + ) + _suppress_ray_logging() + + # Merge with user-provided ray_conf if available + if hasattr(config, "ray_conf") and config.ray_conf: + ray_init_kwargs.update(config.ray_conf) + + # Initialize Ray + ray.init(**ray_init_kwargs) + + # Configure DatasetContext + ctx = DatasetContext.get_current() + ctx.shuffle_strategy = "sort" # type: ignore + ctx.enable_tensor_extension_casting = False + + # Log cluster info + if enable_logging: + cluster_resources = ray.cluster_resources() + logger.info( + f"Ray remote cluster initialized with {cluster_resources.get('CPU', 0)} CPUs, " + f"{cluster_resources.get('memory', 0) / (1024**3):.1f}GB memory" + ) + + +def _initialize_kuberay(config: Any, enable_logging: bool = False) -> None: + """ + Initialize Ray in KubeRay mode using CodeFlare SDK. + + Args: + config: Configuration object with KubeRay settings + enable_logging: Whether to enable Ray logging + """ + global _ray_wrapper + + logger.info("Initializing Ray in KUBERAY mode using CodeFlare SDK") + + if not enable_logging: + _suppress_ray_logging() + + # Get KubeRay configuration + config_manager = RayConfigManager(config) + kuberay_config = config_manager.get_kuberay_config() + + # Initialize CodeFlare Ray wrapper - this connects to the cluster + _ray_wrapper = CodeFlareRayWrapper( + cluster_name=kuberay_config["cluster_name"], + namespace=kuberay_config["namespace"], + auth_token=kuberay_config["auth_token"], + auth_server=kuberay_config["auth_server"], + skip_tls=kuberay_config.get("skip_tls", False), + enable_logging=enable_logging, + ) + + logger.info("KubeRay cluster connection established via CodeFlare SDK") + + +def ensure_ray_initialized( + config: Optional[Any] = None, force_reinit: bool = False +) -> None: + """ + Ensure Ray is initialized with appropriate configuration. + + This is the main entry point for Ray initialization across all Feast components. + It automatically detects the execution mode and initializes Ray accordingly. + + Args: + config: Configuration object (RayOfflineStoreConfig, RayComputeEngineConfig, or RepoConfig) + force_reinit: If True, reinitialize Ray even if already initialized + + Raises: + ValueError: If configuration is invalid or required parameters are missing + """ + global _ray_initialized + + # Check if already initialized + if _ray_initialized and not force_reinit: + logger.debug("Ray already initialized, skipping initialization") + return + + # Extract Ray-specific config if RepoConfig is provided + ray_config = config + if config and hasattr(config, "offline_store"): + ray_config = config.offline_store + elif config and hasattr(config, "batch_engine"): + ray_config = config.batch_engine + + # Determine enable_logging setting + enable_logging = ( + getattr(ray_config, "enable_ray_logging", False) if ray_config else False + ) + + # Use RayConfigManager to determine execution mode + config_manager = RayConfigManager(ray_config) + execution_mode = config_manager.determine_execution_mode() + + logger.info(f"Ray execution mode detected: {execution_mode.value}") + + # Check if Ray is already initialized (from external source) + if ray.is_initialized() and not force_reinit: + logger.info("Ray is already initialized externally, using existing cluster") + # Configure DatasetContext even if Ray is already initialized + ctx = DatasetContext.get_current() + ctx.shuffle_strategy = "sort" # type: ignore + ctx.enable_tensor_extension_casting = False + if not enable_logging: + _suppress_ray_logging() + _ray_initialized = True + return + + # Initialize based on execution mode + try: + if execution_mode == RayExecutionMode.KUBERAY: + _initialize_kuberay(ray_config, enable_logging) + elif execution_mode == RayExecutionMode.REMOTE: + _initialize_remote_ray(ray_config, enable_logging) + else: # LOCAL + _initialize_local_ray(ray_config, enable_logging) + + _ray_initialized = True + logger.info(f"Ray initialized successfully in {execution_mode.value} mode") + + except Exception as e: + logger.error(f"Failed to initialize Ray in {execution_mode.value} mode: {e}") + raise + + +def get_ray_wrapper() -> Union[StandardRayWrapper, CodeFlareRayWrapper]: + """ + Get the appropriate Ray wrapper based on current initialization mode. + + Returns: + StandardRayWrapper for local/remote modes, CodeFlareRayWrapper for KubeRay mode + """ + global _ray_wrapper + + if _ray_wrapper is None: + # Return a standard Ray wrapper for local/remote modes + _ray_wrapper = StandardRayWrapper() + + return _ray_wrapper + + +def is_ray_initialized() -> bool: + """Check if Ray has been initialized via this module.""" + return _ray_initialized + + +def shutdown_ray() -> None: + """Shutdown Ray and reset initialization state.""" + global _ray_initialized, _ray_wrapper + + if ray.is_initialized(): + logger.info("Shutting down Ray") + ray.shutdown() + + _ray_initialized = False + _ray_wrapper = None + logger.info("Ray shutdown complete") diff --git a/sdk/python/feast/infra/ray_shared_utils.py b/sdk/python/feast/infra/ray_shared_utils.py index 9e9254fbfae..9614623294f 100644 --- a/sdk/python/feast/infra/ray_shared_utils.py +++ b/sdk/python/feast/infra/ray_shared_utils.py @@ -49,7 +49,12 @@ def to_arrow(self) -> pa.Table: @ray.remote def _remote_to_arrow(dataset): - return dataset.to_arrow() + arrow_refs = dataset.to_arrow_refs() + if arrow_refs: + tables = ray.get(arrow_refs) + return pa.concat_tables(tables) + else: + return pa.Table.from_pydict({}) result_ref = _remote_to_arrow.remote(self._dataset_ref) return ray.get(result_ref) @@ -124,6 +129,16 @@ def _remote_take(dataset, num): result_ref = _remote_take.remote(self._dataset_ref, n) return ray.get(result_ref) + def size_bytes(self) -> int: + """Execute size_bytes remotely and return result.""" + + @ray.remote + def _remote_size_bytes(dataset): + return dataset.size_bytes() + + result_ref = _remote_size_bytes.remote(self._dataset_ref) + return ray.get(result_ref) + def __getattr__(self, name): """Catch any method calls that we haven't explicitly implemented.""" raise AttributeError(f"RemoteDatasetProxy has no attribute '{name}'") diff --git a/sdk/python/feast/infra/registry/contrib/hdfs/__init__.py b/sdk/python/feast/infra/registry/contrib/hdfs/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/sdk/python/feast/infra/registry/contrib/hdfs/hdfs_registry_store.py b/sdk/python/feast/infra/registry/contrib/hdfs/hdfs_registry_store.py new file mode 100644 index 00000000000..f4c4193d569 --- /dev/null +++ b/sdk/python/feast/infra/registry/contrib/hdfs/hdfs_registry_store.py @@ -0,0 +1,121 @@ +import json +import uuid +from pathlib import Path, PurePosixPath +from typing import Optional +from urllib.parse import urlparse + +from pyarrow import fs + +from feast.infra.registry.registry_store import RegistryStore +from feast.protos.feast.core.Registry_pb2 import Registry as RegistryProto +from feast.repo_config import RegistryConfig +from feast.utils import _utc_now + + +class HDFSRegistryStore(RegistryStore): + """HDFS implementation of RegistryStore. + registryConfig.path should be a hdfs path like hdfs://namenode:8020/path/to/registry.db + """ + + def __init__(self, registry_config: RegistryConfig, repo_path: Path): + try: + from pyarrow.fs import HadoopFileSystem + except ImportError as e: + from feast.errors import FeastExtrasDependencyImportError + + raise FeastExtrasDependencyImportError( + "pyarrow.fs.HadoopFileSystem", str(e) + ) + uri = registry_config.path + self._uri = urlparse(uri) + if self._uri.scheme != "hdfs": + raise ValueError( + f"Unsupported scheme {self._uri.scheme} in HDFS path {uri}" + ) + self._hdfs = HadoopFileSystem(self._uri.hostname, self._uri.port or 8020) + self._path = PurePosixPath(self._uri.path) + + def get_registry_proto(self): + registry_proto = RegistryProto() + if _check_hdfs_path_exists(self._hdfs, str(self._path)): + with self._hdfs.open_input_file(str(self._path)) as f: + registry_proto.ParseFromString(f.read()) + return registry_proto + raise FileNotFoundError( + f'Registry not found at path "{self._uri.geturl()}". Have you run "feast apply"?' + ) + + def update_registry_proto(self, registry_proto: RegistryProto): + self._write_registry(registry_proto) + + def teardown(self): + if _check_hdfs_path_exists(self._hdfs, str(self._path)): + self._hdfs.delete_file(str(self._path)) + else: + # Nothing to do + pass + + def _write_registry(self, registry_proto: RegistryProto): + """Write registry protobuf to HDFS.""" + registry_proto.version_id = str(uuid.uuid4()) + registry_proto.last_updated.FromDatetime(_utc_now()) + + dir_path = self._path.parent + if not _check_hdfs_path_exists(self._hdfs, str(dir_path)): + self._hdfs.create_dir(str(dir_path), recursive=True) + + with self._hdfs.open_output_stream(str(self._path)) as f: + f.write(registry_proto.SerializeToString()) + + def set_project_metadata(self, project: str, key: str, value: str): + """Set a custom project metadata key-value pair in the registry (HDFS backend).""" + registry_proto = self.get_registry_proto() + found = False + + for pm in registry_proto.project_metadata: + if pm.project == project: + # Load JSON metadata from project_uuid + try: + meta = json.loads(pm.project_uuid) if pm.project_uuid else {} + except Exception: + meta = {} + + if not isinstance(meta, dict): + meta = {} + + meta[key] = value + pm.project_uuid = json.dumps(meta) + found = True + break + + if not found: + # Create new ProjectMetadata entry + from feast.project_metadata import ProjectMetadata + + pm = ProjectMetadata(project_name=project) + pm.project_uuid = json.dumps({key: value}) + registry_proto.project_metadata.append(pm.to_proto()) + + # Write back + self.update_registry_proto(registry_proto) + + def get_project_metadata(self, project: str, key: str) -> Optional[str]: + """Get custom project metadata key from registry (HDFS backend).""" + registry_proto = self.get_registry_proto() + + for pm in registry_proto.project_metadata: + if pm.project == project: + try: + meta = json.loads(pm.project_uuid) if pm.project_uuid else {} + except Exception: + meta = {} + + if not isinstance(meta, dict): + return None + return meta.get(key, None) + return None + + +def _check_hdfs_path_exists(hdfs, path: str) -> bool: + info = hdfs.get_file_info([path])[0] + return info.type != fs.FileType.NotFound diff --git a/sdk/python/feast/infra/registry/registry.py b/sdk/python/feast/infra/registry/registry.py index 38f79f84e7a..9a021744dd1 100644 --- a/sdk/python/feast/infra/registry/registry.py +++ b/sdk/python/feast/infra/registry/registry.py @@ -62,12 +62,14 @@ "S3RegistryStore": "feast.infra.registry.s3.S3RegistryStore", "FileRegistryStore": "feast.infra.registry.file.FileRegistryStore", "AzureRegistryStore": "feast.infra.registry.contrib.azure.azure_registry_store.AzBlobRegistryStore", + "HDFSRegistryStore": "feast.infra.registry.contrib.hdfs.hdfs_registry_store.HDFSRegistryStore", } REGISTRY_STORE_CLASS_FOR_SCHEME = { "gs": "GCSRegistryStore", "s3": "S3RegistryStore", "file": "FileRegistryStore", + "hdfs": "HDFSRegistryStore", "": "FileRegistryStore", } @@ -143,7 +145,7 @@ def get_registry_store_class_from_scheme(registry_path: str): if uri.scheme not in REGISTRY_STORE_CLASS_FOR_SCHEME: raise Exception( f"Registry path {registry_path} has unsupported scheme {uri.scheme}. " - f"Supported schemes are file, s3 and gs." + f"Supported schemes are file, s3, gs and hdfs." ) else: registry_store_type = REGISTRY_STORE_CLASS_FOR_SCHEME[uri.scheme] diff --git a/sdk/python/feast/infra/registry/sql.py b/sdk/python/feast/infra/registry/sql.py index fd4119a966f..103b1f6c0a6 100644 --- a/sdk/python/feast/infra/registry/sql.py +++ b/sdk/python/feast/infra/registry/sql.py @@ -234,9 +234,6 @@ class SqlRegistryConfig(RegistryConfig): sqlalchemy_config_kwargs: Dict[str, Any] = {"echo": False} """ Dict[str, Any]: Extra arguments to pass to SQLAlchemy.create_engine. """ - cache_mode: StrictStr = "sync" - """ str: Cache mode type, Possible options are sync and thread(asynchronous caching using threading library)""" - thread_pool_executor_worker_count: StrictInt = 0 """ int: Number of worker threads to use for asynchronous caching in SQL Registry. If set to 0, it doesn't use ThreadPoolExecutor. """ diff --git a/sdk/python/feast/offline_server.py b/sdk/python/feast/offline_server.py index 776a0dfb96d..6bc573888fc 100644 --- a/sdk/python/feast/offline_server.py +++ b/sdk/python/feast/offline_server.py @@ -449,6 +449,17 @@ def get_historical_features(self, command: dict, key: Optional[str] = None): resource=feature_view, actions=[AuthzedAction.READ_OFFLINE] ) + # Extract and deserialize start_date/end_date if present + kwargs = {} + if "start_date" in command and command["start_date"] is not None: + kwargs["start_date"] = utils.make_tzaware( + datetime.fromisoformat(command["start_date"]) + ) + if "end_date" in command and command["end_date"] is not None: + kwargs["end_date"] = utils.make_tzaware( + datetime.fromisoformat(command["end_date"]) + ) + retJob = self.offline_store.get_historical_features( config=self.store.config, feature_views=feature_views, @@ -457,6 +468,7 @@ def get_historical_features(self, command: dict, key: Optional[str] = None): registry=self.store.registry, project=project, full_feature_names=full_feature_names, + **kwargs, ) return retJob diff --git a/sdk/python/feast/protos/feast/core/FeatureView_pb2.py b/sdk/python/feast/protos/feast/core/FeatureView_pb2.py index b7032bc6a00..9a59255375f 100644 --- a/sdk/python/feast/protos/feast/core/FeatureView_pb2.py +++ b/sdk/python/feast/protos/feast/core/FeatureView_pb2.py @@ -19,7 +19,7 @@ from feast.protos.feast.core import Transformation_pb2 as feast_dot_core_dot_Transformation__pb2 -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x1c\x66\x65\x61st/core/FeatureView.proto\x12\nfeast.core\x1a\x1egoogle/protobuf/duration.proto\x1a\x1fgoogle/protobuf/timestamp.proto\x1a\x1b\x66\x65\x61st/core/DataSource.proto\x1a\x18\x66\x65\x61st/core/Feature.proto\x1a\x1f\x66\x65\x61st/core/Transformation.proto\"c\n\x0b\x46\x65\x61tureView\x12)\n\x04spec\x18\x01 \x01(\x0b\x32\x1b.feast.core.FeatureViewSpec\x12)\n\x04meta\x18\x02 \x01(\x0b\x32\x1b.feast.core.FeatureViewMeta\"\xc6\x04\n\x0f\x46\x65\x61tureViewSpec\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07project\x18\x02 \x01(\t\x12\x10\n\x08\x65ntities\x18\x03 \x03(\t\x12+\n\x08\x66\x65\x61tures\x18\x04 \x03(\x0b\x32\x19.feast.core.FeatureSpecV2\x12\x31\n\x0e\x65ntity_columns\x18\x0c \x03(\x0b\x32\x19.feast.core.FeatureSpecV2\x12\x13\n\x0b\x64\x65scription\x18\n \x01(\t\x12\x33\n\x04tags\x18\x05 \x03(\x0b\x32%.feast.core.FeatureViewSpec.TagsEntry\x12\r\n\x05owner\x18\x0b \x01(\t\x12&\n\x03ttl\x18\x06 \x01(\x0b\x32\x19.google.protobuf.Duration\x12,\n\x0c\x62\x61tch_source\x18\x07 \x01(\x0b\x32\x16.feast.core.DataSource\x12-\n\rstream_source\x18\t \x01(\x0b\x32\x16.feast.core.DataSource\x12\x0e\n\x06online\x18\x08 \x01(\x08\x12\x0f\n\x07offline\x18\r \x01(\x08\x12\x31\n\x0csource_views\x18\x0e \x03(\x0b\x32\x1b.feast.core.FeatureViewSpec\x12\x43\n\x16\x66\x65\x61ture_transformation\x18\x0f \x01(\x0b\x32#.feast.core.FeatureTransformationV2\x1a+\n\tTagsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"\xcc\x01\n\x0f\x46\x65\x61tureViewMeta\x12\x35\n\x11\x63reated_timestamp\x18\x01 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12:\n\x16last_updated_timestamp\x18\x02 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x46\n\x19materialization_intervals\x18\x03 \x03(\x0b\x32#.feast.core.MaterializationInterval\"w\n\x17MaterializationInterval\x12.\n\nstart_time\x18\x01 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12,\n\x08\x65nd_time\x18\x02 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\"@\n\x0f\x46\x65\x61tureViewList\x12-\n\x0c\x66\x65\x61tureviews\x18\x01 \x03(\x0b\x32\x17.feast.core.FeatureViewBU\n\x10\x66\x65\x61st.proto.coreB\x10\x46\x65\x61tureViewProtoZ/github.com/feast-dev/feast/go/protos/feast/coreb\x06proto3') +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x1c\x66\x65\x61st/core/FeatureView.proto\x12\nfeast.core\x1a\x1egoogle/protobuf/duration.proto\x1a\x1fgoogle/protobuf/timestamp.proto\x1a\x1b\x66\x65\x61st/core/DataSource.proto\x1a\x18\x66\x65\x61st/core/Feature.proto\x1a\x1f\x66\x65\x61st/core/Transformation.proto\"c\n\x0b\x46\x65\x61tureView\x12)\n\x04spec\x18\x01 \x01(\x0b\x32\x1b.feast.core.FeatureViewSpec\x12)\n\x04meta\x18\x02 \x01(\x0b\x32\x1b.feast.core.FeatureViewMeta\"\xd4\x04\n\x0f\x46\x65\x61tureViewSpec\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07project\x18\x02 \x01(\t\x12\x10\n\x08\x65ntities\x18\x03 \x03(\t\x12+\n\x08\x66\x65\x61tures\x18\x04 \x03(\x0b\x32\x19.feast.core.FeatureSpecV2\x12\x33\n\x04tags\x18\x05 \x03(\x0b\x32%.feast.core.FeatureViewSpec.TagsEntry\x12&\n\x03ttl\x18\x06 \x01(\x0b\x32\x19.google.protobuf.Duration\x12,\n\x0c\x62\x61tch_source\x18\x07 \x01(\x0b\x32\x16.feast.core.DataSource\x12\x0e\n\x06online\x18\x08 \x01(\x08\x12-\n\rstream_source\x18\t \x01(\x0b\x32\x16.feast.core.DataSource\x12\x13\n\x0b\x64\x65scription\x18\n \x01(\t\x12\r\n\x05owner\x18\x0b \x01(\t\x12\x31\n\x0e\x65ntity_columns\x18\x0c \x03(\x0b\x32\x19.feast.core.FeatureSpecV2\x12\x0f\n\x07offline\x18\r \x01(\x08\x12\x31\n\x0csource_views\x18\x0e \x03(\x0b\x32\x1b.feast.core.FeatureViewSpec\x12\x43\n\x16\x66\x65\x61ture_transformation\x18\x0f \x01(\x0b\x32#.feast.core.FeatureTransformationV2\x12\x0c\n\x04mode\x18\x10 \x01(\t\x1a+\n\tTagsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"\xcc\x01\n\x0f\x46\x65\x61tureViewMeta\x12\x35\n\x11\x63reated_timestamp\x18\x01 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12:\n\x16last_updated_timestamp\x18\x02 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x46\n\x19materialization_intervals\x18\x03 \x03(\x0b\x32#.feast.core.MaterializationInterval\"w\n\x17MaterializationInterval\x12.\n\nstart_time\x18\x01 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12,\n\x08\x65nd_time\x18\x02 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\"@\n\x0f\x46\x65\x61tureViewList\x12-\n\x0c\x66\x65\x61tureviews\x18\x01 \x03(\x0b\x32\x17.feast.core.FeatureViewBU\n\x10\x66\x65\x61st.proto.coreB\x10\x46\x65\x61tureViewProtoZ/github.com/feast-dev/feast/go/protos/feast/coreb\x06proto3') _globals = globals() _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) @@ -32,13 +32,13 @@ _globals['_FEATUREVIEW']._serialized_start=197 _globals['_FEATUREVIEW']._serialized_end=296 _globals['_FEATUREVIEWSPEC']._serialized_start=299 - _globals['_FEATUREVIEWSPEC']._serialized_end=881 - _globals['_FEATUREVIEWSPEC_TAGSENTRY']._serialized_start=838 - _globals['_FEATUREVIEWSPEC_TAGSENTRY']._serialized_end=881 - _globals['_FEATUREVIEWMETA']._serialized_start=884 - _globals['_FEATUREVIEWMETA']._serialized_end=1088 - _globals['_MATERIALIZATIONINTERVAL']._serialized_start=1090 - _globals['_MATERIALIZATIONINTERVAL']._serialized_end=1209 - _globals['_FEATUREVIEWLIST']._serialized_start=1211 - _globals['_FEATUREVIEWLIST']._serialized_end=1275 + _globals['_FEATUREVIEWSPEC']._serialized_end=895 + _globals['_FEATUREVIEWSPEC_TAGSENTRY']._serialized_start=852 + _globals['_FEATUREVIEWSPEC_TAGSENTRY']._serialized_end=895 + _globals['_FEATUREVIEWMETA']._serialized_start=898 + _globals['_FEATUREVIEWMETA']._serialized_end=1102 + _globals['_MATERIALIZATIONINTERVAL']._serialized_start=1104 + _globals['_MATERIALIZATIONINTERVAL']._serialized_end=1223 + _globals['_FEATUREVIEWLIST']._serialized_start=1225 + _globals['_FEATUREVIEWLIST']._serialized_end=1289 # @@protoc_insertion_point(module_scope) diff --git a/sdk/python/feast/protos/feast/core/FeatureView_pb2.pyi b/sdk/python/feast/protos/feast/core/FeatureView_pb2.pyi index d90d8b7e419..30a0bcf5679 100644 --- a/sdk/python/feast/protos/feast/core/FeatureView_pb2.pyi +++ b/sdk/python/feast/protos/feast/core/FeatureView_pb2.pyi @@ -58,7 +58,7 @@ class FeatureView(google.protobuf.message.Message): global___FeatureView = FeatureView class FeatureViewSpec(google.protobuf.message.Message): - """Next available id: 16 + """Next available id: 17 TODO(adchia): refactor common fields from this and ODFV into separate metadata proto """ @@ -83,17 +83,18 @@ class FeatureViewSpec(google.protobuf.message.Message): PROJECT_FIELD_NUMBER: builtins.int ENTITIES_FIELD_NUMBER: builtins.int FEATURES_FIELD_NUMBER: builtins.int - ENTITY_COLUMNS_FIELD_NUMBER: builtins.int - DESCRIPTION_FIELD_NUMBER: builtins.int TAGS_FIELD_NUMBER: builtins.int - OWNER_FIELD_NUMBER: builtins.int TTL_FIELD_NUMBER: builtins.int BATCH_SOURCE_FIELD_NUMBER: builtins.int - STREAM_SOURCE_FIELD_NUMBER: builtins.int ONLINE_FIELD_NUMBER: builtins.int + STREAM_SOURCE_FIELD_NUMBER: builtins.int + DESCRIPTION_FIELD_NUMBER: builtins.int + OWNER_FIELD_NUMBER: builtins.int + ENTITY_COLUMNS_FIELD_NUMBER: builtins.int OFFLINE_FIELD_NUMBER: builtins.int SOURCE_VIEWS_FIELD_NUMBER: builtins.int FEATURE_TRANSFORMATION_FIELD_NUMBER: builtins.int + MODE_FIELD_NUMBER: builtins.int name: builtins.str """Name of the feature view. Must be unique. Not updated.""" project: builtins.str @@ -105,15 +106,8 @@ class FeatureViewSpec(google.protobuf.message.Message): def features(self) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[feast.core.Feature_pb2.FeatureSpecV2]: """List of specifications for each feature defined as part of this feature view.""" @property - def entity_columns(self) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[feast.core.Feature_pb2.FeatureSpecV2]: - """List of specifications for each entity defined as part of this feature view.""" - description: builtins.str - """Description of the feature view.""" - @property def tags(self) -> google.protobuf.internal.containers.ScalarMap[builtins.str, builtins.str]: """User defined metadata""" - owner: builtins.str - """Owner of the feature view.""" @property def ttl(self) -> google.protobuf.duration_pb2.Duration: """Features in this feature view can only be retrieved from online serving @@ -124,20 +118,29 @@ class FeatureViewSpec(google.protobuf.message.Message): @property def batch_source(self) -> feast.core.DataSource_pb2.DataSource: """Batch/Offline DataSource where this view can retrieve offline feature data.""" - @property - def stream_source(self) -> feast.core.DataSource_pb2.DataSource: - """Streaming DataSource from where this view can consume "online" feature data.""" online: builtins.bool """Whether these features should be served online or not This is also used to determine whether the features should be written to the online store """ + @property + def stream_source(self) -> feast.core.DataSource_pb2.DataSource: + """Streaming DataSource from where this view can consume "online" feature data.""" + description: builtins.str + """Description of the feature view.""" + owner: builtins.str + """Owner of the feature view.""" + @property + def entity_columns(self) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[feast.core.Feature_pb2.FeatureSpecV2]: + """List of specifications for each entity defined as part of this feature view.""" offline: builtins.bool """Whether these features should be written to the offline store""" @property def source_views(self) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[global___FeatureViewSpec]: ... @property def feature_transformation(self) -> feast.core.Transformation_pb2.FeatureTransformationV2: - """Feature transformation (UDF or Substrait) for batch feature views""" + """Feature transformation for batch feature views""" + mode: builtins.str + """The transformation mode (e.g., "python", "pandas", "ray", "spark", "sql")""" def __init__( self, *, @@ -145,20 +148,21 @@ class FeatureViewSpec(google.protobuf.message.Message): project: builtins.str = ..., entities: collections.abc.Iterable[builtins.str] | None = ..., features: collections.abc.Iterable[feast.core.Feature_pb2.FeatureSpecV2] | None = ..., - entity_columns: collections.abc.Iterable[feast.core.Feature_pb2.FeatureSpecV2] | None = ..., - description: builtins.str = ..., tags: collections.abc.Mapping[builtins.str, builtins.str] | None = ..., - owner: builtins.str = ..., ttl: google.protobuf.duration_pb2.Duration | None = ..., batch_source: feast.core.DataSource_pb2.DataSource | None = ..., - stream_source: feast.core.DataSource_pb2.DataSource | None = ..., online: builtins.bool = ..., + stream_source: feast.core.DataSource_pb2.DataSource | None = ..., + description: builtins.str = ..., + owner: builtins.str = ..., + entity_columns: collections.abc.Iterable[feast.core.Feature_pb2.FeatureSpecV2] | None = ..., offline: builtins.bool = ..., source_views: collections.abc.Iterable[global___FeatureViewSpec] | None = ..., feature_transformation: feast.core.Transformation_pb2.FeatureTransformationV2 | None = ..., + mode: builtins.str = ..., ) -> None: ... def HasField(self, field_name: typing_extensions.Literal["batch_source", b"batch_source", "feature_transformation", b"feature_transformation", "stream_source", b"stream_source", "ttl", b"ttl"]) -> builtins.bool: ... - def ClearField(self, field_name: typing_extensions.Literal["batch_source", b"batch_source", "description", b"description", "entities", b"entities", "entity_columns", b"entity_columns", "feature_transformation", b"feature_transformation", "features", b"features", "name", b"name", "offline", b"offline", "online", b"online", "owner", b"owner", "project", b"project", "source_views", b"source_views", "stream_source", b"stream_source", "tags", b"tags", "ttl", b"ttl"]) -> None: ... + def ClearField(self, field_name: typing_extensions.Literal["batch_source", b"batch_source", "description", b"description", "entities", b"entities", "entity_columns", b"entity_columns", "feature_transformation", b"feature_transformation", "features", b"features", "mode", b"mode", "name", b"name", "offline", b"offline", "online", b"online", "owner", b"owner", "project", b"project", "source_views", b"source_views", "stream_source", b"stream_source", "tags", b"tags", "ttl", b"ttl"]) -> None: ... global___FeatureViewSpec = FeatureViewSpec diff --git a/sdk/python/feast/protos/feast/core/Transformation_pb2.py b/sdk/python/feast/protos/feast/core/Transformation_pb2.py index 9fd11d3026b..c322bc1925c 100644 --- a/sdk/python/feast/protos/feast/core/Transformation_pb2.py +++ b/sdk/python/feast/protos/feast/core/Transformation_pb2.py @@ -14,7 +14,7 @@ -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x1f\x66\x65\x61st/core/Transformation.proto\x12\nfeast.core\"F\n\x15UserDefinedFunctionV2\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0c\n\x04\x62ody\x18\x02 \x01(\x0c\x12\x11\n\tbody_text\x18\x03 \x01(\t\"\xba\x01\n\x17\x46\x65\x61tureTransformationV2\x12\x42\n\x15user_defined_function\x18\x01 \x01(\x0b\x32!.feast.core.UserDefinedFunctionV2H\x00\x12I\n\x18substrait_transformation\x18\x02 \x01(\x0b\x32%.feast.core.SubstraitTransformationV2H\x00\x42\x10\n\x0etransformation\"J\n\x19SubstraitTransformationV2\x12\x16\n\x0esubstrait_plan\x18\x01 \x01(\x0c\x12\x15\n\ribis_function\x18\x02 \x01(\x0c\x42_\n\x10\x66\x65\x61st.proto.coreB\x1a\x46\x65\x61tureTransformationProtoZ/github.com/feast-dev/feast/go/protos/feast/coreb\x06proto3') +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x1f\x66\x65\x61st/core/Transformation.proto\x12\nfeast.core\"T\n\x15UserDefinedFunctionV2\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0c\n\x04\x62ody\x18\x02 \x01(\x0c\x12\x11\n\tbody_text\x18\x03 \x01(\t\x12\x0c\n\x04mode\x18\x04 \x01(\t\"\xba\x01\n\x17\x46\x65\x61tureTransformationV2\x12\x42\n\x15user_defined_function\x18\x01 \x01(\x0b\x32!.feast.core.UserDefinedFunctionV2H\x00\x12I\n\x18substrait_transformation\x18\x02 \x01(\x0b\x32%.feast.core.SubstraitTransformationV2H\x00\x42\x10\n\x0etransformation\"J\n\x19SubstraitTransformationV2\x12\x16\n\x0esubstrait_plan\x18\x01 \x01(\x0c\x12\x15\n\ribis_function\x18\x02 \x01(\x0c\x42_\n\x10\x66\x65\x61st.proto.coreB\x1a\x46\x65\x61tureTransformationProtoZ/github.com/feast-dev/feast/go/protos/feast/coreb\x06proto3') _globals = globals() _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) @@ -23,9 +23,9 @@ _globals['DESCRIPTOR']._options = None _globals['DESCRIPTOR']._serialized_options = b'\n\020feast.proto.coreB\032FeatureTransformationProtoZ/github.com/feast-dev/feast/go/protos/feast/core' _globals['_USERDEFINEDFUNCTIONV2']._serialized_start=47 - _globals['_USERDEFINEDFUNCTIONV2']._serialized_end=117 - _globals['_FEATURETRANSFORMATIONV2']._serialized_start=120 - _globals['_FEATURETRANSFORMATIONV2']._serialized_end=306 - _globals['_SUBSTRAITTRANSFORMATIONV2']._serialized_start=308 - _globals['_SUBSTRAITTRANSFORMATIONV2']._serialized_end=382 + _globals['_USERDEFINEDFUNCTIONV2']._serialized_end=131 + _globals['_FEATURETRANSFORMATIONV2']._serialized_start=134 + _globals['_FEATURETRANSFORMATIONV2']._serialized_end=320 + _globals['_SUBSTRAITTRANSFORMATIONV2']._serialized_start=322 + _globals['_SUBSTRAITTRANSFORMATIONV2']._serialized_end=396 # @@protoc_insertion_point(module_scope) diff --git a/sdk/python/feast/protos/feast/core/Transformation_pb2.pyi b/sdk/python/feast/protos/feast/core/Transformation_pb2.pyi index 1120c447e01..fb56ab5bc73 100644 --- a/sdk/python/feast/protos/feast/core/Transformation_pb2.pyi +++ b/sdk/python/feast/protos/feast/core/Transformation_pb2.pyi @@ -22,20 +22,24 @@ class UserDefinedFunctionV2(google.protobuf.message.Message): NAME_FIELD_NUMBER: builtins.int BODY_FIELD_NUMBER: builtins.int BODY_TEXT_FIELD_NUMBER: builtins.int + MODE_FIELD_NUMBER: builtins.int name: builtins.str """The function name""" body: builtins.bytes """The python-syntax function body (serialized by dill)""" body_text: builtins.str """The string representation of the udf""" + mode: builtins.str + """The transformation mode (e.g., "python", "pandas", "ray", "spark", "sql")""" def __init__( self, *, name: builtins.str = ..., body: builtins.bytes = ..., body_text: builtins.str = ..., + mode: builtins.str = ..., ) -> None: ... - def ClearField(self, field_name: typing_extensions.Literal["body", b"body", "body_text", b"body_text", "name", b"name"]) -> None: ... + def ClearField(self, field_name: typing_extensions.Literal["body", b"body", "body_text", b"body_text", "mode", b"mode", "name", b"name"]) -> None: ... global___UserDefinedFunctionV2 = UserDefinedFunctionV2 diff --git a/sdk/python/feast/repo_config.py b/sdk/python/feast/repo_config.py index 395720304de..895002948f1 100644 --- a/sdk/python/feast/repo_config.py +++ b/sdk/python/feast/repo_config.py @@ -152,6 +152,12 @@ class RegistryConfig(FeastBaseModel): set to infinity by setting TTL to 0 seconds, which means the cache will only be loaded once and will never expire. Users can manually refresh the cache by calling feature_store.refresh_registry() """ + cache_mode: StrictStr = "sync" + """str: Cache mode type. Possible options are 'sync' (immediate refresh after each write operation) and + 'thread' (asynchronous background refresh at cache_ttl_seconds intervals). In 'sync' mode, registry changes + are immediately visible. In 'thread' mode, changes may take up to + cache_ttl_seconds to be visible.""" + s3_additional_kwargs: Optional[Dict[str, str]] = None """ Dict[str, str]: Extra arguments to pass to boto3 when writing the registry file to S3. """ diff --git a/sdk/python/feast/stream_feature_view.py b/sdk/python/feast/stream_feature_view.py index 7225be4b7ec..ff6d603333d 100644 --- a/sdk/python/feast/stream_feature_view.py +++ b/sdk/python/feast/stream_feature_view.py @@ -157,6 +157,7 @@ def __init__( owner=owner, schema=schema, source=source, # type: ignore[arg-type] + mode=mode, sink_source=sink_source, ) diff --git a/sdk/python/feast/templates/ray_rag/README.md b/sdk/python/feast/templates/ray_rag/README.md new file mode 100644 index 00000000000..e1289826c6a --- /dev/null +++ b/sdk/python/feast/templates/ray_rag/README.md @@ -0,0 +1,181 @@ +# Feast Ray RAG Template - Batch Embedding at scale for RAG with Ray + +RAG (Retrieval-Augmented Generation) template using Feast with Ray for distributed processing and Milvus for vector search. + +## 🚀 What This Template Provides + +- **🎬 Sample IMDB Data**: 10 curated movies included for quick demos +- **⚡ Ray Distributed Processing**: Parallel embedding generation across workers +- **🔍 Vector Search**: Milvus integration for semantic similarity +- **🎯 Complete Pipeline**: Data → Embeddings → Search in one workflow +- **📦 Ready to Scale**: Easy upgrade to full dataset (48K+ movies) if needed + +## 📁 Template Structure + +``` +ray_rag/ +├── feature_repo/ +│ ├── feature_store.yaml # Ray + Milvus configuration +│ ├── example_repo.py # Feature definitions with Ray UDF +│ ├── test_workflow.py # End-to-end demo +│ └── data/ +│ └── raw_movies.parquet # Sample IMDB dataset (10 movies) +├── bootstrap.py # Template initialization +└── README.md +``` + +## 🚦 Quick Start + +### 1. Initialize Template + +```bash +feast init -t ray_rag my_rag_project +cd my_rag_project/feature_repo +``` + +The template includes a sample dataset with 10 movies for quick testing. + +### 2. Install Dependencies + +```bash +# Core dependencies +pip install feast[ray] sentence-transformers +``` + +### 3. Apply Feature Definitions + +```bash +feast apply +``` + +### 4. Materialize Features + +```bash +# Generate embeddings for sample movies +feast materialize --disable-event-timestamp +``` + +### 5. Test the Pipeline + +```bash +python test_workflow.py +``` + +Expected output with sample dataset: +- ✅ 10 embeddings materialized +- ✅ Vector search working with relevant results +- ✅ Similarity scores for relevant matches + + +## 📊 Architecture + +``` +Raw Data (IMDB CSV) + ↓ +Ray Offline Store (Distributed I/O) + ↓ +Ray Compute Engine (Parallel Embedding Generation) + ↓ +Milvus Online Store (Vector Search) + ↓ +RAG Application +``` + + +## 🎬 Example Workflow + +```python +from feast import FeatureStore +from sentence_transformers import SentenceTransformer + +# 1. Initialize +store = FeatureStore(repo_path=".") + +# 2. Materialize (embeddings generated in parallel) +store.materialize_incremental(end_date) + +# 3. Search using Feast API +model = SentenceTransformer("sentence-transformers/all-MiniLM-L6-v2") +query_embedding = model.encode(["sci-fi movie about space"])[0].tolist() + +results = store.retrieve_online_documents_v2( + features=[ + "document_embeddings:embedding", + "document_embeddings:movie_name", + "document_embeddings:movie_director", + ], + query=query_embedding, + top_k=5, +).to_dict() + +# Display results with metadata +for i in range(len(results["document_id_pk"])): + print(f"{i+1}. {results['movie_name'][i]}") + print(f" Director: {results['movie_director'][i]}") + print(f" Distance: {results['distance'][i]:.3f}") +``` + +## 📥 Using the Full IMDB Dataset (Optional) + +The template includes a small sample dataset (10 movies) for quick testing. To work with the full dataset containing 48K+ movies: + +### Option 1: Download via Kaggle API + +1. **Setup Kaggle credentials:** + ```bash + # Get API credentials from https://www.kaggle.com/account + # Place kaggle.json in ~/.kaggle/ + chmod 600 ~/.kaggle/kaggle.json + ``` + +2. **Install Kaggle API and download dataset:** + ```bash + pip install kaggle + + # Download to your feature_repo/data directory + cd feature_repo + kaggle datasets download -d yashgupta24/48000-movies-dataset -p ./data --unzip + ``` + +3. **Convert to parquet format:** + ```python + import pandas as pd + import pyarrow as pa + import pyarrow.parquet as pq + from pathlib import Path + + # Read the CSV file (filename may vary) + data_path = Path("./data") + csv_files = list(data_path.glob("*.csv")) + df = pd.read_csv(csv_files[0]) + + # Convert DatePublished to datetime with UTC timezone + df = df.dropna(subset=["DatePublished"]) + df["DatePublished"] = pd.to_datetime(df["DatePublished"], errors="coerce", utc=True) + + # Write to parquet + table = pa.Table.from_pandas(df) + pq.write_table(table, data_path / "raw_movies.parquet") + print(f"✅ Converted {len(df)} movies to parquet format") + ``` + +4. **Run the full pipeline:** + ```bash + feast apply + feast materialize --disable-event-timestamp + python test_workflow.py + ``` + +### Option 2: Use Your Own Dataset + +Replace `feature_repo/data/raw_movies.parquet` with your own dataset. Required schema: + +- `id`: Unique identifier (string) +- `Name`: Movie name (string) +- `Description`: Movie description for embedding (string) +- `Director`: Director name (string) +- `Genres`: Comma-separated genres (string) +- `RatingValue`: Rating score (float) +- `DatePublished`: Publication date (datetime with UTC timezone) + +The Ray embedding pipeline will automatically process your dataset in parallel. diff --git a/sdk/python/feast/templates/ray_rag/__init__.py b/sdk/python/feast/templates/ray_rag/__init__.py new file mode 100644 index 00000000000..e0a1678e04e --- /dev/null +++ b/sdk/python/feast/templates/ray_rag/__init__.py @@ -0,0 +1 @@ +# Ray RAG Template for Feast diff --git a/sdk/python/feast/templates/ray_rag/bootstrap.py b/sdk/python/feast/templates/ray_rag/bootstrap.py new file mode 100644 index 00000000000..752cee0a5c8 --- /dev/null +++ b/sdk/python/feast/templates/ray_rag/bootstrap.py @@ -0,0 +1,90 @@ +import pathlib + +import pandas as pd +import pyarrow as pa +import pyarrow.parquet as pq + +from feast.file_utils import replace_str_in_file + + +def bootstrap(): + repo_path = pathlib.Path(__file__).parent.absolute() / "feature_repo" + project_name = pathlib.Path(__file__).parent.absolute().name + data_path = repo_path / "data" + data_path.mkdir(exist_ok=True) + + print(" 🎬 Setting up sample IMDB movie data for RAG demonstration...") + + parquet_file = data_path / "raw_movies.parquet" + + if parquet_file.exists(): + try: + df = pd.read_parquet(parquet_file) + print(f" ✅ Sample dataset ready with {len(df)} movies") + print(" 💡 For full dataset (48K+ movies), see README.md") + except Exception as e: + print(f" ⚠️ Could not read sample dataset: {e}") + else: + print(" ⚠️ Sample dataset not found, creating minimal example...") + sample_data = pd.DataFrame( + { + "id": ["tt0111161", "tt0068646", "tt0468569", "tt0071562", "tt0050083"], + "Name": [ + "The Shawshank Redemption", + "The Godfather", + "The Dark Knight", + "The Godfather Part II", + "12 Angry Men", + ], + "Description": [ + "Two imprisoned men bond over a number of years, finding solace and eventual redemption through acts of common decency.", + "The aging patriarch of an organized crime dynasty transfers control of his clandestine empire to his reluctant son.", + "When the menace known as the Joker wreaks havoc and chaos on the people of Gotham, Batman must accept one of the greatest psychological and physical tests.", + "The early life and career of Vito Corleone in 1920s New York City is portrayed, while his son, Michael, expands and tightens his grip on the family crime syndicate.", + "A jury holdout attempts to prevent a miscarriage of justice by forcing his colleagues to reconsider the evidence.", + ], + "Director": [ + "Frank Darabont", + "Francis Ford Coppola", + "Christopher Nolan", + "Francis Ford Coppola", + "Sidney Lumet", + ], + "Genres": [ + "Drama", + "Crime, Drama", + "Action, Crime, Drama", + "Crime, Drama", + "Crime, Drama", + ], + "RatingValue": [9.3, 9.2, 9.0, 9.0, 9.0], + "DatePublished": pd.to_datetime( + [ + "1994-09-23", + "1972-03-24", + "2008-07-18", + "1974-12-20", + "1957-04-10", + ], + utc=True, + ), + } + ) + table = pa.Table.from_pandas(sample_data) + pq.write_table(table, parquet_file) + print(f" ✅ Created sample dataset with {len(sample_data)} movies") + + example_py_file = repo_path / "example_repo.py" + replace_str_in_file(example_py_file, "%PROJECT_NAME%", str(project_name)) + + print("🚀 Ray RAG template initialized successfully!") + + print("\n🎯 To get started:") + print(f" 1. cd {project_name}/feature_repo") + print(" 2. feast apply") + print(" 3. feast materialize --disable-event-timestamp") + print(" 4. python test_workflow.py") + + +if __name__ == "__main__": + bootstrap() diff --git a/sdk/python/feast/templates/ray_rag/feature_repo/__init__.py b/sdk/python/feast/templates/ray_rag/feature_repo/__init__.py new file mode 100644 index 00000000000..7349767eb35 --- /dev/null +++ b/sdk/python/feast/templates/ray_rag/feature_repo/__init__.py @@ -0,0 +1 @@ +# Ray RAG Feature Repository diff --git a/sdk/python/feast/templates/ray_rag/feature_repo/example_repo.py b/sdk/python/feast/templates/ray_rag/feature_repo/example_repo.py new file mode 100644 index 00000000000..5312996fd92 --- /dev/null +++ b/sdk/python/feast/templates/ray_rag/feature_repo/example_repo.py @@ -0,0 +1,139 @@ +""" +RAG Feature Repository - Movie Embeddings with Ray Native Processing + +This template demonstrates distributed embedding generation using: +- Ray offline store for scalable data I/O +- Ray compute engine for parallel processing +- Milvus online store with vector search capabilities +""" + +from datetime import timedelta +from pathlib import Path + +import pandas as pd + +from feast import BatchFeatureView, Entity, Field, FileSource, ValueType +from feast.types import Array, Float32, String + +# Configuration +repo_path = Path(__file__).parent +data_path = repo_path / "data" +data_path.mkdir(exist_ok=True) + +EMBED_MODEL_ID = "sentence-transformers/all-MiniLM-L6-v2" + +# Entity definition +document = Entity( + name="document_id", + join_keys=["document_id_pk"], + value_type=ValueType.STRING, + description="Document identifier for RAG retrieval", +) + +# Data source +movies_source = FileSource( + path=str(data_path / "raw_movies.parquet"), + timestamp_field="DatePublished", +) + + +# Embedding processor for distributed Ray processing +class EmbeddingProcessor: + """ + Generate embeddings using SentenceTransformer model. + Model is loaded once per worker and reused for all batches. + """ + + def __init__(self): + """Initialize model once per worker.""" + import torch + from sentence_transformers import SentenceTransformer + + device = "cuda" if torch.cuda.is_available() else "cpu" + self.model = SentenceTransformer(EMBED_MODEL_ID, device=device) + + def __call__(self, batch: pd.DataFrame) -> pd.DataFrame: + """Process batch and generate embeddings.""" + if "id" in batch.columns: + batch["document_id"] = "movie_" + batch["id"].astype(str) + batch["document_id_pk"] = batch["document_id"] + + # Generate embeddings from descriptions + if "Description" in batch.columns: + descriptions = batch["Description"].fillna("").tolist() + model_batch_size = min(128, max(32, len(descriptions))) + embeddings = self.model.encode( + descriptions, + show_progress_bar=False, + batch_size=model_batch_size, + normalize_embeddings=True, + convert_to_numpy=True, + ) + batch["embedding"] = embeddings.tolist() + batch["embedding_model"] = EMBED_MODEL_ID + + # Standardize movie-related metadata + if "Name" in batch.columns: + batch["movie_name"] = batch["Name"].fillna("") + if "Director" in batch.columns: + batch["movie_director"] = batch["Director"].fillna("") + if "Genres" in batch.columns: + batch["movie_genres"] = batch["Genres"].fillna("") + if "RatingValue" in batch.columns: + batch["movie_rating"] = batch["RatingValue"].fillna(0.0) + + return batch + + +# Ray native UDF - Fully adaptive to cluster resources +def generate_embeddings_ray_native(ds): + """ + Distributed embedding generation using Ray Data. + """ + # Ray transformation mode providing control to user over the resources + # Optimize the resources for the transformation + num_blocks = ds.num_blocks() + max_workers = 9 + batch_size = 2500 + try: + sample = ds.take(1) + has_data = len(sample) > 0 + except Exception: + has_data = False + if has_data and num_blocks < max_workers: + ds = ds.repartition(max_workers) + + result = ds.map_batches( + EmbeddingProcessor, + batch_format="pandas", + concurrency=max_workers, + batch_size=batch_size, + ) + return result + + +# Batch feature view for embeddings with metadata +document_embeddings_view = BatchFeatureView( + name="document_embeddings", + entities=[document], + mode="ray", # Native Ray Dataset mode for distributed processing + ttl=timedelta(days=365 * 100), + schema=[ + Field(name="document_id", dtype=String), + Field(name="document_id_pk", dtype=String), + Field(name="embedding", dtype=Array(Float32), vector_index=True), + Field(name="embedding_model", dtype=String), + Field(name="movie_name", dtype=String), + Field(name="movie_director", dtype=String), + Field(name="movie_genres", dtype=String), + Field(name="movie_rating", dtype=Float32), + ], + source=movies_source, + udf=generate_embeddings_ray_native, + online=True, + tags={ + "team": "ml_platform", + "use_case": "rag", + "transformation_mode": "ray_native", + }, +) diff --git a/sdk/python/feast/templates/ray_rag/feature_repo/feature_store.yaml b/sdk/python/feast/templates/ray_rag/feature_repo/feature_store.yaml new file mode 100644 index 00000000000..25aaef1259d --- /dev/null +++ b/sdk/python/feast/templates/ray_rag/feature_repo/feature_store.yaml @@ -0,0 +1,45 @@ +project: my_project +registry: data/registry.db +provider: local + +# Ray offline store configuration for distributed data processing +offline_store: + type: ray + storage_path: data/ray_storage # Path for storing Ray datasets + broadcast_join_threshold_mb: 100 + max_parallelism_multiplier: 2 + target_partition_size_mb: 64 + enable_ray_logging: true + # ray_address: "127.0.0.1:10001" + +# Ray compute engine configuration for batch embedding processing +batch_engine: + type: ray.engine + max_workers: 12 # Maximum parallel workers (adjust based on CPU cores) + enable_optimization: true + broadcast_join_threshold_mb: 100 + target_partition_size_mb: 64 # Size of each data partition for processing + window_size_for_joins: "1H" + enable_ray_logging: true + # ray_address: "127.0.0.1:10001" + +# Online store for serving features with vector search capabilities +online_store: + type: milvus + path: data/online_store.db + vector_enabled: true + embedding_dim: 384 + index_type: "FLAT" + metric_type: "COSINE" + +# For production with remote Milvus, use: +# online_store: +# type: milvus +# host: "localhost" +# port: 19530 +# vector_enabled: true +# embedding_dim: 384 +# index_type: "IVF_FLAT" +# metric_type: "COSINE" + +entity_key_serialization_version: 3 diff --git a/sdk/python/feast/templates/ray_rag/feature_repo/test_workflow.py b/sdk/python/feast/templates/ray_rag/feature_repo/test_workflow.py new file mode 100644 index 00000000000..ba693d7b8b7 --- /dev/null +++ b/sdk/python/feast/templates/ray_rag/feature_repo/test_workflow.py @@ -0,0 +1,89 @@ +""" +Feast-Ray RAG Pipeline Demo + +This script demonstrates: +1. Ray offline store for distributed data I/O +2. Ray compute engine for parallel embedding generation +3. Milvus vector search for semantic similarity +4. Complete RAG pipeline from data to search results + +Usage: + 1. feast apply + 2. feast materialize --disable-event-timestamp + 3. python test_workflow.py +""" + +import sys +from pathlib import Path + +sys.path.append(str(Path(__file__).parent)) + +try: + from sentence_transformers import SentenceTransformer + + from feast import FeatureStore +except ImportError as e: + print(f"Missing dependency: {e}") + print("💡 Install with: pip install feast[ray] sentence-transformers") + sys.exit(1) + + +def main(): + """Run the RAG pipeline demonstration.""" + + store = FeatureStore(repo_path=".") + feature_views = store.list_feature_views() + print(f"Feature views: {len(feature_views)}") + + print("Vector similarity search with Feast ...") + try: + model = SentenceTransformer("sentence-transformers/all-MiniLM-L6-v2") + query = "Crime drama about organized crime families" + + print(f"\n Query: '{query}'") + # Generate query embedding + query_embedding = model.encode([query], normalize_embeddings=True)[0].tolist() + + # Use Feast's retrieve_online_documents_v2 API + # Request all fields we want to display + results = store.retrieve_online_documents_v2( + features=[ + "document_embeddings:embedding", + "document_embeddings:movie_name", + "document_embeddings:movie_director", + "document_embeddings:movie_genres", + "document_embeddings:movie_rating", + ], + query=query_embedding, + top_k=3, + ).to_dict() + + if results and len(results.get("document_id_pk", [])) > 0: + print(" 📊 Top 3 results:") + num_results = len(results["document_id_pk"]) + for i in range(num_results): + name = results.get("movie_name", ["Unknown"] * num_results)[i] + director = results.get("movie_director", ["Unknown"] * num_results)[i] + genres = results.get("movie_genres", ["Unknown"] * num_results)[i] + print(f" {i + 1}. {name}") + print(f" Director: {director} | Genres: {genres}") + else: + print("No results found") + + except Exception as e: + print(f"Search failed: {e}") + return + + print("\n📚 What was demonstrated:") + print(" ✅ Ray-based distributed embedding generation") + print(" ✅ Milvus vector storage and retrieval") + print(" ✅ Similarity search") + print(" ✅ Raw Data to Search workflow") + + print("\n🚀 Next steps:") + print(" • Scale to larger datasets") + print(" • Connect to distributed Ray cluster") + + +if __name__ == "__main__": + main() diff --git a/sdk/python/feast/templates/ray_rag/gitignore b/sdk/python/feast/templates/ray_rag/gitignore new file mode 100644 index 00000000000..e4a6beecf3a --- /dev/null +++ b/sdk/python/feast/templates/ray_rag/gitignore @@ -0,0 +1,109 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +.hypothesis/ +.pytest_cache/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# pyenv +.python-version + +# celery beat schedule file +celerybeat-schedule + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ + +# Feast specific +data/ +.feast/ +ray_storage/ diff --git a/sdk/python/feast/transformation/base.py b/sdk/python/feast/transformation/base.py index 8ff1925d0e0..474f33c962f 100644 --- a/sdk/python/feast/transformation/base.py +++ b/sdk/python/feast/transformation/base.py @@ -90,10 +90,14 @@ def __init__( self.owner = owner def to_proto(self) -> Union[UserDefinedFunctionProto, SubstraitTransformationProto]: + mode_str = ( + self.mode.value if isinstance(self.mode, TransformationMode) else self.mode + ) return UserDefinedFunctionProto( name=self.udf.__name__, body=dill.dumps(self.udf, recurse=True), body_text=self.udf_string, + mode=mode_str, ) def __deepcopy__(self, memo: Optional[Dict[int, Any]] = None) -> "Transformation": diff --git a/sdk/python/feast/transformation/factory.py b/sdk/python/feast/transformation/factory.py index 50c3c665764..16d7a7570d5 100644 --- a/sdk/python/feast/transformation/factory.py +++ b/sdk/python/feast/transformation/factory.py @@ -7,6 +7,7 @@ "sql": "feast.transformation.sql_transformation.SQLTransformation", "spark_sql": "feast.transformation.spark_transformation.SparkTransformation", "spark": "feast.transformation.spark_transformation.SparkTransformation", + "ray": "feast.transformation.ray_transformation.RayTransformation", } diff --git a/sdk/python/feast/transformation/mode.py b/sdk/python/feast/transformation/mode.py index 2b453477b3a..44d38d8e99c 100644 --- a/sdk/python/feast/transformation/mode.py +++ b/sdk/python/feast/transformation/mode.py @@ -6,5 +6,6 @@ class TransformationMode(Enum): PANDAS = "pandas" SPARK_SQL = "spark_sql" SPARK = "spark" + RAY = "ray" SQL = "sql" SUBSTRAIT = "substrait" diff --git a/sdk/python/feast/transformation/python_transformation.py b/sdk/python/feast/transformation/python_transformation.py index 2a19b811abe..68e9eee95f6 100644 --- a/sdk/python/feast/transformation/python_transformation.py +++ b/sdk/python/feast/transformation/python_transformation.py @@ -155,6 +155,13 @@ def __eq__(self, other): return True + def __reduce__(self): + """Support for pickle/dill serialization.""" + return ( + self.__class__, + (self.udf, self.udf_string, self.singleton), + ) + @classmethod def from_proto(cls, user_defined_function_proto: UserDefinedFunctionProto): return PythonTransformation( diff --git a/sdk/python/feast/transformation/ray_transformation.py b/sdk/python/feast/transformation/ray_transformation.py new file mode 100644 index 00000000000..b592ce8b0d7 --- /dev/null +++ b/sdk/python/feast/transformation/ray_transformation.py @@ -0,0 +1,293 @@ +import inspect +from typing import Any, Callable, Optional, cast, get_type_hints + +import dill + +from feast.field import Field, from_value_type +from feast.protos.feast.core.Transformation_pb2 import ( + UserDefinedFunctionV2 as UserDefinedFunctionProto, +) +from feast.transformation.base import Transformation +from feast.transformation.mode import TransformationMode + + +class RayTransformation(Transformation): + """ + Ray transformation for distributed data processing using Ray Datasets. + + Use this for computationally intensive transformations that benefit from + parallel processing, such as: + - Embedding generation (e.g., for RAG applications) + - Image/video processing + - Complex feature engineering + - Large-scale data transformations + + Your UDF should accept a Ray Dataset and return a Ray Dataset, enabling + native Ray operations and distributed processing across workers. + + Example - Basic transformation: + >>> import ray.data + >>> def my_ray_udf(ds: ray.data.Dataset) -> ray.data.Dataset: + ... return ds.map_batches( + ... lambda batch: batch, + ... batch_format="pandas" + ... ) + >>> + >>> from feast.transformation.ray_transformation import RayTransformation + >>> transform = RayTransformation( + ... udf=my_ray_udf, + ... udf_string="def my_ray_udf(ds): return ds.map_batches(...)" + ... ) + + Example - Embedding generation with stateful processing: + >>> class EmbeddingProcessor: + ... def __init__(self): + ... # Model loaded once per worker (efficient!) + ... from sentence_transformers import SentenceTransformer + ... self.model = SentenceTransformer("all-MiniLM-L6-v2") + ... + ... def __call__(self, batch): + ... embeddings = self.model.encode(batch["text"].tolist()) + ... batch["embedding"] = embeddings.tolist() + ... return batch + >>> + >>> def generate_embeddings(ds: ray.data.Dataset) -> ray.data.Dataset: + ... return ds.map_batches( + ... EmbeddingProcessor, + ... batch_format="pandas", + ... concurrency=8 # Use 8 parallel workers + ... ) + + Args: + udf: Function that takes a Ray Dataset and returns a Ray Dataset + udf_string: String representation of the UDF (for serialization) + name: Optional name for the transformation + tags: Optional metadata tags + description: Optional description + owner: Optional owner identifier + + Note: + For best performance, use stateful classes with `map_batches` to avoid + reloading models/resources for each batch. See the embedding example above. + """ + + def __new__( + cls, + udf: Optional[Callable[[Any], Any]] = None, + udf_string: Optional[str] = None, + name: Optional[str] = None, + tags: Optional[dict[str, str]] = None, + description: str = "", + owner: str = "", + ) -> "RayTransformation": + # Handle Ray deserialization where parameters may not be provided + if udf is None and udf_string is None: + # Create a bare instance for deserialization + instance = object.__new__(cls) + return cast("RayTransformation", instance) + + # Ensure required parameters are not None before calling parent constructor + if udf is None: + raise ValueError("udf parameter cannot be None") + if udf_string is None: + raise ValueError("udf_string parameter cannot be None") + + return cast( + "RayTransformation", + super(RayTransformation, cls).__new__( + cls, + mode=TransformationMode.RAY, + udf=udf, + name=name, + udf_string=udf_string, + tags=tags, + description=description, + owner=owner, + ), + ) + + def __init__( + self, + udf: Optional[Callable[[Any], Any]] = None, + udf_string: Optional[str] = None, + name: Optional[str] = None, + tags: Optional[dict[str, str]] = None, + description: str = "", + owner: str = "", + *args, + **kwargs, + ): + if udf is None and udf_string is None: + return + if udf is None: + raise ValueError("udf parameter cannot be None") + if udf_string is None: + raise ValueError("udf_string parameter cannot be None") + + type_hints = get_type_hints(udf) + return_annotation = type_hints.get("return", inspect._empty) + + if return_annotation not in (inspect._empty,): + return_type_str = str(return_annotation) + if "ray.data" not in return_type_str and "Dataset" not in return_type_str: + import warnings + + warnings.warn( + f"Return type for RayTransformation should be ray.data.Dataset, got {return_annotation}. " + f"This may cause issues during execution." + ) + + super().__init__( + mode=TransformationMode.RAY, + udf=udf, + name=name, + udf_string=udf_string, + tags=tags, + description=description, + owner=owner, + ) + + def transform(self, inputs: Any) -> Any: + """ + Apply the transformation to a Ray Dataset. + + This method is called automatically by Feast during materialization. + It applies your user-defined function (UDF) to the input data. + + Args: + inputs: Ray Dataset containing the input data to transform. + The dataset will have columns matching your source schema. + + Returns: + Transformed Ray Dataset with output features. The output schema + will be automatically inferred or should match your explicitly + defined schema in BatchFeatureView. + + Example: + You typically don't call this directly - Feast calls it during: + >>> # store.materialize_incremental(end_date) + + But you can test it manually: + >>> import ray.data + >>> import pandas as pd + >>> test_data = ray.data.from_pandas(pd.DataFrame([{"text": "hello"}])) + >>> # my_transformation = RayTransformation(udf=my_udf, udf_string="...") + >>> # result = my_transformation.transform(test_data) + >>> # result.show() + """ + return self.udf(inputs) + + def infer_features( + self, + random_input: dict[str, list[Any]], + *args, + **kwargs, + ) -> list[Field]: + """ + Infer features from the Ray transformation. + + This method automatically infers the output schema by: + 1. Creating a Ray Dataset from sample input data + 2. Applying your transformation UDF + 3. Extracting the schema from the transformed output + + Args: + random_input: Dictionary mapping column names to sample values. + Should contain representative data for your transformation. + + Returns: + List of Field objects representing the inferred schema. + + Raises: + TypeError: If schema inference fails. In this case, explicitly define + the schema in your BatchFeatureView using the `schema` parameter. + + Example: + If your UDF adds an 'embedding' column, this will automatically + detect it and infer its type from the output data. + """ + try: + import pandas as pd + import ray.data + + # Create a Ray Dataset from the sample input + df = pd.DataFrame.from_dict(random_input) + ds = ray.data.from_pandas([df]) + + # Apply the user's transformation + output_ds = self.transform(ds) + + # Convert result to pandas to extract schema + output_df = output_ds.to_pandas() + + # Infer field types from the output + fields = [] + for feature_name, feature_type in zip(output_df.columns, output_df.dtypes): + feature_value = output_df[feature_name].tolist() + if len(feature_value) <= 0: + raise TypeError( + f"Cannot infer type for feature '{feature_name}': " + f"UDF returned empty output. Ensure your transformation " + f"returns at least one row of data." + ) + from feast.type_map import python_type_to_feast_value_type + + fields.append( + Field( + name=feature_name, + dtype=from_value_type( + python_type_to_feast_value_type( + feature_name, + value=feature_value[0], + type_name=str(feature_type), + ) + ), + ) + ) + return fields + except ImportError as e: + raise TypeError( + f"Failed to import required dependencies for RayTransformation: {e}. " + f"Install Ray with: pip install feast[ray]" + ) + except Exception as e: + error_msg = ( + f"Failed to infer features from RayTransformation: {e}\n\n" + f"💡 To fix this:\n" + f"1. Explicitly define the schema in your BatchFeatureView:\n" + f" BatchFeatureView(\n" + f" schema=[\n" + f" Field(name='your_field', dtype=String),\n" + f" Field(name='embedding', dtype=Array(Float32)),\n" + f" ],\n" + f" ...\n" + f" )\n\n" + f"2. Or ensure your UDF returns valid data when called with sample input:\n" + f" - Input sample: {list(random_input.keys())}\n" + f" - Check that your UDF can process this structure\n" + ) + raise TypeError(error_msg) + + def __eq__(self, other): + if not isinstance(other, RayTransformation): + raise TypeError( + "Comparisons should only involve RayTransformation class objects." + ) + + if ( + self.udf_string != other.udf_string + or self.udf.__code__.co_code != other.udf.__code__.co_code + ): + return False + + return True + + @classmethod + def from_proto(cls, user_defined_function_proto: UserDefinedFunctionProto): + return RayTransformation( + udf=dill.loads(user_defined_function_proto.body), + udf_string=user_defined_function_proto.body_text, + name=user_defined_function_proto.name + if user_defined_function_proto.name + else None, + ) diff --git a/sdk/python/feast/type_map.py b/sdk/python/feast/type_map.py index 9d9921a2bf9..ebf6f0eae19 100644 --- a/sdk/python/feast/type_map.py +++ b/sdk/python/feast/type_map.py @@ -172,6 +172,16 @@ def python_type_to_feast_value_type( if type_name in type_map: return type_map[type_name] + # Handle pandas "object" dtype by inspecting the actual value + if type_name == "object" and value is not None: + # Check the actual type of the value + actual_type = type(value).__name__.lower() + if actual_type == "str": + return ValueType.STRING + # If it's a different type wrapped in object, try to infer from the value + elif actual_type in type_map: + return type_map[actual_type] + if isinstance(value, np.ndarray) and str(value.dtype) in type_map: item_type = type_map[str(value.dtype)] return ValueType[item_type.name + "_LIST"] diff --git a/sdk/python/feast/ui/package.json b/sdk/python/feast/ui/package.json index 72029b5c2dd..6b65265755c 100644 --- a/sdk/python/feast/ui/package.json +++ b/sdk/python/feast/ui/package.json @@ -6,7 +6,7 @@ "@elastic/datemath": "^5.0.3", "@elastic/eui": "^72.0.0", "@emotion/react": "^11.9.0", - "@feast-dev/feast-ui": "0.54.0", + "@feast-dev/feast-ui": "0.55.0", "@testing-library/jest-dom": "^5.16.4", "@testing-library/react": "^13.2.0", "@testing-library/user-event": "^13.5.0", diff --git a/sdk/python/feast/ui/yarn.lock b/sdk/python/feast/ui/yarn.lock index 74db44af9aa..152bf0fb3fd 100644 --- a/sdk/python/feast/ui/yarn.lock +++ b/sdk/python/feast/ui/yarn.lock @@ -1575,10 +1575,10 @@ minimatch "^3.1.2" strip-json-comments "^3.1.1" -"@feast-dev/feast-ui@0.54.0": - version "0.54.0" - resolved "https://registry.yarnpkg.com/@feast-dev/feast-ui/-/feast-ui-0.54.0.tgz#f8d671f0540e84e19dd248803cd24440732f9903" - integrity sha512-WFgz75N+Kmt7FNsbPY5hpIZGDfrw0Sdru/7XLoiHtLPASDfUdalfEyCQbknz2f42RwpIVmbM1/eS6VpRcppa3A== +"@feast-dev/feast-ui@0.55.0": + version "0.55.0" + resolved "https://registry.yarnpkg.com/@feast-dev/feast-ui/-/feast-ui-0.55.0.tgz#dd90ae8a96fdf3a5da5dd12e1d740c4b78faf0a5" + integrity sha512-T+j5ZwPafIsnsUFcSUENh+fR9aKmlfMkJhtKUzKreqg/5bxXRAo0fKIBWkKFSV+KsIAkDCwa8R7V9N5C/eeiKQ== dependencies: "@elastic/datemath" "^5.0.3" "@elastic/eui" "^95.12.0" diff --git a/sdk/python/feast/ui_server.py b/sdk/python/feast/ui_server.py index 6883dc1105e..8e201bcf944 100644 --- a/sdk/python/feast/ui_server.py +++ b/sdk/python/feast/ui_server.py @@ -4,7 +4,7 @@ from typing import Callable, Optional import uvicorn -from fastapi import FastAPI, Response +from fastapi import FastAPI, Response, status from fastapi.middleware.cors import CORSMiddleware from fastapi.staticfiles import StaticFiles from pydantic import BaseModel @@ -61,27 +61,66 @@ def shutdown_event(): with importlib_resources.as_file(ui_dir_ref) as ui_dir: # Initialize with the projects-list.json file with ui_dir.joinpath("projects-list.json").open(mode="w") as f: - projects_dict = { - "projects": [ + # Get all projects from the registry + discovered_projects = [] + registry = store.registry.proto() + + # Use the projects list from the registry + if registry and registry.projects and len(registry.projects) > 0: + for proj in registry.projects: + if proj.spec and proj.spec.name: + discovered_projects.append( + { + "name": proj.spec.name.replace("_", " ").title(), + "description": proj.spec.description + or f"Project: {proj.spec.name}", + "id": proj.spec.name, + "registryPath": f"{root_path}/registry", + } + ) + else: + # If no projects in registry, use the current project from feature_store.yaml + discovered_projects.append( { "name": "Project", "description": "Test project", "id": project_id, "registryPath": f"{root_path}/registry", } - ] - } + ) + + # Add "All Projects" option at the beginning if there are multiple projects + if len(discovered_projects) > 1: + all_projects_entry = { + "name": "All Projects", + "description": "View data across all projects", + "id": "all", + "registryPath": f"{root_path}/registry", + } + discovered_projects.insert(0, all_projects_entry) + + projects_dict = {"projects": discovered_projects} f.write(json.dumps(projects_dict)) @app.get("/registry") def read_registry(): if registry_proto is None: - return Response(status_code=503) # Service Unavailable + return Response( + status_code=status.HTTP_503_SERVICE_UNAVAILABLE + ) # Service Unavailable return Response( content=registry_proto.SerializeToString(), media_type="application/octet-stream", ) + @app.get("/health") + def health(): + return ( + Response(status_code=status.HTTP_200_OK) + if registry_proto + else Response(status_code=status.HTTP_503_SERVICE_UNAVAILABLE) + ) + @app.post("/save-document") async def save_document_endpoint(request: SaveDocumentRequest): try: diff --git a/sdk/python/feast/utils.py b/sdk/python/feast/utils.py index 939adbe933f..01ffa774ccd 100644 --- a/sdk/python/feast/utils.py +++ b/sdk/python/feast/utils.py @@ -33,6 +33,7 @@ RequestDataNotFoundInEntityRowsException, ) from feast.field import Field +from feast.infra.compute_engines.backends.pandas_backend import PandasBackend from feast.infra.key_encoding_utils import deserialize_entity_key from feast.protos.feast.serving.ServingService_pb2 import ( FieldStatus, @@ -561,6 +562,76 @@ def construct_response_feature_vector( ) +def _get_aggregate_operations(agg_specs) -> dict: + """ + Convert Aggregation specs to agg_ops format for PandasBackend. + + Reused from LocalFeatureBuilder logic. + TODO: This logic is duplicated from feast.infra.compute_engines.local.feature_builder.LocalFeatureBuilder._get_aggregate_operations(). + Consider refactoring to a shared utility module in the future. + """ + agg_ops = {} + for agg in agg_specs: + if agg.time_window is not None: + raise ValueError( + "Time window aggregation is not supported in online serving." + ) + alias = f"{agg.function}_{agg.column}" + agg_ops[alias] = (agg.function, agg.column) + return agg_ops + + +def _apply_aggregations_to_response( + response_data: Union[pyarrow.Table, Dict[str, List[Any]]], + aggregations, + group_keys: Optional[List[str]], + mode: str, +) -> Union[pyarrow.Table, Dict[str, List[Any]]]: + """ + Apply aggregations using PandasBackend. + + Args: + response_data: Either a pyarrow.Table or dict of lists containing the data + aggregations: List of Aggregation objects to apply + group_keys: List of column names to group by (optional) + mode: Transformation mode ("python", "pandas", or "substrait") + + Returns: + Aggregated data in the same format as input + + TODO: Consider refactoring to support backends other than pandas in the future. + """ + if not aggregations: + return response_data + + backend = PandasBackend() + + # Convert to pandas DataFrame + if isinstance(response_data, dict): + df = pd.DataFrame(response_data) + else: # pyarrow.Table + df = backend.from_arrow(response_data) + + if df.empty: + return response_data + + # Convert aggregations to agg_ops format + agg_ops = _get_aggregate_operations(aggregations) + + # Apply aggregations using PandasBackend + if group_keys: + result_df = backend.groupby_agg(df, group_keys, agg_ops) + else: + # No grouping - aggregate over entire dataset + result_df = backend.groupby_agg(df, [], agg_ops) + + # Convert back to original format + if mode == "python": + return {col: result_df[col].tolist() for col in result_df.columns} + else: # pandas or substrait + return backend.to_arrow(result_df) + + def _augment_response_with_on_demand_transforms( online_features_response: GetOnlineFeaturesResponse, feature_refs: List[str], @@ -605,7 +676,31 @@ def _augment_response_with_on_demand_transforms( for odfv_name, _feature_refs in odfv_feature_refs.items(): odfv = requested_odfv_map[odfv_name] if not odfv.write_to_online_store: - if odfv.mode == "python": + # Apply aggregations if configured. + if odfv.aggregations: + if odfv.mode == "python": + if initial_response_dict is None: + initial_response_dict = initial_response.to_dict() + initial_response_dict = _apply_aggregations_to_response( + initial_response_dict, + odfv.aggregations, + odfv.entities, + odfv.mode, + ) + elif odfv.mode in {"pandas", "substrait"}: + if initial_response_arrow is None: + initial_response_arrow = initial_response.to_arrow() + initial_response_arrow = _apply_aggregations_to_response( + initial_response_arrow, + odfv.aggregations, + odfv.entities, + odfv.mode, + ) + + # Apply transformation. Note: aggregations and transformation configs are mutually exclusive + # TODO: Fix to make it work for having both aggregation and transformation + # ticket: https://github.com/feast-dev/feast/issues/5689 + elif odfv.mode == "python": if initial_response_dict is None: initial_response_dict = initial_response.to_dict() transformed_features_dict: Dict[str, List[Any]] = odfv.transform_dict( diff --git a/sdk/python/tests/integration/compute_engines/ray_compute/ray_shared_utils.py b/sdk/python/tests/integration/compute_engines/ray_compute/ray_shared_utils.py index 9e9aabc4f90..6b28949f401 100644 --- a/sdk/python/tests/integration/compute_engines/ray_compute/ray_shared_utils.py +++ b/sdk/python/tests/integration/compute_engines/ray_compute/ray_shared_utils.py @@ -9,10 +9,10 @@ import pandas as pd import pytest -import ray from feast import Entity, FileSource from feast.data_source import DataSource +from feast.infra.ray_initializer import shutdown_ray from feast.utils import _utc_now from tests.integration.feature_repos.repo_configuration import ( construct_test_environment, @@ -126,8 +126,7 @@ def cleanup_ray_environment(ray_environment): # Ensure Ray is shut down completely try: - if ray.is_initialized(): - ray.shutdown() + shutdown_ray() time.sleep(0.2) # Brief pause to ensure clean shutdown except Exception as e: print(f"Warning: Ray shutdown failed: {e}") @@ -147,9 +146,8 @@ def create_ray_environment(): def ray_environment() -> Generator: """Pytest fixture to provide a Ray environment for tests with automatic cleanup.""" try: - if ray.is_initialized(): - ray.shutdown() - time.sleep(0.2) + shutdown_ray() + time.sleep(0.2) except Exception: pass diff --git a/sdk/python/tests/integration/compute_engines/ray_compute/test_compute.py b/sdk/python/tests/integration/compute_engines/ray_compute/test_compute.py index e7060b4a756..ef4bfa131da 100644 --- a/sdk/python/tests/integration/compute_engines/ray_compute/test_compute.py +++ b/sdk/python/tests/integration/compute_engines/ray_compute/test_compute.py @@ -19,6 +19,7 @@ from feast.infra.offline_stores.contrib.ray_offline_store.ray import ( RayOfflineStore, ) +from feast.transformation.ray_transformation import RayTransformation from feast.types import Float32, Int32, Int64 from tests.integration.compute_engines.ray_compute.ray_shared_utils import ( driver, @@ -162,10 +163,169 @@ def test_ray_compute_engine_config(): window_size_for_joins="2H", max_workers=4, enable_optimization=True, - execution_timeout_seconds=3600, ) assert config.type == "ray.engine" assert config.ray_address == "ray://localhost:10001" assert config.broadcast_join_threshold_mb == 200 assert config.window_size_timedelta == timedelta(hours=2) + + +@pytest.mark.integration +@pytest.mark.xdist_group(name="ray") +def test_ray_transformation_compute_engine(ray_environment, feature_dataset, entity_df): + """Test Ray compute engine with Ray transformation mode.""" + import ray.data + + fs = ray_environment.feature_store + registry = fs.registry + + def ray_transformation_udf(ds: ray.data.Dataset) -> ray.data.Dataset: + """Ray native transformation that processes data in parallel.""" + + def process_batch(batch: pd.DataFrame) -> pd.DataFrame: + # Simulate some computation (e.g., feature engineering) + if "conv_rate" in batch.columns: + batch["processed_conv_rate"] = batch["conv_rate"] * 2.0 + if "acc_rate" in batch.columns: + batch["processed_acc_rate"] = batch["acc_rate"] * 1.5 + return batch + + return ds.map_batches( + process_batch, + batch_format="pandas", + concurrency=2, # Use 2 parallel workers + ) + + # Create Ray transformation + ray_transform = RayTransformation( + udf=ray_transformation_udf, + udf_string="def ray_transformation_udf(ds): return ds.map_batches(...)", + ) + + driver_stats_fv = BatchFeatureView( + name="driver_hourly_stats_ray", + entities=[driver], + mode="ray", # Use Ray transformation mode + feature_transformation=ray_transform, + ttl=timedelta(days=3), + schema=[ + Field(name="conv_rate", dtype=Float32), + Field(name="acc_rate", dtype=Float32), + Field(name="processed_conv_rate", dtype=Float32), + Field(name="processed_acc_rate", dtype=Float32), + Field(name="avg_daily_trips", dtype=Int64), + Field(name="driver_id", dtype=Int32), + ], + online=False, + offline=False, + source=feature_dataset, + ) + + fs.apply([driver, driver_stats_fv]) + + # Build retrieval task + task = HistoricalRetrievalTask( + project=ray_environment.project, + entity_df=entity_df, + feature_view=driver_stats_fv, + full_feature_name=False, + registry=registry, + ) + engine = RayComputeEngine( + repo_config=ray_environment.config, + offline_store=RayOfflineStore(), + online_store=MagicMock(), + ) + + ray_dag_retrieval_job = engine.get_historical_features(registry, task) + ray_dataset = cast(RayDAGRetrievalJob, ray_dag_retrieval_job).to_ray_dataset() + df_out = ray_dataset.to_pandas().sort_values("driver_id") + + # Verify the transformation was applied + assert df_out.driver_id.to_list() == [1001, 1002] + + # Check that original columns are present + assert "conv_rate" in df_out.columns + assert "acc_rate" in df_out.columns + + # Check that transformed columns are present + assert "processed_conv_rate" in df_out.columns + assert "processed_acc_rate" in df_out.columns + + # Verify the transformation logic was applied + for idx, row in df_out.iterrows(): + assert abs(row["processed_conv_rate"] - row["conv_rate"] * 2.0) < 1e-6 + assert abs(row["processed_acc_rate"] - row["acc_rate"] * 1.5) < 1e-6 + + +@pytest.mark.integration +@pytest.mark.xdist_group(name="ray") +def test_ray_transformation_materialization(ray_environment, feature_dataset): + """Test Ray transformation during materialization.""" + import ray.data + + fs = ray_environment.feature_store + registry = fs.registry + + def ray_embedding_udf(ds: ray.data.Dataset) -> ray.data.Dataset: + """Simulate embedding generation with Ray native processing.""" + + def generate_embeddings(batch: pd.DataFrame) -> pd.DataFrame: + # Simulate embedding generation + if "conv_rate" in batch.columns: + # Create a simple embedding based on conv_rate + batch["embedding"] = batch["conv_rate"].apply( + lambda x: [x * 0.1, x * 0.2, x * 0.3] + ) + return batch + + return ds.map_batches(generate_embeddings, batch_format="pandas", concurrency=2) + + # Create Ray transformation for embeddings + ray_embedding_transform = RayTransformation( + udf=ray_embedding_udf, + udf_string="def ray_embedding_udf(ds): return ds.map_batches(...)", + ) + + driver_embeddings_fv = BatchFeatureView( + name="driver_embeddings", + entities=[driver], + mode="ray", + feature_transformation=ray_embedding_transform, + ttl=timedelta(days=3), + schema=[ + Field(name="conv_rate", dtype=Float32), + Field( + name="embedding", dtype=Float32 + ), # This would be Array(Float32) in real usage + Field(name="driver_id", dtype=Int32), + ], + online=True, + offline=False, + source=feature_dataset, + ) + + def tqdm_builder(length): + return tqdm(length, ncols=100) + + fs.apply([driver, driver_embeddings_fv]) + + task = MaterializationTask( + project=ray_environment.project, + feature_view=driver_embeddings_fv, + start_time=now - timedelta(days=2), + end_time=now, + tqdm_builder=tqdm_builder, + ) + + engine = RayComputeEngine( + repo_config=ray_environment.config, + offline_store=RayOfflineStore(), + online_store=MagicMock(), + ) + + ray_materialize_jobs = engine.materialize(registry, task) + + assert len(ray_materialize_jobs) == 1 + assert ray_materialize_jobs[0].status() == MaterializationJobStatus.SUCCEEDED diff --git a/sdk/python/tests/integration/registration/test_universal_registry.py b/sdk/python/tests/integration/registration/test_universal_registry.py index eb663d8565a..29b31ef1b75 100644 --- a/sdk/python/tests/integration/registration/test_universal_registry.py +++ b/sdk/python/tests/integration/registration/test_universal_registry.py @@ -22,8 +22,12 @@ import grpc_testing import pandas as pd +import pyarrow.fs as fs import pytest from pytest_lazyfixture import lazy_fixture +from testcontainers.core.container import DockerContainer +from testcontainers.core.network import Network +from testcontainers.core.waiting_utils import wait_for_logs from testcontainers.mysql import MySqlContainer from testcontainers.postgres import PostgresContainer @@ -280,6 +284,60 @@ def sqlite_registry(): yield SqlRegistry(registry_config, "project", None) +@pytest.fixture(scope="function") +def hdfs_registry(): + HADOOP_NAMENODE_IMAGE = "bde2020/hadoop-namenode:2.0.0-hadoop3.2.1-java8" + HADOOP_DATANODE_IMAGE = "bde2020/hadoop-datanode:2.0.0-hadoop3.2.1-java8" + HDFS_CLUSTER_NAME = "feast-hdfs-cluster" + HADOOP_NAMENODE_WAIT_LOG = "namenode.NameNode: NameNode RPC up" + HADOOP_DATANODE_WAIT_LOG = "datanode.DataNode: .*successfully registered with NN" + with Network() as network: + namenode = None + datanode = None + + try: + namenode = ( + DockerContainer(HADOOP_NAMENODE_IMAGE) + .with_network(network) + .with_env("CLUSTER_NAME", HDFS_CLUSTER_NAME) + .with_exposed_ports(8020) + .with_network_aliases("namenode") + .with_kwargs(hostname="namenode") + .start() + ) + wait_for_logs(namenode, HADOOP_NAMENODE_WAIT_LOG, timeout=120) + namenode_ip = namenode.get_container_host_ip() + namenode_port = int(namenode.get_exposed_port(8020)) + + datanode = ( + DockerContainer(HADOOP_DATANODE_IMAGE) + .with_network(network) + .with_exposed_ports(9867) + .with_env("CLUSTER_NAME", HDFS_CLUSTER_NAME) + .with_env("CORE_CONF_fs_defaultFS", "hdfs://namenode:8020") + .with_network_aliases("datanode") + .with_kwargs(hostname="datanode") + .start() + ) + + wait_for_logs(datanode, HADOOP_DATANODE_WAIT_LOG, timeout=120) + + hdfs = fs.HadoopFileSystem(host=namenode_ip, port=namenode_port) + hdfs.create_dir("/feast") + registry_path = f"hdfs://{namenode_ip}:{namenode_port}/feast/registry.db" + with hdfs.open_output_stream(registry_path) as f: + f.write(b"") + + registry_config = RegistryConfig(path=registry_path, cache_ttl_seconds=600) + reg = Registry("project", registry_config, None) + yield reg + finally: + if datanode: + datanode.stop() + if namenode: + namenode.stop() + + class GrpcMockChannel: def __init__(self, service, servicer): self.service = service @@ -350,6 +408,10 @@ def mock_remote_registry(): lazy_fixture("mock_remote_registry"), marks=pytest.mark.rbac_remote_integration_test, ), + pytest.param( + lazy_fixture("hdfs_registry"), + marks=pytest.mark.xdist_group(name="hdfs_registry"), + ), ] sql_fixtures = [ diff --git a/sdk/python/tests/integration/registration/test_universal_types.py b/sdk/python/tests/integration/registration/test_universal_types.py index 5ba99b9d7f1..b464cf2f766 100644 --- a/sdk/python/tests/integration/registration/test_universal_types.py +++ b/sdk/python/tests/integration/registration/test_universal_types.py @@ -343,7 +343,6 @@ def offline_types_test_fixtures(request, environment): if ( environment.data_source_creator.__class__.__name__ == "ClickhouseDataSourceCreator" - and config.feature_dtype in {"float", "datetime", "bool"} and config.feature_is_list and not config.has_empty_list ): diff --git a/sdk/python/tests/unit/api/test_api_rest_registry.py b/sdk/python/tests/unit/api/test_api_rest_registry.py index 8d7a939c583..12e22737f93 100644 --- a/sdk/python/tests/unit/api/test_api_rest_registry.py +++ b/sdk/python/tests/unit/api/test_api_rest_registry.py @@ -1528,6 +1528,90 @@ def test_metrics_resource_counts_via_rest(fastapi_test_app): assert isinstance(per_project["demo_project"], dict) +def test_feature_views_all_types_and_resource_counts_match(fastapi_test_app): + """ + Test that verifies: + 1. All types of feature views (regular, on-demand, stream) are returned in /feature_views/all + 2. The count from /metrics/resource_counts matches the count from /feature_views/all + """ + response_all = fastapi_test_app.get("/feature_views/all") + assert response_all.status_code == 200 + data_all = response_all.json() + assert "featureViews" in data_all + + feature_views = data_all["featureViews"] + + # Count should include at least: + # - 3 regular feature views: user_profile, user_behavior, user_preferences + # - 1 on-demand feature view: test_on_demand_feature_view + assert len(feature_views) >= 4, ( + f"Expected at least 4 feature views, got {len(feature_views)}" + ) + + # Verify we have different types of feature views + feature_view_names = {fv["spec"]["name"] for fv in feature_views} + + # Check for regular feature views + assert "user_profile" in feature_view_names, ( + "Regular feature view 'user_profile' not found" + ) + assert "user_behavior" in feature_view_names, ( + "Regular feature view 'user_behavior' not found" + ) + assert "user_preferences" in feature_view_names, ( + "Regular feature view 'user_preferences' not found" + ) + + # Check for on-demand feature view + assert "test_on_demand_feature_view" in feature_view_names, ( + "On-demand feature view 'test_on_demand_feature_view' not found" + ) + + # Verify all have the correct project + for fv in feature_views: + assert fv["project"] == "demo_project", ( + f"Feature view has incorrect project: {fv.get('project')}" + ) + + # Now get resource counts from /metrics/resource_counts endpoint + response_metrics = fastapi_test_app.get( + "/metrics/resource_counts?project=demo_project" + ) + assert response_metrics.status_code == 200 + data_metrics = response_metrics.json() + assert "counts" in data_metrics + + counts = data_metrics["counts"] + assert "featureViews" in counts + + # Verify that the count from metrics matches the count from feature_views/all + feature_views_count_from_all = len(feature_views) + feature_views_count_from_metrics = counts["featureViews"] + + assert feature_views_count_from_all == feature_views_count_from_metrics, ( + f"Feature views count mismatch: /feature_views/all returned {feature_views_count_from_all} " + f"but /metrics/resource_counts returned {feature_views_count_from_metrics}" + ) + + # Test without project parameter (all projects) + response_all_projects = fastapi_test_app.get("/feature_views/all") + assert response_all_projects.status_code == 200 + data_all_projects = response_all_projects.json() + + response_metrics_all = fastapi_test_app.get("/metrics/resource_counts") + assert response_metrics_all.status_code == 200 + data_metrics_all = response_metrics_all.json() + + total_fv_count_from_all = len(data_all_projects["featureViews"]) + total_fv_count_from_metrics = data_metrics_all["total"]["featureViews"] + + assert total_fv_count_from_all == total_fv_count_from_metrics, ( + f"Total feature views count mismatch across all projects: " + f"/feature_views/all returned {total_fv_count_from_all} " + f"but /metrics/resource_counts returned {total_fv_count_from_metrics}" + ) + + def test_metrics_recently_visited_via_rest(fastapi_test_app): """Test the /metrics/recently_visited endpoint.""" # First, make some requests to generate visit data diff --git a/sdk/python/tests/unit/infra/compute_engines/local/test_nodes.py b/sdk/python/tests/unit/infra/compute_engines/local/test_nodes.py index 20e23c35e03..905ea65ae42 100644 --- a/sdk/python/tests/unit/infra/compute_engines/local/test_nodes.py +++ b/sdk/python/tests/unit/infra/compute_engines/local/test_nodes.py @@ -4,9 +4,9 @@ import pandas as pd import pyarrow as pa +from feast.infra.compute_engines.backends.pandas_backend import PandasBackend from feast.infra.compute_engines.dag.context import ColumnInfo, ExecutionContext from feast.infra.compute_engines.local.arrow_table_value import ArrowTableValue -from feast.infra.compute_engines.local.backends.pandas_backend import PandasBackend from feast.infra.compute_engines.local.nodes import ( LocalAggregationNode, LocalDedupNode, diff --git a/sdk/python/tests/unit/infra/compute_engines/ray_compute/test_nodes.py b/sdk/python/tests/unit/infra/compute_engines/ray_compute/test_nodes.py index e8c40d43099..0da4fcb956e 100644 --- a/sdk/python/tests/unit/infra/compute_engines/ray_compute/test_nodes.py +++ b/sdk/python/tests/unit/infra/compute_engines/ray_compute/test_nodes.py @@ -1,4 +1,6 @@ +import os from datetime import datetime, timedelta +from unittest.mock import patch import pandas as pd import pytest @@ -18,6 +20,12 @@ RayReadNode, RayTransformationNode, ) +from feast.infra.ray_initializer import ( + RayConfigManager, + RayExecutionMode, + ensure_ray_initialized, + get_ray_wrapper, +) class DummyInputNode(DAGNode): @@ -317,3 +325,101 @@ def test_ray_config_validation(): # Test invalid window size defaults to 1 hour config_invalid = RayComputeEngineConfig(window_size_for_joins="invalid") assert config_invalid.window_size_timedelta == timedelta(hours=1) + + +def test_ray_initialization_and_kuberay_modes(): + """ + Comprehensive test for Ray initialization modes and KubeRay configuration. + + Tests: Mode detection (LOCAL/REMOTE/KUBERAY), config parsing, defaults, + environment variables, mode precedence, and Ray wrapper instantiation. + """ + # Test LOCAL mode (default) + config_local = RayComputeEngineConfig() + assert ( + RayConfigManager(config_local).determine_execution_mode() + == RayExecutionMode.LOCAL + ) + + # Test REMOTE mode + config_remote = RayComputeEngineConfig(ray_address="ray://localhost:10001") + manager_remote = RayConfigManager(config_remote) + assert manager_remote.determine_execution_mode() == RayExecutionMode.REMOTE + # Test execution mode caching + assert manager_remote.determine_execution_mode() == RayExecutionMode.REMOTE + + # Test KUBERAY mode with full config + config_kuberay = RayComputeEngineConfig( + use_kuberay=True, + kuberay_conf={ + "cluster_name": "feast-cluster", + "namespace": "feast-system", + "auth_token": "test-token", + "auth_server": "https://api.example.com", + "skip_tls": True, + }, + ) + manager_kuberay = RayConfigManager(config_kuberay) + assert manager_kuberay.determine_execution_mode() == RayExecutionMode.KUBERAY + kuberay_config = manager_kuberay.get_kuberay_config() + assert kuberay_config["cluster_name"] == "feast-cluster" + assert kuberay_config["namespace"] == "feast-system" + assert kuberay_config["auth_token"] == "test-token" + assert kuberay_config["skip_tls"] is True + + # Test KubeRay defaults + config_defaults = RayComputeEngineConfig( + use_kuberay=True, kuberay_conf={"cluster_name": "test-cluster"} + ) + defaults_config = RayConfigManager(config_defaults).get_kuberay_config() + assert defaults_config["namespace"] == "default" + assert defaults_config["skip_tls"] is False + + # Test mode precedence - KUBERAY overrides REMOTE + config_precedence = RayComputeEngineConfig( + ray_address="ray://localhost:10001", + use_kuberay=True, + kuberay_conf={"cluster_name": "test-cluster"}, + ) + assert ( + RayConfigManager(config_precedence).determine_execution_mode() + == RayExecutionMode.KUBERAY + ) + + # Test environment variable support + with patch.dict( + os.environ, + { + "FEAST_RAY_CLUSTER_NAME": "env-cluster", + "FEAST_RAY_NAMESPACE": "env-namespace", + "FEAST_RAY_AUTH_TOKEN": "env-token", + }, + ): + env_config = RayConfigManager( + RayComputeEngineConfig(use_kuberay=True, kuberay_conf={}) + ).get_kuberay_config() + assert env_config["cluster_name"] == "env-cluster" + assert env_config["namespace"] == "env-namespace" + assert env_config["auth_token"] == "env-token" + + # Test Ray wrapper instantiation + from feast.infra.ray_initializer import StandardRayWrapper + + wrapper = get_ray_wrapper() + assert isinstance(wrapper, StandardRayWrapper) + + config_custom = RayComputeEngineConfig( + enable_ray_logging=True, + max_workers=4, + broadcast_join_threshold_mb=200, + ray_conf={"num_cpus": 4}, + ) + assert config_custom.enable_ray_logging is True + assert config_custom.max_workers == 4 + assert config_custom.broadcast_join_threshold_mb == 200 + assert config_custom.ray_conf["num_cpus"] == 4 + + with patch("feast.infra.ray_initializer.ray") as mock_ray: + mock_ray.is_initialized.return_value = True + ensure_ray_initialized(config_local) + mock_ray.init.assert_not_called() diff --git a/sdk/python/tests/unit/infra/test_inference_unit_tests.py b/sdk/python/tests/unit/infra/test_inference_unit_tests.py index f1aef20d113..d0df8153c73 100644 --- a/sdk/python/tests/unit/infra/test_inference_unit_tests.py +++ b/sdk/python/tests/unit/infra/test_inference_unit_tests.py @@ -96,14 +96,6 @@ def python_native_test_view(input_dict: dict[str, Any]) -> dict[str, Any]: python_native_test_view.infer_features() - -def test_on_demand_features_invalid_type_inference(): - # Create Feature Views - date_request = RequestSource( - name="date_request", - schema=[Field(name="some_date", dtype=UnixTimestamp)], - ) - @on_demand_feature_view( sources=[date_request], schema=[ @@ -111,14 +103,20 @@ def test_on_demand_features_invalid_type_inference(): Field(name="object_output", dtype=String), ], ) - def invalid_test_view(features_df: pd.DataFrame) -> pd.DataFrame: + def object_string_test_view(features_df: pd.DataFrame) -> pd.DataFrame: data = pd.DataFrame() data["output"] = features_df["some_date"] data["object_output"] = features_df["some_date"].astype(str) return data - with pytest.raises(ValueError, match="Value with native type object"): - invalid_test_view.infer_features() + object_string_test_view.infer_features() + + +def test_on_demand_features_invalid_type_inference(): + date_request = RequestSource( + name="date_request", + schema=[Field(name="some_date", dtype=UnixTimestamp)], + ) @on_demand_feature_view( schema=[ @@ -184,14 +182,13 @@ def test_view(features_df: pd.DataFrame) -> pd.DataFrame: Field(name="object_output", dtype=String), ], ) - def invalid_test_view(features_df: pd.DataFrame) -> pd.DataFrame: + def object_string_view(features_df: pd.DataFrame) -> pd.DataFrame: data = pd.DataFrame() data["output"] = features_df["some_date"] data["object_output"] = features_df["some_date"].astype(str) return data - with pytest.raises(ValueError, match="Value with native type object"): - invalid_test_view.infer_features() + object_string_view.infer_features() @on_demand_feature_view( sources=[date_request], diff --git a/sdk/python/tests/unit/local_feast_tests/test_local_feature_store.py b/sdk/python/tests/unit/local_feast_tests/test_local_feature_store.py index 397ca72e90d..cbb9d3d334a 100644 --- a/sdk/python/tests/unit/local_feast_tests/test_local_feature_store.py +++ b/sdk/python/tests/unit/local_feast_tests/test_local_feature_store.py @@ -1,5 +1,6 @@ from datetime import datetime, timedelta from tempfile import mkstemp +from unittest.mock import patch import pytest from pytest_lazyfixture import lazy_fixture @@ -18,7 +19,7 @@ from feast.permissions.action import AuthzedAction from feast.permissions.permission import Permission from feast.permissions.policy import RoleBasedPolicy -from feast.repo_config import RepoConfig +from feast.repo_config import RegistryConfig, RepoConfig from feast.stream_feature_view import stream_feature_view from feast.types import Array, Bytes, Float32, Int64, String, ValueType, from_value_type from tests.integration.feature_repos.universal.feature_views import TAGS @@ -797,3 +798,67 @@ def feature_store_with_local_registry(): entity_key_serialization_version=3, ) ) + + +@pytest.mark.parametrize( + "test_feature_store", + [lazy_fixture("feature_store_with_local_registry")], +) +def test_apply_refreshes_registry_cache_sync_mode(test_feature_store): + """Test that apply() refreshes registry cache when cache_mode is 'sync' (default)""" + # Create a simple entity (no FeatureView to avoid file path issues) + entity = Entity(name="test_entity", join_keys=["id"]) + + # Mock the refresh_registry method to verify it's called + with patch.object(test_feature_store, "refresh_registry") as mock_refresh: + # Apply the entity + test_feature_store.apply([entity]) + + # Verify refresh_registry was called once (due to sync mode) + mock_refresh.assert_called_once() + + test_feature_store.teardown() + + +@pytest.mark.parametrize( + "test_feature_store", + [lazy_fixture("feature_store_with_local_registry")], +) +def test_apply_skips_refresh_registry_cache_thread_mode(test_feature_store): + """Test that apply() skips registry refresh when cache_mode is 'thread'""" + # Create a simple entity + entity = Entity(name="test_entity", join_keys=["id"]) + + # Temporarily change cache_mode to 'thread' + original_cache_mode = test_feature_store.config.registry.cache_mode + test_feature_store.config.registry.cache_mode = "thread" + + try: + # Mock the refresh_registry method to verify it's NOT called + with patch.object(test_feature_store, "refresh_registry") as mock_refresh: + # Apply the entity + test_feature_store.apply([entity]) + + # Verify refresh_registry was NOT called (due to thread mode) + mock_refresh.assert_not_called() + finally: + # Restore original cache_mode + test_feature_store.config.registry.cache_mode = original_cache_mode + + test_feature_store.teardown() + + +def test_registry_config_cache_mode_default(): + """Test that RegistryConfig has cache_mode with default value 'sync'""" + config = RegistryConfig() + assert hasattr(config, "cache_mode") + assert config.cache_mode == "sync" + + +def test_registry_config_cache_mode_can_be_set(): + """Test that RegistryConfig cache_mode can be set to different values""" + config = RegistryConfig(cache_mode="thread") + assert config.cache_mode == "thread" + + config = RegistryConfig(cache_mode="sync") + assert config.cache_mode == "sync" diff --git a/sdk/python/tests/unit/test_feature_server_async.py b/sdk/python/tests/unit/test_feature_server_async.py new file mode 100644 index 00000000000..641a3c53278 --- /dev/null +++ b/sdk/python/tests/unit/test_feature_server_async.py @@ -0,0 +1,28 @@ +from unittest.mock import AsyncMock, MagicMock + +from fastapi.testclient import TestClient + +from feast.feature_server import get_app +from feast.online_response import OnlineResponse +from feast.protos.feast.serving.ServingService_pb2 import GetOnlineFeaturesResponse + + +def test_async_get_online_features(): + """Test that async get_online_features endpoint works correctly""" + fs = MagicMock() + fs._get_provider.return_value.async_supported.online.read = True + fs.get_online_features_async = AsyncMock( + return_value=OnlineResponse(GetOnlineFeaturesResponse()) + ) + fs.get_feature_service = MagicMock() + fs.initialize = AsyncMock() + fs.close = AsyncMock() + + client = TestClient(get_app(fs)) + response = client.post( + "/get-online-features", + json={"features": ["test:feature"], "entities": {"entity_id": [123]}}, + ) + + assert response.status_code == 200 + assert fs.get_online_features_async.await_count == 1 diff --git a/sdk/python/tests/unit/test_feature_views.py b/sdk/python/tests/unit/test_feature_views.py index db058ed68cf..9030e6e0c69 100644 --- a/sdk/python/tests/unit/test_feature_views.py +++ b/sdk/python/tests/unit/test_feature_views.py @@ -390,3 +390,143 @@ def transform_udf(df: pd.DataFrame) -> pd.DataFrame: assert isinstance(second_deserialized, BatchFeatureView) assert second_deserialized.name == original_bfv.name assert second_deserialized.feature_transformation is not None + + +def test_transformation_mode_serialization(): + """ + Test that transformation mode is properly serialized to proto and deserialized back. + This verifies the fix for the mode field in Transformation.proto. + """ + from feast.transformation.mode import TransformationMode + from feast.transformation.pandas_transformation import PandasTransformation + from feast.transformation.python_transformation import PythonTransformation + + def simple_udf(df: pd.DataFrame) -> pd.DataFrame: + df["output"] = df["input"] * 2 + return df + + file_source = FileSource( + name="test-source", + path="test_data.parquet", + timestamp_field="event_timestamp", + ) + + entity = Entity(name="test_entity", join_keys=["entity_id"]) + + # Test different transformation modes + test_cases = [ + ("python", TransformationMode.PYTHON, PythonTransformation), + ("pandas", TransformationMode.PANDAS, PandasTransformation), + ] + + for mode_str, mode_enum, transformation_class in test_cases: + # Create BatchFeatureView with the transformation + bfv = BatchFeatureView( + name=f"test_bfv_{mode_str}", + entities=[entity], + schema=[ + Field(name="entity_id", dtype=String), + Field(name="input", dtype=Int64), + Field(name="output", dtype=Int64), + ], + source=file_source, + ttl=timedelta(days=1), + mode=mode_str, + udf=simple_udf, + ) + + # Serialize to proto + proto = bfv.to_proto() + + # Verify mode is in the proto + assert proto.spec.HasField("feature_transformation") + assert proto.spec.feature_transformation.HasField("user_defined_function") + udf_proto = proto.spec.feature_transformation.user_defined_function + assert udf_proto.mode == mode_str, ( + f"Expected mode '{mode_str}' in proto, got '{udf_proto.mode}'" + ) + + # Deserialize from proto + deserialized = FeatureView.from_proto(proto) + + # Verify mode is preserved + assert isinstance(deserialized, BatchFeatureView) + # Mode can be either string or enum, so compare values + deserialized_mode_str = ( + deserialized.mode.value + if isinstance(deserialized.mode, TransformationMode) + else deserialized.mode + ) + assert deserialized_mode_str == mode_str, ( + f"Expected mode '{mode_str}' after deserialization, got '{deserialized_mode_str}'" + ) + assert deserialized.feature_transformation is not None + assert deserialized.feature_transformation.mode == mode_enum, ( + f"Expected transformation mode {mode_enum} after deserialization, " + f"got {deserialized.feature_transformation.mode}" + ) + + +def test_mode_serialization_without_transformation(): + """ + Test that mode is properly serialized in FeatureViewSpec proto. + This tests the scenario where mode is set on the FeatureView level, + ensuring it's stored in the proto independently of the transformation. + """ + from feast.transformation.mode import TransformationMode + + def simple_udf(df: pd.DataFrame) -> pd.DataFrame: + df["output"] = df["feature1"] * 2 + return df + + file_source = FileSource( + name="test-source", + path="test_data.parquet", + timestamp_field="event_timestamp", + ) + + entity = Entity(name="test_entity", join_keys=["entity_id"]) + + test_modes = ["python", "pandas"] + + for mode_str in test_modes: + bfv = BatchFeatureView( + name=f"test_bfv_mode_in_spec_{mode_str}", + entities=[entity], + schema=[ + Field(name="entity_id", dtype=String), + Field(name="feature1", dtype=Int64), + Field(name="output", dtype=Int64), + ], + source=file_source, + ttl=timedelta(days=1), + mode=mode_str, + udf=simple_udf, + ) + + assert bfv.mode == mode_str + proto = bfv.to_proto() + assert proto.spec.mode == mode_str, ( + f"Expected mode '{mode_str}' in FeatureViewSpec proto, got '{proto.spec.mode}'" + ) + deserialized = FeatureView.from_proto(proto) + assert isinstance(deserialized, FeatureView) + + # With UDF, should deserialize as BatchFeatureView + assert isinstance(deserialized, BatchFeatureView), ( + f"Expected BatchFeatureView, got {type(deserialized).__name__}" + ) + + # Verify mode is preserved from FeatureViewSpec proto + deserialized_mode_str = ( + deserialized.mode.value + if isinstance(deserialized.mode, TransformationMode) + else deserialized.mode + ) + assert deserialized_mode_str == mode_str, ( + f"Expected mode '{mode_str}' after deserialization, got '{deserialized_mode_str}'" + ) + + assert deserialized.feature_transformation is not None, ( + "Expected transformation to be present" + ) diff --git a/sdk/python/tests/unit/test_on_demand_feature_view_aggregation.py b/sdk/python/tests/unit/test_on_demand_feature_view_aggregation.py new file mode 100644 index 00000000000..3d6199be3a0 --- /dev/null +++ b/sdk/python/tests/unit/test_on_demand_feature_view_aggregation.py @@ -0,0 +1,89 @@ +# Copyright 2025 The Feast Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tests for OnDemandFeatureView aggregations in online serving.""" + +import pyarrow as pa + +from feast.aggregation import Aggregation +from feast.utils import _apply_aggregations_to_response + + +def test_aggregation_python_mode(): + """Test aggregations in Python mode (dict format).""" + data = { + "driver_id": [1, 1, 2, 2], + "trips": [10, 20, 15, 25], + } + aggs = [Aggregation(column="trips", function="sum")] + + result = _apply_aggregations_to_response(data, aggs, ["driver_id"], "python") + + assert result == {"driver_id": [1, 2], "sum_trips": [30, 40]} + + +def test_aggregation_pandas_mode(): + """Test aggregations in Pandas mode (Arrow table format).""" + table = pa.table( + { + "driver_id": [1, 1, 2, 2], + "trips": [10, 20, 15, 25], + } + ) + aggs = [Aggregation(column="trips", function="sum")] + + result = _apply_aggregations_to_response(table, aggs, ["driver_id"], "pandas") + + assert isinstance(result, pa.Table) + result_df = result.to_pandas() + assert list(result_df["driver_id"]) == [1, 2] + assert list(result_df["sum_trips"]) == [30, 40] + + +def test_multiple_aggregations(): + """Test multiple aggregation functions.""" + data = { + "driver_id": [1, 1, 2, 2], + "trips": [10, 20, 15, 25], + "revenue": [100.0, 200.0, 150.0, 250.0], + } + aggs = [ + Aggregation(column="trips", function="sum"), + Aggregation(column="revenue", function="mean"), + ] + + result = _apply_aggregations_to_response(data, aggs, ["driver_id"], "python") + + assert result["driver_id"] == [1, 2] + assert result["sum_trips"] == [30, 40] + assert result["mean_revenue"] == [150.0, 200.0] + + +def test_no_aggregations_returns_original(): + """Test that no aggregations returns original data.""" + data = {"driver_id": [1, 2], "trips": [10, 20]} + + result = _apply_aggregations_to_response(data, [], ["driver_id"], "python") + + assert result == data + + +def test_empty_data_returns_empty(): + """Test that empty data returns empty result.""" + data = {"driver_id": [], "trips": []} + aggs = [Aggregation(column="trips", function="sum")] + + result = _apply_aggregations_to_response(data, aggs, ["driver_id"], "python") + + assert result == data diff --git a/sdk/python/tests/unit/test_ui_server.py b/sdk/python/tests/unit/test_ui_server.py new file mode 100644 index 00000000000..c5a85a85382 --- /dev/null +++ b/sdk/python/tests/unit/test_ui_server.py @@ -0,0 +1,297 @@ +import contextlib +import json +import os +import tempfile +from pathlib import Path +from unittest.mock import MagicMock, patch + +import assertpy +import pytest +from fastapi.testclient import TestClient + +from feast.ui_server import get_app + +# Test constants +EXPECTED_SUCCESS_STATUS = 200 +EXPECTED_ERROR_STATUS = 503 +TEST_PROJECT_NAME = "test_project" +REGISTRY_TTL_SECS = 60 + + +def _create_mock_ui_files(temp_dir): + """Helper function to create required UI files structure""" + ui_dir = os.path.join(temp_dir, "ui", "build") + os.makedirs(ui_dir, exist_ok=True) + + # Create projects-list.json file + projects_file = os.path.join(ui_dir, "projects-list.json") + with open(projects_file, "w") as f: + json.dump({"projects": []}, f) + + # Create index.html file + index_file = os.path.join(ui_dir, "index.html") + with open(index_file, "w") as f: + f.write("Test UI") + + +@contextlib.contextmanager +def _setup_importlib_mocks(temp_dir): + """Helper function to setup importlib resource mocks. + + This function mocks the importlib_resources functionality used by the UI server + to serve static files. It creates a proper context manager that returns the + temporary directory path when used with importlib_resources.as_file(). + """ + mock_path = Path(temp_dir) + + # Create a proper context manager mock + mock_context_manager = MagicMock() + mock_context_manager.__enter__.return_value = mock_path + mock_context_manager.__exit__.return_value = None + + # Mock the files() method to return a mock that supports division + mock_file_ref = MagicMock() + mock_file_ref.__truediv__.return_value = MagicMock() + + with ( + patch("feast.ui_server.importlib_resources.files") as mock_files, + patch("feast.ui_server.importlib_resources.as_file") as mock_as_file, + ): + mock_files.return_value = mock_file_ref + mock_as_file.return_value = mock_context_manager + + yield mock_files, mock_as_file + + +@pytest.fixture +def mock_feature_store(): + """Fixture for creating a mock feature store""" + mock_store = MagicMock() + mock_store.refresh_registry = MagicMock() + return mock_store + + +@pytest.fixture +def ui_app_with_registry(mock_feature_store): + """Fixture for UI app with valid registry data. + + Creates a UI app instance with a properly configured feature store + that has valid registry data available for testing endpoints that + require registry access. + """ + mock_registry = MagicMock() + mock_proto = MagicMock() + mock_proto.SerializeToString.return_value = b"mock_proto_data" + mock_registry.proto.return_value = mock_proto + mock_feature_store.registry = mock_registry + + with tempfile.TemporaryDirectory() as temp_dir: + _create_mock_ui_files(temp_dir) + + with _setup_importlib_mocks(temp_dir): + app = get_app(mock_feature_store, TEST_PROJECT_NAME, REGISTRY_TTL_SECS) + yield app + + +@pytest.fixture +def ui_app_without_registry(mock_feature_store): + """Fixture for UI app with None registry data. + + Creates a UI app instance with a feature store that has no registry + data available, used for testing error conditions and service + unavailable responses. + """ + mock_registry = MagicMock() + mock_registry.proto.return_value = None + mock_feature_store.registry = mock_registry + + with tempfile.TemporaryDirectory() as temp_dir: + _create_mock_ui_files(temp_dir) + + with _setup_importlib_mocks(temp_dir): + app = get_app(mock_feature_store, TEST_PROJECT_NAME, REGISTRY_TTL_SECS) + yield app + + +def test_ui_server_health_endpoint(ui_app_with_registry): + """Test the UI server health endpoint returns 200 when registry is available. + + This test verifies that the /health endpoint correctly returns HTTP 200 + when the feature store registry is properly initialized and contains data. + """ + client = TestClient(ui_app_with_registry) + response = client.get("/health") + assertpy.assert_that(response.status_code).is_equal_to(EXPECTED_SUCCESS_STATUS) + + +def test_ui_server_health_endpoint_with_none_registry(ui_app_without_registry): + """Test the UI server health endpoint returns 503 when registry is None. + + This test verifies that the /health endpoint correctly returns HTTP 503 + (Service Unavailable) when the feature store registry is not available + or contains no data. + """ + client = TestClient(ui_app_without_registry) + response = client.get("/health") + assertpy.assert_that(response.status_code).is_equal_to(EXPECTED_ERROR_STATUS) + + +def test_registry_endpoint_with_valid_data(ui_app_with_registry): + """Test the registry endpoint returns valid data with correct content type. + + This test verifies that the /registry endpoint correctly returns HTTP 200 + with the proper content-type header when registry data is available. + """ + client = TestClient(ui_app_with_registry) + response = client.get("/registry") + assertpy.assert_that(response.status_code).is_equal_to(EXPECTED_SUCCESS_STATUS) + assertpy.assert_that(response.headers["content-type"]).is_equal_to( + "application/octet-stream" + ) + + +def test_registry_endpoint_with_none_data(ui_app_without_registry): + """Test the registry endpoint returns 503 when registry data is None. + + This test verifies that the /registry endpoint correctly returns HTTP 503 + (Service Unavailable) when no registry data is available. + """ + client = TestClient(ui_app_without_registry) + response = client.get("/registry") + assertpy.assert_that(response.status_code).is_equal_to(EXPECTED_ERROR_STATUS) + + +def test_save_document_endpoint_success(ui_app_with_registry): + """Test the save document endpoint successfully saves data to a labels file. + + This test verifies that the /save-document endpoint correctly processes + a valid request, creates a labels file, and returns success confirmation. + """ + client = TestClient(ui_app_with_registry) + + # Create a temporary file in the current working directory for testing + with tempfile.NamedTemporaryFile( + mode="w", suffix=".py", delete=False, dir=os.getcwd() + ) as f: + test_file_path = f.name + f.write("# Test file content") + + try: + request_data = { + "file_path": test_file_path, + "data": {"test": "data", "key": "value"}, + } + + response = client.post("/save-document", json=request_data) + assertpy.assert_that(response.status_code).is_equal_to(EXPECTED_SUCCESS_STATUS) + + response_data = response.json() + assertpy.assert_that(response_data["success"]).is_true() + assertpy.assert_that(response_data).contains_key("saved_to") + + # Verify the file was created + labels_file = response_data["saved_to"] + assertpy.assert_that(os.path.exists(labels_file)).is_true() + + with open(labels_file, "r") as f: + saved_data = json.load(f) + assertpy.assert_that(saved_data).is_equal_to(request_data["data"]) + + finally: + # Cleanup + if os.path.exists(test_file_path): + os.unlink(test_file_path) + labels_file = test_file_path.replace(".py", "-labels.json") + if os.path.exists(labels_file): + os.unlink(labels_file) + + +def test_save_document_endpoint_invalid_path(ui_app_with_registry): + """Test the save document endpoint returns error for invalid file path. + + This test verifies that the /save-document endpoint correctly rejects + file paths that are outside the current working directory for security. + """ + client = TestClient(ui_app_with_registry) + + request_data = { + "file_path": "/invalid/absolute/path/outside/workspace.py", + "data": {"test": "data"}, + } + + response = client.post("/save-document", json=request_data) + assertpy.assert_that(response.status_code).is_equal_to(EXPECTED_SUCCESS_STATUS) + + response_data = response.json() + assertpy.assert_that(response_data).contains_key("error") + assertpy.assert_that(response_data["error"]).contains("Invalid file path") + + +def test_save_document_endpoint_exception_handling(ui_app_with_registry): + """Test the save document endpoint handles exceptions gracefully. + + This test verifies that the /save-document endpoint properly catches + and returns error responses when exceptions occur during processing. + """ + client = TestClient(ui_app_with_registry) + + # Test with a file path outside the current working directory (will cause an exception) + request_data = { + "file_path": "/invalid/absolute/path/outside/workspace.py", + "data": {"test": "data"}, + } + + response = client.post("/save-document", json=request_data) + assertpy.assert_that(response.status_code).is_equal_to(EXPECTED_SUCCESS_STATUS) + + response_data = response.json() + assertpy.assert_that(response_data).contains_key("error") + assertpy.assert_that(response_data["error"]).contains("Invalid file path") + + +@pytest.mark.parametrize( + "registry_available,expected_status", + [(True, EXPECTED_SUCCESS_STATUS), (False, EXPECTED_ERROR_STATUS)], +) +def test_health_endpoint_status( + registry_available, expected_status, mock_feature_store +): + """Test the health endpoint returns correct status based on registry availability. + + This parametrized test verifies that the /health endpoint returns the + appropriate HTTP status code based on whether registry data is available. + """ + if registry_available: + mock_registry = MagicMock() + mock_proto = MagicMock() + mock_proto.SerializeToString.return_value = b"mock_proto_data" + mock_registry.proto.return_value = mock_proto + mock_feature_store.registry = mock_registry + else: + mock_registry = MagicMock() + mock_registry.proto.return_value = None + mock_feature_store.registry = mock_registry + + with tempfile.TemporaryDirectory() as temp_dir: + _create_mock_ui_files(temp_dir) + + with _setup_importlib_mocks(temp_dir): + app = get_app(mock_feature_store, TEST_PROJECT_NAME, REGISTRY_TTL_SECS) + client = TestClient(app) + response = client.get("/health") + assertpy.assert_that(response.status_code).is_equal_to(expected_status) + + +def test_catch_all_route(ui_app_with_registry): + """Test the catch-all route for React router paths. + + This test reveals a bug in the original UI server code where ui_dir + is not in scope for the catch_all function. The ui_dir variable is defined + inside the importlib_resources context manager but used outside of it. + This causes a NameError when the route is accessed. + """ + client = TestClient(ui_app_with_registry) + + # The route will fail due to the scope issue with ui_dir + with pytest.raises(Exception): # Expecting NameError or FileNotFoundError + client.get("/p/some/react/path") diff --git a/ui/package.json b/ui/package.json index 17bff99e1aa..f087d6cad14 100644 --- a/ui/package.json +++ b/ui/package.json @@ -1,6 +1,6 @@ { "name": "@feast-dev/feast-ui", - "version": "0.55.0", + "version": "0.56.0", "private": false, "files": [ "dist" diff --git a/ui/src/components/CommandPalette.tsx b/ui/src/components/CommandPalette.tsx index 9750d73e2d9..8e18898400a 100644 --- a/ui/src/components/CommandPalette.tsx +++ b/ui/src/components/CommandPalette.tsx @@ -149,6 +149,7 @@ const CommandPalette: React.FC = ({ ? String(item.spec.description || "") : "", type: getItemType(item, name), + projectId: "projectId" in item ? String(item.projectId) : undefined, }; }); @@ -158,15 +159,7 @@ const CommandPalette: React.FC = ({ }; }); - console.log( - "CommandPalette isOpen:", - isOpen, - "categories:", - categories.length, - ); // Debug log - if (!isOpen) { - console.log("CommandPalette not rendering due to isOpen=false"); return null; } @@ -227,16 +220,11 @@ const CommandPalette: React.FC = ({ href={item.link} onClick={(e) => { e.preventDefault(); - console.log( - "Search result clicked:", - item.name, - ); onClose(); setSearchText(""); - console.log("Navigating to:", item.link); navigate(item.link); }} style={{ @@ -253,6 +241,17 @@ const CommandPalette: React.FC = ({ {item.description} )} + {item.projectId && ( +
+ Project: {item.projectId} +
+ )} {item.type && ( diff --git a/ui/src/components/GlobalSearchShortcut.tsx b/ui/src/components/GlobalSearchShortcut.tsx index 28e55454f30..aa96abe5b97 100644 --- a/ui/src/components/GlobalSearchShortcut.tsx +++ b/ui/src/components/GlobalSearchShortcut.tsx @@ -9,23 +9,13 @@ const GlobalSearchShortcut: React.FC = ({ }) => { useEffect(() => { const handleKeyDown = (event: KeyboardEvent) => { - console.log( - "Key pressed:", - event.key, - "metaKey:", - event.metaKey, - "ctrlKey:", - event.ctrlKey, - ); if ((event.metaKey || event.ctrlKey) && event.key.toLowerCase() === "k") { - console.log("Cmd+K detected, preventing default and calling onOpen"); event.preventDefault(); event.stopPropagation(); onOpen(); } }; - console.log("Adding keydown event listener to window"); window.addEventListener("keydown", handleKeyDown, true); return () => { window.removeEventListener("keydown", handleKeyDown, true); diff --git a/ui/src/components/ProjectSelector.tsx b/ui/src/components/ProjectSelector.tsx index 1bb7ebf85a7..ac9057bfb00 100644 --- a/ui/src/components/ProjectSelector.tsx +++ b/ui/src/components/ProjectSelector.tsx @@ -1,11 +1,12 @@ import { EuiSelect, useGeneratedHtmlId } from "@elastic/eui"; import React from "react"; -import { useNavigate, useParams } from "react-router-dom"; +import { useNavigate, useParams, useLocation } from "react-router-dom"; import { useLoadProjectsList } from "../contexts/ProjectListContext"; const ProjectSelector = () => { const { projectName } = useParams(); const navigate = useNavigate(); + const location = useLocation(); const { isLoading, data } = useLoadProjectsList(); @@ -22,7 +23,20 @@ const ProjectSelector = () => { const basicSelectId = useGeneratedHtmlId({ prefix: "basicSelect" }); const onChange = (e: React.ChangeEvent) => { - navigate(`/p/${e.target.value}`); + const newProjectId = e.target.value; + + // If we're on a project page, maintain the current path context + if (projectName && location.pathname.startsWith(`/p/${projectName}`)) { + // Replace the old project name with the new one in the current path + const newPath = location.pathname.replace( + `/p/${projectName}`, + `/p/${newProjectId}`, + ); + navigate(newPath); + } else { + // Otherwise, just navigate to the project home + navigate(`/p/${newProjectId}`); + } }; return ( diff --git a/ui/src/components/RegistrySearch.tsx b/ui/src/components/RegistrySearch.tsx index d9d72a20b1a..46d72966713 100644 --- a/ui/src/components/RegistrySearch.tsx +++ b/ui/src/components/RegistrySearch.tsx @@ -112,6 +112,7 @@ const RegistrySearch = forwardRef( ? String(item.spec.description || "") : "", type: getItemType(item, name), + projectId: "projectId" in item ? String(item.projectId) : undefined, }; }); @@ -187,6 +188,17 @@ const RegistrySearch = forwardRef( {item.description} )} + {item.projectId && ( +
+ Project: {item.projectId} +
+ )}
{item.type && ( diff --git a/ui/src/components/RegistryVisualizationTab.tsx b/ui/src/components/RegistryVisualizationTab.tsx index accf02971c6..ebc77604322 100644 --- a/ui/src/components/RegistryVisualizationTab.tsx +++ b/ui/src/components/RegistryVisualizationTab.tsx @@ -1,4 +1,5 @@ import React, { useContext, useState } from "react"; +import { useParams } from "react-router-dom"; import { EuiEmptyPrompt, EuiLoadingSpinner, @@ -16,7 +17,11 @@ import { filterPermissionsByAction } from "../utils/permissionUtils"; const RegistryVisualizationTab = () => { const registryUrl = useContext(RegistryPathContext); - const { isLoading, isSuccess, isError, data } = useLoadRegistry(registryUrl); + const { projectName } = useParams(); + const { isLoading, isSuccess, isError, data } = useLoadRegistry( + registryUrl, + projectName, + ); const [selectedObjectType, setSelectedObjectType] = useState(""); const [selectedObjectName, setSelectedObjectName] = useState(""); const [selectedPermissionAction, setSelectedPermissionAction] = useState(""); diff --git a/ui/src/mocks/handlers.ts b/ui/src/mocks/handlers.ts index 23904787c16..1c32bb2cf87 100644 --- a/ui/src/mocks/handlers.ts +++ b/ui/src/mocks/handlers.ts @@ -8,12 +8,12 @@ const registry = readFileSync( const projectsListWithDefaultProject = http.get("/projects-list.json", () => HttpResponse.json({ - default: "credit_score_project", + default: "credit_scoring_aws", projects: [ { name: "Credit Score Project", description: "Project for credit scoring team and associated models.", - id: "credit_score_project", + id: "credit_scoring_aws", registryPath: "/registry.db", // Changed to match what the test expects }, ], diff --git a/ui/src/pages/Layout.tsx b/ui/src/pages/Layout.tsx index 4a00eb64a37..0e3341b8820 100644 --- a/ui/src/pages/Layout.tsx +++ b/ui/src/pages/Layout.tsx @@ -42,8 +42,19 @@ const Layout = () => { }); const registryPath = currentProject?.registryPath || ""; - const { data } = useLoadRegistry(registryPath); + // For global search, use the first available registry path (typically all projects share the same registry) + // If projects have different registries, we use the first one as the "global" registry + const globalRegistryPath = + projectsData?.projects?.[0]?.registryPath || registryPath; + + // Load filtered data for current project (for sidebar and page-level search) + const { data } = useLoadRegistry(registryPath, projectName); + + // Load unfiltered data for global search (across all projects) + const { data: globalData } = useLoadRegistry(globalRegistryPath); + + // Categories for page-level search (filtered to current project) const categories = data ? [ { @@ -84,31 +95,92 @@ const Layout = () => { ] : []; + // Helper function to extract project ID from an item + const getProjectId = (item: any): string => { + // Try different possible locations for the project field + return item?.spec?.project || item?.project || projectName || "unknown"; + }; + + // Categories for global search (includes all projects) + const globalCategories = globalData + ? [ + { + name: "Data Sources", + data: (globalData.objects.dataSources || []).map((item: any) => ({ + ...item, + projectId: getProjectId(item), + })), + getLink: (item: any) => { + const project = item?.projectId || getProjectId(item); + return `/p/${project}/data-source/${item.name}`; + }, + }, + { + name: "Entities", + data: (globalData.objects.entities || []).map((item: any) => ({ + ...item, + projectId: getProjectId(item), + })), + getLink: (item: any) => { + const project = item?.projectId || getProjectId(item); + return `/p/${project}/entity/${item.name}`; + }, + }, + { + name: "Features", + data: (globalData.allFeatures || []).map((item: any) => ({ + ...item, + projectId: getProjectId(item), + })), + getLink: (item: any) => { + const featureView = item?.featureView; + const project = item?.projectId || getProjectId(item); + return featureView + ? `/p/${project}/feature-view/${featureView}/feature/${item.name}` + : "#"; + }, + }, + { + name: "Feature Views", + data: (globalData.mergedFVList || []).map((item: any) => ({ + ...item, + projectId: getProjectId(item), + })), + getLink: (item: any) => { + const project = item?.projectId || getProjectId(item); + return `/p/${project}/feature-view/${item.name}`; + }, + }, + { + name: "Feature Services", + data: (globalData.objects.featureServices || []).map((item: any) => ({ + ...item, + projectId: getProjectId(item), + })), + getLink: (item: any) => { + const serviceName = item?.name || item?.spec?.name; + const project = item?.projectId || getProjectId(item); + return serviceName + ? `/p/${project}/feature-service/${serviceName}` + : "#"; + }, + }, + ] + : []; + const handleSearchOpen = () => { - console.log("Opening command palette - before state update"); // Debug log setIsCommandPaletteOpen(true); - console.log("Command palette state should be updated to true"); }; useEffect(() => { const handleKeyDown = (event: KeyboardEvent) => { - console.log( - "Layout key pressed:", - event.key, - "metaKey:", - event.metaKey, - "ctrlKey:", - event.ctrlKey, - ); if ((event.metaKey || event.ctrlKey) && event.key.toLowerCase() === "k") { - console.log("Layout detected Cmd+K, preventing default"); event.preventDefault(); event.stopPropagation(); handleSearchOpen(); } }; - console.log("Layout adding keydown event listener"); window.addEventListener("keydown", handleKeyDown, true); return () => { window.removeEventListener("keydown", handleKeyDown, true); @@ -121,7 +193,7 @@ const Layout = () => { setIsCommandPaletteOpen(false)} - categories={categories} + categories={globalCategories} /> { grow={false} style={{ width: "600px", maxWidth: "90%" }} > - + diff --git a/ui/src/pages/ProjectOverviewPage.tsx b/ui/src/pages/ProjectOverviewPage.tsx index aa8d7cd4745..839fbcc5d89 100644 --- a/ui/src/pages/ProjectOverviewPage.tsx +++ b/ui/src/pages/ProjectOverviewPage.tsx @@ -9,6 +9,9 @@ import { EuiSkeletonText, EuiEmptyPrompt, EuiFieldSearch, + EuiPanel, + EuiStat, + EuiCard, } from "@elastic/eui"; import { useDocumentTitle } from "../hooks/useDocumentTitle"; @@ -18,14 +21,191 @@ import useLoadRegistry from "../queries/useLoadRegistry"; import RegistryPathContext from "../contexts/RegistryPathContext"; import RegistryVisualizationTab from "../components/RegistryVisualizationTab"; import RegistrySearch from "../components/RegistrySearch"; -import { useParams } from "react-router-dom"; +import { useParams, useNavigate } from "react-router-dom"; +import { useLoadProjectsList } from "../contexts/ProjectListContext"; + +// Component for "All Projects" view +const AllProjectsDashboard = () => { + const registryUrl = useContext(RegistryPathContext); + const navigate = useNavigate(); + const { data: projectsData } = useLoadProjectsList(); + const { data: registryData } = useLoadRegistry(registryUrl); + + if (!registryData) { + return ; + } + + // Calculate total counts across all projects + const totalCounts = { + featureViews: registryData.objects.featureViews?.length || 0, + entities: registryData.objects.entities?.length || 0, + dataSources: registryData.objects.dataSources?.length || 0, + featureServices: registryData.objects.featureServices?.length || 0, + features: registryData.allFeatures?.length || 0, + }; + + // Get projects from registry and count their objects + const projects = projectsData?.projects.filter((p) => p.id !== "all") || []; + const projectStats = projects.map((project) => { + const projectFVs = + registryData.objects.featureViews?.filter( + (fv: any) => fv?.spec?.project === project.id, + ) || []; + const projectEntities = + registryData.objects.entities?.filter( + (e: any) => e?.spec?.project === project.id, + ) || []; + const projectFeatures = + registryData.allFeatures?.filter((f: any) => f?.project === project.id) || + []; + + return { + ...project, + counts: { + featureViews: projectFVs.length, + entities: projectEntities.length, + features: projectFeatures.length, + }, + }; + }); + + return ( + + + +

All Projects Overview

+
+ + + +

+ View aggregated statistics and explore data across all your Feast + projects. +

+
+ + + {/* Total Stats */} + + +

Total Across All Projects

+
+ + + + + + + + + + + + + + + + + + +
+ + + + {/* Individual Projects */} + +

Projects ({projects.length})

+
+ + + {projectStats.map((project) => ( + + navigate(`/p/${project.id}`)} + style={{ cursor: "pointer" }} + > + + + + + {project.counts.featureViews} +
+ + Feature Views + +
+
+ + + {project.counts.entities} +
+ + Entities + +
+
+ + + {project.counts.features} +
+ + Features + +
+
+
+
+
+ ))} +
+
+
+ ); +}; const ProjectOverviewPage = () => { useDocumentTitle("Feast Home"); const registryUrl = useContext(RegistryPathContext); - const { isLoading, isSuccess, isError, data } = useLoadRegistry(registryUrl); - const { projectName } = useParams<{ projectName: string }>(); + const { isLoading, isSuccess, isError, data } = useLoadRegistry( + registryUrl, + projectName, + ); + + // Show aggregated dashboard for "All Projects" view + if (projectName === "all") { + return ; + } const categories = [ { diff --git a/ui/src/pages/Sidebar.tsx b/ui/src/pages/Sidebar.tsx index d7a5a54cda0..55c8ec805c9 100644 --- a/ui/src/pages/Sidebar.tsx +++ b/ui/src/pages/Sidebar.tsx @@ -17,8 +17,8 @@ import { PermissionsIcon } from "../graphics/PermissionsIcon"; const SideNav = () => { const registryUrl = useContext(RegistryPathContext); - const { isSuccess, data } = useLoadRegistry(registryUrl); const { projectName } = useParams(); + const { isSuccess, data } = useLoadRegistry(registryUrl, projectName); const [isSideNavOpenOnMobile, setisSideNavOpenOnMobile] = useState(false); diff --git a/ui/src/pages/data-sources/DataSourceOverviewTab.tsx b/ui/src/pages/data-sources/DataSourceOverviewTab.tsx index e4931aa7c50..d702034a558 100644 --- a/ui/src/pages/data-sources/DataSourceOverviewTab.tsx +++ b/ui/src/pages/data-sources/DataSourceOverviewTab.tsx @@ -27,9 +27,9 @@ import RequestDataSourceSchemaTable from "./RequestDataSourceSchemaTable"; import useLoadDataSource from "./useLoadDataSource"; const DataSourceOverviewTab = () => { - let { dataSourceName } = useParams(); + let { dataSourceName, projectName } = useParams(); const registryUrl = useContext(RegistryPathContext); - const registryQuery = useLoadRegistry(registryUrl); + const registryQuery = useLoadRegistry(registryUrl, projectName); const dsName = dataSourceName === undefined ? "" : dataSourceName; const { isLoading, isSuccess, isError, data, consumingFeatureViews } = diff --git a/ui/src/pages/data-sources/DataSourcesListingTable.tsx b/ui/src/pages/data-sources/DataSourcesListingTable.tsx index fd1ff73deb7..c314a4dfb94 100644 --- a/ui/src/pages/data-sources/DataSourcesListingTable.tsx +++ b/ui/src/pages/data-sources/DataSourcesListingTable.tsx @@ -18,9 +18,11 @@ const DatasourcesListingTable = ({ name: "Name", field: "name", sortable: true, - render: (name: string) => { + render: (name: string, item: feast.core.IDataSource) => { + // For "All Projects" view, link to the specific project + const itemProject = item?.project || projectName; return ( - + {name} ); @@ -36,6 +38,18 @@ const DatasourcesListingTable = ({ }, ]; + // Add Project column when viewing all projects + if (projectName === "all") { + columns.splice(1, 0, { + name: "Project", + field: "project", + sortable: true, + render: (project: string) => { + return {project || "Unknown"}; + }, + }); + } + const getRowProps = (item: feast.core.IDataSource) => { return { "data-test-subj": `row-${item.name}`, diff --git a/ui/src/pages/data-sources/Index.tsx b/ui/src/pages/data-sources/Index.tsx index 59bdcecd1df..96aef712aec 100644 --- a/ui/src/pages/data-sources/Index.tsx +++ b/ui/src/pages/data-sources/Index.tsx @@ -1,4 +1,5 @@ import React, { useContext } from "react"; +import { useParams } from "react-router-dom"; import { EuiPageTemplate, @@ -22,7 +23,8 @@ import ExportButton from "../../components/ExportButton"; const useLoadDatasources = () => { const registryUrl = useContext(RegistryPathContext); - const registryQuery = useLoadRegistry(registryUrl); + const { projectName } = useParams(); + const registryQuery = useLoadRegistry(registryUrl, projectName); const data = registryQuery.data === undefined diff --git a/ui/src/pages/data-sources/useLoadDataSource.ts b/ui/src/pages/data-sources/useLoadDataSource.ts index aa9f4e731bf..43f697fca03 100644 --- a/ui/src/pages/data-sources/useLoadDataSource.ts +++ b/ui/src/pages/data-sources/useLoadDataSource.ts @@ -1,11 +1,13 @@ import { useContext } from "react"; +import { useParams } from "react-router-dom"; import RegistryPathContext from "../../contexts/RegistryPathContext"; import { FEAST_FCO_TYPES } from "../../parsers/types"; import useLoadRegistry from "../../queries/useLoadRegistry"; const useLoadDataSource = (dataSourceName: string) => { const registryUrl = useContext(RegistryPathContext); - const registryQuery = useLoadRegistry(registryUrl); + const { projectName } = useParams(); + const registryQuery = useLoadRegistry(registryUrl, projectName); const data = registryQuery.data === undefined diff --git a/ui/src/pages/entities/EntitiesListingTable.tsx b/ui/src/pages/entities/EntitiesListingTable.tsx index 06190409b04..51ffb7c8609 100644 --- a/ui/src/pages/entities/EntitiesListingTable.tsx +++ b/ui/src/pages/entities/EntitiesListingTable.tsx @@ -18,9 +18,11 @@ const EntitiesListingTable = ({ entities }: EntitiesListingTableProps) => { name: "Name", field: "spec.name", sortable: true, - render: (name: string) => { + render: (name: string, item: feast.core.IEntity) => { + // For "All Projects" view, link to the specific project + const itemProject = item?.spec?.project || projectName; return ( - + {name} ); @@ -46,6 +48,18 @@ const EntitiesListingTable = ({ entities }: EntitiesListingTableProps) => { }, ]; + // Add Project column when viewing all projects + if (projectName === "all") { + columns.splice(1, 0, { + name: "Project", + field: "spec.project", + sortable: true, + render: (project: string) => { + return {project || "Unknown"}; + }, + }); + } + const getRowProps = (item: feast.core.IEntity) => { return { "data-test-subj": `row-${item?.spec?.name}`, diff --git a/ui/src/pages/entities/EntityOverviewTab.tsx b/ui/src/pages/entities/EntityOverviewTab.tsx index 09d9aaa3446..8a20688d140 100644 --- a/ui/src/pages/entities/EntityOverviewTab.tsx +++ b/ui/src/pages/entities/EntityOverviewTab.tsx @@ -28,9 +28,9 @@ import useFeatureViewEdgesByEntity from "./useFeatureViewEdgesByEntity"; import useLoadEntity from "./useLoadEntity"; const EntityOverviewTab = () => { - let { entityName } = useParams(); + let { entityName, projectName } = useParams(); const registryUrl = useContext(RegistryPathContext); - const registryQuery = useLoadRegistry(registryUrl); + const registryQuery = useLoadRegistry(registryUrl, projectName); const eName = entityName === undefined ? "" : entityName; const { isLoading, isSuccess, isError, data } = useLoadEntity(eName); diff --git a/ui/src/pages/entities/Index.tsx b/ui/src/pages/entities/Index.tsx index bed1bfb762c..070c53d38fa 100644 --- a/ui/src/pages/entities/Index.tsx +++ b/ui/src/pages/entities/Index.tsx @@ -1,4 +1,5 @@ import React, { useContext } from "react"; +import { useParams } from "react-router-dom"; import { EuiPageTemplate, EuiLoadingSpinner } from "@elastic/eui"; @@ -13,7 +14,8 @@ import ExportButton from "../../components/ExportButton"; const useLoadEntities = () => { const registryUrl = useContext(RegistryPathContext); - const registryQuery = useLoadRegistry(registryUrl); + const { projectName } = useParams(); + const registryQuery = useLoadRegistry(registryUrl, projectName); const data = registryQuery.data === undefined diff --git a/ui/src/pages/entities/useLoadEntity.ts b/ui/src/pages/entities/useLoadEntity.ts index e3e2ede8c2c..fdb4a7968f1 100644 --- a/ui/src/pages/entities/useLoadEntity.ts +++ b/ui/src/pages/entities/useLoadEntity.ts @@ -1,10 +1,12 @@ import { useContext } from "react"; +import { useParams } from "react-router-dom"; import RegistryPathContext from "../../contexts/RegistryPathContext"; import useLoadRegistry from "../../queries/useLoadRegistry"; const useLoadEntity = (entityName: string) => { const registryUrl = useContext(RegistryPathContext); - const registryQuery = useLoadRegistry(registryUrl); + const { projectName } = useParams(); + const registryQuery = useLoadRegistry(registryUrl, projectName); const data = registryQuery.data === undefined diff --git a/ui/src/pages/feature-services/FeatureServiceListingTable.tsx b/ui/src/pages/feature-services/FeatureServiceListingTable.tsx index 69d4d1f969d..acc68b6e619 100644 --- a/ui/src/pages/feature-services/FeatureServiceListingTable.tsx +++ b/ui/src/pages/feature-services/FeatureServiceListingTable.tsx @@ -28,9 +28,11 @@ const FeatureServiceListingTable = ({ { name: "Name", field: "spec.name", - render: (name: string) => { + render: (name: string, item: feast.core.IFeatureService) => { + // For "All Projects" view, link to the specific project + const itemProject = item?.spec?.project || projectName; return ( - + {name} ); @@ -56,6 +58,18 @@ const FeatureServiceListingTable = ({ }, ]; + // Add Project column when viewing all projects + if (projectName === "all") { + columns.splice(1, 0, { + name: "Project", + field: "spec.project", + sortable: true, + render: (project: string) => { + return project || "Unknown"; + }, + }); + } + tagKeysSet.forEach((key) => { columns.push({ name: key, diff --git a/ui/src/pages/feature-services/Index.tsx b/ui/src/pages/feature-services/Index.tsx index 0da8986e610..260a9b821dc 100644 --- a/ui/src/pages/feature-services/Index.tsx +++ b/ui/src/pages/feature-services/Index.tsx @@ -1,4 +1,5 @@ import React, { useContext } from "react"; +import { useParams } from "react-router-dom"; import { EuiPageTemplate, @@ -30,7 +31,8 @@ import { feast } from "../../protos"; const useLoadFeatureServices = () => { const registryUrl = useContext(RegistryPathContext); - const registryQuery = useLoadRegistry(registryUrl); + const { projectName } = useParams(); + const registryQuery = useLoadRegistry(registryUrl, projectName); const data = registryQuery.data === undefined diff --git a/ui/src/pages/feature-services/useLoadFeatureService.ts b/ui/src/pages/feature-services/useLoadFeatureService.ts index fe21fe2d36b..004ab35b927 100644 --- a/ui/src/pages/feature-services/useLoadFeatureService.ts +++ b/ui/src/pages/feature-services/useLoadFeatureService.ts @@ -1,5 +1,6 @@ import { FEAST_FCO_TYPES } from "../../parsers/types"; import { useContext } from "react"; +import { useParams } from "react-router-dom"; import RegistryPathContext from "../../contexts/RegistryPathContext"; import useLoadRegistry from "../../queries/useLoadRegistry"; @@ -7,7 +8,8 @@ import { EntityReference } from "../../parsers/parseEntityRelationships"; const useLoadFeatureService = (featureServiceName: string) => { const registryUrl = useContext(RegistryPathContext); - const registryQuery = useLoadRegistry(registryUrl); + const { projectName } = useParams(); + const registryQuery = useLoadRegistry(registryUrl, projectName); const data = registryQuery.data === undefined diff --git a/ui/src/pages/feature-views/FeatureViewInstance.tsx b/ui/src/pages/feature-views/FeatureViewInstance.tsx index 4a0cc6a9129..93d0245b9fa 100644 --- a/ui/src/pages/feature-views/FeatureViewInstance.tsx +++ b/ui/src/pages/feature-views/FeatureViewInstance.tsx @@ -14,9 +14,9 @@ import useLoadRegistry from "../../queries/useLoadRegistry"; import RegistryPathContext from "../../contexts/RegistryPathContext"; const FeatureViewInstance = () => { - const { featureViewName } = useParams(); + const { featureViewName, projectName } = useParams(); const registryUrl = React.useContext(RegistryPathContext); - const registryQuery = useLoadRegistry(registryUrl); + const registryQuery = useLoadRegistry(registryUrl, projectName); const fvName = featureViewName === undefined ? "" : featureViewName; diff --git a/ui/src/pages/feature-views/FeatureViewLineageTab.tsx b/ui/src/pages/feature-views/FeatureViewLineageTab.tsx index 39c31e17dbd..8759935e8ea 100644 --- a/ui/src/pages/feature-views/FeatureViewLineageTab.tsx +++ b/ui/src/pages/feature-views/FeatureViewLineageTab.tsx @@ -22,13 +22,13 @@ interface FeatureViewLineageTabProps { const FeatureViewLineageTab = ({ data }: FeatureViewLineageTabProps) => { const registryUrl = useContext(RegistryPathContext); + const { featureViewName, projectName } = useParams(); const { isLoading, isSuccess, isError, data: registryData, - } = useLoadRegistry(registryUrl); - const { featureViewName } = useParams(); + } = useLoadRegistry(registryUrl, projectName); const [selectedPermissionAction, setSelectedPermissionAction] = useState(""); const filterNode = { diff --git a/ui/src/pages/feature-views/FeatureViewListingTable.tsx b/ui/src/pages/feature-views/FeatureViewListingTable.tsx index cf0fc305f84..e865abe6e74 100644 --- a/ui/src/pages/feature-views/FeatureViewListingTable.tsx +++ b/ui/src/pages/feature-views/FeatureViewListingTable.tsx @@ -30,8 +30,10 @@ const FeatureViewListingTable = ({ field: "name", sortable: true, render: (name: string, item: genericFVType) => { + // For "All Projects" view, link to the specific project + const itemProject = item.object?.spec?.project || projectName; return ( - + {name}{" "} {(item.type === "ondemand" && ondemand) || (item.type === "stream" && stream)} @@ -49,6 +51,16 @@ const FeatureViewListingTable = ({ }, ]; + // Add Project column when viewing all projects + if (projectName === "all") { + columns.splice(1, 0, { + name: "Project", + render: (item: genericFVType) => { + return {item.object?.spec?.project || "Unknown"}; + }, + }); + } + // Add columns if they come up in search tagKeysSet.forEach((key) => { columns.push({ diff --git a/ui/src/pages/feature-views/Index.tsx b/ui/src/pages/feature-views/Index.tsx index 57ac597168b..b1c28895370 100644 --- a/ui/src/pages/feature-views/Index.tsx +++ b/ui/src/pages/feature-views/Index.tsx @@ -1,4 +1,5 @@ import React, { useContext } from "react"; +import { useParams } from "react-router-dom"; import { EuiPageTemplate, @@ -29,7 +30,8 @@ import ExportButton from "../../components/ExportButton"; const useLoadFeatureViews = () => { const registryUrl = useContext(RegistryPathContext); - const registryQuery = useLoadRegistry(registryUrl); + const { projectName } = useParams(); + const registryQuery = useLoadRegistry(registryUrl, projectName); const data = registryQuery.data === undefined diff --git a/ui/src/pages/features/FeatureListPage.tsx b/ui/src/pages/features/FeatureListPage.tsx index 72428dde494..36087f98bc0 100644 --- a/ui/src/pages/features/FeatureListPage.tsx +++ b/ui/src/pages/features/FeatureListPage.tsx @@ -33,6 +33,7 @@ interface Feature { name: string; featureView: string; type: string; + project?: string; permissions?: any[]; } @@ -43,7 +44,10 @@ type FeatureColumn = const FeatureListPage = () => { const { projectName } = useParams(); const registryUrl = useContext(RegistryPathContext); - const { data, isLoading, isError } = useLoadRegistry(registryUrl); + const { data, isLoading, isError } = useLoadRegistry( + registryUrl, + projectName, + ); const [searchText, setSearchText] = useState(""); const [selectedPermissionAction, setSelectedPermissionAction] = useState(""); @@ -95,23 +99,31 @@ const FeatureListPage = () => { name: "Feature Name", field: "name", sortable: true, - render: (name: string, feature: Feature) => ( - - {name} - - ), + render: (name: string, feature: Feature) => { + // For "All Projects" view, link to the specific project + const itemProject = feature.project || projectName; + return ( + + {name} + + ); + }, }, { name: "Feature View", field: "featureView", sortable: true, - render: (featureView: string) => ( - - {featureView} - - ), + render: (featureView: string, feature: Feature) => { + // For "All Projects" view, link to the specific project + const itemProject = feature.project || projectName; + return ( + + {featureView} + + ); + }, }, { name: "Type", field: "type", sortable: true }, { @@ -144,6 +156,18 @@ const FeatureListPage = () => { }, ]; + // Add Project column when viewing all projects + if (projectName === "all") { + columns.splice(1, 0, { + name: "Project", + field: "project", + sortable: true, + render: (project: string) => { + return {project || "Unknown"}; + }, + }); + } + const onTableChange = ({ page, sort }: CriteriaWithPagination) => { if (sort) { setSortField(sort.field as keyof Feature); diff --git a/ui/src/pages/lineage/Index.tsx b/ui/src/pages/lineage/Index.tsx index 24112ea8571..a3a9ca19296 100644 --- a/ui/src/pages/lineage/Index.tsx +++ b/ui/src/pages/lineage/Index.tsx @@ -16,8 +16,44 @@ import { useParams } from "react-router-dom"; const LineagePage = () => { useDocumentTitle("Feast Lineage"); const registryUrl = useContext(RegistryPathContext); - const { isLoading, isSuccess, isError, data } = useLoadRegistry(registryUrl); const { projectName } = useParams<{ projectName: string }>(); + const { isLoading, isSuccess, isError, data } = useLoadRegistry( + registryUrl, + projectName, + ); + + // Show message for "All Projects" view + if (projectName === "all") { + return ( + + + +

Lineage Visualization

+
+ + Project Selection Required} + body={ + <> +

+ Lineage visualization requires a specific project context to + show the relationships between Feature Views, Entities, and + Data Sources. +

+

+ + Please select a specific project from the dropdown above + {" "} + to view its lineage graph. +

+ + } + /> +
+
+ ); + } return ( diff --git a/ui/src/pages/permissions/Index.tsx b/ui/src/pages/permissions/Index.tsx index 683d3dcdba0..76dde026e90 100644 --- a/ui/src/pages/permissions/Index.tsx +++ b/ui/src/pages/permissions/Index.tsx @@ -13,6 +13,7 @@ import { EuiFormRow, } from "@elastic/eui"; import { useContext, useState } from "react"; +import { useParams } from "react-router-dom"; import RegistryPathContext from "../../contexts/RegistryPathContext"; import useLoadRegistry from "../../queries/useLoadRegistry"; import PermissionsDisplay from "../../components/PermissionsDisplay"; @@ -20,7 +21,11 @@ import { filterPermissionsByAction } from "../../utils/permissionUtils"; const PermissionsIndex = () => { const registryUrl = useContext(RegistryPathContext); - const { isLoading, isSuccess, isError, data } = useLoadRegistry(registryUrl); + const { projectName } = useParams(); + const { isLoading, isSuccess, isError, data } = useLoadRegistry( + registryUrl, + projectName, + ); const [selectedPermissionAction, setSelectedPermissionAction] = useState(""); return ( diff --git a/ui/src/queries/useLoadRegistry.ts b/ui/src/queries/useLoadRegistry.ts index 4354ec0e98e..e3f5ac87a1d 100644 --- a/ui/src/queries/useLoadRegistry.ts +++ b/ui/src/queries/useLoadRegistry.ts @@ -22,11 +22,12 @@ interface Feature { name: string; featureView: string; type: string; + project?: string; } -const useLoadRegistry = (url: string) => { +const useLoadRegistry = (url: string, projectName?: string) => { return useQuery( - `registry:${url}`, + `registry:${url}:${projectName || "all"}`, () => { return fetch(url, { headers: { @@ -55,6 +56,78 @@ const useLoadRegistry = (url: string) => { objects.featureViews = []; } + // Filter objects by project if projectName is provided + // Skip filtering if projectName is "all" (All Projects view) + // Only filter if we detect that the registry contains multiple projects + if (projectName && projectName !== "all") { + // Check if the registry actually has multiple projects + const projectsInRegistry = new Set(); + objects.featureViews?.forEach((fv: any) => { + if (fv?.spec?.project) projectsInRegistry.add(fv.spec.project); + }); + objects.entities?.forEach((entity: any) => { + if (entity?.spec?.project) + projectsInRegistry.add(entity.spec.project); + }); + + // Only apply filtering if there are actually multiple projects in the registry + // OR if the projectName matches one of the projects in the registry + const shouldFilter = + projectsInRegistry.size > 1 || + projectsInRegistry.has(projectName); + + if (shouldFilter && projectsInRegistry.has(projectName)) { + if (objects.featureViews) { + objects.featureViews = objects.featureViews.filter( + (fv: any) => fv?.spec?.project === projectName, + ); + } + if (objects.entities) { + objects.entities = objects.entities.filter( + (entity: any) => entity?.spec?.project === projectName, + ); + } + if (objects.dataSources) { + objects.dataSources = objects.dataSources.filter( + (ds: any) => ds?.project === projectName, + ); + } + if (objects.featureServices) { + objects.featureServices = objects.featureServices.filter( + (fs: any) => fs?.spec?.project === projectName, + ); + } + if (objects.onDemandFeatureViews) { + objects.onDemandFeatureViews = + objects.onDemandFeatureViews.filter( + (odfv: any) => odfv?.spec?.project === projectName, + ); + } + if (objects.streamFeatureViews) { + objects.streamFeatureViews = objects.streamFeatureViews.filter( + (sfv: any) => sfv?.spec?.project === projectName, + ); + } + if (objects.savedDatasets) { + objects.savedDatasets = objects.savedDatasets.filter( + (sd: any) => sd?.spec?.project === projectName, + ); + } + if (objects.validationReferences) { + objects.validationReferences = + objects.validationReferences.filter( + (vr: any) => vr?.project === projectName, + ); + } + if (objects.permissions) { + objects.permissions = objects.permissions.filter( + (perm: any) => + perm?.spec?.project === projectName || !perm?.spec?.project, + ); + } + } + } + if ( process.env.NODE_ENV === "test" && objects.featureViews.length === 0 @@ -107,32 +180,42 @@ const useLoadRegistry = (url: string) => { feature.valueType != null ? feast.types.ValueType.Enum[feature.valueType] : "Unknown Type", + project: fv?.spec?.project, // Include project from parent feature view })) || [], ) || []; - let projectName = - process.env.NODE_ENV === "test" - ? "credit_scoring_aws" - : objects.projects && - objects.projects.length > 0 && - objects.projects[0].spec && - objects.projects[0].spec.name - ? objects.projects[0].spec.name - : objects.project - ? objects.project - : "credit_scoring_aws"; + // Use the provided projectName parameter if available, otherwise try to determine from registry + let resolvedProjectName: string = + projectName === "all" + ? "All Projects" + : projectName || + (process.env.NODE_ENV === "test" + ? "credit_scoring_aws" + : objects.projects && + objects.projects.length > 0 && + objects.projects[0].spec && + objects.projects[0].spec.name + ? objects.projects[0].spec.name + : objects.project + ? objects.project + : "credit_scoring_aws"); let projectDescription = undefined; - if ( - objects.projects && - objects.projects.length > 0 && - objects.projects[0].spec - ) { - projectDescription = objects.projects[0].spec.description; + + // Find project description from the projects array + if (projectName === "all") { + projectDescription = "View data across all projects"; + } else if (objects.projects && objects.projects.length > 0) { + const currentProject = objects.projects.find( + (p: any) => p?.spec?.name === resolvedProjectName, + ); + if (currentProject?.spec) { + projectDescription = currentProject.spec.description; + } } return { - project: projectName, + project: resolvedProjectName, description: projectDescription, objects, mergedFVMap, diff --git a/ui/src/queries/useLoadRelationshipsData.ts b/ui/src/queries/useLoadRelationshipsData.ts index 6f65af7e764..c0b7f1c1a28 100644 --- a/ui/src/queries/useLoadRelationshipsData.ts +++ b/ui/src/queries/useLoadRelationshipsData.ts @@ -1,10 +1,12 @@ import { useContext } from "react"; +import { useParams } from "react-router-dom"; import RegistryPathContext from "../contexts/RegistryPathContext"; import useLoadRegistry from "./useLoadRegistry"; const useLoadRelationshipData = () => { const registryUrl = useContext(RegistryPathContext); - const registryQuery = useLoadRegistry(registryUrl); + const { projectName } = useParams(); + const registryQuery = useLoadRegistry(registryUrl, projectName); const data = registryQuery.data === undefined